Mirrors/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-09-22 13:01:16 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/async_memcpy_dw_gdma_backend' into 'master'

Browse Source 
feat(esp_driver_dma): add dw_gdma backend for async_memcpy

Closes IDF-15760

See merge request espressif/esp-idf!50878
This commit is contained in:
morris
2026-08-10 16:16:53 +08:00
parent 1f590b9006 eec76a070e
commit bff6bf5e8b
29 changed files with 1144 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
components/esp_driver_cam/csi/src/esp_cam_ctlr_csi.c
View File

@@ -381,16 +381,16 @@ IRAM_ATTR static bool csi_dma_trans_done_callback(dw_gdma_channel_handle_t chan,
csi_dma_transfer_config = (dw_gdma_block_transfer_config_t) {
.src = {
.addr = MIPI_CSI_BRG_MEM_BASE,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.dst = {
.addr = 0,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.size = ctlr->csi_transfer_size,
@@ -603,16 +603,16 @@ esp_err_t s_ctlr_csi_start(esp_cam_ctlr_handle_t handle)
csi_dma_transfer_config = (dw_gdma_block_transfer_config_t) {
.src = {
.addr = MIPI_CSI_BRG_MEM_BASE,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.dst = {
.addr = (uint32_t)(trans.buffer),
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.size = ctlr->csi_transfer_size,

24
components/esp_driver_cam/isp_dvp/src/esp_cam_ctlr_isp_dvp.c
View File

@@ -387,16 +387,16 @@ static esp_err_t s_isp_dvp_start(esp_cam_ctlr_handle_t handle)
dvp_dma_transfer_config = (dw_gdma_block_transfer_config_t) {
.src = {
.addr = MIPI_CSI_BRG_MEM_BASE,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.dst = {
.addr = (uint32_t)(trans.buffer),
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.size = dvp_ctlr->dvp_transfer_size,
@@ -459,16 +459,16 @@ IRAM_ATTR static bool s_dvp_dma_trans_done_callback(dw_gdma_channel_handle_t cha
dvp_dma_transfer_config = (dw_gdma_block_transfer_config_t) {
.src = {
.addr = MIPI_CSI_BRG_MEM_BASE,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.dst = {
.addr = 0,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.size = dvp_ctlr->dvp_transfer_size,

7
components/esp_driver_dma/CMakeLists.txt
View File

@@ -11,7 +11,7 @@ if(CONFIG_SOC_GDMA_SUPPORTED)
list(APPEND srcs "src/gdma.c")
if(CONFIG_SOC_GDMA_SUPPORT_SLEEP_RETENTION AND CONFIG_SOC_PAU_SUPPORTED)
list(APPEND srcs "src/gdma_sleep.c")
list(APPEND srcs "${target}/gdma_retention.c")
list(APPEND srcs "src/${target}/gdma_retention.c")
endif()
if(CONFIG_SOC_GDMA_SUPPORT_ETM)
list(APPEND srcs "src/gdma_etm.c")
@@ -29,6 +29,9 @@ if(CONFIG_SOC_ASYNC_MEMCPY_SUPPORTED)
if(CONFIG_SOC_CP_DMA_SUPPORTED)
list(APPEND srcs "src/async_memcpy_cp_dma.c")
endif() # CONFIG_SOC_CP_DMA_SUPPORTED
if(CONFIG_SOC_DW_GDMA_SUPPORTED)
list(APPEND srcs "src/async_memcpy_dw_gdma.c")
endif() # CONFIG_SOC_DW_GDMA_SUPPORTED
endif() # CONFIG_SOC_ASYNC_MEMCPY_SUPPORTED
if(CONFIG_SOC_DW_GDMA_SUPPORTED)
@@ -38,7 +41,7 @@ endif()
if(CONFIG_SOC_DMA2D_SUPPORTED)
list(APPEND srcs "src/dma2d.c" "src/esp_async_color_convert.c" "src/async_color_convert_dma2d.c")
if(CONFIG_SOC_PAU_SUPPORTED)
list(APPEND srcs "${target}/dma2d_retention.c")
list(APPEND srcs "src/${target}/dma2d_retention.c")
endif()
endif()

40
components/esp_driver_dma/include/esp_async_memcpy.h
View File

@@ -67,7 +67,7 @@ typedef struct {
.flags = 0, \
}
#if SOC_HAS(AHB_GDMA)
#if SOC_AHB_GDMA_SUPPORTED
/**
* @brief Install async memcpy driver, with AHB-GDMA as the backend
*
@@ -80,9 +80,9 @@ typedef struct {
* - ESP_FAIL: Install async memcpy driver failed because of other error
*/
esp_err_t esp_async_memcpy_install_gdma_ahb(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp);
#endif // SOC_HAS(AHB_GDMA)
#endif // SOC_AHB_GDMA_SUPPORTED
#if SOC_HAS(AXI_GDMA)
#if SOC_AXI_GDMA_SUPPORTED
/**
* @brief Install async memcpy driver, with AXI-GDMA as the backend
*
@@ -95,9 +95,9 @@ esp_err_t esp_async_memcpy_install_gdma_ahb(const async_memcpy_config_t *config,
* - ESP_FAIL: Install async memcpy driver failed because of other error
*/
esp_err_t esp_async_memcpy_install_gdma_axi(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp);
#endif // SOC_HAS(AXI_GDMA)
#endif // SOC_AXI_GDMA_SUPPORTED
#if SOC_HAS(LP_AHB_GDMA)
#if SOC_LP_AHB_GDMA_SUPPORTED
/**
* @brief Install async memcpy driver, with LP AHB-GDMA as the backend
*
@@ -110,7 +110,7 @@ esp_err_t esp_async_memcpy_install_gdma_axi(const async_memcpy_config_t *config,
* - ESP_FAIL: Install async memcpy driver failed because of other error
*/
esp_err_t esp_async_memcpy_install_gdma_lp_ahb(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp);
#endif // SOC_HAS(LP_AHB_GDMA)
#endif // SOC_LP_AHB_GDMA_SUPPORTED
#if SOC_CP_DMA_SUPPORTED
/**
@@ -129,6 +129,22 @@ esp_err_t esp_async_memcpy_install_gdma_lp_ahb(const async_memcpy_config_t *conf
esp_err_t esp_async_memcpy_install_cpdma(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp);
#endif // SOC_CP_DMA_SUPPORTED
#if SOC_DW_GDMA_SUPPORTED
/**
* @brief Install async memcpy driver, with DW_GDMA as the backend
*
* @param[in] config Configuration of async memcpy
* @param[out] mcp Returned driver handle
* @return
* - ESP_OK: Install async memcpy driver successfully
* - ESP_ERR_INVALID_ARG: Install async memcpy driver failed because of invalid argument
* - ESP_ERR_NO_MEM: Install async memcpy driver failed because out of memory
* - ESP_FAIL: Install async memcpy driver failed because of other error
*/
esp_err_t esp_async_memcpy_install_dw_gdma(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp);
#endif // SOC_DW_GDMA_SUPPORTED
/** @cond */
/**
* @brief Install async memcpy driver with the default DMA backend
*
@@ -143,12 +159,14 @@ esp_err_t esp_async_memcpy_install_cpdma(const async_memcpy_config_t *config, as
* - ESP_ERR_NO_MEM: Install async memcpy driver failed because out of memory
* - ESP_FAIL: Install async memcpy driver failed because of other error
*/
esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp);
esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp)
__attribute__((deprecated("Select a DMA backend explicitly with esp_async_memcpy_install_* instead")));
/** @endcond */
/**
* @brief Uninstall async memcpy driver
*
* @param[in] mcp Handle of async memcpy driver that returned from `esp_async_memcpy_install`
* @param[in] mcp Handle of async memcpy driver returned by an install function
* @return
* - ESP_OK: Uninstall async memcpy driver successfully
* - ESP_ERR_INVALID_ARG: Uninstall async memcpy driver failed because of invalid argument
@@ -161,7 +179,7 @@ esp_err_t esp_async_memcpy_uninstall(async_memcpy_handle_t mcp);
*
* @note The callback function is invoked in interrupt context, never do blocking jobs in the callback.
*
* @param[in] mcp Handle of async memcpy driver that returned from `esp_async_memcpy_install`
* @param[in] mcp Handle of async memcpy driver returned by an install function
* @param[in] dst Destination address (copy to)
* @param[in] src Source address (copy from)
* @param[in] n Number of bytes to copy
@@ -180,7 +198,7 @@ esp_err_t esp_async_memcpy(async_memcpy_handle_t mcp, void *dst, void *src, size
* @note This function is blocking and should not be called from interrupt context.
* @note Only `timeout_ms=-1` is supported, which means waiting indefinitely.
*
* @param[in] mcp Handle of async memcpy driver that returned from `esp_async_memcpy_install`
* @param[in] mcp Handle of async memcpy driver returned by an install function
* @param[in] dst Destination address (copy to)
* @param[in] src Source address (copy from)
* @param[in] n Number of bytes to copy
@@ -206,7 +224,7 @@ typedef enum {
*
* @note The created ETM event object can be deleted later by calling `esp_etm_del_event`
*
* @param[in] mcp Handle of async memcpy driver that returned from `esp_async_memcpy_install`
* @param[in] mcp Handle of async memcpy driver returned by an install function
* @param[in] event_type ETM event type
* @param[out] out_event Returned ETM event handle
* @return

53
components/esp_driver_dma/include/esp_private/dw_gdma.h
View File

@@ -100,11 +100,46 @@ esp_err_t dw_gdma_channel_get_id(dw_gdma_channel_handle_t chan, int *channel_id)
* @note By dynamic, we mean these channel end configurations can be changed in each transfer.
*/
typedef struct {
uint32_t addr; /*!< Memory address */
dw_gdma_transfer_width_t width; /*!< Transfer width */
dw_gdma_burst_mode_t burst_mode; /*!< Burst mode */
dw_gdma_burst_items_t burst_items; /*!< Number of data items that are contained in one burst transaction */
uint8_t burst_len; /*!< Burst transaction length, if set to 0, the hardware will apply a possible value as burst length */
uint32_t addr; /*!< Memory address */
/**
* @brief Transfer width, i.e. the bit width of a single data item (a "beat").
*
* This is the atomic unit of transfer. It decides how many bytes are moved per data item,
* and therefore the granularity of both `burst_size` / `axi_burst_len` and the address step when
* `addr_inc_mode` is INCREMENT. E.g. with `DW_GDMA_TRANS_WIDTH_32`, one item is 4 bytes.
*/
dw_gdma_transfer_width_t width;
/**
* @brief Address increment mode: whether the transfer address advances after each data item.
*
* @note It controls the address behaviour item by item:
* - `DW_GDMA_ADDR_INC_MODE_INCREMENT`: address is increased by `width` after each item
* (normal reading/writing of a contiguous buffer).
* - `DW_GDMA_ADDR_INC_MODE_FIXED` : address stays the same after each item
* (used when the target is a FIFO / peripheral register, e.g. DSI/CSI/ISP).
*/
dw_gdma_addr_inc_mode_t addr_inc_mode;
/**
* @brief Burst size: number of data items contained in one (core-level) burst transaction.
*
* This is the "burst size" (the DW_GDMA `MSIZE` field): how many `width`-sized items the DMA
* engine requests in a single burst. It sets the granularity of the transfer and is typically a
* power of two (1, 4, 8, ...). The total bytes of one such burst is `burst_size * width`.
*
* @note To tell it apart from `axi_burst_len`: `burst_size` is the coarse, power-of-two size of
* a burst as configured by the DMA core, while `axi_burst_len` is the fine-grained AXI
* burst length. For most cases keep them consistent (e.g. both 4/8/16), but you may set
* `axi_burst_len` to a value that is not a power of two and let `burst_size` stay coarse.
*/
dw_gdma_burst_size_t burst_size;
/**
* @brief AXI burst transaction length, in number of data items (beats).
*
* This directly programs the AXI `arlen`/`awlen` fields, i.e. how many beats the AXI master
* puts into a single AXI burst. The total bytes transferred in one burst is `axi_burst_len * width`.
* If set to 0, the hardware chooses a possible value for the burst length.
*/
uint8_t axi_burst_len;
struct {
uint32_t en_status_write_back: 1; /*!< Enable peripheral status write back */
} flags;
@@ -116,7 +151,13 @@ typedef struct {
typedef struct {
dw_gdma_channel_dynamic_config_t src; /*!< source configuration */
dw_gdma_channel_dynamic_config_t dst; /*!< destination configuration */
size_t size; /*!< Transfer size */
/**
* @brief Transfer size, in number of data items
*
* The size in bytes is `size * width` (each data item is `width` bytes). For a
* Link-List transfer, this is the block transfer size of the corresponding LLI.
*/
size_t size;
} dw_gdma_block_transfer_config_t;
/**

447
components/esp_driver_dma/src/async_memcpy_dw_gdma.c Normal file
View File

@@ -0,0 +1,447 @@
/*
* SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <string.h>
#include <stdlib.h>
#include <stdatomic.h>
#include <sys/queue.h>
#include <sys/param.h>
#include "sdkconfig.h"
#include "freertos/FreeRTOS.h"
#include "soc/soc_caps.h"
#include "esp_log.h"
#include "esp_check.h"
#include "esp_attr.h"
#include "esp_err.h"
#include "esp_private/dw_gdma.h"
#include "esp_private/esp_dma_utils.h"
#include "esp_private/critical_section.h"
#include "esp_memory_utils.h"
#include "esp_cache.h"
#include "esp_async_memcpy.h"
#include "esp_async_memcpy_priv.h"
#include "hal/cache_hal.h"
#include "hal/cache_ll.h"
ESP_LOG_ATTR_TAG(TAG, "async_mcp.dw_gdma");
/// @brief Maximum number of data items that one DW_GDMA block (link list item) can transfer.
/// @note The DW_GDMA block_ts register field is 22 bits wide and stores (size - 1).
#define MCP_DW_GDMA_MAX_BLOCK_ITEMS (1 << 22)
/// @brief Maximum body transfer width (in bits), capped by the AXI data width.
#define MCP_DW_GDMA_MAX_BODY_WIDTH_BITS 64
/// @brief Transaction object for async memcpy
typedef struct async_memcpy_transaction_t {
dw_gdma_link_list_handle_t link_list; // DW_GDMA link list for this transaction (body only)
async_memcpy_isr_cb_t cb; // user callback
void *cb_args; // user callback args
async_memcpy_split_t split; // cache aligned split, consumed by the deferred cache ops and CPU copy
STAILQ_ENTRY(async_memcpy_transaction_t) idle_queue_entry; // Entry for the idle queue
STAILQ_ENTRY(async_memcpy_transaction_t) ready_queue_entry; // Entry for the ready queue
} async_memcpy_transaction_t;
/// @brief Context of async memcpy driver
/// @note - It saves two queues, one for idle transaction objects, one for ready transaction objects
/// @note - Number of transaction objects are determined by the backlog parameter
typedef struct {
async_memcpy_context_t parent; // Parent IO interface
size_t dma_burst_size; // DMA burst size (in bytes), configured by the user
dw_gdma_channel_handle_t channel; // DW_GDMA channel handle
portMUX_TYPE spin_lock; // spin lock to avoid threads and isr from accessing the same resource simultaneously
_Atomic async_memcpy_fsm_t fsm; // driver state machine, changing state should be atomic
size_t num_trans_objs; // number of transaction objects
async_memcpy_transaction_t *transaction_pool; // transaction object pool
async_memcpy_transaction_t *current_transaction; // current transaction object
STAILQ_HEAD(, async_memcpy_transaction_t) idle_queue_head; // Head of the idle queue
STAILQ_HEAD(, async_memcpy_transaction_t) ready_queue_head; // Head of the ready queue
} async_memcpy_dw_gdma_context_t;
static bool mcp_dw_gdma_full_trans_done_callback(dw_gdma_channel_handle_t dma_chan, const dw_gdma_trans_done_event_data_t *event_data, void *user_data);
static bool mcp_dw_gdma_invalid_block_callback(dw_gdma_channel_handle_t dma_chan, const dw_gdma_break_event_data_t *event_data, void *user_data);
static esp_err_t mcp_dw_gdma_del(async_memcpy_context_t *ctx);
static esp_err_t mcp_dw_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args);
static esp_err_t mcp_dw_gdma_destroy(async_memcpy_dw_gdma_context_t *mcp_dw_gdma)
{
// clean up transaction pool
if (mcp_dw_gdma->transaction_pool) {
for (size_t i = 0; i < mcp_dw_gdma->num_trans_objs; i++) {
async_memcpy_transaction_t *trans = &mcp_dw_gdma->transaction_pool[i];
if (trans->link_list) {
dw_gdma_del_link_list(trans->link_list);
}
}
free(mcp_dw_gdma->transaction_pool);
}
// clean up DW_GDMA channel
if (mcp_dw_gdma->channel) {
dw_gdma_del_channel(mcp_dw_gdma->channel);
}
free(mcp_dw_gdma);
return ESP_OK;
}
/// @brief Pick the largest supported transfer width (in bits) that the given address is aligned to.
/// @note The width is also capped by the cache line size (the destination body is cache aligned) and the AXI data width.
/// @note The width must also divide the body length, so that the (cache line aligned) body can be evenly
/// split into transfer width sized data items without a remainder.
static dw_gdma_transfer_width_t mcp_dw_gdma_get_body_width(size_t cache_line_size, uintptr_t body_addr, size_t body_len)
{
// the body width is capped by the cache line size (the destination body is cache aligned) and the AXI data width
size_t max_width_bits = cache_line_size ? MIN(cache_line_size * 8, MCP_DW_GDMA_MAX_BODY_WIDTH_BITS) : MCP_DW_GDMA_MAX_BODY_WIDTH_BITS;
size_t width_bits = 8;
for (size_t cand = max_width_bits; cand >= 8; cand >>= 1) {
if (((body_addr & ((cand >> 3) - 1)) == 0) && (body_len % (cand >> 3) == 0)) {
width_bits = cand;
break;
}
}
switch (width_bits) {
case 64:
return DW_GDMA_TRANS_WIDTH_64;
case 32:
return DW_GDMA_TRANS_WIDTH_32;
case 16:
return DW_GDMA_TRANS_WIDTH_16;
default:
return DW_GDMA_TRANS_WIDTH_8;
}
}
/// @brief Map a DMA burst size (in bytes) to a supported DW_GDMA burst size enum value.
static dw_gdma_burst_size_t mcp_dw_gdma_get_burst_size(size_t burst_size_bytes, size_t width_bytes)
{
static const dw_gdma_burst_size_t supported[] = {
DW_GDMA_BURST_SIZE_1, DW_GDMA_BURST_SIZE_4, DW_GDMA_BURST_SIZE_8, DW_GDMA_BURST_SIZE_16,
DW_GDMA_BURST_SIZE_32, DW_GDMA_BURST_SIZE_64, DW_GDMA_BURST_SIZE_128, DW_GDMA_BURST_SIZE_256,
DW_GDMA_BURST_SIZE_512, DW_GDMA_BURST_SIZE_1024,
};
static const size_t supported_count[] = {1, 4, 8, 16, 32, 64, 128, 256, 512, 1024};
size_t items = (burst_size_bytes && width_bytes) ? burst_size_bytes / width_bytes : 0;
dw_gdma_burst_size_t pick = DW_GDMA_BURST_SIZE_4;
for (int i = sizeof(supported) / sizeof(supported[0]) - 1; i >= 0; i--) {
if (supported_count[i] <= items) {
pick = supported[i];
break;
}
}
return pick;
}
esp_err_t esp_async_memcpy_install_dw_gdma(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp)
{
esp_err_t ret = ESP_OK;
async_memcpy_dw_gdma_context_t *mcp_dw_gdma = NULL;
ESP_RETURN_ON_FALSE(config && mcp, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
// allocate memory of driver context from internal memory (because it contains atomic variable)
mcp_dw_gdma = heap_caps_calloc(1, sizeof(async_memcpy_dw_gdma_context_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
ESP_GOTO_ON_FALSE(mcp_dw_gdma, ESP_ERR_NO_MEM, err, TAG, "no mem for driver context");
uint32_t trans_queue_len = config->backlog ? config->backlog : DEFAULT_TRANSACTION_QUEUE_LENGTH;
// allocate memory for transaction pool from internal memory
mcp_dw_gdma->transaction_pool = heap_caps_calloc(trans_queue_len, sizeof(async_memcpy_transaction_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
ESP_GOTO_ON_FALSE(mcp_dw_gdma->transaction_pool, ESP_ERR_NO_MEM, err, TAG, "no mem for transaction pool");
// create a single M2M DW_GDMA channel
dw_gdma_channel_static_config_t static_config = {
.block_transfer_type = DW_GDMA_BLOCK_TRANSFER_LIST,
.role = DW_GDMA_ROLE_MEM,
.num_outstanding_requests = 1,
};
dw_gdma_channel_alloc_config_t alloc_config = {
.src = static_config,
.dst = static_config,
.flow_controller = DW_GDMA_FLOW_CTRL_SELF, // DMA as the flow controller
// Fix the channel priority to a constant. The `async_memcpy_config_t.weight` field is intentionally
// ignored here: DW_GDMA has no weight/bandwidth control capability, and `chan_priority`(arbitration
// priority) is a different concept from the GDMA weight, so it can't be mapped 1:1.
.chan_priority = 1,
};
ESP_GOTO_ON_ERROR(dw_gdma_new_channel(&alloc_config, &mcp_dw_gdma->channel), err, TAG, "failed to alloc DW_GDMA channel");
// register event callbacks
dw_gdma_event_callbacks_t cbs = {
.on_full_trans_done = mcp_dw_gdma_full_trans_done_callback,
.on_invalid_block = mcp_dw_gdma_invalid_block_callback,
};
ESP_GOTO_ON_ERROR(dw_gdma_channel_register_event_callbacks(mcp_dw_gdma->channel, &cbs, mcp_dw_gdma),
err, TAG, "failed to register event callbacks");
// initialize transaction queue
STAILQ_INIT(&mcp_dw_gdma->idle_queue_head);
STAILQ_INIT(&mcp_dw_gdma->ready_queue_head);
// pick transactions from the pool and insert to the idle queue
for (int i = 0; i < trans_queue_len; i++) {
STAILQ_INSERT_TAIL(&mcp_dw_gdma->idle_queue_head, &mcp_dw_gdma->transaction_pool[i], idle_queue_entry);
}
// initialize other members
portMUX_INITIALIZE(&mcp_dw_gdma->spin_lock);
atomic_init(&mcp_dw_gdma->fsm, MCP_FSM_IDLE);
mcp_dw_gdma->num_trans_objs = trans_queue_len;
mcp_dw_gdma->dma_burst_size = config->dma_burst_size;
mcp_dw_gdma->parent.del = mcp_dw_gdma_del;
mcp_dw_gdma->parent.memcpy = mcp_dw_gdma_memcpy;
// ETM is not supported by the DW_GDMA backend, leave new_etm_event as NULL
// return base object
*mcp = &mcp_dw_gdma->parent;
return ESP_OK;
err:
if (mcp_dw_gdma) {
mcp_dw_gdma_destroy(mcp_dw_gdma);
}
return ret;
}
static esp_err_t mcp_dw_gdma_del(async_memcpy_context_t *ctx)
{
async_memcpy_dw_gdma_context_t *mcp_dw_gdma = __containerof(ctx, async_memcpy_dw_gdma_context_t, parent);
// check if there are pending transactions
ESP_RETURN_ON_FALSE(STAILQ_EMPTY(&mcp_dw_gdma->ready_queue_head), ESP_ERR_INVALID_STATE, TAG, "there are pending transactions");
// check if the driver is in IDLE state
ESP_RETURN_ON_FALSE(atomic_load(&mcp_dw_gdma->fsm) == MCP_FSM_IDLE, ESP_ERR_INVALID_STATE, TAG, "driver is not in IDLE state");
return mcp_dw_gdma_destroy(mcp_dw_gdma);
}
/// @brief help function to get one transaction from the ready queue
/// @note this function is allowed to be called in ISR
static async_memcpy_transaction_t *try_pop_trans_from_ready_queue(async_memcpy_dw_gdma_context_t *mcp_dw_gdma)
{
async_memcpy_transaction_t *trans = NULL;
esp_os_enter_critical_safe(&mcp_dw_gdma->spin_lock);
trans = STAILQ_FIRST(&mcp_dw_gdma->ready_queue_head);
if (trans) {
STAILQ_REMOVE_HEAD(&mcp_dw_gdma->ready_queue_head, ready_queue_entry);
}
esp_os_exit_critical_safe(&mcp_dw_gdma->spin_lock);
return trans;
}
/// @brief help function to start a pending transaction
/// @note this function is allowed to be called in ISR
static void try_start_pending_transaction(async_memcpy_dw_gdma_context_t *mcp_dw_gdma)
{
async_memcpy_fsm_t expected_fsm = MCP_FSM_IDLE;
async_memcpy_transaction_t *trans = NULL;
if (atomic_compare_exchange_strong(&mcp_dw_gdma->fsm, &expected_fsm, MCP_FSM_WAIT)) {
trans = try_pop_trans_from_ready_queue(mcp_dw_gdma);
if (trans) {
atomic_store(&mcp_dw_gdma->fsm, MCP_FSM_RUN);
mcp_dw_gdma->current_transaction = trans;
// Deferred during submit to avoid racing with the previous (still running) DMA transfer that
// may write to the same/overlapping destination. Now the transfer is actually starting, so the
// CPU copies and cache maintenance are serialized with the DMA engine.
async_memcpy_do_cache_ops_and_cpu_copy(&trans->split);
// apply the link list and kick off the DMA engine
dw_gdma_channel_use_link_list(mcp_dw_gdma->channel, trans->link_list);
dw_gdma_channel_enable_ctrl(mcp_dw_gdma->channel, true);
} else {
atomic_store(&mcp_dw_gdma->fsm, MCP_FSM_IDLE);
}
}
}
/// @brief help function to get one transaction from the idle queue
/// @note this function is allowed to be called in ISR
static async_memcpy_transaction_t *try_pop_trans_from_idle_queue(async_memcpy_dw_gdma_context_t *mcp_dw_gdma)
{
async_memcpy_transaction_t *trans = NULL;
esp_os_enter_critical_safe(&mcp_dw_gdma->spin_lock);
trans = STAILQ_FIRST(&mcp_dw_gdma->idle_queue_head);
if (trans) {
STAILQ_REMOVE_HEAD(&mcp_dw_gdma->idle_queue_head, idle_queue_entry);
}
esp_os_exit_critical_safe(&mcp_dw_gdma->spin_lock);
return trans;
}
static esp_err_t mcp_dw_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args)
{
esp_err_t ret = ESP_OK;
async_memcpy_dw_gdma_context_t *mcp_dw_gdma = __containerof(ctx, async_memcpy_dw_gdma_context_t, parent);
async_memcpy_transaction_t *trans = NULL;
// pick one transaction node from idle queue
trans = try_pop_trans_from_idle_queue(mcp_dw_gdma);
// check if we get the transaction object successfully
ESP_RETURN_ON_FALSE(trans, ESP_ERR_INVALID_STATE, TAG, "no free node in the idle queue");
// clean up the transaction configuration comes from the last one
if (trans->link_list) {
dw_gdma_del_link_list(trans->link_list);
trans->link_list = NULL;
}
// Split the destination buffer into a cache aligned body and (optional) head/tail.
// The DMA engine only handles the cache aligned body; the unaligned head and tail (if any) are
// copied by the CPU later, right before the DMA transfer starts (see try_start_pending_transaction).
// The head/tail and the body never share a cache line, so the CPU and DMA can work concurrently
// without cache coherency conflicts.
async_memcpy_split_t split;
ESP_GOTO_ON_ERROR(async_memcpy_split_cache_aligned(dst, src, n, &split), err, TAG, "failed to split buffer");
size_t body_len = split.body_len;
void *body_src = split.body_src;
void *body_dst = split.body_dst;
size_t cache_line_size = split.cache_line_size;
// pick the body transfer width based on the source body address alignment
// note: the destination body is already cache aligned (so it's aligned to any width <= cache line),
// but the source body address may not be, which limits the usable transfer width.
dw_gdma_transfer_width_t body_width = mcp_dw_gdma_get_body_width(cache_line_size, (uintptr_t)body_src, body_len);
size_t body_width_bytes = 1;
switch (body_width) {
case DW_GDMA_TRANS_WIDTH_64:
body_width_bytes = 8;
break;
case DW_GDMA_TRANS_WIDTH_32:
body_width_bytes = 4;
break;
case DW_GDMA_TRANS_WIDTH_16:
body_width_bytes = 2;
break;
default:
body_width_bytes = 1;
body_width = DW_GDMA_TRANS_WIDTH_8;
break;
}
// the body might need to be split into multiple link list items if it exceeds the maximum block size
size_t max_body_bytes_per_item = MCP_DW_GDMA_MAX_BLOCK_ITEMS * body_width_bytes;
size_t body_alignment = cache_line_size ? cache_line_size : body_width_bytes;
size_t body_num_items = esp_dma_calculate_node_count(body_len, body_alignment, max_body_bytes_per_item);
// align down the max body bytes per item to the body alignment, so each chunk stays aligned
if (cache_line_size) {
max_body_bytes_per_item &= ~(cache_line_size - 1);
}
dw_gdma_link_list_config_t link_list_cfg = {
.num_items = body_num_items,
.link_type = DW_GDMA_LINKED_LIST_TYPE_SINGLY,
};
ESP_GOTO_ON_ERROR(dw_gdma_new_link_list(&link_list_cfg, &trans->link_list), err, TAG, "failed to create link list");
// the burst configuration is derived from the user configured dma_burst_size
dw_gdma_burst_size_t body_burst_size = mcp_dw_gdma_get_burst_size(mcp_dw_gdma->dma_burst_size, body_width_bytes);
// configure each body link list item
uint8_t *cur_src = body_src;
uint8_t *cur_dst = body_dst;
size_t remaining = body_len;
for (size_t i = 0; i < body_num_items; i++) {
size_t chunk = MIN(remaining, max_body_bytes_per_item);
dw_gdma_block_transfer_config_t transfer_config = {
.src = {
.addr = (uint32_t)cur_src,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = body_width,
.burst_size = body_burst_size,
.axi_burst_len = 0,
},
.dst = {
.addr = (uint32_t)cur_dst,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = body_width,
.burst_size = body_burst_size,
.axi_burst_len = 0,
},
.size = chunk / body_width_bytes,
};
dw_gdma_lli_handle_t lli = dw_gdma_link_list_get_item(trans->link_list, i);
ESP_GOTO_ON_FALSE(lli, ESP_ERR_INVALID_STATE, err, TAG, "failed to get link list item");
ESP_GOTO_ON_ERROR(dw_gdma_lli_config_transfer(lli, &transfer_config), err, TAG, "failed to config link list item");
cur_src += chunk;
cur_dst += chunk;
remaining -= chunk;
}
// set block markers: all items are valid, the last item marks the end of the transfer.
// the singly link list already chains the items in order (and terminates at the last item),
// so we only need to mark the last item with is_last and enable the transfer done interrupt on it.
for (size_t i = 0; i < body_num_items; i++) {
dw_gdma_lli_handle_t lli = dw_gdma_link_list_get_item(trans->link_list, i);
dw_gdma_block_markers_t markers = {
.is_valid = true,
.is_last = (i == body_num_items - 1),
.en_trans_done_intr = (i == body_num_items - 1),
};
ESP_GOTO_ON_ERROR(dw_gdma_lli_set_block_markers(lli, markers), err, TAG, "failed to set block markers");
}
// save other transaction context
trans->cb = cb_isr;
trans->cb_args = cb_args;
// save the split info (includes the original buffers and cache aligned body) for the deferred
// cache ops and CPU copy (performed in try_start_pending_transaction when the transfer actually starts)
trans->split = split;
esp_os_enter_critical(&mcp_dw_gdma->spin_lock);
// insert the trans to ready queue
STAILQ_INSERT_TAIL(&mcp_dw_gdma->ready_queue_head, trans, ready_queue_entry);
esp_os_exit_critical(&mcp_dw_gdma->spin_lock);
// check driver state, if there's no running transaction, start a new one
try_start_pending_transaction(mcp_dw_gdma);
return ESP_OK;
err:
if (trans) {
// return back the trans to idle queue
esp_os_enter_critical(&mcp_dw_gdma->spin_lock);
STAILQ_INSERT_TAIL(&mcp_dw_gdma->idle_queue_head, trans, idle_queue_entry);
esp_os_exit_critical(&mcp_dw_gdma->spin_lock);
}
return ret;
}
static bool mcp_dw_gdma_full_trans_done_callback(dw_gdma_channel_handle_t dma_chan, const dw_gdma_trans_done_event_data_t *event_data, void *user_data)
{
bool need_yield = false;
async_memcpy_dw_gdma_context_t *mcp_dw_gdma = (async_memcpy_dw_gdma_context_t *)user_data;
async_memcpy_transaction_t *trans = mcp_dw_gdma->current_transaction;
// switch driver state from RUN to IDLE
async_memcpy_fsm_t expected_fsm = MCP_FSM_RUN;
if (atomic_compare_exchange_strong(&mcp_dw_gdma->fsm, &expected_fsm, MCP_FSM_WAIT)) {
// the head/tail were already copied by the CPU before the transfer started, and the body was written by the
// DMA to memory (the body cache lines were invalidated before the transfer), so nothing to merge here.
// invoked callback registered by user
async_memcpy_isr_cb_t cb = trans->cb;
if (cb) {
async_memcpy_event_t e = {
// No event data for now
};
need_yield = cb(&mcp_dw_gdma->parent, &e, trans->cb_args);
}
trans->cb = NULL;
esp_os_enter_critical_isr(&mcp_dw_gdma->spin_lock);
// insert the trans object to the idle queue
STAILQ_INSERT_TAIL(&mcp_dw_gdma->idle_queue_head, trans, idle_queue_entry);
mcp_dw_gdma->current_transaction = NULL;
esp_os_exit_critical_isr(&mcp_dw_gdma->spin_lock);
atomic_store(&mcp_dw_gdma->fsm, MCP_FSM_IDLE);
}
// try start the next pending transaction
try_start_pending_transaction(mcp_dw_gdma);
return need_yield;
}
static bool mcp_dw_gdma_invalid_block_callback(dw_gdma_channel_handle_t dma_chan, const dw_gdma_break_event_data_t *event_data, void *user_data)
{
// A valid transfer should never hit an invalid block. Terminate instead of leaving callers blocked.
ESP_EARLY_LOGE(TAG, "invalid block detected, terminating");
abort();
}

86
components/esp_driver_dma/src/async_memcpy_gdma.c
View File

@@ -35,10 +35,9 @@ ESP_LOG_ATTR_TAG(TAG, "async_mcp.gdma");
typedef struct async_memcpy_transaction_t {
gdma_link_list_handle_t tx_link_list; // DMA link list for TX direction
gdma_link_list_handle_t rx_link_list; // DMA link list for RX direction
dma_buffer_split_array_t rx_buf_array; // Split the destination buffer into cache aligned ones, save the splits in this array
uint8_t* stash_buffer; // Stash buffer for cache aligned buffer
async_memcpy_isr_cb_t cb; // user callback
void *cb_args; // user callback args
async_memcpy_split_t split; // cache aligned split, consumed by the deferred cache ops and CPU copy
STAILQ_ENTRY(async_memcpy_transaction_t) idle_queue_entry; // Entry for the idle queue
STAILQ_ENTRY(async_memcpy_transaction_t) ready_queue_entry; // Entry for the ready queue
} async_memcpy_transaction_t;
@@ -83,9 +82,6 @@ static esp_err_t mcp_gdma_destroy(async_memcpy_gdma_context_t *mcp_gdma)
if (trans->rx_link_list) {
gdma_del_link_list(trans->rx_link_list);
}
if (trans->stash_buffer) {
free(trans->stash_buffer);
}
}
free(mcp_gdma->transaction_pool);
}
@@ -259,6 +255,10 @@ static void try_start_pending_transaction(async_memcpy_gdma_context_t *mcp_gdma)
if (trans) {
atomic_store(&mcp_gdma->fsm, MCP_FSM_RUN);
mcp_gdma->current_transaction = trans;
// Deferred during submit to avoid racing with the previous (still running) DMA transfer that
// may write to the same/overlapping destination. Now the transfer is actually starting, so the
// CPU copies and cache maintenance are serialized with the DMA engine.
async_memcpy_do_cache_ops_and_cpu_copy(&trans->split);
gdma_start(mcp_gdma->rx_channel, gdma_link_get_head_addr(trans->rx_link_list));
gdma_start(mcp_gdma->tx_channel, gdma_link_get_head_addr(trans->tx_link_list));
} else {
@@ -353,17 +353,23 @@ static esp_err_t mcp_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *s
gdma_del_link_list(trans->rx_link_list);
trans->rx_link_list = NULL;
}
if (trans->stash_buffer) {
free(trans->stash_buffer);
trans->stash_buffer = NULL;
}
// Split the destination buffer into a cache aligned body and (optional) head/tail.
// The DMA engine only handles the cache aligned body; the unaligned head and tail (if any) are
// copied by the CPU later, right before the DMA transfer starts (see try_start_pending_transaction).
// The head/tail and the body never share a cache line, so the CPU and DMA can work concurrently
// without cache coherency conflicts.
// The source buffer is split at the same boundaries because the GDMA TX/RX
// channels are paired and each descriptor carries both the source and destination.
async_memcpy_split_t split;
ESP_GOTO_ON_ERROR(async_memcpy_split_cache_aligned(dst, src, n, &split), err, TAG, "failed to split buffer");
size_t buffer_alignment = 0;
size_t num_dma_nodes = 0;
// allocate gdma TX link
buffer_alignment = esp_ptr_internal(src) ? mcp_gdma->tx_int_mem_alignment : mcp_gdma->tx_ext_mem_alignment;
num_dma_nodes = esp_dma_calculate_node_count(n, buffer_alignment, MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE);
// allocate gdma TX link, only the body is handled by the DMA
buffer_alignment = esp_ptr_internal(split.body_src) ? mcp_gdma->tx_int_mem_alignment : mcp_gdma->tx_ext_mem_alignment;
num_dma_nodes = esp_dma_calculate_node_count(split.body_len, buffer_alignment, MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE);
gdma_link_list_config_t tx_link_cfg = {
.item_alignment = dma_link_item_alignment,
.num_items = num_dma_nodes,
@@ -373,12 +379,12 @@ static esp_err_t mcp_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *s
},
};
ESP_GOTO_ON_ERROR(gdma_new_link_list(&tx_link_cfg, &trans->tx_link_list), err, TAG, "failed to create TX link list");
// mount the source buffer to the TX link list
// mount the source body to the TX link list
gdma_buffer_mount_config_t tx_buf_mount_config[1] = {
[0] = {
.buffer = src,
.buffer = split.body_src,
.buffer_alignment = buffer_alignment,
.length = n,
.length = split.body_len,
.flags = {
.mark_eof = true, // mark the last item as EOF, so the RX channel can also received an EOF list item
.mark_final = GDMA_FINAL_LINK_TO_NULL, // using singly list, so terminate the link here
@@ -387,41 +393,38 @@ static esp_err_t mcp_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *s
};
gdma_link_mount_buffers(trans->tx_link_list, 0, tx_buf_mount_config, 1, NULL);
// read the cache line size of internal and external memory, we use this information to check if a given memory is behind the cache
// write back the source data if it's behind the cache
size_t cache_line_size = esp_cache_get_line_size_by_addr(src);
if (cache_line_size > 0) {
esp_cache_msync(src, n, ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_UNALIGNED);
}
// allocate gdma RX link
buffer_alignment = esp_ptr_internal(dst) ? mcp_gdma->rx_int_mem_alignment : mcp_gdma->rx_ext_mem_alignment;
num_dma_nodes = esp_dma_calculate_node_count(n, buffer_alignment, MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE);
// allocate gdma RX link, only the body is handled by the DMA
buffer_alignment = esp_ptr_internal(split.body_dst) ? mcp_gdma->rx_int_mem_alignment : mcp_gdma->rx_ext_mem_alignment;
num_dma_nodes = esp_dma_calculate_node_count(split.body_len, buffer_alignment, MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE);
gdma_link_list_config_t rx_link_cfg = {
.item_alignment = dma_link_item_alignment,
.num_items = num_dma_nodes + 3, // add 3 extra items for the cache aligned buffers
.num_items = num_dma_nodes,
.flags = {
.check_owner = true,
.items_in_ext_mem = false, // TODO: if the memcopy size is too large, we may need to allocate the link list items from external memory
.items_in_ext_mem = false, // TODO: if the memcopy size is too large, we can consider allocating the link list items from external memory
},
};
ESP_GOTO_ON_ERROR(gdma_new_link_list(&rx_link_cfg, &trans->rx_link_list), err, TAG, "failed to create RX link list");
// if the destination buffer address is not cache line aligned, we need to split the buffer into cache line aligned ones
ESP_GOTO_ON_ERROR(esp_dma_split_rx_buffer_to_cache_aligned(dst, n, &trans->rx_buf_array, &trans->stash_buffer),
err, TAG, "failed to split RX buffer into aligned ones");
// mount the destination buffer to the RX link list
gdma_buffer_mount_config_t rx_buf_mount_config[3] = {0};
for (int i = 0; i < 3; i++) {
rx_buf_mount_config[i].buffer = trans->rx_buf_array.aligned_buffer[i].aligned_buffer;
rx_buf_mount_config[i].buffer_alignment = buffer_alignment;
rx_buf_mount_config[i].length = trans->rx_buf_array.aligned_buffer[i].length;
}
gdma_link_mount_buffers(trans->rx_link_list, 0, rx_buf_mount_config, 3, NULL);
// mount the destination body to the RX link list
gdma_buffer_mount_config_t rx_buf_mount_config[1] = {
[0] = {
.buffer = split.body_dst,
.buffer_alignment = buffer_alignment,
.length = split.body_len,
.flags = {
.mark_eof = true,
.mark_final = GDMA_FINAL_LINK_TO_NULL,
}
}
};
gdma_link_mount_buffers(trans->rx_link_list, 0, rx_buf_mount_config, 1, NULL);
// save other transaction context
trans->cb = cb_isr;
trans->cb_args = cb_args;
// save the split info (includes the original buffers and cache aligned body) for the deferred
// cache ops and CPU copy (performed in try_start_pending_transaction when the transfer actually starts)
trans->split = split;
esp_os_enter_critical(&mcp_gdma->spin_lock);
// insert the trans to ready queue
@@ -448,13 +451,12 @@ static bool mcp_gdma_rx_eof_callback(gdma_channel_handle_t dma_chan, gdma_event_
bool need_yield = false;
async_memcpy_gdma_context_t *mcp_gdma = (async_memcpy_gdma_context_t *)user_data;
async_memcpy_transaction_t *trans = mcp_gdma->current_transaction;
dma_buffer_split_array_t *rx_buf_array = &trans->rx_buf_array;
// switch driver state from RUN to IDLE
async_memcpy_fsm_t expected_fsm = MCP_FSM_RUN;
if (atomic_compare_exchange_strong(&mcp_gdma->fsm, &expected_fsm, MCP_FSM_WAIT)) {
// merge the cache aligned buffers to the original buffer
esp_dma_merge_aligned_rx_buffers(rx_buf_array);
// the head/tail were already copied by the CPU before the transfer started, and the body was written by the
// DMA to memory (the body cache lines were invalidated before the transfer), so nothing to merge here.
// invoked callback registered by user
async_memcpy_isr_cb_t cb = trans->cb;

36
components/esp_driver_dma/src/dw_gdma.c
View File

@@ -478,16 +478,16 @@ esp_err_t dw_gdma_channel_config_transfer(dw_gdma_channel_handle_t chan, const d
// transfer width
dw_gdma_ll_channel_set_src_trans_width(hal->dev, chan_id, config->src.width);
dw_gdma_ll_channel_set_dst_trans_width(hal->dev, chan_id, config->dst.width);
// set burst items
dw_gdma_ll_channel_set_src_burst_items(hal->dev, chan_id, config->src.burst_items);
dw_gdma_ll_channel_set_dst_burst_items(hal->dev, chan_id, config->dst.burst_items);
// set burst mode
dw_gdma_ll_channel_set_src_burst_mode(hal->dev, chan_id, config->src.burst_mode);
dw_gdma_ll_channel_set_dst_burst_mode(hal->dev, chan_id, config->dst.burst_mode);
// set burst size
dw_gdma_ll_channel_set_src_burst_size(hal->dev, chan_id, config->src.burst_size);
dw_gdma_ll_channel_set_dst_burst_size(hal->dev, chan_id, config->dst.burst_size);
// set address increment mode
dw_gdma_ll_channel_set_src_addr_inc_mode(hal->dev, chan_id, config->src.addr_inc_mode);
dw_gdma_ll_channel_set_dst_addr_inc_mode(hal->dev, chan_id, config->dst.addr_inc_mode);
// [Ctrl1] register
// set burst length
dw_gdma_ll_channel_set_src_burst_len(hal->dev, chan_id, config->src.burst_len);
dw_gdma_ll_channel_set_dst_burst_len(hal->dev, chan_id, config->dst.burst_len);
// set AXI burst length
dw_gdma_ll_channel_set_src_axi_burst_len(hal->dev, chan_id, config->src.axi_burst_len);
dw_gdma_ll_channel_set_dst_axi_burst_len(hal->dev, chan_id, config->dst.axi_burst_len);
// whether to enable the peripheral status write back
dw_gdma_ll_channel_enable_src_periph_status_write_back(hal->dev, chan_id, config->src.flags.en_status_write_back);
dw_gdma_ll_channel_enable_dst_periph_status_write_back(hal->dev, chan_id, config->dst.flags.en_status_write_back);
@@ -527,16 +527,16 @@ esp_err_t dw_gdma_lli_config_transfer(dw_gdma_lli_handle_t lli, const dw_gdma_bl
// transfer width
dw_gdma_ll_lli_set_src_trans_width(lli, config->src.width);
dw_gdma_ll_lli_set_dst_trans_width(lli, config->dst.width);
// set burst items
dw_gdma_ll_lli_set_src_burst_items(lli, config->src.burst_items);
dw_gdma_ll_lli_set_dst_burst_items(lli, config->dst.burst_items);
// set burst mode
dw_gdma_ll_lli_set_src_burst_mode(lli, config->src.burst_mode);
dw_gdma_ll_lli_set_dst_burst_mode(lli, config->dst.burst_mode);
// set burst size
dw_gdma_ll_lli_set_src_burst_size(lli, config->src.burst_size);
dw_gdma_ll_lli_set_dst_burst_size(lli, config->dst.burst_size);
// set address increment mode
dw_gdma_ll_lli_set_src_addr_inc_mode(lli, config->src.addr_inc_mode);
dw_gdma_ll_lli_set_dst_addr_inc_mode(lli, config->dst.addr_inc_mode);
// [Ctrl1] register
// set burst length
dw_gdma_ll_lli_set_src_burst_len(lli, config->src.burst_len);
dw_gdma_ll_lli_set_dst_burst_len(lli, config->dst.burst_len);
// set AXI burst length
dw_gdma_ll_lli_set_src_axi_burst_len(lli, config->src.axi_burst_len);
dw_gdma_ll_lli_set_dst_axi_burst_len(lli, config->dst.axi_burst_len);
// whether to enable the peripheral status write back
dw_gdma_ll_lli_enable_src_periph_status_write_back(lli, config->src.flags.en_status_write_back);
dw_gdma_ll_lli_enable_dst_periph_status_write_back(lli, config->dst.flags.en_status_write_back);

0
components/esp_driver_dma/esp32c5/gdma_retention.c → components/esp_driver_dma/src/esp32c5/gdma_retention.c
View File

0
components/esp_driver_dma/esp32c6/gdma_retention.c → components/esp_driver_dma/src/esp32c6/gdma_retention.c
View File

0
components/esp_driver_dma/esp32c61/gdma_retention.c → components/esp_driver_dma/src/esp32c61/gdma_retention.c
View File

0
components/esp_driver_dma/esp32h2/gdma_retention.c → components/esp_driver_dma/src/esp32h2/gdma_retention.c
View File

0
components/esp_driver_dma/esp32h21/gdma_retention.c → components/esp_driver_dma/src/esp32h21/gdma_retention.c
View File

0
components/esp_driver_dma/esp32h4/gdma_retention.c → components/esp_driver_dma/src/esp32h4/gdma_retention.c
View File

0
components/esp_driver_dma/esp32p4/dma2d_retention.c → components/esp_driver_dma/src/esp32p4/dma2d_retention.c
View File

0
components/esp_driver_dma/esp32p4/gdma_retention.c → components/esp_driver_dma/src/esp32p4/gdma_retention.c
View File

0
components/esp_driver_dma/esp32s31/dma2d_retention.c → components/esp_driver_dma/src/esp32s31/dma2d_retention.c
View File

0
components/esp_driver_dma/esp32s31/gdma_retention.c → components/esp_driver_dma/src/esp32s31/gdma_retention.c
View File

23
components/esp_driver_dma/src/esp_async_memcpy.c
View File

@@ -4,15 +4,22 @@
* SPDX-License-Identifier: Apache-2.0
*/
#include <string.h>
#include "freertos/FreeRTOS.h"
#include "freertos/semphr.h"
#include "freertos/task.h"
#include "esp_check.h"
#include "esp_cache.h"
#include "esp_async_memcpy.h"
#include "esp_async_memcpy_priv.h"
ESP_LOG_ATTR_TAG(TAG, "async_mcp");
/// Minimum buffer size for a DMA backed memory copy. A buffer smaller than two cache lines can not
/// guarantee a cache aligned "body" segment for the DMA engine, in which case a CPU memcpy is more
/// efficient anyway. The top level rejects such small buffers uniformly for all DMA backends.
#define ASYNC_MEMCPY_MIN_DMA_SIZE_CACHE_LINES 2
esp_err_t esp_async_memcpy_uninstall(async_memcpy_handle_t asmcp)
{
ESP_RETURN_ON_FALSE(asmcp, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
@@ -22,6 +29,14 @@ esp_err_t esp_async_memcpy_uninstall(async_memcpy_handle_t asmcp)
esp_err_t esp_async_memcpy(async_memcpy_handle_t asmcp, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args)
{
ESP_RETURN_ON_FALSE(asmcp && dst && src && n, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
// Reject buffers that are too small to benefit from a DMA transfer. A buffer smaller than two
// cache lines can not provide a cache aligned body segment, which all DMA backends rely on.
// Use a CPU memcpy for such small buffers instead.
size_t cache_line_size = esp_cache_get_line_size_by_addr(dst);
if (cache_line_size && n < ASYNC_MEMCPY_MIN_DMA_SIZE_CACHE_LINES * cache_line_size) {
ESP_RETURN_ON_FALSE(false, ESP_ERR_INVALID_SIZE, TAG, "buffer size %zu is too small for DMA, minimum is %zu", n,
ASYNC_MEMCPY_MIN_DMA_SIZE_CACHE_LINES * cache_line_size);
}
return asmcp->memcpy(asmcp, dst, src, n, cb_isr, cb_args);
}
@@ -48,6 +63,14 @@ esp_err_t esp_memcpy_blocking(async_memcpy_handle_t asmcp, void *dst, void *src,
ESP_RETURN_ON_FALSE(!xPortInIsrContext(), ESP_ERR_INVALID_STATE, TAG, "called from ISR context is not allowed");
ESP_RETURN_ON_FALSE(timeout_ms == -1, ESP_ERR_INVALID_ARG, TAG, "only timeout_ms=-1 is supported");
// For buffers too small to benefit from a DMA transfer, fall back to a CPU memcpy. The blocking
// API is not bound by the ISR callback contract, so a synchronous CPU copy is safe here.
size_t cache_line_size = esp_cache_get_line_size_by_addr(dst);
if (cache_line_size && n < ASYNC_MEMCPY_MIN_DMA_SIZE_CACHE_LINES * cache_line_size) {
memcpy(dst, src, n);
return ESP_OK;
}
memcpy_blocking_context_t ctx = {};
ctx.semaphore = xSemaphoreCreateBinaryStatic(&ctx.semaphore_buffer);

90
components/esp_driver_dma/src/esp_async_memcpy_priv.h
View File

@@ -8,8 +8,10 @@
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
#include "esp_err.h"
#include "esp_etm.h"
#include "esp_cache.h"
#include "esp_async_memcpy.h"
#include "soc/soc_caps.h"
@@ -38,6 +40,94 @@ struct async_memcpy_context_t {
esp_err_t (*del)(async_memcpy_context_t *ctx);
};
/// @brief Cache aligned split of an async memcpy transaction.
/// @note The DMA engine only handles the cache aligned body. The unaligned head and tail (if any)
/// are copied by the CPU right before the DMA transfer starts (see
/// `async_memcpy_do_cache_ops_and_cpu_copy`). The head/tail and the body never share a cache
/// line, so the CPU and DMA can work concurrently without cache coherency conflicts. This
/// avoids the need for a stash buffer and the merge step in the ISR.
typedef struct {
void *src; ///< Source buffer of the whole transaction (for the deferred CPU copy and cache ops)
void *dst; ///< Destination buffer of the whole transaction (for the deferred CPU copy and cache ops)
size_t head_len; ///< Bytes of the unaligned head (copied by CPU)
size_t body_len; ///< Bytes of the cache aligned body (copied by DMA)
size_t tail_len; ///< Bytes of the unaligned tail (copied by CPU)
void *body_src; ///< Source address of the body
void *body_dst; ///< Destination address of the body
size_t cache_line_size; ///< Cache line size used for the split (0 if the memory is not cached)
} async_memcpy_split_t;
/// @brief Split an async memcpy transaction into a cache aligned body and (optional) head/tail.
/// @note This function only computes the split. It does NOT touch any memory or cache, the caller
/// must invoke `async_memcpy_do_cache_ops_and_cpu_copy` (right before the DMA transfer starts)
/// to actually copy the head/tail and do the required cache maintenance.
/// @note The top level esp_async_memcpy guarantees n >= 2 * cache_line_size, so a non-zero body is
/// always available. This is asserted here defensively.
static inline esp_err_t async_memcpy_split_cache_aligned(void *dst, void *src, size_t n, async_memcpy_split_t *split)
{
split->src = src;
split->dst = dst;
split->head_len = 0;
split->tail_len = 0;
split->body_len = n;
split->body_src = src;
split->body_dst = dst;
split->cache_line_size = esp_cache_get_line_size_by_addr(dst);
if (split->cache_line_size) {
size_t head_overflow = (uintptr_t)dst % split->cache_line_size;
split->head_len = head_overflow ? (split->cache_line_size - head_overflow) : 0;
// the top level esp_async_memcpy guarantees n >= 2 * cache_line_size, so a non-zero body is always available
if (split->head_len >= n) {
return ESP_ERR_INVALID_SIZE;
}
split->tail_len = (n - split->head_len) % split->cache_line_size;
split->body_len = n - split->head_len - split->tail_len;
if (split->body_len == 0) {
return ESP_ERR_INVALID_SIZE;
}
split->body_src = (uint8_t *)src + split->head_len;
split->body_dst = (uint8_t *)dst + split->head_len;
}
return ESP_OK;
}
/// @brief Do the CPU copy of the unaligned head/tail and the cache maintenance for a split.
/// @note This must be called from `try_start_pending_transaction`, right before the DMA transfer
/// actually kicks off. Because the async memcpy allows different requests to use the same or
/// overlapping destination buffers, doing this at submit time would race with a previous
/// (still running) DMA transfer that writes to the same destination. By deferring it to the
/// moment the transfer starts (which, with a single channel FSM, happens only after the
/// previous transfer has completed), the CPU copies and cache operations are serialized with
/// the DMA engine.
/// @note The head/tail cache lines are disjoint from the body's, so the CPU copies don't interfere
/// with the body transferred by the DMA.
static inline esp_err_t async_memcpy_do_cache_ops_and_cpu_copy(const async_memcpy_split_t *split)
{
size_t n = split->head_len + split->body_len + split->tail_len;
// CPU copies the unaligned head and tail. These cache lines are disjoint from the body's.
if (split->head_len) {
memcpy(split->dst, split->src, split->head_len);
}
if (split->tail_len) {
memcpy((uint8_t *)split->body_dst + split->body_len, (uint8_t *)split->body_src + split->body_len, split->tail_len);
}
// write back the source data if it's behind the cache, so that the DMA can read the latest data
if (esp_cache_get_line_size_by_addr(split->src) > 0) {
esp_cache_msync(split->src, n, ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_UNALIGNED);
}
// invalidate the destination body if it's behind the cache, so that the CPU can see the data written
// by the DMA engine later (the head/tail cache lines are kept intact, they hold the CPU data)
if (split->cache_line_size) {
esp_cache_msync(split->body_dst, split->body_len, ESP_CACHE_MSYNC_FLAG_DIR_M2C);
}
return ESP_OK;
}
#ifdef __cplusplus
}
#endif

46
components/esp_driver_dma/test_apps/dma/main/test_async_memcpy.c
View File

@@ -148,6 +148,13 @@ TEST_CASE("memory copy the same buffer with different content", "[async mcp]")
test_memory_copy_with_same_buffer(driver, &config);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(LP_AHB_GDMA)
#if SOC_DW_GDMA_SUPPORTED
printf("Testing memcpy by DW_GDMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_dw_gdma(&config, &driver));
test_memory_copy_with_same_buffer(driver, &config);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_DW_GDMA_SUPPORTED
}
static void test_memory_copy_blocking(async_memcpy_handle_t driver)
@@ -196,12 +203,12 @@ TEST_CASE("memory copy by DMA (blocking)", "[async mcp]")
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(AXI_GDMA)
#if SOC_CP_DMA_SUPPORTED
#if SOC_HAS(CP_DMA)
printf("Testing memcpy by CP DMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_cpdma(&config, &driver));
test_memory_copy_blocking(driver);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_CP_DMA_SUPPORTED
#endif // SOC_HAS(CP_DMA)
#if SOC_HAS(LP_AHB_GDMA)
printf("Testing memcpy by LP AHB GDMA\r\n");
@@ -209,6 +216,13 @@ TEST_CASE("memory copy by DMA (blocking)", "[async mcp]")
test_memory_copy_blocking(driver);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(LP_AHB_GDMA)
#if SOC_HAS(DW_GDMA)
printf("Testing memcpy by DW_GDMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_dw_gdma(&config, &driver));
test_memory_copy_blocking(driver);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(DW_GDMA)
}
[[maybe_unused]] static void test_memcpy_with_dest_addr_unaligned(async_memcpy_handle_t driver, bool src_in_psram, bool dst_in_psram)
@@ -283,6 +297,16 @@ TEST_CASE("memory copy with dest address unaligned", "[async mcp]")
#endif // GDMA_LL_GET(LP_AHB_PSRAM_CAPABLE) && SOC_HAS(SPIRAM)
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(LP_AHB_GDMA)
#if SOC_HAS(DW_GDMA)
printf("Testing memcpy by DW_GDMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_dw_gdma(&driver_config, &driver));
test_memcpy_with_dest_addr_unaligned(driver, false, false);
#if SOC_HAS(SPIRAM)
test_memcpy_with_dest_addr_unaligned(driver, true, true);
#endif // SOC_HAS(SPIRAM)
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(DW_GDMA)
}
#define TEST_ASYNC_MEMCPY_BENCH_COUNTS 16
@@ -366,12 +390,12 @@ TEST_CASE("memory copy performance 40KB: SRAM->SRAM", "[async mcp]")
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(AXI_GDMA)
#if SOC_CP_DMA_SUPPORTED
#if SOC_HAS(CP_DMA)
printf("Testing memcpy by CP DMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_cpdma(&driver_config, &driver));
test_memcpy_performance(driver, 40 * 1024, false, false);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_CP_DMA_SUPPORTED
#endif // SOC_HAS(CP_DMA)
#if SOC_HAS(LP_AHB_GDMA)
printf("Testing memcpy by LP AHB GDMA\r\n");
@@ -379,6 +403,13 @@ TEST_CASE("memory copy performance 40KB: SRAM->SRAM", "[async mcp]")
test_memcpy_performance(driver, 40 * 1024, false, false);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(LP_AHB_GDMA)
#if SOC_HAS(DW_GDMA)
printf("Testing memcpy by DW_GDMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_dw_gdma(&driver_config, &driver));
test_memcpy_performance(driver, 40 * 1024, false, false);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(DW_GDMA)
}
#if SOC_SPIRAM_SUPPORTED
@@ -420,6 +451,13 @@ TEST_CASE("memory copy performance 40KB: PSRAM->PSRAM", "[async mcp]")
}
#endif // GDMA_LL_GET(LP_AHB_PSRAM_CAPABLE)
#endif // SOC_HAS(LP_AHB_GDMA)
#if SOC_HAS(DW_GDMA)
printf("Testing memcpy by DW_GDMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_dw_gdma(&driver_config, &driver));
test_memcpy_performance(driver, 40 * 1024, true, true);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(DW_GDMA)
}
#endif

60
components/esp_driver_dma/test_apps/dma/main/test_dw_gdma.c
View File

@@ -98,17 +98,17 @@ TEST_CASE("DW_GDMA M2M Test: Contiguous Mode", "[DW_GDMA]")
dw_gdma_block_transfer_config_t transfer_config = {
.src = {
.addr = (uint32_t)src_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.dst = {
.addr = (uint32_t)dst_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.size = 256,
};
@@ -190,17 +190,17 @@ TEST_CASE("DW_GDMA M2M Test: Reload Mode", "[DW_GDMA]")
dw_gdma_block_transfer_config_t transfer_config = {
.src = {
.addr = (uint32_t)src_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.dst = {
.addr = (uint32_t)dst_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.size = 256,
};
@@ -306,17 +306,17 @@ TEST_CASE("DW_GDMA M2M Test: Shadow Mode", "[DW_GDMA]")
dw_gdma_block_transfer_config_t transfer_config = {
.src = {
.addr = (uint32_t)src_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.dst = {
.addr = (uint32_t)dst_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.size = 256,
};
@@ -441,17 +441,17 @@ TEST_CASE("DW_GDMA M2M Test: Link-List Mode", "[DW_GDMA]")
dw_gdma_block_transfer_config_t transfer_config = {
.src = {
.addr = (uint32_t)src_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.dst = {
.addr = (uint32_t)dst_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.size = 128,
};
@@ -578,17 +578,17 @@ TEST_CASE("DW_GDMA M2M Test: memory set with fixed address", "[DW_GDMA]")
dw_gdma_block_transfer_config_t transfer_config = {
.src = {
.addr = (uint32_t)src_buf,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 1, // Note for ESP32P4, if the buffer is in PSRAM and the burst mode is fixed, we can't set the burst length larger than 1
.burst_size = 4,
.axi_burst_len = 1, // Note for ESP32P4, if the buffer is in PSRAM and the address increment mode is fixed, we can't set the AXI burst length larger than 1
},
.dst = {
.addr = (uint32_t)dst_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 1,
.burst_size = 4,
.axi_burst_len = 1,
},
.size = 256,
};

2
components/esp_driver_dma/test_apps/dma/main/test_gdma_etm.c
View File

@@ -47,7 +47,7 @@ TEST_CASE("async_memcpy_eof_event", "[GDMA][ETM]")
printf("install async memcpy context\r\n");
async_memcpy_handle_t mcp_ctx = NULL;
async_memcpy_config_t config = ASYNC_MEMCPY_DEFAULT_CONFIG();
TEST_ESP_OK(esp_async_memcpy_install(&config, &mcp_ctx));
TEST_ESP_OK(esp_async_memcpy_install_gdma_ahb(&config, &mcp_ctx));
printf("get async memcpy etm event handle\r\n");
esp_etm_event_handle_t mcp_event = NULL;

30
components/esp_driver_isp/src/isp_dma.c
View File

@@ -47,13 +47,13 @@ static bool IRAM_ATTR s_isp_dma_done_cb(dw_gdma_channel_handle_t chan, const dw_
return high_task_woken == pdTRUE;
}
static dw_gdma_burst_items_t s_isp_dma_burst_len_to_items(uint32_t burst_len)
static dw_gdma_burst_size_t s_isp_dma_burst_len_to_size(uint32_t burst_len)
{
switch (burst_len) {
case 1: return DW_GDMA_BURST_ITEMS_1;
case 4: return DW_GDMA_BURST_ITEMS_4;
case 8: return DW_GDMA_BURST_ITEMS_8;
default: return DW_GDMA_BURST_ITEMS_8;
case 1: return DW_GDMA_BURST_SIZE_1;
case 4: return DW_GDMA_BURST_SIZE_4;
case 8: return DW_GDMA_BURST_SIZE_8;
default: return DW_GDMA_BURST_SIZE_8;
}
}
@@ -199,29 +199,29 @@ esp_err_t isp_dma_new_frame_ctx(isp_proc_handle_t proc)
.src = {
.addr = MIPI_CSI_BRG_MEM_BASE,
.width = DW_GDMA_TRANS_WIDTH_64,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = output_burst_len,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = output_burst_len,
},
.dst = {
.width = DW_GDMA_TRANS_WIDTH_64,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = output_burst_len,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = output_burst_len,
},
.size = ctx->output_frame_size_64bit,
};
ctx->dma_in_trans = (dw_gdma_block_transfer_config_t) {
.src = {
.width = DW_GDMA_TRANS_WIDTH_64,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_32,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_32,
},
.dst = {
.addr = MIPI_CSI_BRG_MEM_BASE,
.width = DW_GDMA_TRANS_WIDTH_64,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = s_isp_dma_burst_len_to_items(proc->dma_in_burst_len),
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = s_isp_dma_burst_len_to_size(proc->dma_in_burst_len),
},
.size = ctx->input_frame_size_64bit,
};

88
components/esp_hal_dma/esp32p4/include/hal/dw_gdma_ll.h
View File

@@ -423,31 +423,31 @@ static inline void dw_gdma_ll_channel_set_dst_master_port(dw_gdma_dev_t *dev, ui
}
/**
* @brief Enable the source address burst mode
* @brief Set the source address increment mode
*
* @note Increase the source address by the data width after each transfer
* @note Controls whether the source address is increased by the data width after each transfer
*
* @param dev Pointer to the DW_GDMA registers
* @param channel Channel number
* @param mode Address burst mode
* @param mode Address increment mode
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_channel_set_src_burst_mode(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_burst_mode_t mode)
static inline void dw_gdma_ll_channel_set_src_addr_inc_mode(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_addr_inc_mode_t mode)
{
dev->ch[channel].ctl0.sinc = mode;
}
/**
* @brief Enable the destination address burst mode
* @brief Set the destination address increment mode
*
* @note Increase the destination address by the data width after each transfer
* @note Controls whether the destination address is increased by the data width after each transfer
*
* @param dev Pointer to the DW_GDMA registers
* @param channel Channel number
* @param mode Address burst mode
* @param mode Address increment mode
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_channel_set_dst_burst_mode(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_burst_mode_t mode)
static inline void dw_gdma_ll_channel_set_dst_addr_inc_mode(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_addr_inc_mode_t mode)
{
dev->ch[channel].ctl0.dinc = mode;
}
@@ -483,12 +483,12 @@ static inline void dw_gdma_ll_channel_set_dst_trans_width(dw_gdma_dev_t *dev, ui
*
* @param dev Pointer to the DW_GDMA registers
* @param channel Channel number
* @param items Number of data items
* @param size Burst size (number of data items)
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_channel_set_src_burst_items(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_burst_items_t items)
static inline void dw_gdma_ll_channel_set_src_burst_size(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_burst_size_t size)
{
dev->ch[channel].ctl0.src_msize = items;
dev->ch[channel].ctl0.src_msize = size;
}
/**
@@ -496,39 +496,39 @@ static inline void dw_gdma_ll_channel_set_src_burst_items(dw_gdma_dev_t *dev, ui
*
* @param dev Pointer to the DW_GDMA registers
* @param channel Channel number
* @param items Number of data items
* @param size Burst size (number of data items)
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_channel_set_dst_burst_items(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_burst_items_t items)
static inline void dw_gdma_ll_channel_set_dst_burst_size(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_burst_size_t size)
{
dev->ch[channel].ctl0.dst_msize = items;
dev->ch[channel].ctl0.dst_msize = size;
}
/**
* @brief Set the source burst length
* @brief Set the source AXI burst length
*
* @note This controls how many times the DMA controller will ask for data from the source device in a single burst transaction.
* @note This controls how many times the DMA controller will ask for data from the source device in a single AXI burst transaction.
*
* @param dev Pointer to the DW_GDMA registers
* @param channel Channel number
* @param len Burst length
* @param len AXI burst length
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_channel_set_src_burst_len(dw_gdma_dev_t *dev, uint8_t channel, uint8_t len)
static inline void dw_gdma_ll_channel_set_src_axi_burst_len(dw_gdma_dev_t *dev, uint8_t channel, uint8_t len)
{
dev->ch[channel].ctl1.arlen_en = len > 0;
HAL_FORCE_MODIFY_U32_REG_FIELD(dev->ch[channel].ctl1, arlen, len);
}
/**
* @brief Set the destination burst length
* @brief Set the destination AXI burst length
*
* @param dev Pointer to the DW_GDMA registers
* @param channel Channel number
* @param len Burst length
* @param len AXI burst length
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_channel_set_dst_burst_len(dw_gdma_dev_t *dev, uint8_t channel, uint8_t len)
static inline void dw_gdma_ll_channel_set_dst_axi_burst_len(dw_gdma_dev_t *dev, uint8_t channel, uint8_t len)
{
dev->ch[channel].ctl1.awlen_en = len > 0;
HAL_FORCE_MODIFY_U32_REG_FIELD(dev->ch[channel].ctl1, awlen, len);
@@ -1072,80 +1072,80 @@ static inline void dw_gdma_ll_lli_set_trans_block_size(dw_gdma_link_list_item_t
}
/**
* @brief Enable the source address burst mode
* @brief Set the source address increment mode
*
* @note Increase the source address by the data width after each transfer
* @note Controls whether the source address is increased by the data width after each transfer
*
* @param lli Link list item
* @param mode Address burst mode
* @param mode Address increment mode
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_lli_set_src_burst_mode(dw_gdma_link_list_item_t *lli, dw_gdma_burst_mode_t mode)
static inline void dw_gdma_ll_lli_set_src_addr_inc_mode(dw_gdma_link_list_item_t *lli, dw_gdma_addr_inc_mode_t mode)
{
lli->ctrl_lo.sinc = mode;
}
/**
* @brief Enable the destination address burst mode
* @brief Set the destination address increment mode
*
* @note Increase the destination address by the data width after each transfer
* @note Controls whether the destination address is increased by the data width after each transfer
*
* @param lli Link list item
* @param mode Address burst mode
* @param mode Address increment mode
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_lli_set_dst_burst_mode(dw_gdma_link_list_item_t *lli, dw_gdma_burst_mode_t mode)
static inline void dw_gdma_ll_lli_set_dst_addr_inc_mode(dw_gdma_link_list_item_t *lli, dw_gdma_addr_inc_mode_t mode)
{
lli->ctrl_lo.dinc = mode;
}
/**
* @brief Set the number of data items that can be transferred in a single burst transaction for the source master port
* @brief Set the burst size for the source master port
*
* @param lli Link list item
* @param items Number of data items
* @param size Burst size (number of data items)
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_lli_set_src_burst_items(dw_gdma_link_list_item_t *lli, dw_gdma_burst_items_t items)
static inline void dw_gdma_ll_lli_set_src_burst_size(dw_gdma_link_list_item_t *lli, dw_gdma_burst_size_t size)
{
lli->ctrl_lo.src_msize = items;
lli->ctrl_lo.src_msize = size;
}
/**
* @brief Set the number of data items that can be transferred in a single burst transaction for the destination master port
* @brief Set the burst size for the destination master port
*
* @param lli Link list item
* @param items Number of data items
* @param size Burst size (number of data items)
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_lli_set_dst_burst_items(dw_gdma_link_list_item_t *lli, dw_gdma_burst_items_t items)
static inline void dw_gdma_ll_lli_set_dst_burst_size(dw_gdma_link_list_item_t *lli, dw_gdma_burst_size_t size)
{
lli->ctrl_lo.dst_msize = items;
lli->ctrl_lo.dst_msize = size;
}
/**
* @brief Set the source burst length
* @brief Set the source AXI burst length
*
* @note This controls how many times the DMA controller will ask for data from the source device in a single burst transaction.
* @note This controls how many times the DMA controller will ask for data from the source device in a single AXI burst transaction.
*
* @param lli Link list item
* @param len Burst length
* @param len AXI burst length
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_lli_set_src_burst_len(dw_gdma_link_list_item_t *lli, uint8_t len)
static inline void dw_gdma_ll_lli_set_src_axi_burst_len(dw_gdma_link_list_item_t *lli, uint8_t len)
{
lli->ctrl_hi.arlen_en = len > 0;
HAL_FORCE_MODIFY_U32_REG_FIELD(lli->ctrl_hi, arlen, len);
}
/**
* @brief Set the destination burst length
* @brief Set the destination AXI burst length
*
* @param lli Link list item
* @param len Burst length
* @param len AXI burst length
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_lli_set_dst_burst_len(dw_gdma_link_list_item_t *lli, uint8_t len)
static inline void dw_gdma_ll_lli_set_dst_axi_burst_len(dw_gdma_link_list_item_t *lli, uint8_t len)
{
lli->ctrl_hi.awlen_en = len > 0;
HAL_FORCE_MODIFY_U32_REG_FIELD(lli->ctrl_hi, awlen, len);

36
components/esp_hal_dma/include/hal/dw_gdma_types.h
View File

@@ -69,30 +69,32 @@ typedef enum {
} dw_gdma_transfer_width_t;
/**
* @brief DW_GDMA burst mode
* @brief DW_GDMA address increment mode
*/
typedef enum {
DW_GDMA_BURST_MODE_INCREMENT, /*!< The address is increased after each transfer */
DW_GDMA_BURST_MODE_FIXED, /*!< The address remains the same after each transfer */
} dw_gdma_burst_mode_t;
DW_GDMA_ADDR_INC_MODE_INCREMENT, /*!< The address is increased by the data width after each data item */
DW_GDMA_ADDR_INC_MODE_FIXED, /*!< The address remains the same after each data item (e.g. FIFO/peripheral target) */
} dw_gdma_addr_inc_mode_t;
/**
* @brief Number of data items that are contained in one burst transaction
* @brief DW_GDMA burst size: number of data items contained in one (core-level) burst transaction
*
* @note One item's bit width is set by `dw_gdma_transfer_width_t`
* @note This maps to the DW_GDMA `MSIZE` field. It is the coarse, usually power-of-two, size of a
* single burst request. One item's bit width is set by `dw_gdma_transfer_width_t`.
* @note This is different from `axi_burst_len`, which is the fine-grained AXI burst length.
*/
typedef enum {
DW_GDMA_BURST_ITEMS_1, /*!< 1 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_4, /*!< 4 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_8, /*!< 8 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_16, /*!< 16 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_32, /*!< 32 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_64, /*!< 64 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_128, /*!< 128 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_256, /*!< 256 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_512, /*!< 512 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_1024, /*!< 1024 data items in a burst transaction */
} dw_gdma_burst_items_t;
DW_GDMA_BURST_SIZE_1, /*!< 1 data item in a burst transaction */
DW_GDMA_BURST_SIZE_4, /*!< 4 data items in a burst transaction */
DW_GDMA_BURST_SIZE_8, /*!< 8 data items in a burst transaction */
DW_GDMA_BURST_SIZE_16, /*!< 16 data items in a burst transaction */
DW_GDMA_BURST_SIZE_32, /*!< 32 data items in a burst transaction */
DW_GDMA_BURST_SIZE_64, /*!< 64 data items in a burst transaction */
DW_GDMA_BURST_SIZE_128, /*!< 128 data items in a burst transaction */
DW_GDMA_BURST_SIZE_256, /*!< 256 data items in a burst transaction */
DW_GDMA_BURST_SIZE_512, /*!< 512 data items in a burst transaction */
DW_GDMA_BURST_SIZE_1024, /*!< 1024 data items in a burst transaction */
} dw_gdma_burst_size_t;
#ifdef __cplusplus
}

12
components/esp_lcd/dsi/esp_lcd_panel_dpi.c
View File

@@ -433,16 +433,16 @@ static esp_err_t dpi_panel_init(esp_lcd_panel_t *panel)
dw_gdma_block_transfer_config_t dma_transfer_config = {
.src = {
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.dst = {
.addr = MIPI_DSI_BRG_MEM_BASE,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = DW_GDMA_BURST_ITEMS_256,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = DW_GDMA_BURST_SIZE_256,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.size = dpi_panel->fb_size * 8 / 64,