mirror of
https://github.com/espressif/esp-idf.git
synced 2026-10-02 11:10:54 +03:00
feat(dma): graduate the dma driver from esp_hw_support to esp_driver_dma
This commit is contained in:
morris
@@ -0,0 +1,330 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
#include <stdatomic.h>
|
||||
#include <sys/queue.h>
|
||||
#include <sys/param.h>
|
||||
#include "sdkconfig.h"
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "soc/interrupts.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_check.h"
|
||||
#include "esp_attr.h"
|
||||
#include "esp_err.h"
|
||||
#include "esp_intr_alloc.h"
|
||||
#include "esp_memory_utils.h"
|
||||
#include "esp_async_memcpy.h"
|
||||
#include "esp_async_memcpy_priv.h"
|
||||
#include "esp_private/gdma_link.h"
|
||||
#include "esp_private/esp_dma_utils.h"
|
||||
#include "esp_private/critical_section.h"
|
||||
#include "hal/cp_dma_hal.h"
|
||||
#include "hal/cp_dma_ll.h"
|
||||
|
||||
ESP_LOG_ATTR_TAG(TAG, "async_mcp.cpdma");
|
||||
|
||||
#define MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE 4095
|
||||
|
||||
/// @brief Transaction object for async memcpy
|
||||
typedef struct async_memcpy_transaction_t {
|
||||
gdma_link_list_handle_t tx_link_list; // DMA link list for TX direction
|
||||
gdma_link_list_handle_t rx_link_list; // DMA link list for RX direction
|
||||
async_memcpy_isr_cb_t cb; // user callback
|
||||
void *cb_args; // user callback args
|
||||
STAILQ_ENTRY(async_memcpy_transaction_t) idle_queue_entry; // Entry for the idle queue
|
||||
STAILQ_ENTRY(async_memcpy_transaction_t) ready_queue_entry; // Entry for the ready queue
|
||||
} async_memcpy_transaction_t;
|
||||
|
||||
/// @brief Context of async memcpy driver
|
||||
/// @note - It saves two queues, one for idle transaction objects, one for ready transaction objects
|
||||
/// @note - Number of transaction objects are determined by the backlog parameter
|
||||
typedef struct {
|
||||
async_memcpy_context_t parent; // Parent IO interface
|
||||
cp_dma_hal_context_t hal; // CPDMA hal
|
||||
intr_handle_t intr; // CPDMA interrupt handle
|
||||
portMUX_TYPE spin_lock; // spin lock to avoid threads and isr from accessing the same resource simultaneously
|
||||
_Atomic async_memcpy_fsm_t fsm;// driver state machine, changing state should be atomic
|
||||
size_t num_trans_objs; // number of transaction objects
|
||||
async_memcpy_transaction_t *transaction_pool; // transaction object pool
|
||||
async_memcpy_transaction_t *current_transaction; // current transaction object
|
||||
STAILQ_HEAD(, async_memcpy_transaction_t) idle_queue_head; // Head of the idle queue
|
||||
STAILQ_HEAD(, async_memcpy_transaction_t) ready_queue_head; // Head of the ready queue
|
||||
} async_memcpy_cpdma_context_t;
|
||||
|
||||
static void mcp_default_isr_handler(void *args);
|
||||
static esp_err_t mcp_cpdma_del(async_memcpy_context_t *ctx);
|
||||
static esp_err_t mcp_cpdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args);
|
||||
|
||||
static esp_err_t mcp_cpdma_destroy(async_memcpy_cpdma_context_t *mcp_dma)
|
||||
{
|
||||
if (mcp_dma->transaction_pool) {
|
||||
for (size_t i = 0; i < mcp_dma->num_trans_objs; i++) {
|
||||
async_memcpy_transaction_t* trans = &mcp_dma->transaction_pool[i];
|
||||
if (trans->tx_link_list) {
|
||||
gdma_del_link_list(trans->tx_link_list);
|
||||
}
|
||||
if (trans->rx_link_list) {
|
||||
gdma_del_link_list(trans->rx_link_list);
|
||||
}
|
||||
}
|
||||
free(mcp_dma->transaction_pool);
|
||||
}
|
||||
if (mcp_dma->intr) {
|
||||
esp_intr_free(mcp_dma->intr);
|
||||
}
|
||||
if (mcp_dma->hal.dev) { // this is for check if the hal is initialized
|
||||
cp_dma_hal_stop(&mcp_dma->hal);
|
||||
cp_dma_hal_deinit(&mcp_dma->hal);
|
||||
}
|
||||
free(mcp_dma);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t esp_async_memcpy_install_cpdma(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp)
|
||||
{
|
||||
esp_err_t ret = ESP_OK;
|
||||
async_memcpy_cpdma_context_t *mcp_dma = NULL;
|
||||
ESP_RETURN_ON_FALSE(config && mcp, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
// allocate memory of driver context from internal memory (because it contains atomic variable)
|
||||
mcp_dma = heap_caps_calloc(1, sizeof(async_memcpy_cpdma_context_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
|
||||
ESP_GOTO_ON_FALSE(mcp_dma, ESP_ERR_NO_MEM, err, TAG, "no mem for driver context");
|
||||
uint32_t trans_queue_len = config->backlog ? config->backlog : DEFAULT_TRANSACTION_QUEUE_LENGTH;
|
||||
// allocate memory for transaction pool from internal memory
|
||||
mcp_dma->transaction_pool = heap_caps_calloc(trans_queue_len, sizeof(async_memcpy_transaction_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
|
||||
ESP_GOTO_ON_FALSE(mcp_dma->transaction_pool, ESP_ERR_NO_MEM, err, TAG, "no mem for transaction pool");
|
||||
|
||||
// Init hal context
|
||||
cp_dma_hal_config_t hal_config = {};
|
||||
cp_dma_hal_init(&mcp_dma->hal, &hal_config);
|
||||
|
||||
ESP_GOTO_ON_ERROR(esp_intr_alloc(ETS_DMA_COPY_INTR_SOURCE, 0, mcp_default_isr_handler, mcp_dma, &mcp_dma->intr),
|
||||
err, TAG, "install isr failed");
|
||||
|
||||
// initialize transaction queue
|
||||
STAILQ_INIT(&mcp_dma->idle_queue_head);
|
||||
STAILQ_INIT(&mcp_dma->ready_queue_head);
|
||||
// pick transactions from the pool and insert to the idle queue
|
||||
for (int i = 0; i < trans_queue_len; i++) {
|
||||
STAILQ_INSERT_TAIL(&mcp_dma->idle_queue_head, &mcp_dma->transaction_pool[i], idle_queue_entry);
|
||||
}
|
||||
|
||||
// initialize other members
|
||||
portMUX_INITIALIZE(&mcp_dma->spin_lock);
|
||||
atomic_init(&mcp_dma->fsm, MCP_FSM_IDLE);
|
||||
mcp_dma->num_trans_objs = trans_queue_len;
|
||||
|
||||
mcp_dma->parent.del = mcp_cpdma_del;
|
||||
mcp_dma->parent.memcpy = mcp_cpdma_memcpy;
|
||||
// return driver object
|
||||
*mcp = &mcp_dma->parent;
|
||||
return ESP_OK;
|
||||
|
||||
err:
|
||||
if (mcp_dma) {
|
||||
mcp_cpdma_destroy(mcp_dma);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_memcpy_handle_t *asmcp)
|
||||
__attribute__((alias("esp_async_memcpy_install_cpdma")));
|
||||
|
||||
static esp_err_t mcp_cpdma_del(async_memcpy_context_t *ctx)
|
||||
{
|
||||
async_memcpy_cpdma_context_t *mcp_dma = __containerof(ctx, async_memcpy_cpdma_context_t, parent);
|
||||
// check if there are pending transactions
|
||||
ESP_RETURN_ON_FALSE(STAILQ_EMPTY(&mcp_dma->ready_queue_head), ESP_ERR_INVALID_STATE, TAG, "there are pending transactions");
|
||||
// check if the driver is in IDLE state
|
||||
ESP_RETURN_ON_FALSE(atomic_load(&mcp_dma->fsm) == MCP_FSM_IDLE, ESP_ERR_INVALID_STATE, TAG, "driver is not in IDLE state");
|
||||
return mcp_cpdma_destroy(mcp_dma);
|
||||
}
|
||||
|
||||
/// @brief help function to get one transaction from the ready queue
|
||||
/// @note this function is allowed to be called in ISR
|
||||
static async_memcpy_transaction_t *try_pop_trans_from_ready_queue(async_memcpy_cpdma_context_t *mcp_dma)
|
||||
{
|
||||
async_memcpy_transaction_t *trans = NULL;
|
||||
esp_os_enter_critical_safe(&mcp_dma->spin_lock);
|
||||
trans = STAILQ_FIRST(&mcp_dma->ready_queue_head);
|
||||
if (trans) {
|
||||
STAILQ_REMOVE_HEAD(&mcp_dma->ready_queue_head, ready_queue_entry);
|
||||
}
|
||||
esp_os_exit_critical_safe(&mcp_dma->spin_lock);
|
||||
return trans;
|
||||
}
|
||||
|
||||
/// @brief help function to start a pending transaction
|
||||
/// @note this function is allowed to be called in ISR
|
||||
static void try_start_pending_transaction(async_memcpy_cpdma_context_t *mcp_dma)
|
||||
{
|
||||
async_memcpy_fsm_t expected_fsm = MCP_FSM_IDLE;
|
||||
async_memcpy_transaction_t *trans = NULL;
|
||||
if (atomic_compare_exchange_strong(&mcp_dma->fsm, &expected_fsm, MCP_FSM_RUN_WAIT)) {
|
||||
trans = try_pop_trans_from_ready_queue(mcp_dma);
|
||||
if (trans) {
|
||||
atomic_store(&mcp_dma->fsm, MCP_FSM_RUN);
|
||||
mcp_dma->current_transaction = trans;
|
||||
cp_dma_hal_set_desc_base_addr(&mcp_dma->hal,
|
||||
gdma_link_get_head_addr(trans->tx_link_list),
|
||||
gdma_link_get_head_addr(trans->rx_link_list));
|
||||
cp_dma_hal_start(&mcp_dma->hal); // enable DMA and interrupt
|
||||
} else {
|
||||
atomic_store(&mcp_dma->fsm, MCP_FSM_IDLE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// @brief help function to get one transaction from the idle queue
|
||||
/// @note this function is allowed to be called in ISR
|
||||
static async_memcpy_transaction_t *try_pop_trans_from_idle_queue(async_memcpy_cpdma_context_t *mcp_dma)
|
||||
{
|
||||
async_memcpy_transaction_t *trans = NULL;
|
||||
esp_os_enter_critical_safe(&mcp_dma->spin_lock);
|
||||
trans = STAILQ_FIRST(&mcp_dma->idle_queue_head);
|
||||
if (trans) {
|
||||
STAILQ_REMOVE_HEAD(&mcp_dma->idle_queue_head, idle_queue_entry);
|
||||
}
|
||||
esp_os_exit_critical_safe(&mcp_dma->spin_lock);
|
||||
return trans;
|
||||
}
|
||||
|
||||
static esp_err_t mcp_cpdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args)
|
||||
{
|
||||
esp_err_t ret = ESP_OK;
|
||||
async_memcpy_cpdma_context_t *mcp_dma = __containerof(ctx, async_memcpy_cpdma_context_t, parent);
|
||||
ESP_RETURN_ON_FALSE(esp_ptr_internal(src) && esp_ptr_internal(dst), ESP_ERR_INVALID_ARG, TAG, "CP_DMA can only access SRAM");
|
||||
async_memcpy_transaction_t *trans = NULL;
|
||||
// pick one transaction node from idle queue
|
||||
trans = try_pop_trans_from_idle_queue(mcp_dma);
|
||||
// check if we get the transaction object successfully
|
||||
ESP_RETURN_ON_FALSE(trans, ESP_ERR_INVALID_STATE, TAG, "no free node in the idle queue");
|
||||
|
||||
// clean up the transaction configuration comes from the last one
|
||||
if (trans->tx_link_list) {
|
||||
gdma_del_link_list(trans->tx_link_list);
|
||||
trans->tx_link_list = NULL;
|
||||
}
|
||||
if (trans->rx_link_list) {
|
||||
gdma_del_link_list(trans->rx_link_list);
|
||||
trans->rx_link_list = NULL;
|
||||
}
|
||||
|
||||
size_t num_dma_nodes = esp_dma_calculate_node_count(n, 1, MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE);
|
||||
|
||||
// allocate gdma TX link
|
||||
gdma_link_list_config_t tx_link_cfg = {
|
||||
.item_alignment = 4, // CP_DMA requires 4 bytes alignment for each descriptor
|
||||
.num_items = num_dma_nodes,
|
||||
.flags = {
|
||||
.check_owner = true,
|
||||
.items_in_ext_mem = false,
|
||||
},
|
||||
};
|
||||
ESP_GOTO_ON_ERROR(gdma_new_link_list(&tx_link_cfg, &trans->tx_link_list), err, TAG, "failed to create TX link list");
|
||||
// mount the source buffer to the TX link list
|
||||
gdma_buffer_mount_config_t tx_buf_mount_config[1] = {
|
||||
[0] = {
|
||||
.buffer = src,
|
||||
.buffer_alignment = 1, // CP_DMA doesn't have alignment requirement for internal memory
|
||||
.length = n,
|
||||
.flags = {
|
||||
.mark_eof = true, // mark the last item as EOF, so the RX channel can also received an EOF list item
|
||||
.mark_final = GDMA_FINAL_LINK_TO_NULL, // using singly list, so terminate the link here
|
||||
}
|
||||
}
|
||||
};
|
||||
gdma_link_mount_buffers(trans->tx_link_list, 0, tx_buf_mount_config, 1, NULL);
|
||||
|
||||
// allocate gdma RX link
|
||||
gdma_link_list_config_t rx_link_cfg = {
|
||||
.item_alignment = 4, // CP_DMA requires 4 bytes alignment for each descriptor
|
||||
.num_items = num_dma_nodes,
|
||||
.flags = {
|
||||
.check_owner = true,
|
||||
.items_in_ext_mem = false,
|
||||
},
|
||||
};
|
||||
ESP_GOTO_ON_ERROR(gdma_new_link_list(&rx_link_cfg, &trans->rx_link_list), err, TAG, "failed to create RX link list");
|
||||
// mount the destination buffer to the RX link list
|
||||
gdma_buffer_mount_config_t rx_buf_mount_config[1] = {
|
||||
[0] = {
|
||||
.buffer = dst,
|
||||
.buffer_alignment = 1, // CP_DMA doesn't have alignment requirement for internal memory
|
||||
.length = n,
|
||||
.flags = {
|
||||
.mark_eof = false, // EOF is set by TX side
|
||||
.mark_final = GDMA_FINAL_LINK_TO_NULL, // using singly list, so terminate the link here
|
||||
}
|
||||
}
|
||||
};
|
||||
gdma_link_mount_buffers(trans->rx_link_list, 0, rx_buf_mount_config, 1, NULL);
|
||||
|
||||
// save other transaction context
|
||||
trans->cb = cb_isr;
|
||||
trans->cb_args = cb_args;
|
||||
|
||||
esp_os_enter_critical(&mcp_dma->spin_lock);
|
||||
// insert the trans to ready queue
|
||||
STAILQ_INSERT_TAIL(&mcp_dma->ready_queue_head, trans, ready_queue_entry);
|
||||
esp_os_exit_critical(&mcp_dma->spin_lock);
|
||||
|
||||
// check driver state, if there's no running transaction, start a new one
|
||||
try_start_pending_transaction(mcp_dma);
|
||||
|
||||
return ESP_OK;
|
||||
|
||||
err:
|
||||
if (trans) {
|
||||
// return back the trans to idle queue
|
||||
esp_os_enter_critical(&mcp_dma->spin_lock);
|
||||
STAILQ_INSERT_TAIL(&mcp_dma->idle_queue_head, trans, idle_queue_entry);
|
||||
esp_os_exit_critical(&mcp_dma->spin_lock);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void mcp_default_isr_handler(void *args)
|
||||
{
|
||||
bool need_yield = false;
|
||||
async_memcpy_cpdma_context_t *mcp_dma = (async_memcpy_cpdma_context_t *)args;
|
||||
// get the interrupt status and clear it
|
||||
uint32_t status = cp_dma_ll_get_intr_status(mcp_dma->hal.dev);
|
||||
cp_dma_ll_clear_intr_status(mcp_dma->hal.dev, status);
|
||||
|
||||
// End-Of-Frame on RX side
|
||||
if (status & CP_DMA_LL_EVENT_RX_EOF) {
|
||||
async_memcpy_transaction_t *trans = mcp_dma->current_transaction;
|
||||
|
||||
// switch driver state from RUN to IDLE
|
||||
async_memcpy_fsm_t expected_fsm = MCP_FSM_RUN;
|
||||
if (atomic_compare_exchange_strong(&mcp_dma->fsm, &expected_fsm, MCP_FSM_IDLE_WAIT)) {
|
||||
// invoked callback registered by user
|
||||
async_memcpy_isr_cb_t cb = trans->cb;
|
||||
if (cb) {
|
||||
async_memcpy_event_t e = {
|
||||
// No event data for now
|
||||
};
|
||||
need_yield = cb(&mcp_dma->parent, &e, trans->cb_args);
|
||||
}
|
||||
trans->cb = NULL;
|
||||
|
||||
esp_os_enter_critical_isr(&mcp_dma->spin_lock);
|
||||
// insert the trans object to the idle queue
|
||||
STAILQ_INSERT_TAIL(&mcp_dma->idle_queue_head, trans, idle_queue_entry);
|
||||
esp_os_exit_critical_isr(&mcp_dma->spin_lock);
|
||||
|
||||
atomic_store(&mcp_dma->fsm, MCP_FSM_IDLE);
|
||||
}
|
||||
|
||||
// try start the next pending transaction
|
||||
try_start_pending_transaction(mcp_dma);
|
||||
}
|
||||
|
||||
if (need_yield) {
|
||||
portYIELD_FROM_ISR();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,482 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
#include <stdatomic.h>
|
||||
#include <sys/queue.h>
|
||||
#include <sys/param.h>
|
||||
#include "sdkconfig.h"
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "soc/soc_caps.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_check.h"
|
||||
#include "esp_attr.h"
|
||||
#include "esp_err.h"
|
||||
#include "esp_private/gdma.h"
|
||||
#include "esp_private/gdma_link.h"
|
||||
#include "esp_private/esp_dma_utils.h"
|
||||
#include "esp_private/critical_section.h"
|
||||
#include "esp_memory_utils.h"
|
||||
#include "esp_cache.h"
|
||||
#include "esp_async_memcpy.h"
|
||||
#include "esp_async_memcpy_priv.h"
|
||||
#include "hal/cache_hal.h"
|
||||
#include "hal/cache_ll.h"
|
||||
#include "hal/gdma_ll.h"
|
||||
|
||||
ESP_LOG_ATTR_TAG(TAG, "async_mcp.gdma");
|
||||
|
||||
#define MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE 4095
|
||||
|
||||
/// @brief Transaction object for async memcpy
|
||||
typedef struct async_memcpy_transaction_t {
|
||||
gdma_link_list_handle_t tx_link_list; // DMA link list for TX direction
|
||||
gdma_link_list_handle_t rx_link_list; // DMA link list for RX direction
|
||||
dma_buffer_split_array_t rx_buf_array; // Split the destination buffer into cache aligned ones, save the splits in this array
|
||||
uint8_t* stash_buffer; // Stash buffer for cache aligned buffer
|
||||
async_memcpy_isr_cb_t cb; // user callback
|
||||
void *cb_args; // user callback args
|
||||
STAILQ_ENTRY(async_memcpy_transaction_t) idle_queue_entry; // Entry for the idle queue
|
||||
STAILQ_ENTRY(async_memcpy_transaction_t) ready_queue_entry; // Entry for the ready queue
|
||||
} async_memcpy_transaction_t;
|
||||
|
||||
/// @brief Context of async memcpy driver
|
||||
/// @note - It saves two queues, one for idle transaction objects, one for ready transaction objects
|
||||
/// @note - Number of transaction objects are determined by the backlog parameter
|
||||
typedef struct {
|
||||
async_memcpy_context_t parent; // Parent IO interface
|
||||
size_t rx_int_mem_alignment; // Required DMA buffer alignment for internal RX memory
|
||||
size_t rx_ext_mem_alignment; // Required DMA buffer alignment for external RX memory
|
||||
size_t tx_int_mem_alignment; // Required DMA buffer alignment for internal TX memory
|
||||
size_t tx_ext_mem_alignment; // Required DMA buffer alignment for external TX memory
|
||||
int gdma_bus_id; // GDMA bus id (AHB, AXI, etc.)
|
||||
gdma_channel_handle_t tx_channel; // GDMA TX channel handle
|
||||
gdma_channel_handle_t rx_channel; // GDMA RX channel handle
|
||||
portMUX_TYPE spin_lock; // spin lock to avoid threads and isr from accessing the same resource simultaneously
|
||||
_Atomic async_memcpy_fsm_t fsm; // driver state machine, changing state should be atomic
|
||||
size_t num_trans_objs; // number of transaction objects
|
||||
async_memcpy_transaction_t *transaction_pool; // transaction object pool
|
||||
async_memcpy_transaction_t *current_transaction; // current transaction object
|
||||
STAILQ_HEAD(, async_memcpy_transaction_t) idle_queue_head; // Head of the idle queue
|
||||
STAILQ_HEAD(, async_memcpy_transaction_t) ready_queue_head; // Head of the ready queue
|
||||
} async_memcpy_gdma_context_t;
|
||||
|
||||
static bool mcp_gdma_rx_eof_callback(gdma_channel_handle_t dma_chan, gdma_event_data_t *event_data, void *user_data);
|
||||
static esp_err_t mcp_gdma_del(async_memcpy_context_t *ctx);
|
||||
static esp_err_t mcp_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args);
|
||||
#if SOC_GDMA_SUPPORT_ETM
|
||||
static esp_err_t mcp_new_etm_event(async_memcpy_context_t *ctx, async_memcpy_etm_event_t event_type, esp_etm_event_handle_t *out_event);
|
||||
#endif // SOC_GDMA_SUPPORT_ETM
|
||||
|
||||
static esp_err_t mcp_gdma_destroy(async_memcpy_gdma_context_t *mcp_gdma)
|
||||
{
|
||||
// clean up transaction pool
|
||||
if (mcp_gdma->transaction_pool) {
|
||||
for (size_t i = 0; i < mcp_gdma->num_trans_objs; i++) {
|
||||
async_memcpy_transaction_t* trans = &mcp_gdma->transaction_pool[i];
|
||||
if (trans->tx_link_list) {
|
||||
gdma_del_link_list(trans->tx_link_list);
|
||||
}
|
||||
if (trans->rx_link_list) {
|
||||
gdma_del_link_list(trans->rx_link_list);
|
||||
}
|
||||
if (trans->stash_buffer) {
|
||||
free(trans->stash_buffer);
|
||||
}
|
||||
}
|
||||
free(mcp_gdma->transaction_pool);
|
||||
}
|
||||
// clean up GDMA channels
|
||||
if (mcp_gdma->tx_channel) {
|
||||
gdma_disconnect(mcp_gdma->tx_channel);
|
||||
gdma_del_channel(mcp_gdma->tx_channel);
|
||||
}
|
||||
if (mcp_gdma->rx_channel) {
|
||||
gdma_disconnect(mcp_gdma->rx_channel);
|
||||
gdma_del_channel(mcp_gdma->rx_channel);
|
||||
}
|
||||
free(mcp_gdma);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
static esp_err_t esp_async_memcpy_install_gdma_template(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp,
|
||||
esp_err_t (*new_channel_func)(const gdma_channel_alloc_config_t *, gdma_channel_handle_t *, gdma_channel_handle_t *),
|
||||
int gdma_bus_id)
|
||||
{
|
||||
esp_err_t ret = ESP_OK;
|
||||
async_memcpy_gdma_context_t *mcp_gdma = NULL;
|
||||
ESP_RETURN_ON_FALSE(config && mcp, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
|
||||
// allocate memory of driver context from internal memory (because it contains atomic variable)
|
||||
mcp_gdma = heap_caps_calloc(1, sizeof(async_memcpy_gdma_context_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
|
||||
ESP_GOTO_ON_FALSE(mcp_gdma, ESP_ERR_NO_MEM, err, TAG, "no mem for driver context");
|
||||
uint32_t trans_queue_len = config->backlog ? config->backlog : DEFAULT_TRANSACTION_QUEUE_LENGTH;
|
||||
// allocate memory for transaction pool from internal memory
|
||||
mcp_gdma->transaction_pool = heap_caps_calloc(trans_queue_len, sizeof(async_memcpy_transaction_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
|
||||
ESP_GOTO_ON_FALSE(mcp_gdma->transaction_pool, ESP_ERR_NO_MEM, err, TAG, "no mem for transaction pool");
|
||||
|
||||
// create TX and RX channels in the same pair with a single call
|
||||
gdma_channel_alloc_config_t channel_config = {0};
|
||||
ESP_GOTO_ON_ERROR(new_channel_func(&channel_config, &mcp_gdma->tx_channel, &mcp_gdma->rx_channel), err, TAG, "failed to alloc GDMA channels");
|
||||
|
||||
gdma_reset(mcp_gdma->tx_channel);
|
||||
gdma_reset(mcp_gdma->rx_channel);
|
||||
// get a free DMA trigger ID for memory copy
|
||||
gdma_trigger_t m2m_trigger = GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_M2M, 0);
|
||||
uint32_t free_m2m_id_mask = 0;
|
||||
gdma_get_free_m2m_trig_id_mask(mcp_gdma->tx_channel, &free_m2m_id_mask);
|
||||
m2m_trigger.instance_id = __builtin_ctz(free_m2m_id_mask);
|
||||
ESP_GOTO_ON_ERROR(gdma_connect(mcp_gdma->rx_channel, m2m_trigger), err, TAG, "GDMA rx connect failed");
|
||||
ESP_GOTO_ON_ERROR(gdma_connect(mcp_gdma->tx_channel, m2m_trigger), err, TAG, "GDMA tx connect failed");
|
||||
|
||||
gdma_strategy_config_t strategy_cfg = {
|
||||
.owner_check = true,
|
||||
.auto_update_desc = true,
|
||||
.eof_till_data_popped = true,
|
||||
};
|
||||
gdma_apply_strategy(mcp_gdma->tx_channel, &strategy_cfg);
|
||||
gdma_apply_strategy(mcp_gdma->rx_channel, &strategy_cfg);
|
||||
|
||||
#if SOC_GDMA_SUPPORT_WEIGHTED_ARBITRATION
|
||||
if (config->weight) {
|
||||
ESP_GOTO_ON_ERROR(gdma_set_weight(mcp_gdma->rx_channel, config->weight), err, TAG, "Set GDMA rx channel weight failed");
|
||||
ESP_GOTO_ON_ERROR(gdma_set_weight(mcp_gdma->tx_channel, config->weight), err, TAG, "Set GDMA tx channel weight failed");
|
||||
}
|
||||
#endif
|
||||
gdma_transfer_config_t transfer_cfg = {
|
||||
.max_data_burst_size = config->dma_burst_size,
|
||||
.access_ext_mem = true, // allow to do memory copy from/to external memory
|
||||
};
|
||||
ESP_GOTO_ON_ERROR(gdma_config_transfer(mcp_gdma->tx_channel, &transfer_cfg), err, TAG, "config transfer for tx channel failed");
|
||||
ESP_GOTO_ON_ERROR(gdma_config_transfer(mcp_gdma->rx_channel, &transfer_cfg), err, TAG, "config transfer for rx channel failed");
|
||||
|
||||
// get the buffer alignment required by the GDMA channel
|
||||
gdma_get_alignment_constraints(mcp_gdma->rx_channel, &mcp_gdma->rx_int_mem_alignment, &mcp_gdma->rx_ext_mem_alignment);
|
||||
gdma_get_alignment_constraints(mcp_gdma->tx_channel, &mcp_gdma->tx_int_mem_alignment, &mcp_gdma->tx_ext_mem_alignment);
|
||||
|
||||
// register rx eof callback
|
||||
gdma_rx_event_callbacks_t cbs = {
|
||||
.on_recv_eof = mcp_gdma_rx_eof_callback,
|
||||
};
|
||||
ESP_GOTO_ON_ERROR(gdma_register_rx_event_callbacks(mcp_gdma->rx_channel, &cbs, mcp_gdma), err, TAG, "failed to register RX EOF callback");
|
||||
|
||||
// initialize transaction queue
|
||||
STAILQ_INIT(&mcp_gdma->idle_queue_head);
|
||||
STAILQ_INIT(&mcp_gdma->ready_queue_head);
|
||||
// pick transactions from the pool and insert to the idle queue
|
||||
for (int i = 0; i < trans_queue_len; i++) {
|
||||
STAILQ_INSERT_TAIL(&mcp_gdma->idle_queue_head, &mcp_gdma->transaction_pool[i], idle_queue_entry);
|
||||
}
|
||||
|
||||
// initialize other members
|
||||
portMUX_INITIALIZE(&mcp_gdma->spin_lock);
|
||||
atomic_init(&mcp_gdma->fsm, MCP_FSM_IDLE);
|
||||
mcp_gdma->gdma_bus_id = gdma_bus_id;
|
||||
mcp_gdma->num_trans_objs = trans_queue_len;
|
||||
|
||||
mcp_gdma->parent.del = mcp_gdma_del;
|
||||
mcp_gdma->parent.memcpy = mcp_gdma_memcpy;
|
||||
#if SOC_GDMA_SUPPORT_ETM
|
||||
mcp_gdma->parent.new_etm_event = mcp_new_etm_event;
|
||||
#endif
|
||||
// return base object
|
||||
*mcp = &mcp_gdma->parent;
|
||||
return ESP_OK;
|
||||
|
||||
err:
|
||||
if (mcp_gdma) {
|
||||
mcp_gdma_destroy(mcp_gdma);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
#if SOC_HAS(AHB_GDMA)
|
||||
esp_err_t esp_async_memcpy_install_gdma_ahb(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp)
|
||||
{
|
||||
return esp_async_memcpy_install_gdma_template(config, mcp, gdma_new_ahb_channel, SOC_GDMA_BUS_AHB);
|
||||
}
|
||||
#endif // SOC_HAS(AHB_GDMA)
|
||||
|
||||
#if SOC_HAS(AXI_GDMA)
|
||||
esp_err_t esp_async_memcpy_install_gdma_axi(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp)
|
||||
{
|
||||
return esp_async_memcpy_install_gdma_template(config, mcp, gdma_new_axi_channel, SOC_GDMA_BUS_AXI);
|
||||
}
|
||||
#endif // SOC_HAS(AXI_GDMA)
|
||||
|
||||
#if SOC_HAS(AHB_GDMA)
|
||||
/// default installation falls back to use the AHB GDMA
|
||||
esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_memcpy_handle_t *asmcp)
|
||||
__attribute__((alias("esp_async_memcpy_install_gdma_ahb")));
|
||||
#elif SOC_HAS(AXI_GDMA)
|
||||
/// default installation falls back to use the AXI GDMA
|
||||
esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_memcpy_handle_t *asmcp)
|
||||
__attribute__((alias("esp_async_memcpy_install_gdma_axi")));
|
||||
#endif
|
||||
|
||||
static esp_err_t mcp_gdma_del(async_memcpy_context_t *ctx)
|
||||
{
|
||||
async_memcpy_gdma_context_t *mcp_gdma = __containerof(ctx, async_memcpy_gdma_context_t, parent);
|
||||
// check if there are pending transactions
|
||||
ESP_RETURN_ON_FALSE(STAILQ_EMPTY(&mcp_gdma->ready_queue_head), ESP_ERR_INVALID_STATE, TAG, "there are pending transactions");
|
||||
// check if the driver is in IDLE state
|
||||
ESP_RETURN_ON_FALSE(atomic_load(&mcp_gdma->fsm) == MCP_FSM_IDLE, ESP_ERR_INVALID_STATE, TAG, "driver is not in IDLE state");
|
||||
return mcp_gdma_destroy(mcp_gdma);
|
||||
}
|
||||
|
||||
/// @brief help function to get one transaction from the ready queue
|
||||
/// @note this function is allowed to be called in ISR
|
||||
static async_memcpy_transaction_t *try_pop_trans_from_ready_queue(async_memcpy_gdma_context_t *mcp_gdma)
|
||||
{
|
||||
async_memcpy_transaction_t *trans = NULL;
|
||||
esp_os_enter_critical_safe(&mcp_gdma->spin_lock);
|
||||
trans = STAILQ_FIRST(&mcp_gdma->ready_queue_head);
|
||||
if (trans) {
|
||||
STAILQ_REMOVE_HEAD(&mcp_gdma->ready_queue_head, ready_queue_entry);
|
||||
}
|
||||
esp_os_exit_critical_safe(&mcp_gdma->spin_lock);
|
||||
return trans;
|
||||
}
|
||||
|
||||
/// @brief help function to start a pending transaction
|
||||
/// @note this function is allowed to be called in ISR
|
||||
static void try_start_pending_transaction(async_memcpy_gdma_context_t *mcp_gdma)
|
||||
{
|
||||
async_memcpy_fsm_t expected_fsm = MCP_FSM_IDLE;
|
||||
async_memcpy_transaction_t *trans = NULL;
|
||||
if (atomic_compare_exchange_strong(&mcp_gdma->fsm, &expected_fsm, MCP_FSM_RUN_WAIT)) {
|
||||
trans = try_pop_trans_from_ready_queue(mcp_gdma);
|
||||
if (trans) {
|
||||
atomic_store(&mcp_gdma->fsm, MCP_FSM_RUN);
|
||||
mcp_gdma->current_transaction = trans;
|
||||
gdma_start(mcp_gdma->rx_channel, gdma_link_get_head_addr(trans->rx_link_list));
|
||||
gdma_start(mcp_gdma->tx_channel, gdma_link_get_head_addr(trans->tx_link_list));
|
||||
} else {
|
||||
atomic_store(&mcp_gdma->fsm, MCP_FSM_IDLE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// @brief help function to get one transaction from the idle queue
|
||||
/// @note this function is allowed to be called in ISR
|
||||
static async_memcpy_transaction_t *try_pop_trans_from_idle_queue(async_memcpy_gdma_context_t *mcp_gdma)
|
||||
{
|
||||
async_memcpy_transaction_t *trans = NULL;
|
||||
esp_os_enter_critical_safe(&mcp_gdma->spin_lock);
|
||||
trans = STAILQ_FIRST(&mcp_gdma->idle_queue_head);
|
||||
if (trans) {
|
||||
STAILQ_REMOVE_HEAD(&mcp_gdma->idle_queue_head, idle_queue_entry);
|
||||
}
|
||||
esp_os_exit_critical_safe(&mcp_gdma->spin_lock);
|
||||
return trans;
|
||||
}
|
||||
|
||||
/// @brief Check if the address and size can meet the requirement of the DMA engine
|
||||
static bool check_buffer_alignment(async_memcpy_gdma_context_t *mcp_gdma, void *src, void *dst, size_t n)
|
||||
{
|
||||
bool valid = true;
|
||||
|
||||
if (esp_ptr_external_ram(dst)) {
|
||||
valid = valid && (((uint32_t)dst & (mcp_gdma->rx_ext_mem_alignment - 1)) == 0);
|
||||
valid = valid && ((n & (mcp_gdma->rx_ext_mem_alignment - 1)) == 0);
|
||||
} else {
|
||||
valid = valid && (((uint32_t)dst & (mcp_gdma->rx_int_mem_alignment - 1)) == 0);
|
||||
valid = valid && ((n & (mcp_gdma->rx_int_mem_alignment - 1)) == 0);
|
||||
}
|
||||
|
||||
if (esp_ptr_external_ram(src)) {
|
||||
valid = valid && (((uint32_t)src & (mcp_gdma->tx_ext_mem_alignment - 1)) == 0);
|
||||
valid = valid && ((n & (mcp_gdma->tx_ext_mem_alignment - 1)) == 0);
|
||||
} else {
|
||||
valid = valid && (((uint32_t)src & (mcp_gdma->tx_int_mem_alignment - 1)) == 0);
|
||||
valid = valid && ((n & (mcp_gdma->tx_int_mem_alignment - 1)) == 0);
|
||||
}
|
||||
|
||||
return valid;
|
||||
}
|
||||
|
||||
static esp_err_t mcp_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args)
|
||||
{
|
||||
esp_err_t ret = ESP_OK;
|
||||
async_memcpy_gdma_context_t *mcp_gdma = __containerof(ctx, async_memcpy_gdma_context_t, parent);
|
||||
size_t dma_link_item_alignment = 4;
|
||||
// buffer location check
|
||||
#if SOC_HAS(AHB_GDMA)
|
||||
if (mcp_gdma->gdma_bus_id == SOC_GDMA_BUS_AHB) {
|
||||
#if !GDMA_LL_GET(AHB_PSRAM_CAPABLE)
|
||||
ESP_RETURN_ON_FALSE(esp_ptr_internal(src) && esp_ptr_internal(dst), ESP_ERR_INVALID_ARG, TAG, "AHB GDMA can only access SRAM");
|
||||
#endif // !GDMA_LL_GET(AHB_PSRAM_CAPABLE)
|
||||
dma_link_item_alignment = GDMA_LL_AHB_DESC_ALIGNMENT;
|
||||
}
|
||||
#endif // SOC_HAS(AHB_GDMA)
|
||||
#if SOC_HAS(AXI_GDMA)
|
||||
if (mcp_gdma->gdma_bus_id == SOC_GDMA_BUS_AXI) {
|
||||
#if !GDMA_LL_GET(AXI_PSRAM_CAPABLE)
|
||||
ESP_RETURN_ON_FALSE(esp_ptr_internal(src) && esp_ptr_internal(dst), ESP_ERR_INVALID_ARG, TAG, "AXI GDMA can only access SRAM");
|
||||
#endif // !GDMA_LL_GET(AXI_PSRAM_CAPABLE)
|
||||
dma_link_item_alignment = GDMA_LL_AXI_DESC_ALIGNMENT;
|
||||
}
|
||||
#endif // SOC_HAS(AXI_GDMA)
|
||||
// alignment check
|
||||
ESP_RETURN_ON_FALSE(check_buffer_alignment(mcp_gdma, src, dst, n), ESP_ERR_INVALID_ARG, TAG, "address|size not aligned: %p -> %p, sz=%zu", src, dst, n);
|
||||
|
||||
async_memcpy_transaction_t *trans = NULL;
|
||||
// pick one transaction node from idle queue
|
||||
trans = try_pop_trans_from_idle_queue(mcp_gdma);
|
||||
// check if we get the transaction object successfully
|
||||
ESP_RETURN_ON_FALSE(trans, ESP_ERR_INVALID_STATE, TAG, "no free node in the idle queue");
|
||||
|
||||
// clean up the transaction configuration comes from the last one
|
||||
if (trans->tx_link_list) {
|
||||
gdma_del_link_list(trans->tx_link_list);
|
||||
trans->tx_link_list = NULL;
|
||||
}
|
||||
if (trans->rx_link_list) {
|
||||
gdma_del_link_list(trans->rx_link_list);
|
||||
trans->rx_link_list = NULL;
|
||||
}
|
||||
if (trans->stash_buffer) {
|
||||
free(trans->stash_buffer);
|
||||
trans->stash_buffer = NULL;
|
||||
}
|
||||
|
||||
size_t buffer_alignment = 0;
|
||||
size_t num_dma_nodes = 0;
|
||||
|
||||
// allocate gdma TX link
|
||||
buffer_alignment = esp_ptr_internal(src) ? mcp_gdma->tx_int_mem_alignment : mcp_gdma->tx_ext_mem_alignment;
|
||||
num_dma_nodes = esp_dma_calculate_node_count(n, buffer_alignment, MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE);
|
||||
gdma_link_list_config_t tx_link_cfg = {
|
||||
.item_alignment = dma_link_item_alignment,
|
||||
.num_items = num_dma_nodes,
|
||||
.flags = {
|
||||
.check_owner = true,
|
||||
.items_in_ext_mem = false, // TODO: if the memcopy size is too large, we may need to allocate the link list items from external memory
|
||||
},
|
||||
};
|
||||
ESP_GOTO_ON_ERROR(gdma_new_link_list(&tx_link_cfg, &trans->tx_link_list), err, TAG, "failed to create TX link list");
|
||||
// mount the source buffer to the TX link list
|
||||
gdma_buffer_mount_config_t tx_buf_mount_config[1] = {
|
||||
[0] = {
|
||||
.buffer = src,
|
||||
.buffer_alignment = buffer_alignment,
|
||||
.length = n,
|
||||
.flags = {
|
||||
.mark_eof = true, // mark the last item as EOF, so the RX channel can also received an EOF list item
|
||||
.mark_final = GDMA_FINAL_LINK_TO_NULL, // using singly list, so terminate the link here
|
||||
}
|
||||
}
|
||||
};
|
||||
gdma_link_mount_buffers(trans->tx_link_list, 0, tx_buf_mount_config, 1, NULL);
|
||||
|
||||
// read the cache line size of internal and external memory, we use this information to check if a given memory is behind the cache
|
||||
// write back the source data if it's behind the cache
|
||||
size_t cache_line_size = esp_cache_get_line_size_by_addr(src);
|
||||
if (cache_line_size > 0) {
|
||||
esp_cache_msync(src, n, ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_UNALIGNED);
|
||||
}
|
||||
|
||||
// allocate gdma RX link
|
||||
buffer_alignment = esp_ptr_internal(dst) ? mcp_gdma->rx_int_mem_alignment : mcp_gdma->rx_ext_mem_alignment;
|
||||
num_dma_nodes = esp_dma_calculate_node_count(n, buffer_alignment, MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE);
|
||||
gdma_link_list_config_t rx_link_cfg = {
|
||||
.item_alignment = dma_link_item_alignment,
|
||||
.num_items = num_dma_nodes + 3, // add 3 extra items for the cache aligned buffers
|
||||
.flags = {
|
||||
.check_owner = true,
|
||||
.items_in_ext_mem = false, // TODO: if the memcopy size is too large, we may need to allocate the link list items from external memory
|
||||
},
|
||||
};
|
||||
ESP_GOTO_ON_ERROR(gdma_new_link_list(&rx_link_cfg, &trans->rx_link_list), err, TAG, "failed to create RX link list");
|
||||
|
||||
// if the destination buffer address is not cache line aligned, we need to split the buffer into cache line aligned ones
|
||||
ESP_GOTO_ON_ERROR(esp_dma_split_rx_buffer_to_cache_aligned(dst, n, &trans->rx_buf_array, &trans->stash_buffer),
|
||||
err, TAG, "failed to split RX buffer into aligned ones");
|
||||
// mount the destination buffer to the RX link list
|
||||
gdma_buffer_mount_config_t rx_buf_mount_config[3] = {0};
|
||||
for (int i = 0; i < 3; i++) {
|
||||
rx_buf_mount_config[i].buffer = trans->rx_buf_array.aligned_buffer[i].aligned_buffer;
|
||||
rx_buf_mount_config[i].buffer_alignment = buffer_alignment;
|
||||
rx_buf_mount_config[i].length = trans->rx_buf_array.aligned_buffer[i].length;
|
||||
}
|
||||
gdma_link_mount_buffers(trans->rx_link_list, 0, rx_buf_mount_config, 3, NULL);
|
||||
|
||||
// save other transaction context
|
||||
trans->cb = cb_isr;
|
||||
trans->cb_args = cb_args;
|
||||
|
||||
esp_os_enter_critical(&mcp_gdma->spin_lock);
|
||||
// insert the trans to ready queue
|
||||
STAILQ_INSERT_TAIL(&mcp_gdma->ready_queue_head, trans, ready_queue_entry);
|
||||
esp_os_exit_critical(&mcp_gdma->spin_lock);
|
||||
|
||||
// check driver state, if there's no running transaction, start a new one
|
||||
try_start_pending_transaction(mcp_gdma);
|
||||
|
||||
return ESP_OK;
|
||||
|
||||
err:
|
||||
if (trans) {
|
||||
// return back the trans to idle queue
|
||||
esp_os_enter_critical(&mcp_gdma->spin_lock);
|
||||
STAILQ_INSERT_TAIL(&mcp_gdma->idle_queue_head, trans, idle_queue_entry);
|
||||
esp_os_exit_critical(&mcp_gdma->spin_lock);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static bool mcp_gdma_rx_eof_callback(gdma_channel_handle_t dma_chan, gdma_event_data_t *event_data, void *user_data)
|
||||
{
|
||||
bool need_yield = false;
|
||||
async_memcpy_gdma_context_t *mcp_gdma = (async_memcpy_gdma_context_t *)user_data;
|
||||
async_memcpy_transaction_t *trans = mcp_gdma->current_transaction;
|
||||
dma_buffer_split_array_t *rx_buf_array = &trans->rx_buf_array;
|
||||
|
||||
// switch driver state from RUN to IDLE
|
||||
async_memcpy_fsm_t expected_fsm = MCP_FSM_RUN;
|
||||
if (atomic_compare_exchange_strong(&mcp_gdma->fsm, &expected_fsm, MCP_FSM_IDLE_WAIT)) {
|
||||
// merge the cache aligned buffers to the original buffer
|
||||
esp_dma_merge_aligned_rx_buffers(rx_buf_array);
|
||||
|
||||
// invoked callback registered by user
|
||||
async_memcpy_isr_cb_t cb = trans->cb;
|
||||
if (cb) {
|
||||
async_memcpy_event_t e = {
|
||||
// No event data for now
|
||||
};
|
||||
need_yield = cb(&mcp_gdma->parent, &e, trans->cb_args);
|
||||
}
|
||||
trans->cb = NULL;
|
||||
|
||||
esp_os_enter_critical_isr(&mcp_gdma->spin_lock);
|
||||
// insert the trans object to the idle queue
|
||||
STAILQ_INSERT_TAIL(&mcp_gdma->idle_queue_head, trans, idle_queue_entry);
|
||||
esp_os_exit_critical_isr(&mcp_gdma->spin_lock);
|
||||
|
||||
atomic_store(&mcp_gdma->fsm, MCP_FSM_IDLE);
|
||||
}
|
||||
|
||||
// try start the next pending transaction
|
||||
try_start_pending_transaction(mcp_gdma);
|
||||
|
||||
return need_yield;
|
||||
}
|
||||
|
||||
#if SOC_GDMA_SUPPORT_ETM
|
||||
static esp_err_t mcp_new_etm_event(async_memcpy_context_t *ctx, async_memcpy_etm_event_t event_type, esp_etm_event_handle_t *out_event)
|
||||
{
|
||||
async_memcpy_gdma_context_t *mcp_gdma = __containerof(ctx, async_memcpy_gdma_context_t, parent);
|
||||
if (event_type == ASYNC_MEMCPY_ETM_EVENT_COPY_DONE) {
|
||||
// use the RX EOF to indicate the async memcpy done event
|
||||
gdma_etm_event_config_t etm_event_conf = {
|
||||
.event_type = GDMA_ETM_EVENT_EOF,
|
||||
};
|
||||
return gdma_new_etm_event(mcp_gdma->rx_channel, &etm_event_conf, out_event);
|
||||
} else {
|
||||
return ESP_ERR_NOT_SUPPORTED;
|
||||
}
|
||||
}
|
||||
#endif // SOC_GDMA_SUPPORT_ETM
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,95 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <sys/queue.h>
|
||||
#include "sdkconfig.h"
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "esp_intr_alloc.h"
|
||||
#include "esp_heap_caps.h"
|
||||
#include "soc/soc_caps.h"
|
||||
#include "hal/dma2d_hal.h"
|
||||
#include "hal/dma2d_ll.h"
|
||||
#include "esp_private/dma2d.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#if CONFIG_DMA2D_OPERATION_FUNC_IN_IRAM || CONFIG_DMA2D_ISR_IRAM_SAFE
|
||||
#define DMA2D_MEM_ALLOC_CAPS (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)
|
||||
#else
|
||||
#define DMA2D_MEM_ALLOC_CAPS MALLOC_CAP_DEFAULT
|
||||
#endif
|
||||
|
||||
#if CONFIG_DMA2D_ISR_IRAM_SAFE
|
||||
#define DMA2D_INTR_ALLOC_FLAGS (ESP_INTR_FLAG_SHARED | ESP_INTR_FLAG_IRAM)
|
||||
#else
|
||||
#define DMA2D_INTR_ALLOC_FLAGS ESP_INTR_FLAG_SHARED
|
||||
#endif
|
||||
|
||||
#define DMA2D_RX_DEFAULT_INTR_FLAG (DMA2D_LL_EVENT_RX_SUC_EOF | DMA2D_LL_EVENT_RX_ERR_EOF | DMA2D_LL_EVENT_RX_DESC_ERROR)
|
||||
|
||||
typedef struct dma2d_channel_t dma2d_channel_t;
|
||||
typedef struct dma2d_tx_channel_t dma2d_tx_channel_t;
|
||||
typedef struct dma2d_rx_channel_t dma2d_rx_channel_t;
|
||||
typedef struct dma2d_group_t dma2d_group_t;
|
||||
|
||||
struct dma2d_trans_s {
|
||||
TAILQ_ENTRY(dma2d_trans_s) entry; // Link entry
|
||||
const dma2d_trans_config_t *desc; // Pointer to the structure containing all configuration items of a transaction
|
||||
dma2d_channel_handle_t rx_chan; // Pointer to the RX channel handle that will be used to do the transaction
|
||||
};
|
||||
|
||||
struct dma2d_group_t {
|
||||
int group_id; // Group ID, index from 0
|
||||
dma2d_hal_context_t hal; // HAL instance is at group level
|
||||
portMUX_TYPE spinlock; // Group level spinlock
|
||||
TAILQ_HEAD(pending_trans, dma2d_trans_s) pending_trans_tailq; // Link head of pending 2D-DMA transactions
|
||||
uint8_t tx_channel_free_mask; // Bit mask indicating the free TX channels at the moment
|
||||
uint8_t rx_channel_free_mask; // Bit mask indicating the free RX channels at the moment
|
||||
uint8_t tx_channel_reserved_mask; // Bit mask indicating the being reserved TX channels
|
||||
uint8_t rx_channel_reserved_mask; // Bit mask indicating the being reserved RX channels
|
||||
uint32_t tx_periph_m2m_free_id_mask; // Bit mask indicating the available TX M2M peripheral selelction IDs at the moment
|
||||
uint32_t rx_periph_m2m_free_id_mask; // Bit mask indicating the available RX M2M peripheral selelction IDs at the moment
|
||||
dma2d_tx_channel_t *tx_chans[DMA2D_LL_GET(TX_CHANS_PER_INST)]; // Handles of 2D-DMA TX channels
|
||||
dma2d_rx_channel_t *rx_chans[DMA2D_LL_GET(RX_CHANS_PER_INST)]; // Handles of 2D-DMA RX channels
|
||||
int intr_priority; // All channels in the same group should share the same interrupt priority
|
||||
};
|
||||
|
||||
struct dma2d_channel_t {
|
||||
dma2d_group_t *group; // Which group the channel belongs to
|
||||
int channel_id; // Channel ID
|
||||
dma2d_channel_direction_t direction; // Channel direction, TX or RX
|
||||
intr_handle_t intr; // Per-channel interrupt handle
|
||||
portMUX_TYPE spinlock; // Channel level spinlock
|
||||
struct {
|
||||
dma2d_trans_t *transaction; // Pointer to the 2D-DMA transaction context that is currently being processed on the channel
|
||||
uint32_t reorder_en : 1; // This flag indicates the channel will enable reorder functionality
|
||||
int periph_sel_id : (DMA2D_LL_CHANNEL_PERIPH_SEL_BIT_WIDTH + 1); // This is used to record the periph_sel_id of each channel
|
||||
} status;
|
||||
};
|
||||
|
||||
struct dma2d_tx_channel_t {
|
||||
dma2d_channel_t base; // 2D-DMA channel base class
|
||||
void *user_data; // User registered DMA event data
|
||||
dma2d_event_callback_t on_desc_done; // TX DONE event callback
|
||||
};
|
||||
|
||||
struct dma2d_rx_channel_t {
|
||||
dma2d_channel_t base; // 2D-DMA channel base class
|
||||
void *user_data; // User registered DMA event data
|
||||
dma2d_event_callback_t on_recv_eof; // RX EOF event callback
|
||||
dma2d_event_callback_t on_desc_done; // RX DONE event callback
|
||||
dma2d_event_callback_t on_desc_empty; // RX desc empty callback, trigger when buffer on dma is not sufficient.
|
||||
uint32_t bundled_tx_channel_mask; // Bit mask indicating the TX channels together with the RX channel to do the transaction
|
||||
};
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,702 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdatomic.h>
|
||||
#include <sys/cdefs.h>
|
||||
#include <sys/lock.h>
|
||||
#include "sdkconfig.h"
|
||||
#if CONFIG_DW_GDMA_ENABLE_DEBUG_LOG
|
||||
// The local log level must be defined before including esp_log.h
|
||||
// Set the maximum log level for this source file
|
||||
#define LOG_LOCAL_LEVEL ESP_LOG_DEBUG
|
||||
#endif
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/task.h"
|
||||
#include "soc/soc_caps.h"
|
||||
#include "soc/interrupts.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_check.h"
|
||||
#include "esp_intr_alloc.h"
|
||||
#include "esp_memory_utils.h"
|
||||
#include "esp_private/periph_ctrl.h"
|
||||
#include "esp_private/dw_gdma.h"
|
||||
#include "esp_private/critical_section.h"
|
||||
#include "hal/dw_gdma_hal.h"
|
||||
#include "hal/dw_gdma_ll.h"
|
||||
#include "hal/cache_hal.h"
|
||||
#include "hal/cache_ll.h"
|
||||
#include "esp_cache.h"
|
||||
|
||||
ESP_LOG_ATTR_TAG(TAG, "dw-gdma");
|
||||
|
||||
#if !SOC_RCC_IS_INDEPENDENT
|
||||
// Reset and Clock Control registers are mixing with other peripherals, so we need to use a critical section
|
||||
#define DW_GDMA_RCC_ATOMIC() PERIPH_RCC_ATOMIC()
|
||||
#else
|
||||
#define DW_GDMA_RCC_ATOMIC()
|
||||
#endif
|
||||
|
||||
#if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE
|
||||
#define DW_GDMA_GET_NON_CACHE_ADDR(addr) ((addr) ? CACHE_LL_L2MEM_NON_CACHE_ADDR(addr) : 0)
|
||||
#define DW_GDMA_GET_CACHE_ADDRESS(nc_addr) ((nc_addr) ? CACHE_LL_L2MEM_CACHE_ADDR(nc_addr) : 0)
|
||||
#else
|
||||
#define DW_GDMA_GET_NON_CACHE_ADDR(addr) (addr)
|
||||
#define DW_GDMA_GET_CACHE_ADDRESS(nc_addr) (nc_addr)
|
||||
#endif
|
||||
|
||||
#if CONFIG_DW_GDMA_OBJ_DRAM_SAFE
|
||||
#define DW_GDMA_MEM_ALLOC_CAPS (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)
|
||||
#else
|
||||
#define DW_GDMA_MEM_ALLOC_CAPS MALLOC_CAP_DEFAULT
|
||||
#endif
|
||||
|
||||
#if CONFIG_DW_GDMA_ISR_IRAM_SAFE
|
||||
#define DW_GDMA_INTR_ALLOC_FLAGS (ESP_INTR_FLAG_IRAM)
|
||||
#else
|
||||
#define DW_GDMA_INTR_ALLOC_FLAGS 0
|
||||
#endif
|
||||
|
||||
#define DW_GDMA_ALLOW_INTR_PRIORITY_MASK ESP_INTR_FLAG_LOWMED
|
||||
|
||||
#define ALIGN_UP(num, align) (((num) + ((align) - 1)) & ~((align) - 1))
|
||||
|
||||
typedef struct dw_gdma_group_t dw_gdma_group_t;
|
||||
typedef struct dw_gdma_channel_t dw_gdma_channel_t;
|
||||
|
||||
typedef struct dw_gdma_link_list_t {
|
||||
uint32_t num_items; // number of items in the link list
|
||||
dw_gdma_link_list_item_t *items; // pointer to the link list items
|
||||
dw_gdma_link_list_item_t *items_nc; // pointer to the link list items, non-cached
|
||||
} dw_gdma_link_list_t;
|
||||
|
||||
typedef struct {
|
||||
_lock_t mutex; // platform level mutex lock
|
||||
dw_gdma_group_t *groups[DW_GDMA_LL_GROUPS]; // array of DMA group instances
|
||||
int group_ref_counts[DW_GDMA_LL_GROUPS]; // reference count used to protect group install/uninstall
|
||||
} dw_gdma_platform_t;
|
||||
|
||||
struct dw_gdma_group_t {
|
||||
int group_id; // Group ID, index from 0
|
||||
dw_gdma_hal_context_t hal; // HAL instance is at group level
|
||||
int intr_priority; // all channels in the same group should share the same interrupt priority
|
||||
portMUX_TYPE spinlock; // group level spinlock, protect group level stuffs, e.g. hal object, pair handle slots and reference count of each pair
|
||||
dw_gdma_channel_t *channels[DW_GDMA_LL_CHANNELS_PER_GROUP]; // handles of DMA channels
|
||||
};
|
||||
|
||||
struct dw_gdma_channel_t {
|
||||
int chan_id; // channel ID, index from 0
|
||||
intr_handle_t intr; // per-channel interrupt handle
|
||||
portMUX_TYPE spinlock; // channel level spinlock
|
||||
dw_gdma_group_t *group; // pointer to the group which the channel belongs to
|
||||
void *user_data; // user registered DMA event data
|
||||
dw_gdma_event_callbacks_t cbs; // Event callbacks
|
||||
dw_gdma_block_transfer_type_t src_transfer_type; // transfer type for source
|
||||
dw_gdma_block_transfer_type_t dst_transfer_type; // transfer type for destination
|
||||
};
|
||||
|
||||
// dw_gdma driver platform
|
||||
static dw_gdma_platform_t s_platform;
|
||||
|
||||
static dw_gdma_group_t *dw_gdma_acquire_group_handle(int group_id)
|
||||
{
|
||||
bool new_group = false;
|
||||
dw_gdma_group_t *group = NULL;
|
||||
|
||||
// prevent install dw_gdma group concurrently
|
||||
_lock_acquire(&s_platform.mutex);
|
||||
if (!s_platform.groups[group_id]) {
|
||||
// The group is handle is not created yet
|
||||
group = heap_caps_calloc(1, sizeof(dw_gdma_group_t), DW_GDMA_MEM_ALLOC_CAPS);
|
||||
if (group) {
|
||||
new_group = true;
|
||||
s_platform.groups[group_id] = group;
|
||||
// enable APB to access DMA registers
|
||||
DW_GDMA_RCC_ATOMIC() {
|
||||
dw_gdma_ll_enable_bus_clock(group_id, true);
|
||||
dw_gdma_ll_reset_register(group_id);
|
||||
}
|
||||
// initialize the HAL context
|
||||
dw_gdma_hal_config_t hal_config = {};
|
||||
dw_gdma_hal_init(&group->hal, &hal_config);
|
||||
}
|
||||
} else {
|
||||
// the group is installed, we just retrieve it and increase the reference count
|
||||
group = s_platform.groups[group_id];
|
||||
}
|
||||
if (group) {
|
||||
// someone acquired the group handle means we have a new object that refer to this group
|
||||
s_platform.group_ref_counts[group_id]++;
|
||||
}
|
||||
_lock_release(&s_platform.mutex);
|
||||
|
||||
if (new_group) {
|
||||
portMUX_INITIALIZE(&group->spinlock);
|
||||
group->group_id = group_id;
|
||||
group->intr_priority = -1; // interrupt priority not assigned yet
|
||||
ESP_LOGD(TAG, "new group (%d) at %p", group_id, group);
|
||||
}
|
||||
|
||||
return group;
|
||||
}
|
||||
|
||||
static void dw_gdma_release_group_handle(dw_gdma_group_t *group)
|
||||
{
|
||||
int group_id = group->group_id;
|
||||
bool del_group = false;
|
||||
|
||||
_lock_acquire(&s_platform.mutex);
|
||||
s_platform.group_ref_counts[group_id]--;
|
||||
if (s_platform.group_ref_counts[group_id] == 0) {
|
||||
del_group = true;
|
||||
// the group now is not used by any channel, unregister it from the platform
|
||||
s_platform.groups[group_id] = NULL;
|
||||
// deinitialize the HAL context
|
||||
dw_gdma_hal_deinit(&group->hal);
|
||||
DW_GDMA_RCC_ATOMIC() {
|
||||
dw_gdma_ll_enable_bus_clock(group_id, false);
|
||||
}
|
||||
}
|
||||
_lock_release(&s_platform.mutex);
|
||||
|
||||
if (del_group) {
|
||||
free(group);
|
||||
ESP_LOGD(TAG, "delete group (%d)", group_id);
|
||||
}
|
||||
}
|
||||
|
||||
static esp_err_t channel_register_to_group(dw_gdma_channel_t *chan)
|
||||
{
|
||||
dw_gdma_group_t *group = NULL;
|
||||
int chan_id = -1;
|
||||
for (int i = 0; i < DW_GDMA_LL_GROUPS; i++) {
|
||||
group = dw_gdma_acquire_group_handle(i);
|
||||
ESP_RETURN_ON_FALSE(group, ESP_ERR_NO_MEM, TAG, "no mem for group(%d)", i);
|
||||
// loop to search free channel in the group
|
||||
esp_os_enter_critical(&group->spinlock);
|
||||
for (int j = 0; j < DW_GDMA_LL_CHANNELS_PER_GROUP; j++) {
|
||||
if (group->channels[j] == NULL) {
|
||||
group->channels[j] = chan;
|
||||
chan_id = j;
|
||||
break;
|
||||
}
|
||||
}
|
||||
esp_os_exit_critical(&group->spinlock);
|
||||
if (chan_id < 0) {
|
||||
dw_gdma_release_group_handle(group);
|
||||
} else {
|
||||
chan->group = group;
|
||||
chan->chan_id = chan_id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
ESP_RETURN_ON_FALSE(chan_id >= 0, ESP_ERR_NOT_FOUND, TAG, "no free channels");
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
static void channel_unregister_from_group(dw_gdma_channel_t *chan)
|
||||
{
|
||||
dw_gdma_group_t *group = chan->group;
|
||||
int chan_id = chan->chan_id;
|
||||
esp_os_enter_critical(&group->spinlock);
|
||||
group->channels[chan_id] = NULL;
|
||||
esp_os_exit_critical(&group->spinlock);
|
||||
// channel has a reference on group, release it now
|
||||
dw_gdma_release_group_handle(group);
|
||||
}
|
||||
|
||||
static esp_err_t channel_destroy(dw_gdma_channel_t *chan)
|
||||
{
|
||||
if (chan->group) {
|
||||
channel_unregister_from_group(chan);
|
||||
}
|
||||
if (chan->intr) {
|
||||
esp_intr_free(chan->intr);
|
||||
}
|
||||
free(chan);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_new_channel(const dw_gdma_channel_alloc_config_t *config, dw_gdma_channel_handle_t *ret_chan)
|
||||
{
|
||||
#if CONFIG_DW_GDMA_ENABLE_DEBUG_LOG
|
||||
esp_log_level_set(TAG, ESP_LOG_DEBUG);
|
||||
#endif
|
||||
esp_err_t ret = ESP_OK;
|
||||
dw_gdma_channel_t *chan = NULL;
|
||||
ESP_RETURN_ON_FALSE(config && ret_chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
ESP_RETURN_ON_FALSE(config->src.num_outstanding_requests >= 1 && config->src.num_outstanding_requests <= DW_GDMA_LL_MAX_OUTSTANDING_REQUESTS,
|
||||
ESP_ERR_INVALID_ARG, TAG, "invalid num_outstanding_requests");
|
||||
ESP_RETURN_ON_FALSE(config->dst.num_outstanding_requests >= 1 && config->dst.num_outstanding_requests <= DW_GDMA_LL_MAX_OUTSTANDING_REQUESTS,
|
||||
ESP_ERR_INVALID_ARG, TAG, "invalid num_outstanding_request");
|
||||
ESP_RETURN_ON_FALSE(config->chan_priority >= 0 && config->chan_priority < DW_GDMA_LL_CHANNELS_PER_GROUP,
|
||||
ESP_ERR_INVALID_ARG, TAG, "invalid channel priority");
|
||||
if (config->intr_priority) {
|
||||
ESP_RETURN_ON_FALSE(1 << (config->intr_priority) & DW_GDMA_ALLOW_INTR_PRIORITY_MASK, ESP_ERR_INVALID_ARG,
|
||||
TAG, "invalid interrupt priority:%d", config->intr_priority);
|
||||
}
|
||||
|
||||
chan = heap_caps_calloc(1, sizeof(dw_gdma_channel_t), DW_GDMA_MEM_ALLOC_CAPS);
|
||||
ESP_RETURN_ON_FALSE(chan, ESP_ERR_NO_MEM, TAG, "no mem for channel");
|
||||
// register channel to the group
|
||||
ESP_GOTO_ON_ERROR(channel_register_to_group(chan), err, TAG, "register to group failed");
|
||||
dw_gdma_group_t *group = chan->group;
|
||||
dw_gdma_hal_context_t *hal = &group->hal;
|
||||
int group_id = group->group_id;
|
||||
int chan_id = chan->chan_id;
|
||||
|
||||
// all channels in the same group should use the same interrupt priority
|
||||
bool intr_priority_conflict = false;
|
||||
esp_os_enter_critical(&group->spinlock);
|
||||
if (group->intr_priority == -1) {
|
||||
group->intr_priority = config->intr_priority;
|
||||
} else if (config->intr_priority != 0) {
|
||||
intr_priority_conflict = (group->intr_priority != config->intr_priority);
|
||||
}
|
||||
esp_os_exit_critical(&group->spinlock);
|
||||
ESP_GOTO_ON_FALSE(!intr_priority_conflict, ESP_ERR_INVALID_STATE, err, TAG, "intr_priority conflict, already is %d but attempt to %d", group->intr_priority, config->intr_priority);
|
||||
|
||||
// basic initialization
|
||||
portMUX_INITIALIZE(&chan->spinlock);
|
||||
chan->src_transfer_type = config->src.block_transfer_type;
|
||||
chan->dst_transfer_type = config->dst.block_transfer_type;
|
||||
// set transfer flow type
|
||||
dw_gdma_ll_channel_set_trans_flow(hal->dev, chan_id, config->src.role, config->dst.role, config->flow_controller);
|
||||
// set the transfer type for source and destination
|
||||
dw_gdma_ll_channel_set_src_multi_block_type(hal->dev, chan_id, config->src.block_transfer_type);
|
||||
dw_gdma_ll_channel_set_dst_multi_block_type(hal->dev, chan_id, config->dst.block_transfer_type);
|
||||
// set handshake interface
|
||||
dw_gdma_ll_channel_set_src_handshake_interface(hal->dev, chan_id, config->src.handshake_type);
|
||||
dw_gdma_ll_channel_set_dst_handshake_interface(hal->dev, chan_id, config->dst.handshake_type);
|
||||
// set handshake peripheral
|
||||
if (config->src.role != DW_GDMA_ROLE_MEM) {
|
||||
dw_gdma_ll_channel_set_src_handshake_periph(hal->dev, chan_id, config->src.role);
|
||||
}
|
||||
if (config->dst.role != DW_GDMA_ROLE_MEM) {
|
||||
dw_gdma_ll_channel_set_dst_handshake_periph(hal->dev, chan_id, config->dst.role);
|
||||
}
|
||||
// set channel priority
|
||||
dw_gdma_ll_channel_set_priority(hal->dev, chan_id, config->chan_priority);
|
||||
// set the outstanding request number
|
||||
dw_gdma_ll_channel_set_src_outstanding_limit(hal->dev, chan_id, config->src.num_outstanding_requests);
|
||||
dw_gdma_ll_channel_set_dst_outstanding_limit(hal->dev, chan_id, config->dst.num_outstanding_requests);
|
||||
// set the status fetch address
|
||||
dw_gdma_ll_channel_set_src_periph_status_addr(hal->dev, chan_id, config->src.status_fetch_addr);
|
||||
dw_gdma_ll_channel_set_dst_periph_status_addr(hal->dev, chan_id, config->dst.status_fetch_addr);
|
||||
|
||||
// enable all channel events (notes, they can't trigger an interrupt until `dw_gdma_ll_channel_enable_intr_propagation` is called)
|
||||
dw_gdma_ll_channel_enable_intr_generation(hal->dev, chan_id, UINT32_MAX, true);
|
||||
|
||||
ESP_LOGD(TAG, "new channel (%d,%d) at %p", group_id, chan_id, chan);
|
||||
*ret_chan = chan;
|
||||
return ESP_OK;
|
||||
err:
|
||||
if (chan) {
|
||||
channel_destroy(chan);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_del_channel(dw_gdma_channel_handle_t chan)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
dw_gdma_group_t *group = chan->group;
|
||||
int group_id = group->group_id;
|
||||
int chan_id = chan->chan_id;
|
||||
ESP_LOGD(TAG, "del channel (%d,%d)", group_id, chan_id);
|
||||
// recycle memory resource
|
||||
ESP_RETURN_ON_ERROR(channel_destroy(chan), TAG, "destroy channel failed");
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_channel_enable_ctrl(dw_gdma_channel_handle_t chan, bool en_or_dis)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
dw_gdma_hal_context_t *hal = &chan->group->hal;
|
||||
int chan_id = chan->chan_id;
|
||||
// the atomic is ensured by the hardware, so no lock is needed here
|
||||
dw_gdma_ll_channel_enable(hal->dev, chan_id, en_or_dis);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_channel_suspend_ctrl(dw_gdma_channel_handle_t chan, bool enter_or_exit)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
dw_gdma_hal_context_t *hal = &chan->group->hal;
|
||||
int chan_id = chan->chan_id;
|
||||
// the atomic is ensured by the hardware, so no lock is needed here
|
||||
dw_gdma_ll_channel_suspend(hal->dev, chan_id, enter_or_exit);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_channel_abort(dw_gdma_channel_handle_t chan)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
dw_gdma_hal_context_t *hal = &chan->group->hal;
|
||||
int chan_id = chan->chan_id;
|
||||
// the atomic is ensured by the hardware, so no lock is needed here
|
||||
dw_gdma_ll_channel_abort(hal->dev, chan_id);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_channel_lock(dw_gdma_channel_handle_t chan, dw_gdma_lock_level_t level)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
dw_gdma_hal_context_t *hal = &chan->group->hal;
|
||||
int chan_id = chan->chan_id;
|
||||
|
||||
// the lock control bit is located in a cfg register, with other configuration bits
|
||||
esp_os_enter_critical(&chan->spinlock);
|
||||
dw_gdma_ll_channel_lock(hal->dev, chan_id, level);
|
||||
esp_os_exit_critical(&chan->spinlock);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_channel_unlock(dw_gdma_channel_handle_t chan)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
dw_gdma_hal_context_t *hal = &chan->group->hal;
|
||||
int chan_id = chan->chan_id;
|
||||
|
||||
// the lock control bit is located in a cfg register, with other configuration bits
|
||||
esp_os_enter_critical(&chan->spinlock);
|
||||
dw_gdma_ll_channel_unlock(hal->dev, chan_id);
|
||||
esp_os_exit_critical(&chan->spinlock);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_channel_continue(dw_gdma_channel_handle_t chan)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
dw_gdma_hal_context_t *hal = &chan->group->hal;
|
||||
int chan_id = chan->chan_id;
|
||||
// the atomic is ensured by the hardware, so no lock is needed here
|
||||
dw_gdma_ll_channel_resume_multi_block_transfer(hal->dev, chan_id);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_new_link_list(const dw_gdma_link_list_config_t *config, dw_gdma_link_list_handle_t *ret_list)
|
||||
{
|
||||
esp_err_t ret = ESP_OK;
|
||||
ESP_RETURN_ON_FALSE(config && ret_list, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
dw_gdma_link_list_item_t *items = NULL;
|
||||
dw_gdma_link_list_t *list = NULL;
|
||||
uint32_t num_items = config->num_items;
|
||||
list = heap_caps_calloc(1, sizeof(dw_gdma_link_list_t), DW_GDMA_MEM_ALLOC_CAPS);
|
||||
ESP_GOTO_ON_FALSE(list, ESP_ERR_NO_MEM, err, TAG, "no mem for link list");
|
||||
// allocate memory for link list items, from internal memory
|
||||
// the link list items has its own alignment requirement, the heap allocator can help handle the cache alignment as well
|
||||
items = heap_caps_aligned_calloc(DW_GDMA_LL_LINK_LIST_ALIGNMENT, num_items, sizeof(dw_gdma_link_list_item_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA);
|
||||
ESP_GOTO_ON_FALSE(items, ESP_ERR_NO_MEM, err, TAG, "no mem for link list items");
|
||||
// do memory sync when the link list items are cached
|
||||
uint32_t data_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA);
|
||||
if (data_cache_line_size) {
|
||||
// write back and then invalidate the cache, because later we will read/write the link list items by non-cacheable address
|
||||
ESP_GOTO_ON_ERROR(esp_cache_msync(items, num_items * sizeof(dw_gdma_link_list_item_t),
|
||||
ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_INVALIDATE | ESP_CACHE_MSYNC_FLAG_UNALIGNED),
|
||||
err, TAG, "cache sync failed");
|
||||
}
|
||||
|
||||
list->num_items = num_items;
|
||||
list->items = items;
|
||||
// want to use non-cached address to operate the link list items
|
||||
list->items_nc = (dw_gdma_link_list_item_t *)DW_GDMA_GET_NON_CACHE_ADDR(items);
|
||||
|
||||
// set up the link list
|
||||
for (size_t i = 0; i < num_items; i++) {
|
||||
dw_gdma_ll_lli_set_next_item_addr(list->items_nc + i, (uint32_t)(list->items + i + 1));
|
||||
// set master port for the link list
|
||||
dw_gdma_ll_lli_set_link_list_master_port(list->items_nc + i, DW_GDMA_LL_MASTER_PORT_MEMORY);
|
||||
}
|
||||
switch (config->link_type) {
|
||||
case DW_GDMA_LINKED_LIST_TYPE_CIRCULAR:
|
||||
dw_gdma_ll_lli_set_next_item_addr(list->items_nc + num_items - 1, (uint32_t)(list->items));
|
||||
break;
|
||||
case DW_GDMA_LINKED_LIST_TYPE_SINGLY:
|
||||
dw_gdma_ll_lli_set_next_item_addr(list->items_nc + num_items - 1, 0);
|
||||
break;
|
||||
}
|
||||
|
||||
ESP_LOGD(TAG, "new link list @%p, items @%p", list, items);
|
||||
*ret_list = list;
|
||||
return ESP_OK;
|
||||
err:
|
||||
if (list) {
|
||||
free(list);
|
||||
}
|
||||
if (items) {
|
||||
free(items);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_del_link_list(dw_gdma_link_list_handle_t list)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(list, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
ESP_LOGD(TAG, "del link list at %p", list);
|
||||
free(list->items);
|
||||
free(list);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_channel_use_link_list(dw_gdma_channel_handle_t chan, dw_gdma_link_list_handle_t list)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(chan && list, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
ESP_RETURN_ON_FALSE(chan->src_transfer_type == DW_GDMA_BLOCK_TRANSFER_LIST ||
|
||||
chan->dst_transfer_type == DW_GDMA_BLOCK_TRANSFER_LIST,
|
||||
ESP_ERR_INVALID_STATE, TAG, "invalid transfer type");
|
||||
dw_gdma_hal_context_t *hal = &chan->group->hal;
|
||||
int chan_id = chan->chan_id;
|
||||
|
||||
// set master port for the link list
|
||||
dw_gdma_ll_channel_set_link_list_master_port(hal->dev, chan_id, DW_GDMA_LL_MASTER_PORT_MEMORY);
|
||||
// set the link list head address
|
||||
dw_gdma_ll_channel_set_link_list_head_addr(hal->dev, chan_id, (uint32_t)(list->items));
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
dw_gdma_lli_handle_t dw_gdma_link_list_get_item(dw_gdma_link_list_handle_t list, int item_index)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE_ISR(list, NULL, TAG, "invalid argument");
|
||||
ESP_RETURN_ON_FALSE_ISR(item_index < list->num_items, NULL, TAG, "invalid item index");
|
||||
// Note: the returned address is non-cached
|
||||
dw_gdma_link_list_item_t *lli = list->items_nc + item_index;
|
||||
return lli;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_lli_set_next(dw_gdma_lli_handle_t lli, dw_gdma_lli_handle_t next)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(lli && next, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
|
||||
// the next field must use a cached address, so convert it to a cached address
|
||||
dw_gdma_ll_lli_set_next_item_addr(lli, DW_GDMA_GET_CACHE_ADDRESS(next));
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_channel_config_transfer(dw_gdma_channel_handle_t chan, const dw_gdma_block_transfer_config_t *config)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(chan && config, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
ESP_RETURN_ON_FALSE(chan->src_transfer_type != DW_GDMA_BLOCK_TRANSFER_LIST &&
|
||||
chan->dst_transfer_type != DW_GDMA_BLOCK_TRANSFER_LIST,
|
||||
ESP_ERR_INVALID_STATE, TAG, "invalid transfer type");
|
||||
dw_gdma_hal_context_t *hal = &chan->group->hal;
|
||||
int chan_id = chan->chan_id;
|
||||
|
||||
// set memory address
|
||||
dw_gdma_ll_channel_set_src_addr(hal->dev, chan_id, config->src.addr);
|
||||
dw_gdma_ll_channel_set_dst_addr(hal->dev, chan_id, config->dst.addr);
|
||||
// transfer size
|
||||
dw_gdma_ll_channel_set_trans_block_size(hal->dev, chan_id, config->size);
|
||||
// [Ctrl0] register
|
||||
// set master port for the source and destination target
|
||||
dw_gdma_ll_channel_set_src_master_port(hal->dev, chan_id, config->src.addr);
|
||||
dw_gdma_ll_channel_set_dst_master_port(hal->dev, chan_id, config->dst.addr);
|
||||
// transfer width
|
||||
dw_gdma_ll_channel_set_src_trans_width(hal->dev, chan_id, config->src.width);
|
||||
dw_gdma_ll_channel_set_dst_trans_width(hal->dev, chan_id, config->dst.width);
|
||||
// set burst items
|
||||
dw_gdma_ll_channel_set_src_burst_items(hal->dev, chan_id, config->src.burst_items);
|
||||
dw_gdma_ll_channel_set_dst_burst_items(hal->dev, chan_id, config->dst.burst_items);
|
||||
// set burst mode
|
||||
dw_gdma_ll_channel_set_src_burst_mode(hal->dev, chan_id, config->src.burst_mode);
|
||||
dw_gdma_ll_channel_set_dst_burst_mode(hal->dev, chan_id, config->dst.burst_mode);
|
||||
// [Ctrl1] register
|
||||
// set burst length
|
||||
dw_gdma_ll_channel_set_src_burst_len(hal->dev, chan_id, config->src.burst_len);
|
||||
dw_gdma_ll_channel_set_dst_burst_len(hal->dev, chan_id, config->dst.burst_len);
|
||||
// whether to enable the peripheral status write back
|
||||
dw_gdma_ll_channel_enable_src_periph_status_write_back(hal->dev, chan_id, config->src.flags.en_status_write_back);
|
||||
dw_gdma_ll_channel_enable_dst_periph_status_write_back(hal->dev, chan_id, config->dst.flags.en_status_write_back);
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_channel_set_block_markers(dw_gdma_channel_handle_t chan, dw_gdma_block_markers_t markers)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE_ISR(chan, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
ESP_RETURN_ON_FALSE_ISR(chan->src_transfer_type != DW_GDMA_BLOCK_TRANSFER_LIST &&
|
||||
chan->dst_transfer_type != DW_GDMA_BLOCK_TRANSFER_LIST,
|
||||
ESP_ERR_INVALID_STATE, TAG, "invalid transfer type");
|
||||
dw_gdma_hal_context_t *hal = &chan->group->hal;
|
||||
int chan_id = chan->chan_id;
|
||||
|
||||
// [Ctrl1] register
|
||||
// set the block markers
|
||||
dw_gdma_ll_channel_set_block_markers(hal->dev, chan_id, markers.en_trans_done_intr, markers.is_last, markers.is_valid);
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_lli_config_transfer(dw_gdma_lli_handle_t lli, const dw_gdma_block_transfer_config_t *config)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(lli && config, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
|
||||
// set memory address
|
||||
dw_gdma_ll_lli_set_src_addr(lli, config->src.addr);
|
||||
dw_gdma_ll_lli_set_dst_addr(lli, config->dst.addr);
|
||||
// transfer size
|
||||
dw_gdma_ll_lli_set_trans_block_size(lli, config->size);
|
||||
// [Ctrl0] register
|
||||
// set master port for the source and destination target
|
||||
dw_gdma_ll_lli_set_src_master_port(lli, config->src.addr);
|
||||
dw_gdma_ll_lli_set_dst_master_port(lli, config->dst.addr);
|
||||
// transfer width
|
||||
dw_gdma_ll_lli_set_src_trans_width(lli, config->src.width);
|
||||
dw_gdma_ll_lli_set_dst_trans_width(lli, config->dst.width);
|
||||
// set burst items
|
||||
dw_gdma_ll_lli_set_src_burst_items(lli, config->src.burst_items);
|
||||
dw_gdma_ll_lli_set_dst_burst_items(lli, config->dst.burst_items);
|
||||
// set burst mode
|
||||
dw_gdma_ll_lli_set_src_burst_mode(lli, config->src.burst_mode);
|
||||
dw_gdma_ll_lli_set_dst_burst_mode(lli, config->dst.burst_mode);
|
||||
// [Ctrl1] register
|
||||
// set burst length
|
||||
dw_gdma_ll_lli_set_src_burst_len(lli, config->src.burst_len);
|
||||
dw_gdma_ll_lli_set_dst_burst_len(lli, config->dst.burst_len);
|
||||
// whether to enable the peripheral status write back
|
||||
dw_gdma_ll_lli_enable_src_periph_status_write_back(lli, config->src.flags.en_status_write_back);
|
||||
dw_gdma_ll_lli_enable_dst_periph_status_write_back(lli, config->dst.flags.en_status_write_back);
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_lli_set_block_markers(dw_gdma_lli_handle_t lli, dw_gdma_block_markers_t markers)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE_ISR(lli, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
|
||||
// [Ctrl1] register
|
||||
// set the block markers
|
||||
dw_gdma_ll_lli_set_block_markers(lli, markers.en_trans_done_intr, markers.is_last, markers.is_valid);
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
void dw_gdma_channel_default_isr(void *args)
|
||||
{
|
||||
dw_gdma_channel_t *chan = (dw_gdma_channel_t *)args;
|
||||
dw_gdma_group_t *group = chan->group;
|
||||
dw_gdma_hal_context_t *hal = &group->hal;
|
||||
int chan_id = chan->chan_id;
|
||||
bool need_yield = false;
|
||||
// clear pending interrupt event
|
||||
uint32_t intr_status = dw_gdma_ll_channel_get_intr_status(hal->dev, chan_id);
|
||||
dw_gdma_ll_channel_clear_intr(hal->dev, chan_id, intr_status);
|
||||
|
||||
// call user callbacks
|
||||
if (intr_status & DW_GDMA_LL_CHANNEL_EVENT_SHADOWREG_OR_LLI_INVALID_ERR) {
|
||||
if (chan->cbs.on_invalid_block) {
|
||||
intptr_t invalid_lli_addr = dw_gdma_ll_channel_get_current_link_list_item_addr(hal->dev, chan_id);
|
||||
dw_gdma_break_event_data_t edata = {
|
||||
.invalid_lli = (dw_gdma_lli_handle_t)DW_GDMA_GET_NON_CACHE_ADDR(invalid_lli_addr),
|
||||
};
|
||||
if (chan->cbs.on_invalid_block(chan, &edata, chan->user_data)) {
|
||||
need_yield = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (intr_status & DW_GDMA_LL_CHANNEL_EVENT_BLOCK_TFR_DONE) {
|
||||
if (chan->cbs.on_block_trans_done) {
|
||||
dw_gdma_trans_done_event_data_t edata = {};
|
||||
if (chan->cbs.on_block_trans_done(chan, &edata, chan->user_data)) {
|
||||
need_yield = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (intr_status & DW_GDMA_LL_CHANNEL_EVENT_DMA_TFR_DONE) {
|
||||
if (chan->cbs.on_full_trans_done) {
|
||||
dw_gdma_trans_done_event_data_t edata = {};
|
||||
if (chan->cbs.on_full_trans_done(chan, &edata, chan->user_data)) {
|
||||
need_yield = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (need_yield) {
|
||||
portYIELD_FROM_ISR();
|
||||
}
|
||||
}
|
||||
|
||||
static esp_err_t dw_gdma_install_channel_interrupt(dw_gdma_channel_t *chan)
|
||||
{
|
||||
esp_err_t ret = ESP_OK;
|
||||
dw_gdma_group_t *group = chan->group;
|
||||
dw_gdma_hal_context_t *hal = &group->hal;
|
||||
int chan_id = chan->chan_id;
|
||||
// clear pending events
|
||||
dw_gdma_ll_channel_enable_intr_propagation(hal->dev, chan_id, UINT32_MAX, false);
|
||||
dw_gdma_ll_channel_clear_intr(hal->dev, chan_id, UINT32_MAX);
|
||||
|
||||
// pre-alloc a interrupt handle, with handler disabled
|
||||
// DW_GDMA multiple channels share the same interrupt source, so we use a shared interrupt handle
|
||||
intr_handle_t intr = NULL;
|
||||
int isr_flags = DW_GDMA_INTR_ALLOC_FLAGS | ESP_INTR_FLAG_SHARED;
|
||||
if (group->intr_priority) {
|
||||
isr_flags |= 1 << (group->intr_priority);
|
||||
} else {
|
||||
isr_flags |= DW_GDMA_ALLOW_INTR_PRIORITY_MASK;
|
||||
}
|
||||
ret = esp_intr_alloc_intrstatus(ETS_DW_GDMA_INTR_SOURCE, isr_flags,
|
||||
(uint32_t)dw_gdma_ll_get_intr_status_reg(hal->dev), DW_GDMA_LL_CHANNEL_EVENT_MASK(chan_id),
|
||||
dw_gdma_channel_default_isr, chan, &intr);
|
||||
ESP_RETURN_ON_ERROR(ret, TAG, "alloc interrupt failed");
|
||||
|
||||
ESP_LOGD(TAG, "install interrupt service for channel (%d,%d)", group->group_id, chan_id);
|
||||
chan->intr = intr;
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_channel_register_event_callbacks(dw_gdma_channel_handle_t chan, dw_gdma_event_callbacks_t *cbs, void *user_data)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(chan && cbs, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
dw_gdma_group_t *group = chan->group;
|
||||
dw_gdma_hal_context_t *hal = &group->hal;
|
||||
int chan_id = chan->chan_id;
|
||||
|
||||
#if CONFIG_DW_GDMA_ISR_IRAM_SAFE
|
||||
if (cbs->on_block_trans_done) {
|
||||
ESP_RETURN_ON_FALSE(esp_ptr_in_iram(cbs->on_block_trans_done), ESP_ERR_INVALID_ARG,
|
||||
TAG, "on_block_trans_done not in IRAM");
|
||||
}
|
||||
if (cbs->on_full_trans_done) {
|
||||
ESP_RETURN_ON_FALSE(esp_ptr_in_iram(cbs->on_full_trans_done), ESP_ERR_INVALID_ARG,
|
||||
TAG, "on_full_trans_done not in IRAM");
|
||||
}
|
||||
if (cbs->on_invalid_block) {
|
||||
ESP_RETURN_ON_FALSE(esp_ptr_in_iram(cbs->on_invalid_block), ESP_ERR_INVALID_ARG,
|
||||
TAG, "on_invalid_block not in IRAM");
|
||||
}
|
||||
if (user_data) {
|
||||
ESP_RETURN_ON_FALSE(esp_ptr_internal(user_data), ESP_ERR_INVALID_ARG,
|
||||
TAG, "user context not in internal RAM");
|
||||
}
|
||||
#endif // CONFIG_DW_GDMA_ISR_IRAM_SAFE
|
||||
|
||||
// lazy install interrupt service
|
||||
if (!chan->intr) {
|
||||
ESP_RETURN_ON_ERROR(dw_gdma_install_channel_interrupt(chan), TAG, "install interrupt service failed");
|
||||
}
|
||||
|
||||
// enable the event to be able to trigger an interrupt
|
||||
dw_gdma_ll_channel_enable_intr_propagation(hal->dev, chan_id, DW_GDMA_LL_CHANNEL_EVENT_BLOCK_TFR_DONE, cbs->on_block_trans_done != NULL);
|
||||
dw_gdma_ll_channel_enable_intr_propagation(hal->dev, chan_id, DW_GDMA_LL_CHANNEL_EVENT_DMA_TFR_DONE, cbs->on_full_trans_done != NULL);
|
||||
dw_gdma_ll_channel_enable_intr_propagation(hal->dev, chan_id, DW_GDMA_LL_CHANNEL_EVENT_SHADOWREG_OR_LLI_INVALID_ERR, cbs->on_invalid_block != NULL);
|
||||
|
||||
chan->user_data = user_data;
|
||||
memcpy(&chan->cbs, cbs, sizeof(dw_gdma_event_callbacks_t));
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t dw_gdma_channel_get_id(dw_gdma_channel_handle_t chan, int *channel_id)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(chan && channel_id, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
*channel_id = chan->chan_id;
|
||||
return ESP_OK;
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include "esp_check.h"
|
||||
#include "esp_async_memcpy.h"
|
||||
#include "esp_async_memcpy_priv.h"
|
||||
|
||||
ESP_LOG_ATTR_TAG(TAG, "async_mcp");
|
||||
|
||||
esp_err_t esp_async_memcpy_uninstall(async_memcpy_handle_t asmcp)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(asmcp, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
return asmcp->del(asmcp);
|
||||
}
|
||||
|
||||
esp_err_t esp_async_memcpy(async_memcpy_handle_t asmcp, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(asmcp && dst && src && n, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
return asmcp->memcpy(asmcp, dst, src, n, cb_isr, cb_args);
|
||||
}
|
||||
|
||||
#if SOC_ETM_SUPPORTED
|
||||
esp_err_t esp_async_memcpy_new_etm_event(async_memcpy_handle_t asmcp, async_memcpy_etm_event_t event_type, esp_etm_event_handle_t *out_event)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(asmcp && out_event, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
ESP_RETURN_ON_FALSE(asmcp->new_etm_event, ESP_ERR_NOT_SUPPORTED, TAG, "ETM is not supported");
|
||||
return asmcp->new_etm_event(asmcp, event_type, out_event);
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,44 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020-2023 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include "esp_err.h"
|
||||
#include "esp_etm.h"
|
||||
#include "esp_async_memcpy.h"
|
||||
#include "soc/soc_caps.h"
|
||||
|
||||
#define DEFAULT_TRANSACTION_QUEUE_LENGTH 4
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef enum {
|
||||
MCP_FSM_IDLE_WAIT, /// intermediate state, for state changes from others to IDLE
|
||||
MCP_FSM_IDLE,
|
||||
MCP_FSM_RUN_WAIT, /// intermediate state, for state changes from others to RUN
|
||||
MCP_FSM_RUN,
|
||||
} async_memcpy_fsm_t;
|
||||
|
||||
typedef struct async_memcpy_context_t async_memcpy_context_t;
|
||||
|
||||
struct async_memcpy_context_t {
|
||||
/// @brief Start a new async memcpy transaction
|
||||
esp_err_t (*memcpy)(async_memcpy_context_t *ctx, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args);
|
||||
#if SOC_ETM_SUPPORTED
|
||||
/// @brief Create ETM event handle of specific event type
|
||||
esp_err_t (*new_etm_event)(async_memcpy_context_t *ctx, async_memcpy_etm_event_t event_type, esp_etm_event_handle_t *out_event);
|
||||
#endif // SOC_ETM_SUPPORTED
|
||||
/// @brief Delete async memcpy driver context
|
||||
esp_err_t (*del)(async_memcpy_context_t *ctx);
|
||||
};
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,160 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <sys/param.h>
|
||||
#include <inttypes.h>
|
||||
#include <string.h>
|
||||
#include "sdkconfig.h"
|
||||
#include "esp_check.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_heap_caps.h"
|
||||
#include "esp_memory_utils.h"
|
||||
#include "esp_dma_utils.h"
|
||||
#include "esp_private/esp_dma_utils.h"
|
||||
#include "esp_private/esp_cache_private.h"
|
||||
#include "soc/soc_caps.h"
|
||||
#include "hal/hal_utils.h"
|
||||
#include "hal/cache_hal.h"
|
||||
#include "hal/cache_ll.h"
|
||||
#include "esp_cache.h"
|
||||
#if CONFIG_SPIRAM
|
||||
#include "esp_private/esp_psram_mspi.h"
|
||||
#endif
|
||||
|
||||
ESP_LOG_ATTR_TAG(TAG, "dma_utils");
|
||||
|
||||
#define ALIGN_UP_BY(num, align) (((num) + ((align) - 1)) & ~((align) - 1))
|
||||
|
||||
esp_err_t esp_dma_split_rx_buffer_to_cache_aligned(void *rx_buffer, size_t buffer_len, dma_buffer_split_array_t *align_buf_array, uint8_t** ret_stash_buffer)
|
||||
{
|
||||
esp_err_t ret = ESP_OK;
|
||||
uint8_t* stash_buffer = NULL;
|
||||
ESP_RETURN_ON_FALSE_ISR(rx_buffer && buffer_len && align_buf_array, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
|
||||
// read the cache line size of internal and external memory, we also use this information to check if a given memory is behind the cache
|
||||
size_t int_mem_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA);
|
||||
size_t ext_mem_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_DATA);
|
||||
|
||||
size_t split_line_size = 0;
|
||||
if (esp_ptr_external_ram(rx_buffer)) {
|
||||
split_line_size = ext_mem_cache_line_size;
|
||||
} else if (esp_ptr_internal(rx_buffer)) {
|
||||
split_line_size = int_mem_cache_line_size;
|
||||
}
|
||||
bool align_required = split_line_size > 0;
|
||||
ESP_EARLY_LOGV(TAG, "split_line_size:%zu", split_line_size);
|
||||
|
||||
if (*ret_stash_buffer == NULL) {
|
||||
// If the stash buffer is not offered by the caller, allocate the stash buffer from internal RAM
|
||||
// Note, the split_line_size can be 0, in this case, the stash_buffer is also NULL, which is fine
|
||||
stash_buffer = heap_caps_calloc(2, split_line_size, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
|
||||
ESP_RETURN_ON_FALSE_ISR(!(split_line_size && !stash_buffer), ESP_ERR_NO_MEM, TAG, "no mem for stash buffer");
|
||||
} else {
|
||||
// If the stash buffer is offered by the caller, check if it is aligned
|
||||
ESP_RETURN_ON_FALSE_ISR(split_line_size == 0 || (uintptr_t)(*ret_stash_buffer) % split_line_size == 0,
|
||||
ESP_ERR_INVALID_ARG, TAG, "the offered stash buffer is not aligned");
|
||||
// If the stash buffer is offered by the caller, use it
|
||||
stash_buffer = *ret_stash_buffer;
|
||||
}
|
||||
|
||||
// clear align_array to avoid garbage data
|
||||
memset(align_buf_array, 0, sizeof(dma_buffer_split_array_t));
|
||||
bool need_cache_sync[3] = {false};
|
||||
|
||||
// if align_required, split the buffer into head, body and tail
|
||||
if (align_required) {
|
||||
// calculate head_overflow_len
|
||||
size_t head_overflow_len = (uintptr_t)rx_buffer % split_line_size;
|
||||
head_overflow_len = head_overflow_len ? split_line_size - head_overflow_len : 0;
|
||||
ESP_EARLY_LOGV(TAG, "head_addr:%p head_overflow_len:%zu", rx_buffer, head_overflow_len);
|
||||
// calculate tail_overflow_len
|
||||
size_t tail_overflow_len = ((uintptr_t)rx_buffer + buffer_len) % split_line_size;
|
||||
ESP_EARLY_LOGV(TAG, "tail_addr:%p tail_overflow_len:%zu", rx_buffer + buffer_len - tail_overflow_len, tail_overflow_len);
|
||||
|
||||
// special handling when input_buffer length is no more than buffer alignment
|
||||
bool is_small_buf = head_overflow_len >= buffer_len || tail_overflow_len >= buffer_len;
|
||||
uint8_t extra_buf_count = 0;
|
||||
uint8_t* input_buffer = (uint8_t*)rx_buffer;
|
||||
if (head_overflow_len || is_small_buf) {
|
||||
align_buf_array->buf.head.recovery_address = input_buffer;
|
||||
align_buf_array->buf.head.aligned_buffer = stash_buffer + split_line_size * extra_buf_count++;
|
||||
align_buf_array->buf.head.length = is_small_buf ? buffer_len : head_overflow_len;
|
||||
need_cache_sync[0] = int_mem_cache_line_size > 0;
|
||||
}
|
||||
int body_len = (int)buffer_len - (int)head_overflow_len - (int)tail_overflow_len;
|
||||
if (body_len > 0) {
|
||||
align_buf_array->buf.body.recovery_address = input_buffer + head_overflow_len;
|
||||
align_buf_array->buf.body.aligned_buffer = input_buffer + head_overflow_len;
|
||||
align_buf_array->buf.body.length = body_len;
|
||||
need_cache_sync[1] = true;
|
||||
}
|
||||
if (tail_overflow_len && !is_small_buf) {
|
||||
align_buf_array->buf.tail.recovery_address = input_buffer + buffer_len - tail_overflow_len;
|
||||
align_buf_array->buf.tail.aligned_buffer = stash_buffer + split_line_size * extra_buf_count++;
|
||||
align_buf_array->buf.tail.length = tail_overflow_len;
|
||||
need_cache_sync[2] = int_mem_cache_line_size > 0;
|
||||
}
|
||||
} else {
|
||||
align_buf_array->buf.body.aligned_buffer = rx_buffer;
|
||||
align_buf_array->buf.body.recovery_address = rx_buffer;
|
||||
align_buf_array->buf.body.length = buffer_len;
|
||||
}
|
||||
|
||||
// invalidate the aligned buffer if necessary
|
||||
for (int i = 0; i < 3; i++) {
|
||||
size_t sync_size = align_buf_array->aligned_buffer[i].length;
|
||||
if (need_cache_sync[i] && sync_size > 0) {
|
||||
if (sync_size < split_line_size) {
|
||||
// If the buffer is smaller than the cache line size, we need to sync the whole buffer
|
||||
sync_size = split_line_size;
|
||||
}
|
||||
esp_err_t res = esp_cache_msync(align_buf_array->aligned_buffer[i].aligned_buffer, sync_size, ESP_CACHE_MSYNC_FLAG_DIR_M2C);
|
||||
ESP_GOTO_ON_ERROR_ISR(res, err, TAG, "failed to do cache sync");
|
||||
}
|
||||
}
|
||||
|
||||
*ret_stash_buffer = stash_buffer;
|
||||
return ESP_OK;
|
||||
err:
|
||||
// Only free the stash buffer if it is not offered by the caller
|
||||
if (stash_buffer && *ret_stash_buffer == NULL) {
|
||||
free(stash_buffer);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
esp_err_t esp_dma_merge_aligned_rx_buffers(dma_buffer_split_array_t *align_array)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE_ISR(align_array, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
|
||||
#if CONFIG_SPIRAM
|
||||
// check if the original buffer is in external RAM, if so, add memory barrier
|
||||
if (esp_ptr_external_ram(align_array->buf.head.recovery_address) ||
|
||||
esp_ptr_external_ram(align_array->buf.body.recovery_address) ||
|
||||
esp_ptr_external_ram(align_array->buf.tail.recovery_address)) {
|
||||
esp_psram_mspi_mb();
|
||||
}
|
||||
#endif
|
||||
|
||||
// only need to copy the head and tail buffer
|
||||
if (align_array->buf.head.length) {
|
||||
memcpy(align_array->buf.head.recovery_address, align_array->buf.head.aligned_buffer, align_array->buf.head.length);
|
||||
}
|
||||
if (align_array->buf.tail.length) {
|
||||
memcpy(align_array->buf.tail.recovery_address, align_array->buf.tail.aligned_buffer, align_array->buf.tail.length);
|
||||
}
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
size_t esp_dma_calculate_node_count(size_t buffer_size, size_t buffer_alignment, size_t max_buffer_size_per_node)
|
||||
{
|
||||
// buffer_alignment should be power of 2
|
||||
ESP_RETURN_ON_FALSE(buffer_alignment && ((buffer_alignment & (buffer_alignment - 1)) == 0), 0, TAG, "invalid buffer alignment");
|
||||
// align down the max_buffer_size_per_node
|
||||
max_buffer_size_per_node = max_buffer_size_per_node & ~(buffer_alignment - 1);
|
||||
// calculate the number of nodes
|
||||
return (buffer_size + max_buffer_size_per_node - 1) / max_buffer_size_per_node;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,58 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include "gdma_priv.h"
|
||||
|
||||
ESP_LOG_ATTR_TAG(TAG, "gdma");
|
||||
|
||||
esp_err_t gdma_config_crc_calculator(gdma_channel_handle_t dma_chan, const gdma_crc_calculator_config_t *config)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(dma_chan && config, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
gdma_pair_t *pair = dma_chan->pair;
|
||||
gdma_group_t *group = pair->group;
|
||||
gdma_hal_context_t *hal = &group->hal;
|
||||
switch (group->bus_id) {
|
||||
#if SOC_HAS(AHB_GDMA)
|
||||
case SOC_GDMA_BUS_AHB:
|
||||
ESP_RETURN_ON_FALSE(config->crc_bit_width <= GDMA_LL_AHB_MAX_CRC_BIT_WIDTH, ESP_ERR_INVALID_ARG, TAG, "invalid crc bit width");
|
||||
break;
|
||||
#endif // SOC_HAS(AHB_GDMA)
|
||||
#if SOC_HAS(AXI_GDMA)
|
||||
case SOC_GDMA_BUS_AXI:
|
||||
ESP_RETURN_ON_FALSE(config->crc_bit_width <= GDMA_LL_AXI_MAX_CRC_BIT_WIDTH, ESP_ERR_INVALID_ARG, TAG, "invalid crc bit width");
|
||||
break;
|
||||
#endif // SOC_HAS(AXI_GDMA)
|
||||
default:
|
||||
ESP_LOGE(TAG, "invalid bus id: %d", group->bus_id);
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
|
||||
// clear the previous CRC result
|
||||
gdma_hal_clear_crc(hal, pair->pair_id, dma_chan->direction);
|
||||
|
||||
// set polynomial and initial value
|
||||
gdma_hal_crc_config_t hal_config = {
|
||||
.crc_bit_width = config->crc_bit_width,
|
||||
.poly_hex = config->poly_hex,
|
||||
.init_value = config->init_value,
|
||||
.reverse_data_mask = config->reverse_data_mask,
|
||||
};
|
||||
gdma_hal_set_crc_poly(hal, pair->pair_id, dma_chan->direction, &hal_config);
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t gdma_crc_get_result(gdma_channel_handle_t dma_chan, uint32_t *result)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(dma_chan && result, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
gdma_pair_t *pair = dma_chan->pair;
|
||||
gdma_group_t *group = pair->group;
|
||||
gdma_hal_context_t *hal = &group->hal;
|
||||
|
||||
*result = gdma_hal_get_crc_result(hal, pair->pair_id, dma_chan->direction);
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include "gdma_priv.h"
|
||||
#include "esp_private/etm_interface.h"
|
||||
|
||||
#define ETM_MEM_ALLOC_CAPS MALLOC_CAP_DEFAULT
|
||||
|
||||
ESP_LOG_ATTR_TAG(TAG, "gdma");
|
||||
|
||||
typedef struct gdma_etm_task_t {
|
||||
esp_etm_task_t base;
|
||||
gdma_channel_t *chan;
|
||||
} gdma_etm_task_t;
|
||||
|
||||
static esp_err_t gdma_del_etm_event(esp_etm_event_t *event)
|
||||
{
|
||||
free(event);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
static esp_err_t gdma_del_etm_task(esp_etm_task_t *task)
|
||||
{
|
||||
gdma_etm_task_t *gdma_task = __containerof(task, gdma_etm_task_t, base);
|
||||
gdma_channel_t *dma_chan = gdma_task->chan;
|
||||
gdma_pair_t *pair = dma_chan->pair;
|
||||
gdma_group_t *group = pair->group;
|
||||
gdma_hal_context_t* hal = &group->hal;
|
||||
|
||||
gdma_hal_enable_etm_task(hal, pair->pair_id, dma_chan->direction, false);
|
||||
free(gdma_task);
|
||||
dma_chan->flags.start_stop_by_etm = false;
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t gdma_new_etm_event(gdma_channel_handle_t dma_chan, const gdma_etm_event_config_t *config, esp_etm_event_handle_t *out_event)
|
||||
{
|
||||
esp_etm_event_t *event = NULL;
|
||||
esp_err_t ret = ESP_OK;
|
||||
ESP_GOTO_ON_FALSE(dma_chan && config && out_event, ESP_ERR_INVALID_ARG, err, TAG, "invalid argument");
|
||||
ESP_GOTO_ON_FALSE(config->event_type < GDMA_ETM_EVENT_MAX, ESP_ERR_INVALID_ARG, err, TAG, "invalid event type");
|
||||
event = heap_caps_calloc(1, sizeof(esp_etm_event_t), ETM_MEM_ALLOC_CAPS);
|
||||
ESP_GOTO_ON_FALSE(event, ESP_ERR_NO_MEM, err, TAG, "no memory for ETM event");
|
||||
|
||||
gdma_pair_t *pair = dma_chan->pair;
|
||||
gdma_group_t *group = pair->group;
|
||||
uint32_t event_id = 0;
|
||||
|
||||
if (dma_chan->direction == GDMA_CHANNEL_DIRECTION_RX) {
|
||||
event_id = GDMA_LL_RX_ETM_EVENT_TABLE(group->group_id, pair->pair_id, config->event_type);
|
||||
} else {
|
||||
event_id = GDMA_LL_TX_ETM_EVENT_TABLE(group->group_id, pair->pair_id, config->event_type);
|
||||
}
|
||||
ESP_GOTO_ON_FALSE(event_id != 0, ESP_ERR_NOT_SUPPORTED, err, TAG, "not supported event type");
|
||||
|
||||
// fill the ETM event object
|
||||
event->event_id = event_id;
|
||||
event->trig_periph = ETM_TRIG_PERIPH_GDMA;
|
||||
event->del = gdma_del_etm_event;
|
||||
*out_event = event;
|
||||
return ESP_OK;
|
||||
|
||||
err:
|
||||
if (event) {
|
||||
gdma_del_etm_event(event);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
esp_err_t gdma_new_etm_task(gdma_channel_handle_t dma_chan, const gdma_etm_task_config_t *config, esp_etm_task_handle_t *out_task)
|
||||
{
|
||||
gdma_etm_task_t *task = NULL;
|
||||
esp_err_t ret = ESP_OK;
|
||||
ESP_GOTO_ON_FALSE(dma_chan && config && out_task, ESP_ERR_INVALID_ARG, err, TAG, "invalid argument");
|
||||
ESP_GOTO_ON_FALSE(config->task_type < GDMA_ETM_TASK_MAX, ESP_ERR_INVALID_ARG, err, TAG, "invalid task type");
|
||||
task = heap_caps_calloc(1, sizeof(gdma_etm_task_t), ETM_MEM_ALLOC_CAPS);
|
||||
ESP_GOTO_ON_FALSE(task, ESP_ERR_NO_MEM, err, TAG, "no memory for ETM task");
|
||||
|
||||
gdma_pair_t *pair = dma_chan->pair;
|
||||
gdma_group_t *group = pair->group;
|
||||
gdma_hal_context_t* hal = &group->hal;
|
||||
uint32_t task_id = 0;
|
||||
|
||||
gdma_hal_enable_etm_task(hal, pair->pair_id, dma_chan->direction, true);
|
||||
if (dma_chan->direction == GDMA_CHANNEL_DIRECTION_RX) {
|
||||
task_id = GDMA_LL_RX_ETM_TASK_TABLE(group->group_id, pair->pair_id, config->task_type);
|
||||
} else {
|
||||
task_id = GDMA_LL_TX_ETM_TASK_TABLE(group->group_id, pair->pair_id, config->task_type);
|
||||
}
|
||||
ESP_GOTO_ON_FALSE(task_id != 0, ESP_ERR_NOT_SUPPORTED, err, TAG, "not supported task type");
|
||||
|
||||
// set a flag, now the GDMA channel is start/stop by ETM subsystem
|
||||
dma_chan->flags.start_stop_by_etm = true;
|
||||
// fill the ETM task object
|
||||
task->chan = dma_chan;
|
||||
task->base.task_id = task_id;
|
||||
task->base.trig_periph = ETM_TRIG_PERIPH_GDMA;
|
||||
task->base.del = gdma_del_etm_task;
|
||||
*out_task = &(task->base);
|
||||
return ESP_OK;
|
||||
|
||||
err:
|
||||
if (task) {
|
||||
gdma_del_etm_task(&task->base);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <inttypes.h>
|
||||
#include <sys/cdefs.h>
|
||||
#include "soc/soc_caps.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_check.h"
|
||||
#include "esp_memory_utils.h"
|
||||
#include "esp_heap_caps.h"
|
||||
#include "esp_private/gdma_link.h"
|
||||
#include "hal/cache_hal.h"
|
||||
#include "hal/cache_ll.h"
|
||||
#include "esp_cache.h"
|
||||
|
||||
ESP_LOG_ATTR_TAG(TAG, "gdma-link");
|
||||
|
||||
#if SOC_NON_CACHEABLE_OFFSET
|
||||
#define GDMA_CACHE_ADDR_TO_NON_CACHE_ADDR(addr) ((addr) + SOC_NON_CACHEABLE_OFFSET)
|
||||
#else
|
||||
#define GDMA_CACHE_ADDR_TO_NON_CACHE_ADDR(addr) (addr)
|
||||
#endif
|
||||
|
||||
#define ALIGN_UP(num, align) (((num) + ((align) - 1)) & ~((align) - 1))
|
||||
#define ALIGN_DOWN(num, align) ((num) & ~((align) - 1))
|
||||
|
||||
// GDMA link list item definition
|
||||
// TODO: this type will eventually become target specific, we need to move it to the LL layer or soc layer
|
||||
typedef struct gdma_link_list_item_t gdma_link_list_item_t;
|
||||
struct gdma_link_list_item_t {
|
||||
struct {
|
||||
uint32_t size : 12; /*!< Buffer size */
|
||||
uint32_t length : 12; /*!< Number of valid bytes in the buffer */
|
||||
uint32_t reversed24 : 4; /*!< Reserved */
|
||||
uint32_t err_eof : 1; /*!< Whether the received buffer contains error, the error was reported by the peripheral */
|
||||
uint32_t reserved29 : 1; /*!< Reserved */
|
||||
uint32_t suc_eof : 1; /*!< Whether the list item should notify the peripheral an "EOF" event */
|
||||
uint32_t owner : 1; /*!< Who is allowed to access the buffer */
|
||||
} dw0; /*!< list item Word 0 */
|
||||
void *buffer; /*!< Pointer to the buffer */
|
||||
gdma_link_list_item_t *next; /*!< Pointer to the next list item (set to NULL if the list item is the last one of the link) */
|
||||
};
|
||||
|
||||
///< Maximum size of the buffer that can be carried by a DMA link list item
|
||||
#define GDMA_MAX_BUFFER_SIZE_PER_LINK_ITEM 4095
|
||||
|
||||
typedef struct gdma_link_list_t {
|
||||
uint32_t num_items; // number of items in the link list
|
||||
size_t item_size; // size of each item
|
||||
uint8_t *items; // pointer to the link list items
|
||||
uint8_t *items_nc; // pointer to the link list items, non-cached
|
||||
struct {
|
||||
uint32_t check_owner: 1; // Whether the link list is responsible for checking the ownership when mount data buffers
|
||||
} flags;
|
||||
} gdma_link_list_t;
|
||||
|
||||
esp_err_t gdma_new_link_list(const gdma_link_list_config_t *config, gdma_link_list_handle_t *ret_list)
|
||||
{
|
||||
esp_err_t ret = ESP_OK;
|
||||
uint8_t *items = NULL;
|
||||
gdma_link_list_t *list = NULL;
|
||||
ESP_RETURN_ON_FALSE(config && ret_list, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
ESP_RETURN_ON_FALSE(config->num_items, ESP_ERR_INVALID_ARG, TAG, "invalid number of items");
|
||||
|
||||
// the link list container is allocated from internal memory
|
||||
list = heap_caps_calloc(1, sizeof(gdma_link_list_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
|
||||
ESP_GOTO_ON_FALSE(list, ESP_ERR_NO_MEM, err, TAG, "no mem for link list");
|
||||
|
||||
uint32_t num_items = config->num_items;
|
||||
size_t item_alignment = config->item_alignment ? config->item_alignment : 4;
|
||||
// each list item should align to the specified alignment
|
||||
size_t item_size = ALIGN_UP(sizeof(gdma_link_list_item_t), item_alignment);
|
||||
// guard against overflow when calculating total bytes for descriptors
|
||||
ESP_GOTO_ON_FALSE(num_items <= SIZE_MAX / item_size, ESP_ERR_INVALID_SIZE, err, TAG, "list too big");
|
||||
|
||||
uint32_t list_items_mem_caps = MALLOC_CAP_8BIT | MALLOC_CAP_DMA;
|
||||
if (config->flags.items_in_ext_mem) {
|
||||
list_items_mem_caps |= MALLOC_CAP_SPIRAM;
|
||||
} else {
|
||||
list_items_mem_caps |= MALLOC_CAP_INTERNAL;
|
||||
}
|
||||
items = heap_caps_aligned_calloc(item_alignment, num_items, item_size, list_items_mem_caps);
|
||||
ESP_GOTO_ON_FALSE(items, ESP_ERR_NO_MEM, err, TAG, "no mem for link list items");
|
||||
|
||||
// do memory sync if the list items are in the cache
|
||||
uint32_t data_cache_line_size = 0;
|
||||
if (config->flags.items_in_ext_mem) {
|
||||
data_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_DATA);
|
||||
} else {
|
||||
data_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA);
|
||||
}
|
||||
if (data_cache_line_size) {
|
||||
// write back and then invalidate the cache, because later we will read/write the link list items by non-cached address
|
||||
ESP_GOTO_ON_ERROR(esp_cache_msync(items, ALIGN_UP(num_items * item_size, data_cache_line_size),
|
||||
ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_INVALIDATE),
|
||||
err, TAG, "cache sync failed");
|
||||
}
|
||||
|
||||
list->num_items = num_items;
|
||||
list->item_size = item_size;
|
||||
list->items = items;
|
||||
// calculate the non-cached address
|
||||
list->items_nc = GDMA_CACHE_ADDR_TO_NON_CACHE_ADDR(items);
|
||||
list->flags.check_owner = config->flags.check_owner;
|
||||
|
||||
ESP_LOGD(TAG, "new link list @%p, items @%p", list, items);
|
||||
*ret_list = list;
|
||||
return ESP_OK;
|
||||
|
||||
err:
|
||||
if (list) {
|
||||
free(list);
|
||||
}
|
||||
if (items) {
|
||||
free(items);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
esp_err_t gdma_del_link_list(gdma_link_list_handle_t list)
|
||||
{
|
||||
ESP_RETURN_ON_FALSE(list, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
|
||||
ESP_LOGD(TAG, "del link list at %p", list);
|
||||
free(list->items);
|
||||
free(list);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t gdma_link_mount_buffers(gdma_link_list_handle_t list, int start_item_index, const gdma_buffer_mount_config_t *buf_config_array, size_t num_buf, int *end_item_index)
|
||||
{
|
||||
if (!list || !buf_config_array || !num_buf) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
size_t item_size = list->item_size;
|
||||
uint32_t list_item_capacity = list->num_items;
|
||||
// ensure the start_item_index is between 0 and `list_item_capacity - 1`
|
||||
start_item_index = (start_item_index % list_item_capacity + list_item_capacity) % list_item_capacity;
|
||||
gdma_link_list_item_t *lli_nc = NULL;
|
||||
|
||||
uint32_t num_items_avail = 0;
|
||||
// if the link list is responsible for checking the ownership, we need to skip the items that are owned by the DMA
|
||||
if (list->flags.check_owner) {
|
||||
for (int i = 0; i < list_item_capacity; i++) {
|
||||
lli_nc = (gdma_link_list_item_t *)(list->items_nc + (i + start_item_index) % list_item_capacity * item_size);
|
||||
if (lli_nc->dw0.owner == GDMA_LLI_OWNER_CPU) {
|
||||
num_items_avail++;
|
||||
} else {
|
||||
// if the DMA descriptor "write back" feature is not enabled, descriptor is always owned by DMA after being used
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
num_items_avail = list_item_capacity;
|
||||
}
|
||||
|
||||
// check alignment and length for each buffer
|
||||
uint32_t remaining = num_items_avail;
|
||||
for (size_t bi = 0; bi < num_buf; bi++) {
|
||||
const gdma_buffer_mount_config_t *config = &buf_config_array[bi];
|
||||
uint8_t *buf = (uint8_t *)config->buffer;
|
||||
size_t len = config->length;
|
||||
// zero-length/NULL buffers don't consume a slot in pre-check
|
||||
if (len == 0 || buf == NULL) {
|
||||
continue;
|
||||
}
|
||||
size_t buffer_alignment = config->buffer_alignment;
|
||||
if (buffer_alignment == 0) {
|
||||
buffer_alignment = 1;
|
||||
}
|
||||
// alignment must be a power of 2
|
||||
ESP_RETURN_ON_FALSE_ISR((buffer_alignment & (buffer_alignment - 1)) == 0, ESP_ERR_INVALID_ARG, TAG, "align err idx=%"PRIu32" align=%"PRIu32, bi, buffer_alignment);
|
||||
size_t max_buffer_mount_length = ALIGN_DOWN(GDMA_MAX_BUFFER_SIZE_PER_LINK_ITEM, buffer_alignment);
|
||||
if (!config->flags.bypass_buffer_align_check) {
|
||||
ESP_RETURN_ON_FALSE_ISR(((uintptr_t)buf & (buffer_alignment - 1)) == 0, ESP_ERR_INVALID_ARG, TAG, "buf misalign idx=%"PRIu32" align=%"PRIu32, bi, buffer_alignment);
|
||||
}
|
||||
size_t num_items_need = (len + max_buffer_mount_length - 1) / max_buffer_mount_length;
|
||||
ESP_RETURN_ON_FALSE_ISR(num_items_need <= remaining, ESP_ERR_INVALID_ARG, TAG,
|
||||
"lli full start=%d need=%"PRIu32" avail=%"PRIu32, start_item_index, num_items_need, remaining);
|
||||
remaining -= num_items_need;
|
||||
}
|
||||
|
||||
// link_nodes[start_item_index-1] --> link_nodes[start_item_index]
|
||||
lli_nc = (gdma_link_list_item_t *)(list->items_nc + (start_item_index + list_item_capacity - 1) % list_item_capacity * item_size);
|
||||
lli_nc->next = (gdma_link_list_item_t *)(list->items + start_item_index * item_size);
|
||||
|
||||
int begin_item_idx = start_item_index;
|
||||
for (size_t bi = 0; bi < num_buf; bi++) {
|
||||
const gdma_buffer_mount_config_t *config = &buf_config_array[bi];
|
||||
uint8_t *buf = (uint8_t *)config->buffer;
|
||||
size_t len = config->length;
|
||||
size_t buffer_alignment = config->buffer_alignment;
|
||||
if (buffer_alignment == 0) {
|
||||
buffer_alignment = 1;
|
||||
}
|
||||
size_t max_buffer_mount_length = ALIGN_DOWN(GDMA_MAX_BUFFER_SIZE_PER_LINK_ITEM, buffer_alignment);
|
||||
// skip zero-length buffer but scrub any stale descriptor to keep ring clean; no slot consumption
|
||||
if (len == 0 || buf == NULL) {
|
||||
lli_nc = (gdma_link_list_item_t *)(list->items_nc + begin_item_idx % list_item_capacity * item_size);
|
||||
// reset the descriptor, especially the owner and next fields
|
||||
memset(lli_nc, 0, item_size);
|
||||
continue;
|
||||
}
|
||||
size_t num_items_need = (len + max_buffer_mount_length - 1) / max_buffer_mount_length;
|
||||
// mount the buffer to the link list
|
||||
for (size_t i = 0; i < num_items_need; i++) {
|
||||
lli_nc = (gdma_link_list_item_t *)(list->items_nc + (i + begin_item_idx) % list_item_capacity * item_size);
|
||||
lli_nc->buffer = buf;
|
||||
lli_nc->dw0.length = len > max_buffer_mount_length ? max_buffer_mount_length : len;
|
||||
// in fact the DMA doesn't check the "size" field, but we still set it to "length" for consistency
|
||||
// it's the user's responsibility to make sure the buffer size is sufficient
|
||||
lli_nc->dw0.size = lli_nc->dw0.length;
|
||||
// mark the EOF node
|
||||
lli_nc->dw0.suc_eof = (config->flags.mark_eof == 1) && (i == num_items_need - 1);
|
||||
if (i == num_items_need - 1) {
|
||||
// mark the final node
|
||||
switch (config->flags.mark_final) {
|
||||
case GDMA_FINAL_LINK_TO_NULL:
|
||||
lli_nc->next = NULL;
|
||||
break;
|
||||
case GDMA_FINAL_LINK_TO_HEAD:
|
||||
lli_nc->next = (gdma_link_list_item_t *)(list->items);
|
||||
break;
|
||||
case GDMA_FINAL_LINK_TO_START:
|
||||
lli_nc->next = (gdma_link_list_item_t *)(list->items + start_item_index * item_size);
|
||||
break;
|
||||
default:
|
||||
// DMA expects cached addresses in `next`
|
||||
lli_nc->next = (gdma_link_list_item_t *)(list->items + (i + begin_item_idx + 1) % list_item_capacity * item_size);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// DMA expects cached addresses in `next`
|
||||
lli_nc->next = (gdma_link_list_item_t *)(list->items + (i + begin_item_idx + 1) % list_item_capacity * item_size);
|
||||
}
|
||||
lli_nc->dw0.owner = GDMA_LLI_OWNER_DMA;
|
||||
buf += max_buffer_mount_length;
|
||||
len -= max_buffer_mount_length;
|
||||
}
|
||||
begin_item_idx += num_items_need;
|
||||
}
|
||||
|
||||
// return the index of the last modified list item
|
||||
if (end_item_index) {
|
||||
*end_item_index = (begin_item_idx - 1 + list_item_capacity) % list_item_capacity;
|
||||
}
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
uintptr_t gdma_link_get_head_addr(gdma_link_list_handle_t list)
|
||||
{
|
||||
if (!list) {
|
||||
return 0;
|
||||
}
|
||||
return (uintptr_t)(list->items);
|
||||
}
|
||||
|
||||
esp_err_t gdma_link_concat(gdma_link_list_handle_t first_link, int first_link_item_index, gdma_link_list_handle_t second_link, int second_link_item_index)
|
||||
{
|
||||
if (!first_link) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
gdma_link_list_item_t *lli_nc = NULL;
|
||||
// ensure the first_link_item_index is between 0 and `num_items - 1`
|
||||
int num_items = first_link->num_items;
|
||||
first_link_item_index = (first_link_item_index % num_items + num_items) % num_items;
|
||||
lli_nc = (gdma_link_list_item_t *)(first_link->items_nc + first_link_item_index * first_link->item_size);
|
||||
if (second_link == NULL) {
|
||||
lli_nc->next = NULL;
|
||||
} else {
|
||||
// ensure the second_link_item_index is between 0 and `num_items - 1`
|
||||
num_items = second_link->num_items;
|
||||
second_link_item_index = (second_link_item_index % num_items + num_items) % num_items;
|
||||
// concatenate the two link lists
|
||||
lli_nc->next = (gdma_link_list_item_t *)(second_link->items + second_link_item_index * second_link->item_size);
|
||||
}
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t gdma_link_set_owner(gdma_link_list_handle_t list, int item_index, gdma_lli_owner_t owner)
|
||||
{
|
||||
if (!list) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
int num_items = list->num_items;
|
||||
// ensure the item_index is between 0 and `num_items - 1`
|
||||
item_index = (item_index % num_items + num_items) % num_items;
|
||||
gdma_link_list_item_t *lli = (gdma_link_list_item_t *)(list->items_nc + item_index * list->item_size);
|
||||
lli->dw0.owner = owner;
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t gdma_link_get_owner(gdma_link_list_handle_t list, int item_index, gdma_lli_owner_t *owner)
|
||||
{
|
||||
if (!list || !owner) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
int num_items = list->num_items;
|
||||
// ensure the item_index is between 0 and `num_items - 1`
|
||||
item_index = (item_index % num_items + num_items) % num_items;
|
||||
gdma_link_list_item_t *lli = (gdma_link_list_item_t *)(list->items_nc + item_index * list->item_size);
|
||||
*owner = lli->dw0.owner;
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
size_t gdma_link_count_buffer_size_till_eof(gdma_link_list_handle_t list, int start_item_index)
|
||||
{
|
||||
if (!list) {
|
||||
return 0;
|
||||
}
|
||||
int num_items = list->num_items;
|
||||
// ensure the start_item_index is between 0 and `num_items - 1`
|
||||
start_item_index = (start_item_index % num_items + num_items) % num_items;
|
||||
size_t buf_size = 0;
|
||||
gdma_link_list_item_t *lli_nc = NULL;
|
||||
for (int i = 0; i < num_items; i++) {
|
||||
lli_nc = (gdma_link_list_item_t *)(list->items_nc + (start_item_index + i) % num_items * list->item_size);
|
||||
buf_size += lli_nc->dw0.length;
|
||||
// break if the current item is the last one or the EOF item
|
||||
if (lli_nc->dw0.suc_eof || lli_nc->next == NULL) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return buf_size;
|
||||
}
|
||||
|
||||
void* gdma_link_get_buffer(gdma_link_list_handle_t list, int item_index)
|
||||
{
|
||||
if (!list) {
|
||||
return NULL;
|
||||
}
|
||||
int num_items = list->num_items;
|
||||
// ensure the item_index is between 0 and `num_items - 1`
|
||||
item_index = (item_index % num_items + num_items) % num_items;
|
||||
gdma_link_list_item_t *lli = (gdma_link_list_item_t *)(list->items_nc + item_index * list->item_size);
|
||||
return lli->buffer;
|
||||
}
|
||||
|
||||
size_t gdma_link_get_length(gdma_link_list_handle_t list, int item_index)
|
||||
{
|
||||
if (!list) {
|
||||
return 0;
|
||||
}
|
||||
int num_items = list->num_items;
|
||||
// ensure the item_index is between 0 and `num_items - 1`
|
||||
item_index = (item_index % num_items + num_items) % num_items;
|
||||
gdma_link_list_item_t *lli = (gdma_link_list_item_t *)(list->items_nc + item_index * list->item_size);
|
||||
return lli->dw0.length;
|
||||
}
|
||||
|
||||
esp_err_t gdma_link_set_length(gdma_link_list_handle_t list, int item_index, size_t length)
|
||||
{
|
||||
if (!list) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
int num_items = list->num_items;
|
||||
// ensure the item_index is between 0 and `num_items - 1`
|
||||
item_index = (item_index % num_items + num_items) % num_items;
|
||||
gdma_link_list_item_t *lli = (gdma_link_list_item_t *)(list->items_nc + item_index * list->item_size);
|
||||
lli->dw0.length = length;
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
bool gdma_link_check_end(gdma_link_list_handle_t list, int item_index)
|
||||
{
|
||||
if (!list) {
|
||||
return false;
|
||||
}
|
||||
int num_items = list->num_items;
|
||||
// ensure the item_index is between 0 and `num_items - 1`
|
||||
item_index = (item_index % num_items + num_items) % num_items;
|
||||
gdma_link_list_item_t *lli = (gdma_link_list_item_t *)(list->items_nc + item_index * list->item_size);
|
||||
return lli->next == NULL;
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/cdefs.h>
|
||||
#include <sys/param.h>
|
||||
#include "sdkconfig.h"
|
||||
#if CONFIG_GDMA_ENABLE_DEBUG_LOG
|
||||
// The local log level must be defined before including esp_log.h
|
||||
// Set the maximum log level for gdma driver
|
||||
#define LOG_LOCAL_LEVEL ESP_LOG_VERBOSE
|
||||
#endif
|
||||
#include "soc/soc_caps.h"
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/task.h"
|
||||
#include "esp_err.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_check.h"
|
||||
#include "esp_intr_alloc.h"
|
||||
#include "esp_heap_caps.h"
|
||||
#include "hal/gdma_hal.h"
|
||||
#include "hal/gdma_ll.h"
|
||||
#include "hal/gdma_hal_ahb.h"
|
||||
#include "hal/gdma_hal_axi.h"
|
||||
#include "hal/gdma_periph.h"
|
||||
#include "soc/periph_defs.h"
|
||||
#include "esp_private/gdma.h"
|
||||
#include "esp_private/periph_ctrl.h"
|
||||
#include "esp_private/critical_section.h"
|
||||
#include "esp_private/sleep_retention.h"
|
||||
|
||||
#if CONFIG_GDMA_OBJ_DRAM_SAFE
|
||||
#define GDMA_MEM_ALLOC_CAPS (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)
|
||||
#else
|
||||
#define GDMA_MEM_ALLOC_CAPS MALLOC_CAP_DEFAULT
|
||||
#endif
|
||||
|
||||
#define GDMA_ACCESS_ENCRYPTION_MEM_ALIGNMENT 16 /*!< The alignment of the memory and size when DMA accesses the encryption memory */
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct gdma_pair_t gdma_pair_t;
|
||||
typedef struct gdma_channel_t gdma_channel_t;
|
||||
typedef struct gdma_tx_channel_t gdma_tx_channel_t;
|
||||
typedef struct gdma_rx_channel_t gdma_rx_channel_t;
|
||||
|
||||
typedef struct gdma_group_t {
|
||||
int group_id; // Group ID, index from 0
|
||||
int bus_id; // which system does the GDMA instance attached to
|
||||
gdma_hal_context_t hal; // HAL instance is at group level
|
||||
portMUX_TYPE spinlock; // group level spinlock, protect group level stuffs, e.g. hal object, pair handle slots and reference count of each pair
|
||||
uint32_t tx_periph_in_use_mask; // each bit indicates which peripheral (TX direction) has been occupied
|
||||
uint32_t rx_periph_in_use_mask; // each bit indicates which peripheral (RX direction) has been occupied
|
||||
gdma_pair_t *pairs[GDMA_LL_GET(PAIRS_PER_INST)]; // handles of GDMA pairs
|
||||
} gdma_group_t;
|
||||
|
||||
struct gdma_pair_t {
|
||||
gdma_group_t *group; // which group the pair belongs to
|
||||
int pair_id; // Pair ID, index from 0
|
||||
gdma_tx_channel_t *tx_chan; // pointer of tx channel in the pair
|
||||
gdma_rx_channel_t *rx_chan; // pointer of rx channel in the pair
|
||||
int occupy_code; // each bit indicates which channel has been occupied (an occupied channel will be skipped during channel search)
|
||||
portMUX_TYPE spinlock; // pair level spinlock, protect pair level stuffs, e.g. channel handle slots, occupy code
|
||||
};
|
||||
|
||||
struct gdma_channel_t {
|
||||
gdma_pair_t *pair; // which pair the channel belongs to
|
||||
intr_handle_t intr; // per-channel interrupt handle
|
||||
portMUX_TYPE spinlock; // channel level spinlock
|
||||
gdma_channel_direction_t direction; // channel direction
|
||||
int periph_id; // Peripheral instance ID, indicates which peripheral is connected to this GDMA channel
|
||||
size_t int_mem_alignment; // alignment for memory in internal memory
|
||||
size_t ext_mem_alignment; // alignment for memory in external memory
|
||||
esp_err_t (*del)(gdma_channel_t *channel); // channel deletion function, it's polymorphic, see `gdma_del_tx_channel` or `gdma_del_rx_channel`
|
||||
struct {
|
||||
uint32_t start_stop_by_etm: 1; // whether the channel is started/stopped by ETM
|
||||
uint32_t isr_cache_safe: 1; // whether the interrupt of this channel need to be cache safe
|
||||
} flags;
|
||||
};
|
||||
|
||||
struct gdma_tx_channel_t {
|
||||
gdma_channel_t base; // GDMA channel, base class
|
||||
void *user_data; // user registered DMA event data
|
||||
gdma_tx_event_callbacks_t cbs; // TX event callbacks
|
||||
};
|
||||
|
||||
struct gdma_rx_channel_t {
|
||||
gdma_channel_t base; // GDMA channel, base class
|
||||
void *user_data; // user registered DMA event data
|
||||
gdma_rx_event_callbacks_t cbs; // RX event callbacks
|
||||
};
|
||||
|
||||
void gdma_acquire_sleep_retention(gdma_pair_t* pair);
|
||||
void gdma_release_sleep_retention(gdma_pair_t* pair);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <sys/lock.h>
|
||||
#include "gdma_priv.h"
|
||||
|
||||
// Note!: the sleep retention APIs are using OS locks, so here we have to use a lock rather than a light-weight critical section
|
||||
static _lock_t gdma_sleep_retention_lock;
|
||||
static uint8_t pair_ref_counts[GDMA_LL_GET(INST_NUM)][GDMA_LL_GET(PAIRS_PER_INST)];
|
||||
|
||||
ESP_LOG_ATTR_TAG(TAG, "gdma");
|
||||
|
||||
static esp_err_t sleep_gdma_channel_retention_init(void *arg)
|
||||
{
|
||||
gdma_pair_t *pair = (gdma_pair_t *)arg;
|
||||
int group_id = pair->group->group_id;
|
||||
int pair_id = pair->pair_id;
|
||||
|
||||
sleep_retention_module_t module = gdma_chx_regs_retention[group_id][pair_id].module_id;
|
||||
esp_err_t err = sleep_retention_entries_create(gdma_chx_regs_retention[group_id][pair_id].link_list,
|
||||
gdma_chx_regs_retention[group_id][pair_id].link_num,
|
||||
REGDMA_LINK_PRI_GDMA, module);
|
||||
if (err == ESP_OK) {
|
||||
ESP_LOGD(TAG, "retention link created for pair (%d, %d)", group_id, pair_id);
|
||||
} else {
|
||||
ESP_LOGE(TAG, "failed to create retention link for pair (%d, %d)", group_id, pair_id);
|
||||
}
|
||||
return err;
|
||||
}
|
||||
|
||||
void gdma_acquire_sleep_retention(gdma_pair_t* pair)
|
||||
{
|
||||
int group_id = pair->group->group_id;
|
||||
int pair_id = pair->pair_id;
|
||||
sleep_retention_module_init_param_t init_param = {
|
||||
.cbs = { .create = { .handle = sleep_gdma_channel_retention_init, .arg = pair } },
|
||||
.depends = RETENTION_MODULE_BITMAP_INIT(CLOCK_SYSTEM)
|
||||
};
|
||||
sleep_retention_module_t module = gdma_chx_regs_retention[group_id][pair_id].module_id;
|
||||
|
||||
_lock_acquire(&gdma_sleep_retention_lock);
|
||||
// First time acquiring this pair, initialize the module
|
||||
if (pair_ref_counts[group_id][pair_id] == 0) {
|
||||
esp_err_t err = sleep_retention_module_init(module, &init_param);
|
||||
if (err != ESP_OK) {
|
||||
ESP_LOGW(TAG, "init retention module failed for pair (%d, %d), power domain may be turned off during sleep", group_id, pair_id);
|
||||
} else {
|
||||
err = sleep_retention_module_allocate(module);
|
||||
if (err != ESP_OK) {
|
||||
ESP_LOGW(TAG, "fail to allocate retention link list for pair (%d, %d)", group_id, pair_id);
|
||||
// don't call sleep_retention_module_deinit here, otherwise GDMA peripheral may be powered off during sleep
|
||||
}
|
||||
}
|
||||
}
|
||||
pair_ref_counts[group_id][pair_id]++;
|
||||
_lock_release(&gdma_sleep_retention_lock);
|
||||
}
|
||||
|
||||
void gdma_release_sleep_retention(gdma_pair_t* pair)
|
||||
{
|
||||
int group_id = pair->group->group_id;
|
||||
int pair_id = pair->pair_id;
|
||||
sleep_retention_module_t module = gdma_chx_regs_retention[group_id][pair_id].module_id;
|
||||
|
||||
_lock_acquire(&gdma_sleep_retention_lock);
|
||||
pair_ref_counts[group_id][pair_id]--;
|
||||
// Last reference, free the module
|
||||
if (pair_ref_counts[group_id][pair_id] == 0) {
|
||||
esp_err_t err = sleep_retention_module_free(module);
|
||||
if (err != ESP_OK) {
|
||||
ESP_LOGW(TAG, "fail to free the retention link list for pair (%d, %d)", group_id, pair_id);
|
||||
}
|
||||
err = sleep_retention_module_deinit(module);
|
||||
if (err != ESP_OK) {
|
||||
ESP_LOGW(TAG, "fail to deinit the retention module for pair (%d, %d)", group_id, pair_id);
|
||||
}
|
||||
}
|
||||
_lock_release(&gdma_sleep_retention_lock);
|
||||
}
|
||||
Reference in New Issue
Block a user