diff --git a/components/esp_driver_parlio/include/driver/parlio_rx.h b/components/esp_driver_parlio/include/driver/parlio_rx.h index 9774d871b4..bb0ac5c0d0 100644 --- a/components/esp_driver_parlio/include/driver/parlio_rx.h +++ b/components/esp_driver_parlio/include/driver/parlio_rx.h @@ -22,6 +22,7 @@ extern "C" { typedef struct { size_t trans_queue_depth; /*!< Depth of internal transaction queue */ size_t max_recv_size; /*!< Maximum receive size in one transaction, in bytes. This decides the number of DMA nodes will be used for each transaction */ + size_t dma_burst_size; /*!< DMA burst size, in bytes */ size_t data_width; /*!< Parallel IO data width, can set to 1/2/4/8/..., but can't be greater than PARLIO_RX_UNIT_MAX_DATA_WIDTH */ parlio_clock_source_t clk_src; /*!< Parallel IO clock source */ uint32_t ext_clk_freq_hz; /*!< The external source clock frequency. Only be valid when select PARLIO_CLK_SRC_EXTERNAL as clock source */ diff --git a/components/esp_driver_parlio/linker.lf b/components/esp_driver_parlio/linker.lf index 76b5b7f0cf..3c61d44c7f 100644 --- a/components/esp_driver_parlio/linker.lf +++ b/components/esp_driver_parlio/linker.lf @@ -25,3 +25,15 @@ entries: if SOC_PARLIO_TX_SUPPORT_LOOP_TRANSMISSION = y: gdma_link: gdma_link_get_buffer (noflash) + if PARLIO_RX_ISR_HANDLER_IN_IRAM = y: + gdma_link: gdma_link_mount_buffers (noflash) + gdma_link: gdma_link_get_buffer (noflash) + gdma_link: gdma_link_get_length (noflash) + esp_dma_utils: esp_dma_split_rx_buffer_to_cache_aligned (noflash) + esp_dma_utils: esp_dma_merge_aligned_rx_buffers (noflash) + +[mapping:parlio_driver_soc_periph] +archive: libsoc.a +entries: + if PARLIO_RX_ISR_HANDLER_IN_IRAM = y: + parlio_periph: parlio_periph_signals (noflash) diff --git a/components/esp_driver_parlio/src/parlio_rx.c b/components/esp_driver_parlio/src/parlio_rx.c index aa6b2da262..cce214885e 100644 --- a/components/esp_driver_parlio/src/parlio_rx.c +++ b/components/esp_driver_parlio/src/parlio_rx.c @@ -26,10 +26,11 @@ */ typedef struct { parlio_rx_delimiter_handle_t delimiter; /*!< Delimiter of this transaction */ - void *payload; /*!< The payload of this transaction, will be mounted to DMA link list node */ - size_t size; /*!< The payload size in byte */ + dma_buffer_split_array_t aligned_payload; /*!< The aligned payload of this transaction, will be mounted to DMA link list node */ + size_t tot_trans_size; /*!< The total size of the transaction */ size_t recv_bytes; /*!< The received bytes of this transaction will be reset when all data filled in the infinite transaction */ + size_t alignment; /*!< The alignment of the payload buffer */ struct { uint32_t infinite : 1; /*!< Whether this is an infinite transaction */ uint32_t indirect_mount : 1; /*!< Whether the user payload mount to the link list node indirectly via an internal DMA buffer */ @@ -64,14 +65,17 @@ typedef struct parlio_rx_unit_t { /* DMA Resources */ gdma_channel_handle_t dma_chan; /*!< DMA channel */ size_t max_recv_size; /*!< Maximum receive size for a normal transaction */ + size_t dma_burst_size; /*!< DMA burst size, in bytes */ gdma_link_list_handle_t dma_link; /*!< DMA link list handle */ uint32_t node_num; /*!< The number of nodes in the DMA link list */ - size_t int_mem_align; /*!< Alignment for internal memory */ - size_t ext_mem_align; /*!< Alignment for external memory */ + size_t dma_mem_align; /*!< Alignment for DMA memory */ uint32_t curr_node_id; /*!< The index of the current node in the DMA link list */ void *usr_recv_buf; /*!< The point to the user's receiving buffer */ /* Infinite transaction specific */ void *dma_buf; /*!< Additional internal DMA buffer only for infinite transactions */ + /* Unaligned DMA buffer management */ + uint8_t *stash_buf[2]; /*!< The ping-pong stash buffer for unaligned DMA buffer */ + uint8_t stash_buf_idx; /*!< The index of the current stash buffer */ /* Callback */ parlio_rx_event_callbacks_t cbs; /*!< The group of callback function pointers */ @@ -111,7 +115,8 @@ typedef struct parlio_rx_delimiter_t { } flags; } parlio_rx_delimiter_t; -#define PRALIO_RX_MOUNT_SIZE_CALC(total_size, div, align) ((((total_size) / (align)) / (div)) * (align)) +#define PARLIO_RX_MOUNT_SIZE_CALC(total_size, div, align) ((((total_size) / (align)) / (div)) * (align)) +#define PARLIO_RX_CHECK_ISR(condition, err) if (!(condition)) { return err; } static portMUX_TYPE s_rx_spinlock = (portMUX_TYPE)portMUX_INITIALIZER_UNLOCKED; @@ -125,48 +130,62 @@ size_t parlio_rx_mount_transaction_buffer(parlio_rx_unit_handle_t rx_unit, parli } portEXIT_CRITICAL_SAFE(&s_rx_spinlock); - uint32_t required_node_num = trans->size / PARLIO_MAX_ALIGNED_DMA_BUF_SIZE; - uint32_t remain_node_num = trans->size % PARLIO_MAX_ALIGNED_DMA_BUF_SIZE; + /* Calculate the number of nodes needed for the transaction */ + uint32_t body_node_num = trans->aligned_payload.buf.body.length / PARLIO_MAX_ALIGNED_DMA_BUF_SIZE; + uint32_t body_remain_size = trans->aligned_payload.buf.body.length % PARLIO_MAX_ALIGNED_DMA_BUF_SIZE; /* If there are still data remained, need one more node */ - required_node_num += remain_node_num ? 1 : 0; + body_node_num += body_remain_size ? 1 : 0; + + uint32_t head_node_num = trans->aligned_payload.buf.head.length ? 1 : 0; + uint32_t tail_node_num = trans->aligned_payload.buf.tail.length ? 1 : 0; + uint32_t required_node_num = body_node_num + head_node_num + tail_node_num; + if (trans->flags.infinite && required_node_num < 2) { /* At least 2 nodes needed */ required_node_num = 2; } rx_unit->node_num = required_node_num; + + gdma_buffer_mount_config_t mount_config[required_node_num] = {}; + /* Mount head buffer */ + if (head_node_num) { + mount_config[0].buffer = trans->aligned_payload.buf.head.aligned_buffer; + mount_config[0].buffer_alignment = trans->alignment; + mount_config[0].length = trans->aligned_payload.buf.head.length; + mount_config[0].flags.bypass_buffer_align_check = false; + mount_config[0].flags.mark_eof = false; + mount_config[0].flags.mark_final = GDMA_FINAL_LINK_TO_DEFAULT; + } + /* Mount body buffer */ size_t mount_size = 0; size_t offset = 0; - // TODO: update DMA alignment -#if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE - uint32_t alignment = rx_unit->base.group->dma_align; -#else - uint32_t alignment = 4; -#endif - - gdma_buffer_mount_config_t mount_config[required_node_num]; - /* Loop the nodes to assign the data */ - for (int i = 0; i < required_node_num; i++) { - size_t rest_size = trans->size - offset; + for (int i = head_node_num; i < required_node_num - tail_node_num; i++) { + size_t rest_size = trans->aligned_payload.buf.body.length - offset; if (rest_size >= 2 * PARLIO_MAX_ALIGNED_DMA_BUF_SIZE) { - mount_size = PARLIO_RX_MOUNT_SIZE_CALC(trans->size, required_node_num, alignment); + mount_size = PARLIO_RX_MOUNT_SIZE_CALC(trans->aligned_payload.buf.body.length, body_node_num, trans->alignment); } else if (rest_size <= PARLIO_MAX_ALIGNED_DMA_BUF_SIZE) { - mount_size = (required_node_num == 2) && (i == 0) ? PARLIO_RX_MOUNT_SIZE_CALC(rest_size, 2, alignment) : rest_size; + mount_size = (required_node_num == 2) && (i == 0) ? PARLIO_RX_MOUNT_SIZE_CALC(rest_size, 2, trans->alignment) : rest_size; } else { - mount_size = PRALIO_RX_MOUNT_SIZE_CALC(rest_size, 2, alignment); + mount_size = PARLIO_RX_MOUNT_SIZE_CALC(rest_size, 2, trans->alignment); } - mount_config[i].buffer = (void *)((uint8_t *)trans->payload + offset); + mount_config[i].buffer = (void *)((uint8_t *)trans->aligned_payload.buf.body.aligned_buffer + offset); + mount_config[i].buffer_alignment = trans->alignment; mount_config[i].length = mount_size; mount_config[i].flags.bypass_buffer_align_check = false; mount_config[i].flags.mark_eof = false; mount_config[i].flags.mark_final = GDMA_FINAL_LINK_TO_DEFAULT; - // Link the node - if (i == required_node_num - 1) { - /* For infinite transaction, link the node as a ring */ - mount_config[i].flags.mark_final = !trans->flags.infinite ? GDMA_FINAL_LINK_TO_NULL : GDMA_FINAL_LINK_TO_HEAD; - mount_config[i].flags.mark_eof = true; - } offset += mount_size; } + /* Mount tail buffer */ + if (tail_node_num) { + mount_config[required_node_num - 1].buffer = trans->aligned_payload.buf.tail.aligned_buffer; + mount_config[required_node_num - 1].buffer_alignment = trans->alignment; + mount_config[required_node_num - 1].length = trans->aligned_payload.buf.tail.length; + mount_config[required_node_num - 1].flags.bypass_buffer_align_check = false; + } + /* For infinite transaction, link the node as a ring */ + mount_config[required_node_num - 1].flags.mark_final = !trans->flags.infinite ? GDMA_FINAL_LINK_TO_NULL : GDMA_FINAL_LINK_TO_HEAD; + mount_config[required_node_num - 1].flags.mark_eof = true; gdma_link_mount_buffers(rx_unit->dma_link, 0, mount_config, required_node_num, NULL); /* Reset the current DMA node */ rx_unit->curr_node_id = 0; @@ -319,6 +338,7 @@ static bool parlio_rx_default_eof_callback(gdma_channel_handle_t dma_chan, gdma_ need_yield |= rx_unit->cbs.on_timeout(rx_unit, &evt_data, rx_unit->user_data); } } else { + esp_dma_merge_aligned_rx_buffers(&rx_unit->curr_trans.aligned_payload); /* If received a normal EOF, it's a receive done event on parlio RX */ if (rx_unit->cbs.on_receive_done) { evt_data.data = rx_unit->usr_recv_buf; @@ -403,7 +423,7 @@ static bool parlio_rx_default_desc_done_callback(gdma_channel_handle_t dma_chan, portEXIT_CRITICAL_ISR(&s_rx_spinlock); } /* Update received bytes */ - if (rx_unit->curr_trans.recv_bytes >= rx_unit->curr_trans.size) { + if (rx_unit->curr_trans.recv_bytes >= rx_unit->curr_trans.tot_trans_size) { rx_unit->curr_trans.recv_bytes = 0; } rx_unit->curr_trans.recv_bytes += evt_data.recv_bytes; @@ -419,24 +439,21 @@ static esp_err_t parlio_rx_create_dma_link(parlio_rx_unit_handle_t rx_unit, uint ESP_RETURN_ON_FALSE(rx_unit, ESP_ERR_INVALID_ARG, TAG, "invalid param"); esp_err_t ret = ESP_OK; - // create DMA link list - size_t buffer_alignment = MAX(rx_unit->int_mem_align, rx_unit->ext_mem_align); - size_t tot_node_num = esp_dma_calculate_node_count(max_recv_size, buffer_alignment, PARLIO_DMA_DESCRIPTOR_BUFFER_MAX_SIZE); + // calculated the total node number, add 2 for the aligned stash buffer + size_t tot_node_num = esp_dma_calculate_node_count(max_recv_size, rx_unit->dma_mem_align, PARLIO_DMA_DESCRIPTOR_BUFFER_MAX_SIZE) + 2; gdma_link_list_config_t dma_link_config = { - .buffer_alignment = buffer_alignment, - .item_alignment = PARLIO_DMA_DESC_ALIGNMENT, .num_items = tot_node_num, + .item_alignment = PARLIO_DMA_DESC_ALIGNMENT, }; - // throw the error to the caller + // create DMA link list, throw the error to the caller if failed ESP_RETURN_ON_ERROR(gdma_new_link_list(&dma_link_config, &rx_unit->dma_link), TAG, "create DMA link list failed"); rx_unit->max_recv_size = max_recv_size; - return ret; } -static esp_err_t parlio_rx_unit_init_dma(parlio_rx_unit_handle_t rx_unit) +static esp_err_t parlio_rx_unit_init_dma(parlio_rx_unit_handle_t rx_unit, size_t dma_burst_size) { /* Allocate and connect the GDMA channel */ gdma_channel_alloc_config_t dma_chan_config = { @@ -456,13 +473,12 @@ static esp_err_t parlio_rx_unit_init_dma(parlio_rx_unit_handle_t rx_unit) gdma_apply_strategy(rx_unit->dma_chan, &gdma_strategy_conf); // configure DMA transfer parameters + rx_unit->dma_burst_size = dma_burst_size ? dma_burst_size : 16; gdma_transfer_config_t trans_cfg = { - // TODO: add burst size - .max_data_burst_size = 4, // Enable DMA burst transfer for better performance, - .access_ext_mem = true, // support transmit PSRAM buffer + .max_data_burst_size = rx_unit->dma_burst_size, // Enable DMA burst transfer for better performance, }; ESP_RETURN_ON_ERROR(gdma_config_transfer(rx_unit->dma_chan, &trans_cfg), TAG, "config DMA transfer failed"); - ESP_RETURN_ON_ERROR(gdma_get_alignment_constraints(rx_unit->dma_chan, &rx_unit->int_mem_align, &rx_unit->ext_mem_align), TAG, "get alignment constraints failed"); + ESP_RETURN_ON_ERROR(gdma_get_alignment_constraints(rx_unit->dma_chan, &rx_unit->dma_mem_align, NULL), TAG, "get alignment constraints failed"); /* Register callbacks */ gdma_rx_event_callbacks_t cbs = { @@ -571,6 +587,13 @@ static esp_err_t parlio_destroy_rx_unit(parlio_rx_unit_handle_t rx_unit) if (rx_unit->dma_buf) { free(rx_unit->dma_buf); } + /* Free the stash buffer */ + for (uint8_t i = 0; i < 2; i++) { + if (rx_unit->stash_buf[i]) { + free(rx_unit->stash_buf[i]); + rx_unit->stash_buf[i] = NULL; + } + } /* Unregister the RX unit from the PARLIO group */ if (rx_unit->base.group) { parlio_unregister_unit_from_group(&rx_unit->base); @@ -632,9 +655,14 @@ esp_err_t parlio_new_rx_unit(const parlio_rx_unit_config_t *config, parlio_rx_un /* Initialize GPIO */ ESP_GOTO_ON_ERROR(parlio_rx_unit_set_gpio(unit, config), err, TAG, "failed to set GPIO"); /* Install DMA service */ - ESP_GOTO_ON_ERROR(parlio_rx_unit_init_dma(unit), err, TAG, "install rx DMA failed"); + ESP_GOTO_ON_ERROR(parlio_rx_unit_init_dma(unit, config->dma_burst_size), err, TAG, "install rx DMA failed"); ESP_GOTO_ON_ERROR(parlio_rx_create_dma_link(unit, config->max_recv_size), err, TAG, "create dma link list failed"); + for (uint8_t i = 0; i < 2; i++) { + unit->stash_buf[i] = heap_caps_aligned_calloc(unit->dma_mem_align, 2, unit->dma_mem_align, PARLIO_MEM_ALLOC_CAPS | MALLOC_CAP_DMA); + ESP_GOTO_ON_FALSE(unit->stash_buf[i], ESP_ERR_NO_MEM, err, TAG, "no memory for stash buffer"); + } + /* Reset RX module */ PARLIO_RCC_ATOMIC() { parlio_ll_rx_reset_clock(hal->regs); @@ -719,7 +747,16 @@ esp_err_t parlio_rx_unit_enable(parlio_rx_unit_handle_t rx_unit, bool reset_queu xSemaphoreGive(rx_unit->trans_sem); } else if (xQueueReceive(rx_unit->trans_que, &trans, 0) == pdTRUE) { // The semaphore always supposed to be taken successfully - assert(xSemaphoreTake(rx_unit->trans_sem, 0) == pdTRUE); + BaseType_t res = xSemaphoreTake(rx_unit->trans_sem, 0); + (void)res; + assert(res == pdTRUE); + + if (trans.flags.indirect_mount && trans.flags.infinite && rx_unit->dma_buf == NULL) { + rx_unit->dma_buf = heap_caps_aligned_calloc(rx_unit->dma_mem_align, 1, trans.aligned_payload.buf.body.length, PARLIO_DMA_MEM_ALLOC_CAPS); + ESP_GOTO_ON_FALSE(rx_unit->dma_buf, ESP_ERR_NO_MEM, err, TAG, "No memory for the internal DMA buffer"); + trans.aligned_payload.buf.body.aligned_buffer = rx_unit->dma_buf; + trans.aligned_payload.buf.body.recovery_address = rx_unit->dma_buf; + } if (rx_unit->cfg.flags.free_clk) { PARLIO_CLOCK_SRC_ATOMIC() { parlio_ll_rx_enable_clock(hal->regs, false); @@ -940,13 +977,11 @@ esp_err_t parlio_rx_unit_receive(parlio_rx_unit_handle_t rx_unit, ESP_RETURN_ON_FALSE(rx_unit && payload && recv_cfg, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); ESP_RETURN_ON_FALSE(recv_cfg->delimiter, ESP_ERR_INVALID_ARG, TAG, "no delimiter specified"); ESP_RETURN_ON_FALSE(payload_size <= rx_unit->max_recv_size, ESP_ERR_INVALID_ARG, TAG, "trans length too large"); - uint32_t alignment = rx_unit->base.group->dma_align; -#if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE - ESP_RETURN_ON_FALSE(payload_size % alignment == 0, ESP_ERR_INVALID_ARG, TAG, "The payload size should align with %"PRIu32, alignment); + size_t alignment = rx_unit->dma_mem_align; if (recv_cfg->flags.partial_rx_en) { ESP_RETURN_ON_FALSE(payload_size >= 2 * alignment, ESP_ERR_INVALID_ARG, TAG, "The payload size should greater than %"PRIu32, 2 * alignment); } -#endif + #if CONFIG_PARLIO_RX_ISR_CACHE_SAFE ESP_RETURN_ON_FALSE(esp_ptr_internal(payload), ESP_ERR_INVALID_ARG, TAG, "payload not in internal RAM"); #else @@ -967,8 +1002,8 @@ esp_err_t parlio_rx_unit_receive(parlio_rx_unit_handle_t rx_unit, rx_units[rx_unit->base.unit_id]. data_sigs[recv_cfg->delimiter->valid_sig_line_id]; } - void *p_buffer = payload; + dma_buffer_split_array_t dma_buf_array = {0}; /* Create the internal DMA buffer for the infinite transaction if indirect_mount is set */ if (recv_cfg->flags.partial_rx_en && recv_cfg->flags.indirect_mount) { ESP_RETURN_ON_FALSE(!rx_unit->dma_buf, ESP_ERR_INVALID_STATE, TAG, "infinite transaction is using the internal DMA buffer"); @@ -976,14 +1011,21 @@ esp_err_t parlio_rx_unit_receive(parlio_rx_unit_handle_t rx_unit, rx_unit->dma_buf = heap_caps_aligned_calloc(alignment, 1, payload_size, PARLIO_DMA_MEM_ALLOC_CAPS); ESP_RETURN_ON_FALSE(rx_unit->dma_buf, ESP_ERR_NO_MEM, TAG, "No memory for the internal DMA buffer"); /* Use the internal DMA buffer so that the user buffer can always be available */ - p_buffer = rx_unit->dma_buf; + dma_buf_array.buf.body.aligned_buffer = rx_unit->dma_buf; + dma_buf_array.buf.body.recovery_address = rx_unit->dma_buf; + dma_buf_array.buf.body.length = payload_size; + } else { + ESP_RETURN_ON_ERROR(esp_dma_split_rx_buffer_to_cache_aligned(payload, payload_size, &dma_buf_array, &rx_unit->stash_buf[rx_unit->stash_buf_idx]), + TAG, "failed to split the unaligned DMA buffer"); + rx_unit->stash_buf_idx = !rx_unit->stash_buf_idx; } /* Create the transaction */ parlio_rx_transaction_t transaction = { .delimiter = recv_cfg->delimiter, - .payload = p_buffer, - .size = payload_size, + .aligned_payload = dma_buf_array, + .tot_trans_size = payload_size, + .alignment = alignment, .recv_bytes = 0, .flags.infinite = recv_cfg->flags.partial_rx_en, .flags.indirect_mount = recv_cfg->flags.indirect_mount, diff --git a/components/esp_driver_parlio/src/parlio_tx.c b/components/esp_driver_parlio/src/parlio_tx.c index 86d6c22afb..329fcdfcd5 100644 --- a/components/esp_driver_parlio/src/parlio_tx.c +++ b/components/esp_driver_parlio/src/parlio_tx.c @@ -481,7 +481,7 @@ static void parlio_mount_buffer(parlio_tx_unit_t *tx_unit, parlio_tx_trans_desc_ .flags = { // if transmission is loop, we don't need to generate the EOF for 1-bit data width, DIG-559 .mark_eof = tx_unit->data_width == 1 ? !t->flags.loop_transmission : true, - .mark_final = !t->flags.loop_transmission ? GDMA_FINAL_LINK_TO_NULL : GDMA_FINAL_LINK_TO_DEFAULT, + .mark_final = t->flags.loop_transmission ? GDMA_FINAL_LINK_TO_START : GDMA_FINAL_LINK_TO_NULL, } }; diff --git a/components/esp_driver_parlio/test_apps/parlio/main/test_parlio_rx.c b/components/esp_driver_parlio/test_apps/parlio/main/test_parlio_rx.c index a5494f1f9f..e9cddeb20f 100644 --- a/components/esp_driver_parlio/test_apps/parlio/main/test_parlio_rx.c +++ b/components/esp_driver_parlio/test_apps/parlio/main/test_parlio_rx.c @@ -477,7 +477,7 @@ TEST_CASE("parallel_rx_unit_receive_transaction_test", "[parlio_rx]") TEST_ESP_OK(parlio_rx_unit_receive(rx_unit, payload, TEST_PAYLOAD_SIZE, &recv_config)); TEST_ESP_OK(parlio_rx_unit_wait_all_done(rx_unit, 5000)); TEST_ESP_OK(parlio_rx_soft_delimiter_start_stop(rx_unit, deli, false)); - TEST_ASSERT_EQUAL_UINT32(2, test_data.partial_recv_cnt); + TEST_ASSERT_GREATER_OR_EQUAL_UINT32(2, test_data.partial_recv_cnt); TEST_ASSERT_EQUAL_UINT32(1, test_data.recv_done_cnt); memset(&test_data, 0, sizeof(test_data_t)); @@ -489,7 +489,7 @@ TEST_CASE("parallel_rx_unit_receive_transaction_test", "[parlio_rx]") } TEST_ESP_OK(parlio_rx_unit_wait_all_done(rx_unit, 5000)); TEST_ESP_OK(parlio_rx_soft_delimiter_start_stop(rx_unit, deli, false)); - TEST_ASSERT_EQUAL_UINT32(10, test_data.partial_recv_cnt); + TEST_ASSERT_GREATER_OR_EQUAL_UINT32(10, test_data.partial_recv_cnt); TEST_ASSERT_EQUAL_UINT32(5, test_data.recv_done_cnt); memset(&test_data, 0, sizeof(test_data_t)); diff --git a/components/esp_hw_support/dma/esp_dma_utils.c b/components/esp_hw_support/dma/esp_dma_utils.c index ad97b84e7e..a820291bcd 100644 --- a/components/esp_hw_support/dma/esp_dma_utils.c +++ b/components/esp_hw_support/dma/esp_dma_utils.c @@ -29,7 +29,7 @@ esp_err_t esp_dma_split_rx_buffer_to_cache_aligned(void *rx_buffer, size_t buffe { esp_err_t ret = ESP_OK; uint8_t* stash_buffer = NULL; - ESP_RETURN_ON_FALSE(rx_buffer && buffer_len && align_buf_array, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); + ESP_RETURN_ON_FALSE_ISR(rx_buffer && buffer_len && align_buf_array, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); // read the cache line size of internal and external memory, we also use this information to check if a given memory is behind the cache size_t int_mem_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA); @@ -41,80 +41,83 @@ esp_err_t esp_dma_split_rx_buffer_to_cache_aligned(void *rx_buffer, size_t buffe } else if (esp_ptr_internal(rx_buffer)) { split_line_size = int_mem_cache_line_size; } - ESP_LOGV(TAG, "split_line_size:%zu", split_line_size); + bool align_required = split_line_size > 0; + ESP_EARLY_LOGV(TAG, "split_line_size:%zu", split_line_size); - // allocate the stash buffer from internal RAM - // Note, the split_line_size can be 0, in this case, the stash_buffer is also NULL, which is fine - stash_buffer = heap_caps_calloc(2, split_line_size, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); - ESP_RETURN_ON_FALSE(!(split_line_size && !stash_buffer), ESP_ERR_NO_MEM, TAG, "no mem for stash buffer"); + if (*ret_stash_buffer == NULL) { + // If the stash buffer is not offered by the caller, allocate the stash buffer from internal RAM + // Note, the split_line_size can be 0, in this case, the stash_buffer is also NULL, which is fine + stash_buffer = heap_caps_calloc(2, split_line_size, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + ESP_RETURN_ON_FALSE_ISR(!(split_line_size && !stash_buffer), ESP_ERR_NO_MEM, TAG, "no mem for stash buffer"); + } else { + // If the stash buffer is offered by the caller, check if it is aligned + ESP_RETURN_ON_FALSE_ISR(split_line_size == 0 || (uintptr_t)(*ret_stash_buffer) % split_line_size == 0, + ESP_ERR_INVALID_ARG, TAG, "the offered stash buffer is not aligned"); + // If the stash buffer is offered by the caller, use it + stash_buffer = *ret_stash_buffer; + } // clear align_array to avoid garbage data memset(align_buf_array, 0, sizeof(dma_buffer_split_array_t)); bool need_cache_sync[3] = {false}; - // if split_line_size is non-zero, split the buffer into head, body and tail - if (split_line_size > 0) { + // if align_required, split the buffer into head, body and tail + if (align_required) { // calculate head_overflow_len size_t head_overflow_len = (uintptr_t)rx_buffer % split_line_size; head_overflow_len = head_overflow_len ? split_line_size - head_overflow_len : 0; - ESP_LOGV(TAG, "head_addr:%p head_overflow_len:%zu", rx_buffer, head_overflow_len); + ESP_EARLY_LOGV(TAG, "head_addr:%p head_overflow_len:%zu", rx_buffer, head_overflow_len); // calculate tail_overflow_len size_t tail_overflow_len = ((uintptr_t)rx_buffer + buffer_len) % split_line_size; - ESP_LOGV(TAG, "tail_addr:%p tail_overflow_len:%zu", rx_buffer + buffer_len - tail_overflow_len, tail_overflow_len); - - uint8_t extra_buf_count = 0; - uint8_t* input_buffer = (uint8_t*)rx_buffer; - align_buf_array->buf.head.recovery_address = input_buffer; - align_buf_array->buf.head.aligned_buffer = stash_buffer + split_line_size * extra_buf_count++; - align_buf_array->buf.head.length = head_overflow_len; - need_cache_sync[0] = int_mem_cache_line_size > 0; - align_buf_array->buf.body.recovery_address = input_buffer + head_overflow_len; - align_buf_array->buf.body.aligned_buffer = input_buffer + head_overflow_len; - align_buf_array->buf.body.length = buffer_len - head_overflow_len - tail_overflow_len; - need_cache_sync[1] = true; - align_buf_array->buf.tail.recovery_address = input_buffer + buffer_len - tail_overflow_len; - align_buf_array->buf.tail.aligned_buffer = stash_buffer + split_line_size * extra_buf_count++; - align_buf_array->buf.tail.length = tail_overflow_len; - need_cache_sync[2] = int_mem_cache_line_size > 0; + ESP_EARLY_LOGV(TAG, "tail_addr:%p tail_overflow_len:%zu", rx_buffer + buffer_len - tail_overflow_len, tail_overflow_len); // special handling when input_buffer length is no more than buffer alignment - if (head_overflow_len >= buffer_len || tail_overflow_len >= buffer_len) { - align_buf_array->buf.head.length = buffer_len ; - align_buf_array->buf.body.length = 0 ; - align_buf_array->buf.tail.length = 0 ; + bool is_small_buf = head_overflow_len >= buffer_len || tail_overflow_len >= buffer_len; + uint8_t extra_buf_count = 0; + uint8_t* input_buffer = (uint8_t*)rx_buffer; + if (head_overflow_len || is_small_buf) { + align_buf_array->buf.head.recovery_address = input_buffer; + align_buf_array->buf.head.aligned_buffer = stash_buffer + split_line_size * extra_buf_count++; + align_buf_array->buf.head.length = is_small_buf ? buffer_len : head_overflow_len; + need_cache_sync[0] = int_mem_cache_line_size > 0; + } + int body_len = (int)buffer_len - (int)head_overflow_len - (int)tail_overflow_len; + if (body_len > 0) { + align_buf_array->buf.body.recovery_address = input_buffer + head_overflow_len; + align_buf_array->buf.body.aligned_buffer = input_buffer + head_overflow_len; + align_buf_array->buf.body.length = body_len; + need_cache_sync[1] = true; + } + if (tail_overflow_len && !is_small_buf) { + align_buf_array->buf.tail.recovery_address = input_buffer + buffer_len - tail_overflow_len; + align_buf_array->buf.tail.aligned_buffer = stash_buffer + split_line_size * extra_buf_count++; + align_buf_array->buf.tail.length = tail_overflow_len; + need_cache_sync[2] = int_mem_cache_line_size > 0; } } else { align_buf_array->buf.body.aligned_buffer = rx_buffer; align_buf_array->buf.body.recovery_address = rx_buffer; align_buf_array->buf.body.length = buffer_len; - need_cache_sync[1] = false; - } - - for (int i = 0; i < 3; i++) { - if (align_buf_array->aligned_buffer[i].length == 0) { - align_buf_array->aligned_buffer[i].aligned_buffer = NULL; - align_buf_array->aligned_buffer[i].recovery_address = NULL; - need_cache_sync[i] = false; - } } // invalidate the aligned buffer if necessary for (int i = 0; i < 3; i++) { - if (need_cache_sync[i]) { - size_t sync_size = align_buf_array->aligned_buffer[i].length; + size_t sync_size = align_buf_array->aligned_buffer[i].length; + if (need_cache_sync[i] && sync_size > 0) { if (sync_size < split_line_size) { // If the buffer is smaller than the cache line size, we need to sync the whole buffer sync_size = split_line_size; } esp_err_t res = esp_cache_msync(align_buf_array->aligned_buffer[i].aligned_buffer, sync_size, ESP_CACHE_MSYNC_FLAG_DIR_M2C); - ESP_GOTO_ON_ERROR(res, err, TAG, "failed to do cache sync"); + ESP_GOTO_ON_ERROR_ISR(res, err, TAG, "failed to do cache sync"); } } *ret_stash_buffer = stash_buffer; return ESP_OK; err: - if (stash_buffer) { + // Only free the stash buffer if it is not offered by the caller + if (stash_buffer && *ret_stash_buffer == NULL) { free(stash_buffer); } return ret; diff --git a/components/esp_hw_support/dma/gdma_link.c b/components/esp_hw_support/dma/gdma_link.c index 41b29a16e4..0caf76db92 100644 --- a/components/esp_hw_support/dma/gdma_link.c +++ b/components/esp_hw_support/dma/gdma_link.c @@ -203,20 +203,24 @@ esp_err_t gdma_link_mount_buffers(gdma_link_list_handle_t list, int start_item_i lli_nc->dw0.size = lli_nc->dw0.length; // mark the EOF node lli_nc->dw0.suc_eof = (config->flags.mark_eof == 1) && (i == num_items_need - 1); - // mark the final node - switch (config->flags.mark_final) { - case GDMA_FINAL_LINK_TO_NULL: - lli_nc->next = NULL; - break; - case GDMA_FINAL_LINK_TO_HEAD: - lli_nc->next = (gdma_link_list_item_t *)(list->items); - break; - case GDMA_FINAL_LINK_TO_START: - lli_nc->next = (gdma_link_list_item_t *)(list->items + begin_item_idx * item_size); - break; - default: - lli_nc->next = (gdma_link_list_item_t *)(list->items + (i + begin_item_idx + 1) % list_item_capacity * item_size); - break; + if (i == num_items_need - 1) { + // mark the final node + switch (config->flags.mark_final) { + case GDMA_FINAL_LINK_TO_NULL: + lli_nc->next = NULL; + break; + case GDMA_FINAL_LINK_TO_HEAD: + lli_nc->next = (gdma_link_list_item_t *)(list->items); + break; + case GDMA_FINAL_LINK_TO_START: + lli_nc->next = (gdma_link_list_item_t *)(list->items + start_item_index * item_size); + break; + default: + lli_nc->next = (gdma_link_list_item_t *)(list->items + (i + begin_item_idx + 1) % list_item_capacity * item_size); + break; + } + } else { + lli_nc->next = (gdma_link_list_item_t *)(list->items + (i + begin_item_idx + 1) % list_item_capacity * item_size); } lli_nc->dw0.owner = GDMA_LLI_OWNER_DMA; buf += max_buffer_mount_length; diff --git a/components/esp_hw_support/dma/include/esp_private/gdma_link.h b/components/esp_hw_support/dma/include/esp_private/gdma_link.h index 3e3d4be22c..37d9964a3a 100644 --- a/components/esp_hw_support/dma/include/esp_private/gdma_link.h +++ b/components/esp_hw_support/dma/include/esp_private/gdma_link.h @@ -62,7 +62,7 @@ esp_err_t gdma_del_link_list(gdma_link_list_handle_t list); */ typedef enum { GDMA_FINAL_LINK_TO_DEFAULT = 0, /*!< The next node is linked to the default next item in the link list */ - GDMA_FINAL_LINK_TO_NULL = 1, /*!< The next node is linked to the final item in the link list */ + GDMA_FINAL_LINK_TO_NULL = 1, /*!< No next node is linked */ GDMA_FINAL_LINK_TO_HEAD = 2, /*!< The next node is linked to the head item in the link list */ GDMA_FINAL_LINK_TO_START = 3, /*!< The next node is linked to the start item in the link list */ } gdma_final_node_link_type_t; @@ -77,9 +77,11 @@ typedef struct { struct gdma_buffer_mount_flags { uint32_t mark_eof: 1; /*!< Whether to mark the list item as the "EOF" item. Note, an "EOF" descriptor can be interrupted differently by peripheral. - But it doesn't mean to terminate a DMA link (use `mark_final` instead). + But it doesn't mean to terminate a DMA link (set `mark_final` to GDMA_FINAL_LINK_TO_NULL instead). EOF link list item can also trigger an interrupt. */ - gdma_final_node_link_type_t mark_final: 2; /*!< The next node of the final item in the link list */ + gdma_final_node_link_type_t mark_final: 2; /*!< Specify the next item of the final item of this mount. + For the other items that not the final one, it will be linked to the next item automatically and this field takes no effect. + Note, the final item here does not mean the last item in the link list. It is `start_item_index + num_items - 1` */ uint32_t bypass_buffer_align_check: 1; /*!< Whether to bypass the buffer alignment check. Only enable it when you know what you are doing. */ } flags; //!< Flags for buffer mount configurations diff --git a/components/esp_lcd/rgb/esp_lcd_panel_rgb.c b/components/esp_lcd/rgb/esp_lcd_panel_rgb.c index 6d3ed816b4..616e68bb6b 100644 --- a/components/esp_lcd/rgb/esp_lcd_panel_rgb.c +++ b/components/esp_lcd/rgb/esp_lcd_panel_rgb.c @@ -1013,7 +1013,7 @@ static esp_err_t lcd_rgb_panel_init_trans_link(esp_rgb_panel_t *rgb_panel) mount_cfgs[i].buffer = rgb_panel->bounce_buffer[i]; mount_cfgs[i].buffer_alignment = buffer_alignment; mount_cfgs[i].length = rgb_panel->bb_size; - mount_cfgs[i].flags.mark_eof = GDMA_FINAL_LINK_TO_NULL; // we use the DMA EOF interrupt to copy the frame buffer (partially) to the bounce buffer + mount_cfgs[i].flags.mark_eof = true; // we use the DMA EOF interrupt to copy the frame buffer (partially) to the bounce buffer } ESP_RETURN_ON_ERROR(gdma_link_mount_buffers(rgb_panel->dma_bb_link, 0, mount_cfgs, RGB_LCD_PANEL_BOUNCE_BUF_NUM, NULL), TAG, "mount DMA bounce buffers failed");