fix(mbedtls/port): Align AES and SHA DMA buffers to 16 when SPIRAM encryption is enabled

- Targets that support GDMA and MSPI encryption module need data and addresses aligned to 16
This commit is contained in:
harshal.patil
2025-11-11 17:45:11 +05:30
parent 16158b40f7
commit 317a6f074d
10 changed files with 101 additions and 56 deletions
-3
View File
@@ -233,9 +233,6 @@ endif()
if((SHA_PERIPHERAL_TYPE STREQUAL "core" AND CONFIG_SOC_SHA_SUPPORT_DMA) OR AES_PERIPHERAL_TYPE STREQUAL "dma")
target_link_libraries(mbedcrypto PRIVATE idf::esp_mm)
if(CONFIG_SOC_SHA_GDMA OR CONFIG_SOC_AES_GDMA)
if(CONFIG_SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT)
target_link_libraries(mbedcrypto PRIVATE idf::bootloader_support)
endif()
target_sources(mbedcrypto PRIVATE "${COMPONENT_DIR}/port/crypto_shared_gdma/esp_crypto_shared_gdma.c")
endif()
endif()
@@ -41,9 +41,9 @@
#include "aes/esp_aes_gcm.h"
#endif
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
#include "esp_flash_encrypt.h"
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
#ifdef SOC_GDMA_EXT_MEM_ENC_ALIGNMENT
#include "hal/efuse_hal.h"
#endif /* SOC_GDMA_EXT_MEM_ENC_ALIGNMENT */
/* Max size of each chunk to process when output buffer is in unaligned external ram
must be a multiple of block size
@@ -241,17 +241,17 @@ static int esp_aes_process_dma_ext_ram(esp_aes_context *ctx, const unsigned char
/* When AES-DMA operations are carried out using external memory with external memory encryption enabled,
we need to make sure that the addresses and the sizes of the buffers on which the DMA operates are 16 byte-aligned. */
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
if (esp_flash_encryption_enabled()) {
#ifdef SOC_GDMA_EXT_MEM_ENC_ALIGNMENT
if (efuse_hal_flash_encryption_enabled()) {
if (esp_ptr_external_ram(input) || esp_ptr_external_ram(output) || esp_ptr_in_drom(input) || esp_ptr_in_drom(output)) {
input_alignment = MAX(get_cache_line_size(input), SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT);
output_alignment = MAX(get_cache_line_size(output), SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT);
input_alignment = MAX(get_cache_line_size(input), SOC_GDMA_EXT_MEM_ENC_ALIGNMENT);
output_alignment = MAX(get_cache_line_size(output), SOC_GDMA_EXT_MEM_ENC_ALIGNMENT);
input_heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(input) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
output_heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(output) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
}
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
#endif /* SOC_GDMA_EXT_MEM_ENC_ALIGNMENT */
if (realloc_input) {
input_buf = heap_caps_aligned_alloc(input_alignment, chunk_len, input_heap_caps);
@@ -537,19 +537,19 @@ int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsign
return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH;
}
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
if (esp_flash_encryption_enabled()) {
#ifdef SOC_GDMA_EXT_MEM_ENC_ALIGNMENT
if (efuse_hal_flash_encryption_enabled()) {
if (esp_ptr_external_ram(input) || esp_ptr_external_ram(output) || esp_ptr_in_drom(input) || esp_ptr_in_drom(output)) {
if (((intptr_t)(input) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
if (((intptr_t)(input) & (SOC_GDMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
input_needs_realloc = true;
}
if (((intptr_t)(output) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
if (((intptr_t)(output) & (SOC_GDMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
output_needs_realloc = true;
}
}
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
#endif /* SOC_GDMA_EXT_MEM_ENC_ALIGNMENT */
/* DMA cannot access memory in the iCache range, copy input to internal ram */
if (!s_check_dma_capable(input)) {
@@ -1002,6 +1002,20 @@ int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsign
if (block_bytes > 0) {
/* Flush cache if input in external ram */
#if (CONFIG_SPIRAM && SOC_PSRAM_DMA_CAPABLE)
#ifdef SOC_GDMA_EXT_MEM_ENC_ALIGNMENT
if (efuse_hal_flash_encryption_enabled()) {
if (esp_ptr_external_ram(input) || esp_ptr_external_ram(output) || esp_ptr_in_drom(input) || esp_ptr_in_drom(output)) {
if (((intptr_t)(input) & (SOC_GDMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
input_needs_realloc = true;
}
if (((intptr_t)(output) & (SOC_GDMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
output_needs_realloc = true;
}
}
}
#endif /* SOC_GDMA_EXT_MEM_ENC_ALIGNMENT */
if (esp_ptr_external_ram(input)) {
if (esp_cache_msync((void *)input, len, ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_UNALIGNED) != ESP_OK) {
mbedtls_platform_zeroize(output, len);
@@ -1049,12 +1063,18 @@ int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsign
block_in_desc = block_desc;
block_out_desc = block_desc + crypto_dma_desc_num;
#if SOC_AHB_GDMA_VERSION == 2
// Limit max inlink descriptor length to be 16 byte aligned, as buffer sizes need to be 16 byte aligned
// when Flash Encryption is enabled.
dma_desc_setup_link(block_in_desc, input, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_16B_ALIGNED, 0);
#else
// the size field has 12 bits, but 0 not for 4096.
// to avoid possible problem when the size is not word-aligned, we only use 4096-4 per desc.
// Maximum size of data in the buffer that a DMA descriptor can hold.
dma_desc_setup_link(block_in_desc, input, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_4B_ALIGNED, 0);
#endif
//Limit max inlink descriptor length to be 16 byte aligned, require for EDMA
//Limit max outlink descriptor length to be 16 byte aligned, require for EDMA
dma_desc_setup_link(block_out_desc, output, block_bytes, DMA_DESCRIPTOR_BUFFER_MAX_SIZE_16B_ALIGNED, 0);
/* Setup in/out start descriptors */
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@@ -17,14 +17,16 @@
#include "soc/soc_caps.h"
#include "sdkconfig.h"
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
#include "esp_flash_encrypt.h"
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
#ifdef SOC_GDMA_EXT_MEM_ENC_ALIGNMENT
#include "hal/efuse_hal.h"
#endif /* SOC_GDMA_EXT_MEM_ENC_ALIGNMENT */
#if SOC_AHB_GDMA_VERSION == 1
#include "hal/gdma_ll.h"
#elif SOC_AXI_GDMA_SUPPORTED
#if SOC_AXI_GDMA_SUPPORTED
#include "hal/axi_dma_ll.h"
#elif SOC_AHB_GDMA_VERSION == 1
#include "hal/gdma_ll.h"
#elif SOC_AHB_GDMA_VERSION == 2
#include "hal/ahb_dma_ll.h"
#endif /* SOC_AHB_GDMA_VERSION */
#define NEW_CHANNEL_TIMEOUT_MS 1000
@@ -89,7 +91,16 @@ static esp_err_t crypto_shared_gdma_init(void)
.access_ext_mem = true, // crypto peripheral may want to access PSRAM
};
gdma_config_transfer(tx_channel, &transfer_cfg);
/* When using AHB-GDMA version 1, the max data burst size must be 0, otherwise buffers need to be aligned as well.
* Whereas, in case of the other GDMA versions, the RX max burst size is default enabled, but with default burst size of 4,
* but it case of Flash Encryption, the buffers can be allocated from the external memory, which requires 16 byte alignment.
* Thus, we set the max data burst size to 16, similar to the TX channel.
*/
#if SOC_AHB_GDMA_VERSION == 1 || SOC_AXI_GDMA_SUPPORTED // IDF-14335: SOC_AXI_GDMA_SUPPORTED might not be needed here
transfer_cfg.max_data_burst_size = 0;
#endif
gdma_config_transfer(rx_channel, &transfer_cfg);
#ifdef SOC_AES_SUPPORTED
@@ -156,7 +167,7 @@ esp_err_t esp_crypto_shared_gdma_start(const lldesc_t *input, const lldesc_t *ou
/* The external memory ecc-aes access must be enabled when there exists
at least one buffer in the DMA descriptors that resides in external memory. */
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
#if (SOC_GDMA_EXT_MEM_ENC_ALIGNMENT && SOC_AXI_GDMA_SUPPORTED)
static bool check_dma_descs_need_ext_mem_ecc_aes_access(const crypto_dma_desc_t *dmadesc)
{
crypto_dma_desc_t* desc = (crypto_dma_desc_t*) dmadesc;
@@ -168,7 +179,7 @@ static bool check_dma_descs_need_ext_mem_ecc_aes_access(const crypto_dma_desc_t
}
return false;
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
#endif /* (SOC_GDMA_EXT_MEM_ENC_ALIGNMENT && SOC_AXI_GDMA_SUPPORTED) */
esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, const crypto_dma_desc_t *output, gdma_trigger_peripheral_t peripheral)
{
@@ -203,16 +214,19 @@ esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, c
/* tx channel is reset by gdma_connect(), also reset rx to ensure a known state */
gdma_get_channel_id(rx_channel, &rx_ch_id);
#if SOC_AHB_GDMA_VERSION == 1
gdma_ll_rx_reset_channel(&GDMA, rx_ch_id);
#elif SOC_AXI_GDMA_SUPPORTED
// IDF-14335: Use gdma_reset() instead
#if SOC_AXI_GDMA_SUPPORTED
axi_dma_ll_rx_reset_channel(&AXI_DMA, rx_ch_id);
#endif /* SOC_AHB_GDMA_VERSION */
#elif SOC_AHB_GDMA_VERSION == 1
gdma_ll_rx_reset_channel(&GDMA, rx_ch_id);
#elif SOC_AHB_GDMA_VERSION == 2
ahb_dma_ll_rx_reset_channel(&AHB_DMA, rx_ch_id);
#endif /* SOC_AXI_GDMA_SUPPORTED */
/* When GDMA operations are carried out using external memory with external memory encryption enabled,
we need to enable AXI-DMA's AES-ECC mean access bit. */
#if (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT)
if (esp_flash_encryption_enabled()) {
#if (SOC_GDMA_EXT_MEM_ENC_ALIGNMENT && SOC_AXI_GDMA_SUPPORTED)
if (efuse_hal_flash_encryption_enabled()) {
int tx_ch_id = 0;
gdma_get_channel_id(tx_channel, &tx_ch_id);
@@ -224,7 +238,7 @@ esp_err_t esp_crypto_shared_gdma_start_axi_ahb(const crypto_dma_desc_t *input, c
axi_dma_ll_tx_enable_ext_mem_ecc_aes_access(&AXI_DMA, tx_ch_id, false);
}
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
#endif /* SOC_GDMA_EXT_MEM_ENC_ALIGNMENT */
gdma_start(tx_channel, (intptr_t)input);
gdma_start(rx_channel, (intptr_t)output);
+25 -21
View File
@@ -43,9 +43,9 @@
#include "esp_sha_dma_priv.h"
#include "sdkconfig.h"
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
#include "esp_flash_encrypt.h"
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
#ifdef SOC_GDMA_EXT_MEM_ENC_ALIGNMENT
#include "hal/efuse_hal.h"
#endif /* SOC_GDMA_EXT_MEM_ENC_ALIGNMENT */
#if SOC_SHA_CRYPTO_DMA
#include "hal/crypto_dma_ll.h"
@@ -154,7 +154,7 @@ static DRAM_ATTR crypto_dma_desc_t s_dma_descr_buf;
static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, uint32_t ilen,
const void *buf, uint32_t buf_len, bool is_first_block);
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
#ifdef SOC_GDMA_EXT_MEM_ENC_ALIGNMENT
static esp_err_t esp_sha_dma_process_ext(esp_sha_type sha_type, const void *input, uint32_t ilen,
const void *buf, uint32_t buf_len, bool is_first_block,
bool realloc_input, bool realloc_buf)
@@ -170,7 +170,7 @@ static esp_err_t esp_sha_dma_process_ext(esp_sha_type sha_type, const void *inpu
if (realloc_input) {
heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(input) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
input_copy = heap_caps_aligned_alloc(SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT, ilen, heap_caps);
input_copy = heap_caps_aligned_alloc(SOC_GDMA_EXT_MEM_ENC_ALIGNMENT, ilen, heap_caps);
if (input_copy == NULL) {
ESP_LOGE(TAG, "Failed to allocate aligned SPIRAM memory");
return ret;
@@ -183,7 +183,7 @@ static esp_err_t esp_sha_dma_process_ext(esp_sha_type sha_type, const void *inpu
if (realloc_buf) {
heap_caps = MALLOC_CAP_8BIT | (esp_ptr_external_ram(buf) ? MALLOC_CAP_SPIRAM : MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
buf_copy = heap_caps_aligned_alloc(SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT, buf_len, heap_caps);
buf_copy = heap_caps_aligned_alloc(SOC_GDMA_EXT_MEM_ENC_ALIGNMENT, buf_len, heap_caps);
if (buf_copy == NULL) {
ESP_LOGE(TAG, "Failed to allocate aligned internal memory");
return ret;
@@ -206,7 +206,7 @@ static esp_err_t esp_sha_dma_process_ext(esp_sha_type sha_type, const void *inpu
return ret;
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
#endif /* SOC_GDMA_EXT_MEM_ENC_ALIGNMENT */
/* Performs SHA on multiple blocks at a time */
static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, uint32_t ilen,
@@ -227,17 +227,17 @@ static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, u
/* When SHA-DMA operations are carried out using external memory with external memory encryption enabled,
we need to make sure that the addresses and the sizes of the buffers on which the DMA operates are 16 byte-aligned. */
#ifdef SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
if (esp_flash_encryption_enabled()) {
#ifdef SOC_GDMA_EXT_MEM_ENC_ALIGNMENT
if (efuse_hal_flash_encryption_enabled()) {
if (esp_ptr_external_ram(input) || esp_ptr_external_ram(buf) || esp_ptr_in_drom(input) || esp_ptr_in_drom(buf)) {
bool input_needs_realloc = false;
bool buf_needs_realloc = false;
if (ilen && ((intptr_t)(input) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
if (ilen && ((intptr_t)(input) & (SOC_GDMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
input_needs_realloc = true;
}
if (buf_len && ((intptr_t)(buf) & (SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
if (buf_len && ((intptr_t)(buf) & (SOC_GDMA_EXT_MEM_ENC_ALIGNMENT - 1)) != 0) {
buf_needs_realloc = true;
}
@@ -246,7 +246,7 @@ static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, u
}
}
}
#endif /* SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT */
#endif /* SOC_GDMA_EXT_MEM_ENC_ALIGNMENT */
/* DMA descriptor for Memory to DMA-SHA transfer */
if (ilen) {
@@ -273,6 +273,19 @@ static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, u
s_dma_descr_buf.next = (&s_dma_descr_input);
}
/* Write back buffers to memory if they are in external RAM
* The writeback needs to be performed in esp_sha_dma_process() instead of esp_sha_dma() to make
* sure that if the buffers are reallocated, then the writeback is performed on the new buffers.
*/
#if (CONFIG_SPIRAM && SOC_PSRAM_DMA_CAPABLE)
if (esp_ptr_external_ram(input)) {
esp_cache_msync((void *)input, ilen, ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_UNALIGNED);
}
if (esp_ptr_external_ram(buf)) {
esp_cache_msync((void *)buf, buf_len, ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_UNALIGNED);
}
#endif
#if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE
if (ilen) {
ESP_ERROR_CHECK(esp_cache_msync(&s_dma_descr_input, sizeof(crypto_dma_desc_t), ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_UNALIGNED));
@@ -317,15 +330,6 @@ int esp_sha_dma(esp_sha_type sha_type, const void *input, uint32_t ilen,
return 0;
}
#if (CONFIG_SPIRAM && SOC_PSRAM_DMA_CAPABLE)
if (esp_ptr_external_ram(input)) {
esp_cache_msync((void *)input, ilen, ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_UNALIGNED);
}
if (esp_ptr_external_ram(buf)) {
esp_cache_msync((void *)buf, buf_len, ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_UNALIGNED);
}
#endif
/* Copy to internal buf if buf is in non DMA capable memory */
if (!s_check_dma_capable(buf) && (buf_len != 0)) {
dma_cap_buf = heap_caps_malloc(sizeof(unsigned char) * buf_len, MALLOC_CAP_8BIT|MALLOC_CAP_DMA|MALLOC_CAP_INTERNAL);
@@ -519,6 +519,10 @@ config SOC_AHB_GDMA_SUPPORT_PSRAM
bool
default y
config SOC_GDMA_EXT_MEM_ENC_ALIGNMENT
int
default 16
config SOC_ETM_GROUPS
int
default 1
@@ -196,6 +196,7 @@
#define SOC_GDMA_SUPPORT_ETM 1
#define SOC_GDMA_SUPPORT_SLEEP_RETENTION 1
#define SOC_AHB_GDMA_SUPPORT_PSRAM 1
#define SOC_GDMA_EXT_MEM_ENC_ALIGNMENT (16)
/*-------------------------- ETM CAPS --------------------------------------*/
#define SOC_ETM_GROUPS 1U // Number of ETM groups
@@ -399,6 +399,10 @@ config SOC_AHB_GDMA_SUPPORT_PSRAM
bool
default y
config SOC_GDMA_EXT_MEM_ENC_ALIGNMENT
int
default 16
config SOC_ETM_GROUPS
int
default 1
@@ -153,6 +153,7 @@
#define SOC_GDMA_SUPPORT_ETM 1 // Support ETM submodule
#define SOC_GDMA_SUPPORT_SLEEP_RETENTION 1
#define SOC_AHB_GDMA_SUPPORT_PSRAM 1
#define SOC_GDMA_EXT_MEM_ENC_ALIGNMENT (16)
/*-------------------------- ETM CAPS --------------------------------------*/
#define SOC_ETM_GROUPS 1U // Number of ETM groups
@@ -623,7 +623,7 @@ config SOC_GDMA_SUPPORT_SLEEP_RETENTION
bool
default y
config SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT
config SOC_GDMA_EXT_MEM_ENC_ALIGNMENT
int
default 16
@@ -222,7 +222,7 @@
#define SOC_AXI_GDMA_SUPPORT_PSRAM 1
#define SOC_GDMA_SUPPORT_ETM 1
#define SOC_GDMA_SUPPORT_SLEEP_RETENTION 1
#define SOC_AXI_DMA_EXT_MEM_ENC_ALIGNMENT (16)
#define SOC_GDMA_EXT_MEM_ENC_ALIGNMENT (16)
/*-------------------------- 2D-DMA CAPS -------------------------------------*/
#define SOC_DMA2D_GROUPS (1U) // Number of 2D-DMA groups