pioduino/esp-idf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'fix/make_scratch_buffer_dynamic' into 'master'

Browse Source 
fix(esp_http_server): Make the http server scratch buffer dynamic

Closes IDF-11879

See merge request espressif/esp-idf!36651
This commit is contained in:
Mahavir Jain
2025-02-14 14:00:10 +08:00
parent 90bad97f01 9846584def
commit e3e49d7463
17 changed files with 129 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/esp_http_server/Kconfig
+9 -4
View File
@@ -1,11 +1,16 @@
menu "HTTP Server"
config HTTPD_MAX_REQ_HDR_LEN
int "Max HTTP Request Header Length"
default 512
int "HTTP Request Header Length limit"
default 1024
range 128 65536
help
This sets the maximum supported size of headers section in HTTP request packet to be processed by the
server
This sets the default limit for the HTTP request header length. The limit can be
configured at run time by setting max_req_hdr_len member of httpd_config_t structure.
The memory allocated will depend on the actual header length. Hence keeping a sufficiently
large max header length is recommended.
config HTTPD_MAX_URI_LEN
int "Max HTTP URI Length"
components/esp_http_server/include/esp_http_server.h
+13 -10
View File
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@@ -55,6 +55,8 @@ initializer that should be kept in sync
.stack_size = 4096, \
.core_id = tskNO_AFFINITY, \
.task_caps = (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT), \
.max_req_hdr_len = CONFIG_HTTPD_MAX_REQ_HDR_LEN, \
.max_uri_len = CONFIG_HTTPD_MAX_URI_LEN, \
.server_port = 80, \
.ctrl_port = ESP_HTTPD_DEF_CTRL_PORT, \
.max_open_sockets = 7, \
@@ -173,6 +175,13 @@ typedef struct httpd_config {
BaseType_t core_id; /*!< The core the HTTP server task will run on */
uint32_t task_caps; /*!< The memory capabilities to use when allocating the HTTP server task's stack */
/**
* Size limits for the header and URI buffers respectively.
* These are just limits, allocation would depend upon actual size of URI/header.
*/
size_t max_req_hdr_len; /*!< Size limit for the header buffer (By default this value is set to CONFIG_HTTPD_MAX_REQ_HDR_LEN, overwrite is possible) */
size_t max_uri_len; /*!< Size limit for the URI buffer By default this value is set to CONFIG_HTTPD_MAX_URI_LEN, overwrite is possible) */
/**
* TCP Port number for receiving and transmitting HTTP traffic
*/
@@ -358,19 +367,13 @@ esp_err_t httpd_stop(httpd_handle_t handle);
* @{
*/
/* Max supported HTTP request header length */
#define HTTPD_MAX_REQ_HDR_LEN CONFIG_HTTPD_MAX_REQ_HDR_LEN
/* Max supported HTTP request URI length */
#define HTTPD_MAX_URI_LEN CONFIG_HTTPD_MAX_URI_LEN
/**
* @brief HTTP Request Data Structure
*/
typedef struct httpd_req {
httpd_handle_t handle; /*!< Handle to server instance */
int method; /*!< The type of HTTP request, -1 if unsupported method, HTTP_ANY for wildcard method to support every method */
const char uri[HTTPD_MAX_URI_LEN + 1]; /*!< The URI of this request (1 byte extra for null termination) */
const char uri[CONFIG_HTTPD_MAX_URI_LEN + 1]; /*!< The URI of this request (1 byte extra for null termination) */
size_t content_len; /*!< Length of the request body */
void *aux; /*!< Internally used members */
@@ -611,10 +614,10 @@ typedef enum {
/* Incoming payload is too large */
HTTPD_413_CONTENT_TOO_LARGE,
/* URI length greater than CONFIG_HTTPD_MAX_URI_LEN */
/* URI length greater than CONFIG_CONFIG_HTTPD_MAX_URI_LEN */
HTTPD_414_URI_TOO_LONG,
/* Headers section larger than CONFIG_HTTPD_MAX_REQ_HDR_LEN */
/* Headers section larger than CONFIG_CONFIG_HTTPD_MAX_REQ_HDR_LEN */
HTTPD_431_REQ_HDR_FIELDS_TOO_LARGE,
/* Used internally for retrieving the total count of errors */
components/esp_http_server/src/esp_httpd_priv.h
+7 -7
View File
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@@ -31,13 +31,9 @@ extern "C" {
#endif
/* Size of request data block/chunk (not to be confused with chunked encoded data)
* that is received and parsed in one turn of the parsing process. This should not
* exceed the scratch buffer size and should at least be 8 bytes */
* that is received and parsed in one turn of the parsing process. */
#define PARSER_BLOCK_SIZE 128
/* Calculate the maximum size needed for the scratch buffer */
#define HTTPD_SCRATCH_BUF MAX(HTTPD_MAX_REQ_HDR_LEN, HTTPD_MAX_URI_LEN)
/* Formats a log string to prepend context function name */
#define LOG_FMT(x) "%s: " x, __func__
@@ -88,7 +84,11 @@ struct sock_db {
*/
struct httpd_req_aux {
struct sock_db *sd; /*!< Pointer to socket database */
char scratch[HTTPD_SCRATCH_BUF + 1]; /*!< Temporary buffer for our operations (1 byte extra for null termination) */
char *scratch; /*!< Temporary buffer for our operations (1 byte extra for null termination) */
size_t scratch_size_limit; /*!< Scratch buffer size limit (By default this value is set to CONFIG_HTTPD_MAX_REQ_HDR_LEN, overwrite is possible) */
size_t scratch_cur_size; /*!< Scratch buffer cur size (By default this value is set to CONFIG_HTTPD_MAX_URI_LEN, overwrite is possible) */
size_t max_req_hdr_len; /*!< Header buffer size limit */
size_t max_uri_len; /*!< URI buffer size limit */
size_t remaining_len; /*!< Amount of data remaining to be fetched */
char *status; /*!< HTTP response's status code */
char *content_type; /*!< HTTP response's content type */
components/esp_http_server/src/httpd_parse.c
+37 -8
View File
@@ -112,6 +112,8 @@ static esp_err_t cb_url(http_parser *parser,
const char *at, size_t length)
{
parser_data_t *parser_data = (parser_data_t *) parser->data;
httpd_req_t *req = parser_data->req;
struct httpd_req_aux *raux = req->aux;
if (parser_data->status == PARSING_IDLE) {
ESP_LOGD(TAG, LOG_FMT("message begin"));
@@ -130,9 +132,9 @@ static esp_err_t cb_url(http_parser *parser,
ESP_LOGD(TAG, LOG_FMT("processing url = %.*s"), (int)length, at);
/* Update length of URL string */
if ((parser_data->last.length += length) > HTTPD_MAX_URI_LEN) {
if ((parser_data->last.length += length) > raux->max_uri_len) {
ESP_LOGW(TAG, LOG_FMT("URI length (%"NEWLIB_NANO_COMPAT_FORMAT") greater than supported (%d)"),
NEWLIB_NANO_COMPAT_CAST(parser_data->last.length), HTTPD_MAX_URI_LEN);
NEWLIB_NANO_COMPAT_CAST(parser_data->last.length), raux->max_uri_len);
parser_data->error = HTTPD_414_URI_TOO_LONG;
parser_data->status = PARSING_FAILED;
return ESP_FAIL;
@@ -215,6 +217,7 @@ static esp_err_t cb_header_field(http_parser *parser, const char *at, size_t len
parser_data->last.at = ra->scratch;
parser_data->last.length = 0;
parser_data->status = PARSING_HDR_FIELD;
ra->scratch_size_limit = ra->max_req_hdr_len;
/* Stop parsing for now and give control to process */
if (pause_parsing(parser, at) != ESP_OK) {
@@ -232,6 +235,7 @@ static esp_err_t cb_header_field(http_parser *parser, const char *at, size_t len
parser_data->last.at = at;
parser_data->last.length = 0;
parser_data->status = PARSING_HDR_FIELD;
ra->scratch_size_limit = ra->max_req_hdr_len;
/* Increment header count */
ra->req_hdrs_count++;
@@ -480,16 +484,32 @@ static esp_err_t cb_no_body(http_parser *parser)
return ESP_OK;
}
static int read_block(httpd_req_t *req, size_t offset, size_t length)
static int read_block(httpd_req_t *req, http_parser *parser, size_t offset, size_t length)
{
struct httpd_req_aux *raux = req->aux;
parser_data_t *parser_data = (parser_data_t *) parser->data;
/* Limits the read to scratch buffer size */
ssize_t buf_len = MIN(length, (sizeof(raux->scratch) - offset));
ssize_t buf_len = MIN(length, (raux->scratch_size_limit - offset));
if (buf_len <= 0) {
return 0;
}
/* Calculate the offset of the current position from the start of the buffer,
* as after reallocating the buffer, the base address of the buffer may change.
*/
size_t at_offset = parser_data->last.at - raux->scratch;
/* Allocate the buffer according to offset and buf_len. Offset is
from where the reading will start and buf_len is till what length
the buffer will be read.
*/
raux->scratch = (char*) realloc(raux->scratch, offset + buf_len);
if (raux->scratch == NULL) {
ESP_LOGE(TAG, "Unable to allocate the scratch buffer");
return 0;
}
parser_data->last.at = raux->scratch + at_offset;
raux->scratch_cur_size = offset + buf_len;
ESP_LOGD(TAG, "scratch buf qsize = %d", raux->scratch_cur_size);
/* Receive data into buffer. If data is pending (from unrecv) then return
* immediately after receiving pending data, as pending data may just complete
* this request packet. */
@@ -533,13 +553,14 @@ static int parse_block(http_parser *parser, size_t offset, size_t length)
* parse means no more space left on buffer,
* therefore it can be inferred that the
* request URI/header must be too long */
ESP_LOGW(TAG, LOG_FMT("request URI/header too long"));
switch (data->status) {
case PARSING_URL:
ESP_LOGW(TAG, LOG_FMT("request URI too long"));
data->error = HTTPD_414_URI_TOO_LONG;
break;
case PARSING_HDR_FIELD:
case PARSING_HDR_VALUE:
ESP_LOGW(TAG, LOG_FMT("request header too long"));
data->error = HTTPD_431_REQ_HDR_FIELDS_TOO_LARGE;
break;
default:
@@ -632,7 +653,7 @@ static esp_err_t httpd_parse_req(struct httpd_data *hd)
offset = 0;
do {
/* Read block into scratch buffer */
if ((blk_len = read_block(r, offset, PARSER_BLOCK_SIZE)) < 0) {
if ((blk_len = read_block(r, &parser, offset, PARSER_BLOCK_SIZE)) < 0) {
if (blk_len == HTTPD_SOCK_ERR_TIMEOUT) {
/* Retry read in case of non-fatal timeout error.
* read_block() ensures that the timeout error is
@@ -679,13 +700,17 @@ static void init_req(httpd_req_t *r, httpd_config_t *config)
static void init_req_aux(struct httpd_req_aux *ra, httpd_config_t *config)
{
ra->sd = 0;
memset(ra->scratch, 0, sizeof(ra->scratch));
ra->remaining_len = 0;
ra->status = 0;
ra->content_type = 0;
ra->first_chunk_sent = 0;
ra->req_hdrs_count = 0;
ra->resp_hdrs_count = 0;
ra->scratch = NULL;
ra->scratch_cur_size = 0;
ra->max_req_hdr_len = (config->max_req_hdr_len > 0) ? config->max_req_hdr_len : CONFIG_HTTPD_MAX_REQ_HDR_LEN;
ra->max_uri_len = (config->max_uri_len > 0) ? config->max_uri_len : CONFIG_HTTPD_MAX_URI_LEN;
ra->scratch_size_limit = ra->max_uri_len;
#if CONFIG_HTTPD_WS_SUPPORT
ra->ws_handshake_detect = false;
#endif
@@ -716,6 +741,10 @@ static void httpd_req_cleanup(httpd_req_t *r)
/* Clear out the request and request_aux structures */
ra->sd = NULL;
free(ra->scratch);
ra->scratch = NULL;
ra->scratch_size_limit = 0;
ra->scratch_cur_size = 0;
r->handle = NULL;
r->aux = NULL;
r->user_ctx = NULL;
components/esp_http_server/src/httpd_txrx.c
+40 -13
View File
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@@ -247,14 +247,26 @@ esp_err_t httpd_resp_send(httpd_req_t *r, const char *buf, ssize_t buf_len)
/* Request headers are no longer available */
ra->req_hdrs_count = 0;
/* Size of essential headers is limited by scratch buffer size */
if (snprintf(ra->scratch, sizeof(ra->scratch), httpd_hdr_str,
ra->status, ra->content_type, buf_len) >= sizeof(ra->scratch)) {
/* Calculate the size of the headers. +1 for the null terminator */
size_t required_size = snprintf(NULL, 0, httpd_hdr_str, ra->status, ra->content_type, buf_len) + 1;
if (required_size > ra->max_req_hdr_len) {
return ESP_ERR_HTTPD_RESP_HDR;
}
char *res_buf = malloc(required_size); /* Temporary buffer to store the headers */
if (res_buf == NULL) {
ESP_LOGE(TAG, "Unable to allocate httpd send buffer");
return ESP_ERR_HTTPD_ALLOC_MEM;
}
ESP_LOGD(TAG, "httpd send buffer size = %d", strlen(res_buf));
/* Sending essential headers */
if (httpd_send_all(r, ra->scratch, strlen(ra->scratch)) != ESP_OK) {
esp_err_t ret = snprintf(res_buf, required_size, httpd_hdr_str, ra->status, ra->content_type, buf_len);
if (ret < 0 || ret >= required_size) {
free(res_buf);
return ESP_ERR_HTTPD_RESP_HDR;
}
ret = httpd_send_all(r, res_buf, strlen(res_buf));
free(res_buf);
if (ret != ESP_OK) {
return ESP_ERR_HTTPD_RESP_SEND;
}
@@ -320,15 +332,27 @@ esp_err_t httpd_resp_send_chunk(httpd_req_t *r, const char *buf, ssize_t buf_len
/* Request headers are no longer available */
ra->req_hdrs_count = 0;
/* Calculate the size of the headers. +1 for the null terminator */
size_t required_size = snprintf(NULL, 0, httpd_chunked_hdr_str, ra->status, ra->content_type) + 1;
if (required_size > ra->max_req_hdr_len) {
return ESP_ERR_HTTPD_RESP_HDR;
}
char *res_buf = malloc(required_size); /* Temporary buffer to store the headers */
if (res_buf == NULL) {
ESP_LOGE(TAG, "Unable to allocate httpd send chunk buffer");
return ESP_ERR_HTTPD_ALLOC_MEM;
}
ESP_LOGD(TAG, "httpd send chunk buffer size = %d", strlen(res_buf));
if (!ra->first_chunk_sent) {
/* Size of essential headers is limited by scratch buffer size */
if (snprintf(ra->scratch, sizeof(ra->scratch), httpd_chunked_hdr_str,
ra->status, ra->content_type) >= sizeof(ra->scratch)) {
esp_err_t ret = snprintf(res_buf, required_size, httpd_chunked_hdr_str, ra->status, ra->content_type);
if (ret < 0 || ret >= required_size) {
free(res_buf);
return ESP_ERR_HTTPD_RESP_HDR;
}
/* Sending essential headers */
if (httpd_send_all(r, ra->scratch, strlen(ra->scratch)) != ESP_OK) {
/* Size of essential headers is limited by scratch buffer size */
ret = httpd_send_all(r, res_buf, strlen(res_buf));
free(res_buf);
if (ret != ESP_OK) {
return ESP_ERR_HTTPD_RESP_SEND;
}
@@ -650,7 +674,10 @@ esp_err_t httpd_req_async_handler_complete(httpd_req_t *r)
struct httpd_req_aux *ra = r->aux;
ra->sd->for_async_req = false;
free(ra->scratch);
ra->scratch = NULL;
ra->scratch_cur_size = 0;
ra->scratch_size_limit = 0;
free(ra->resp_hdrs);
free(r->aux);
free(r);
components/esp_https_server/include/esp_https_server.h
+2
View File
@@ -150,6 +150,8 @@ typedef struct httpd_ssl_config httpd_ssl_config_t;
.stack_size = 10240, \
.core_id = tskNO_AFFINITY, \
.task_caps = (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT), \
.max_req_hdr_len = CONFIG_HTTPD_MAX_REQ_HDR_LEN, \
.max_uri_len = CONFIG_HTTPD_MAX_URI_LEN, \
.server_port = 0, \
.ctrl_port = ESP_HTTPD_DEF_CTRL_PORT+1, \
.max_open_sockets = 4, \