3.3.7

tools/espota.py

@@ -0,0 +1,583 @@
+#!/usr/bin/env python
+#
+# Original espota.py by Ivan Grokhotkov:
+# https://gist.github.com/igrr/d35ab8446922179dc58c
+#
+# Modified since 2015-09-18 from Pascal Gollor (https://github.com/pgollor)
+# Modified since 2015-11-09 from Hristo Gochkov (https://github.com/me-no-dev)
+# Modified since 2016-01-03 from Matthew O'Gorman (https://githumb.com/mogorman)
+# Modified since 2025-09-04 from Lucas Saavedra Vaz (https://github.com/lucasssvaz)
+#
+# This script will push an OTA update to the ESP
+# use it like:
+# python espota.py -i <ESP_IP_addr> -I <Host_IP_addr> -p <ESP_port> -P <Host_port> [-a password] -f <sketch.bin>
+# Or to upload SPIFFS image:
+# python espota.py -i <ESP_IP_addr> -I <Host_IP_addr> -p <ESP_port> -P <HOST_port> [-a password] -s -f <spiffs.bin>
+#
+# Changes
+# 2015-09-18:
+# - Add option parser.
+# - Add logging.
+# - Send command to controller to differ between flashing and transmitting SPIFFS image.
+#
+# Changes
+# 2015-11-09:
+# - Added digest authentication
+# - Enhanced error tracking and reporting
+#
+# Changes
+# 2016-01-03:
+# - Added more options to parser.
+#
+# Changes
+# 2023-05-22:
+# - Replaced the deprecated optparse module with argparse.
+# - Adjusted the code style to conform to PEP 8 guidelines.
+# - Used with statement for file handling to ensure proper resource cleanup.
+# - Incorporated exception handling to catch and handle potential errors.
+# - Made variable names more descriptive for better readability.
+# - Introduced constants for better code maintainability.
+#
+# Changes
+# 2025-09-04:
+# - Changed authentication to use PBKDF2-HMAC-SHA256 for challenge/response
+#
+# Changes
+# 2025-09-18:
+# - Fixed authentication when using old images with MD5 passwords
+#
+# Changes
+# 2025-10-07:
+# - Fixed authentication when images might use old MD5 hashes stored in the firmware
+
+
+from __future__ import print_function
+import socket
+import sys
+import os
+import argparse
+import logging
+import hashlib
+import random
+
+# Commands
+FLASH = 0
+SPIFFS = 100
+AUTH = 200
+
+# Constants
+PROGRESS_BAR_LENGTH = 60
+
+
+# update_progress(): Displays or updates a console progress bar
+def update_progress(progress):
+    if PROGRESS:
+        status = ""
+        if isinstance(progress, int):
+            progress = float(progress)
+        if not isinstance(progress, float):
+            progress = 0
+            status = "Error: progress var must be float\r\n"
+        if progress < 0:
+            progress = 0
+            status = "Halt...\r\n"
+        if progress >= 1:
+            progress = 1
+            status = "Done...\r\n"
+        block = int(round(PROGRESS_BAR_LENGTH * progress))
+        text = "\rUploading: [{0}] {1}% {2}".format(
+            "=" * block + " " * (PROGRESS_BAR_LENGTH - block), int(progress * 100), status
+        )
+        sys.stderr.write(text)
+        sys.stderr.flush()
+    else:
+        sys.stderr.write(".")
+        sys.stderr.flush()
+
+
+def send_invitation_and_get_auth_challenge(remote_addr, remote_port, message):
+    """
+    Send invitation to ESP device and get authentication challenge.
+    Returns (success, auth_data, error_message) tuple.
+    """
+    remote_address = (remote_addr, int(remote_port))
+    inv_tries = 0
+    data = ""
+
+    msg = "Sending invitation to %s " % remote_addr
+    sys.stderr.write(msg)
+    sys.stderr.flush()
+
+    while inv_tries < 10:
+        inv_tries += 1
+        sock2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        try:
+            sent = sock2.sendto(message.encode(), remote_address)  # noqa: F841
+        except:  # noqa: E722
+            sys.stderr.write("failed\n")
+            sys.stderr.flush()
+            sock2.close()
+            return False, None, "Host %s Not Found" % remote_addr
+
+        sock2.settimeout(TIMEOUT)
+        try:
+            # Try to read up to 69 bytes for new protocol (SHA256)
+            # If device sends less (37 bytes), it's using old MD5 protocol
+            data = sock2.recv(69).decode()
+            sock2.close()
+            break
+        except:  # noqa: E722
+            sys.stderr.write(".")
+            sys.stderr.flush()
+            sock2.close()
+
+    sys.stderr.write("\n")
+    sys.stderr.flush()
+
+    if inv_tries == 10:
+        return False, None, "No response from the ESP"
+
+    return True, data, None
+
+
+def authenticate(
+    remote_addr, remote_port, password, use_md5_password, use_old_protocol, filename, content_size, file_md5, nonce
+):
+    """
+    Perform authentication with the ESP device.
+
+    Args:
+        use_md5_password: If True, hash password with MD5 instead of SHA256
+        use_old_protocol: If True, use old MD5 challenge/response protocol (pre-3.3.1)
+
+    Returns (success, error_message) tuple.
+    """
+    cnonce_text = "%s%u%s%s" % (filename, content_size, file_md5, remote_addr)
+    remote_address = (remote_addr, int(remote_port))
+
+    if use_old_protocol:
+        # Generate client nonce (cnonce)
+        cnonce = hashlib.md5(cnonce_text.encode()).hexdigest()
+
+        # Old MD5 challenge/response protocol (pre-3.3.1)
+        # 1. Hash the password with MD5
+        password_hash = hashlib.md5(password.encode()).hexdigest()
+
+        # 2. Create challenge response
+        challenge = "%s:%s:%s" % (password_hash, nonce, cnonce)
+        response = hashlib.md5(challenge.encode()).hexdigest()
+        expected_response_length = 32
+    else:
+        # Generate client nonce (cnonce) using SHA256 for new protocol
+        cnonce = hashlib.sha256(cnonce_text.encode()).hexdigest()
+
+        # New PBKDF2-HMAC-SHA256 challenge/response protocol (3.3.1+)
+        # The password can be hashed with either MD5 or SHA256
+        if use_md5_password:
+            # Use MD5 for password hash (for devices that stored MD5 hashes)
+            password_hash = hashlib.md5(password.encode()).hexdigest()
+        else:
+            # Use SHA256 for password hash (recommended)
+            password_hash = hashlib.sha256(password.encode()).hexdigest()
+
+        # 2. Derive key using PBKDF2-HMAC-SHA256 with the password hash
+        salt = nonce + ":" + cnonce
+        derived_key = hashlib.pbkdf2_hmac("sha256", password_hash.encode(), salt.encode(), 10000)
+        derived_key_hex = derived_key.hex()
+
+        # 3. Create challenge response
+        challenge = derived_key_hex + ":" + nonce + ":" + cnonce
+        response = hashlib.sha256(challenge.encode()).hexdigest()
+        expected_response_length = 64
+
+    # Send authentication response
+    sock2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    try:
+        message = "%d %s %s\n" % (AUTH, cnonce, response)
+        sock2.sendto(message.encode(), remote_address)
+        sock2.settimeout(10)
+        try:
+            data = sock2.recv(expected_response_length).decode()
+        except:  # noqa: E722
+            sock2.close()
+            return False, "No Answer to our Authentication"
+
+        if data != "OK":
+            sock2.close()
+            return False, data
+
+        sock2.close()
+        return True, None
+    except Exception as e:
+        sock2.close()
+        return False, str(e)
+
+
+def serve(  # noqa: C901
+    remote_addr, local_addr, remote_port, local_port, password, md5_target, filename, command=FLASH
+):
+    # Create a TCP/IP socket
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    server_address = (local_addr, local_port)
+    logging.info("Starting on %s:%s", str(server_address[0]), str(server_address[1]))
+    try:
+        sock.bind(server_address)
+        sock.listen(1)
+    except Exception as e:
+        logging.error("Listen Failed: %s", str(e))
+        return 1
+
+    content_size = os.path.getsize(filename)
+    with open(filename, "rb") as f:
+        file_md5 = hashlib.md5(f.read()).hexdigest()
+    logging.info("Upload size: %d", content_size)
+    message = "%d %d %d %s\n" % (command, local_port, content_size, file_md5)
+
+    # Send invitation and get authentication challenge
+    success, data, error = send_invitation_and_get_auth_challenge(remote_addr, remote_port, message)
+    if not success:
+        logging.error(error)
+        return 1
+
+    if data != "OK":
+        if data.startswith("AUTH"):
+            nonce = data.split()[1]
+            nonce_length = len(nonce)
+
+            # Detect protocol version based on nonce length:
+            # - 32 chars = Old MD5 protocol (pre-3.3.1)
+            # - 64 chars = New SHA256 protocol (3.3.1+)
+
+            if nonce_length == 32:
+                # Scenario 1: Old device (pre-3.3.1) using MD5 protocol
+                logging.info("Detected old MD5 protocol (pre-3.3.1)")
+                sys.stderr.write("Authenticating (MD5 protocol)...")
+                sys.stderr.flush()
+                auth_success, auth_error = authenticate(
+                    remote_addr,
+                    remote_port,
+                    password,
+                    use_md5_password=True,
+                    use_old_protocol=True,
+                    filename=filename,
+                    content_size=content_size,
+                    file_md5=file_md5,
+                    nonce=nonce,
+                )
+
+                if not auth_success:
+                    sys.stderr.write("FAIL\n")
+                    logging.error("Authentication Failed: %s", auth_error)
+                    logging.error("Please check your password and try again")
+                    return 1
+
+                sys.stderr.write("OK\n")
+                logging.warning("====================================================================")
+                logging.warning("WARNING: Device is using old MD5 authentication protocol (pre-3.3.1)")
+                logging.warning("Please update to ESP32 Arduino Core 3.3.1+ for improved security.")
+                logging.warning("======================================================================")
+
+            elif nonce_length == 64:
+                # New protocol (3.3.1+) - try SHA256 password first, then MD5 if it fails
+
+                # Scenario 2: Try SHA256 password hash first (recommended for new devices)
+                if md5_target:
+                    # User explicitly requested MD5 password hash
+                    logging.info("Using MD5 password hash as requested")
+                    sys.stderr.write("Authenticating (SHA256 protocol with MD5 password)...")
+                    sys.stderr.flush()
+                    auth_success, auth_error = authenticate(
+                        remote_addr,
+                        remote_port,
+                        password,
+                        use_md5_password=True,
+                        use_old_protocol=False,
+                        filename=filename,
+                        content_size=content_size,
+                        file_md5=file_md5,
+                        nonce=nonce,
+                    )
+
+                    if auth_success:
+                        logging.warning("Using insecure MD5 hash for password due to legacy device support")
+                        logging.warning("Please upgrade devices to ESP32 Arduino Core 3.3.1+ for improved security")
+                else:
+                    # Try SHA256 password hash first
+                    sys.stderr.write("Authenticating (PBKDF2-HMAC-SHA256)...\n")
+                    sys.stderr.flush()
+                    auth_success, auth_error = authenticate(
+                        remote_addr,
+                        remote_port,
+                        password,
+                        use_md5_password=False,
+                        use_old_protocol=False,
+                        filename=filename,
+                        content_size=content_size,
+                        file_md5=file_md5,
+                        nonce=nonce,
+                    )
+
+                    # Scenario 3: If SHA256 fails, try MD5 password hash (for devices with stored MD5 passwords)
+                    if not auth_success:
+                        sys.stderr.write("FAIL\n")
+                        logging.info("SHA256 password failed, trying MD5 password hash")
+                        sys.stderr.write("Retrying with MD5 password...\n")
+                        sys.stderr.flush()
+
+                        # Device is back in OTA_IDLE after auth failure, need to send new invitation
+                        success, data, error = send_invitation_and_get_auth_challenge(remote_addr, remote_port, message)
+                        if not success:
+                            sys.stderr.write("FAIL\n")
+                            logging.error("Failed to get new challenge for MD5 retry: %s", error)
+                            return 1
+
+                        if not data.startswith("AUTH"):
+                            sys.stderr.write("FAIL\n")
+                            logging.error("Expected AUTH challenge for MD5 retry, got: %s", data)
+                            return 1
+
+                        # Get new nonce for second attempt
+                        nonce = data.split()[1]
+
+                        sys.stderr.write("Authenticating (MD5)...\n")
+                        sys.stderr.flush()
+
+                        auth_success, auth_error = authenticate(
+                            remote_addr,
+                            remote_port,
+                            password,
+                            use_md5_password=True,
+                            use_old_protocol=False,
+                            filename=filename,
+                            content_size=content_size,
+                            file_md5=file_md5,
+                            nonce=nonce,
+                        )
+
+                        if auth_success:
+                            logging.warning("====================================================================")
+                            logging.warning("WARNING: Device authenticated with MD5 password hash (deprecated)")
+                            logging.warning("MD5 is cryptographically broken and should not be used.")
+                            logging.warning(
+                                "Please update your sketch to use either setPassword() or setPasswordHash()"
+                            )
+                            logging.warning(
+                                "with SHA256, then upload again to migrate to the new secure SHA256 protocol."
+                            )
+                            logging.warning("======================================================================")
+
+                if not auth_success:
+                    sys.stderr.write("FAIL\n")
+                    logging.error("Authentication Failed: %s", auth_error)
+                    logging.error("Please check your password and try again")
+                    return 1
+
+                sys.stderr.write("OK\n")
+            else:
+                logging.error("Invalid nonce length: %d (expected 32 or 64)", nonce_length)
+                return 1
+        else:
+            logging.error("Bad Answer: %s", data)
+            return 1
+
+    logging.info("Waiting for device...")
+
+    try:
+        sock.settimeout(10)
+        connection, client_address = sock.accept()
+        sock.settimeout(None)
+        connection.settimeout(None)
+    except:  # noqa: E722
+        logging.error("No response from device")
+        sock.close()
+        return 1
+
+    try:
+        with open(filename, "rb") as f:
+            if PROGRESS:
+                update_progress(0)
+            else:
+                sys.stderr.write("Uploading")
+                sys.stderr.flush()
+            offset = 0
+            while True:
+                chunk = f.read(1024)
+                if not chunk:
+                    break
+                offset += len(chunk)
+                update_progress(offset / float(content_size))
+                connection.settimeout(10)
+                try:
+                    connection.sendall(chunk)
+                    res = connection.recv(10)
+                    response_text = res.decode().strip()
+                    last_response_contained_ok = "OK" in response_text
+                    logging.debug("Chunk response: '%s'", response_text)
+                except Exception as e:
+                    sys.stderr.write("\n")
+                    logging.error("Error Uploading: %s", str(e))
+                    connection.close()
+                    return 1
+
+            if last_response_contained_ok:
+                logging.info("Success")
+                connection.close()
+                return 0
+
+            sys.stderr.write("\n")
+            logging.info("Waiting for result...")
+            count = 0
+            received_any_response = False
+            while count < 10:  # Increased from 5 to 10 attempts
+                count += 1
+                connection.settimeout(30)  # Reduced from 60s to 30s per attempt
+                try:
+                    data = connection.recv(32).decode().strip()
+                    received_any_response = True
+                    logging.info("Result attempt %d: '%s'", count, data)
+
+                    if "OK" in data:
+                        logging.info("Success")
+                        connection.close()
+                        return 0
+                    elif data:  # Got some response but not OK
+                        logging.warning("Unexpected response from device: '%s'", data)
+
+                except socket.timeout:
+                    logging.debug("Timeout waiting for result (attempt %d/10)", count)
+                    continue
+                except Exception as e:
+                    logging.debug("Error receiving result (attempt %d/10): %s", count, str(e))
+                    # Don't return error here, continue trying
+                    continue
+
+            # After all attempts, provide detailed error information
+            if received_any_response:
+                logging.warning(
+                    "Upload completed but device sent unexpected response(s). This may still be successful."
+                )
+                logging.warning("Device might be rebooting to apply firmware - this is normal.")
+                connection.close()
+                return 0  # Consider it successful if we got any response and upload completed
+            else:
+                logging.error("No response from device after upload completion")
+                logging.error("This could indicate device reboot (normal) or network issues")
+                connection.close()
+                return 1
+    except Exception as e:  # noqa: E722
+        logging.error("Error: %s", str(e))
+    finally:
+        connection.close()
+
+    sock.close()
+    return 1
+
+
+def parse_args(unparsed_args):
+    parser = argparse.ArgumentParser(description="Transmit image over the air to the ESP32 module with OTA support.")
+
+    # destination ip and port
+    parser.add_argument("-i", "--ip", dest="esp_ip", action="store", help="ESP32 IP Address.", default=False)
+    parser.add_argument("-I", "--host_ip", dest="host_ip", action="store", help="Host IP Address.", default="0.0.0.0")
+    parser.add_argument("-p", "--port", dest="esp_port", type=int, help="ESP32 OTA Port. Default: 3232", default=3232)
+    parser.add_argument(
+        "-P",
+        "--host_port",
+        dest="host_port",
+        type=int,
+        help="Host server OTA Port. Default: random 10000-60000",
+        default=random.randint(10000, 60000),
+    )
+
+    # authentication
+    parser.add_argument("-a", "--auth", dest="auth", help="Set authentication password.", action="store", default="")
+    parser.add_argument(
+        "-m",
+        "--md5-target",
+        dest="md5_target",
+        help=(
+            "Use MD5 for password hashing (for devices with stored MD5 passwords). "
+            "By default, SHA256 is tried first, then MD5 as fallback."
+        ),
+        action="store_true",
+        default=False,
+    )
+
+    # image
+    parser.add_argument("-f", "--file", dest="image", help="Image file.", metavar="FILE", default=None)
+    parser.add_argument(
+        "-s",
+        "--spiffs",
+        dest="spiffs",
+        action="store_true",
+        help="Transmit a SPIFFS image and do not flash the module.",
+        default=False,
+    )
+
+    # output
+    parser.add_argument(
+        "-d",
+        "--debug",
+        dest="debug",
+        action="store_true",
+        help="Show debug output. Overrides loglevel with debug.",
+        default=False,
+    )
+    parser.add_argument(
+        "-r",
+        "--progress",
+        dest="progress",
+        action="store_true",
+        help="Show progress output. Does not work for Arduino IDE.",
+        default=False,
+    )
+    parser.add_argument(
+        "-t",
+        "--timeout",
+        dest="timeout",
+        type=int,
+        help="Timeout to wait for the ESP32 to accept invitation.",
+        default=10,
+    )
+
+    return parser.parse_args(unparsed_args)
+
+
+def main(args):
+    options = parse_args(args)
+    log_level = logging.WARNING
+    if options.debug:
+        log_level = logging.DEBUG
+
+    logging.basicConfig(level=log_level, format="%(asctime)-8s [%(levelname)s]: %(message)s", datefmt="%H:%M:%S")
+    logging.debug("Options: %s", str(options))
+
+    # check options
+    global PROGRESS
+    PROGRESS = options.progress
+
+    global TIMEOUT
+    TIMEOUT = options.timeout
+
+    if not options.esp_ip or not options.image:
+        logging.critical("Not enough arguments.")
+        return 1
+
+    command = FLASH
+    if options.spiffs:
+        command = SPIFFS
+
+    return serve(
+        options.esp_ip,
+        options.host_ip,
+        options.esp_port,
+        options.host_port,
+        options.auth,
+        options.md5_target,
+        options.image,
+        command,
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

tools/gen_esp32part.py

@@ -0,0 +1,789 @@
+#!/usr/bin/env python
+#
+# ESP32 partition table generation tool
+#
+# Converts partition tables to/from CSV and binary formats.
+#
+# See https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/partition-tables.html
+# for explanation of partition table structure and uses.
+#
+# SPDX-FileCopyrightText: 2016-2025 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import argparse
+import binascii
+import errno
+import hashlib
+import os
+import re
+import struct
+import sys
+
+MAX_PARTITION_LENGTH = 0xC00  # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature
+MD5_PARTITION_BEGIN = b"\xeb\xeb" + b"\xff" * 14  # The first 2 bytes are like magic numbers for MD5 sum
+PARTITION_TABLE_SIZE = 0x1000  # Size of partition table
+
+MIN_PARTITION_SUBTYPE_APP_OTA = 0x10
+NUM_PARTITION_SUBTYPE_APP_OTA = 16
+MIN_PARTITION_SUBTYPE_APP_TEE = 0x30
+NUM_PARTITION_SUBTYPE_APP_TEE = 2
+
+SECURE_NONE = None
+SECURE_V1 = "v1"
+SECURE_V2 = "v2"
+
+__version__ = "1.5"
+
+APP_TYPE = 0x00
+DATA_TYPE = 0x01
+BOOTLOADER_TYPE = 0x02
+PARTITION_TABLE_TYPE = 0x03
+
+TYPES = {
+    "bootloader": BOOTLOADER_TYPE,
+    "partition_table": PARTITION_TABLE_TYPE,
+    "app": APP_TYPE,
+    "data": DATA_TYPE,
+}
+
+NVS_RW_MIN_PARTITION_SIZE = 0x3000
+
+
+def get_ptype_as_int(ptype):
+    """Convert a string which might be numeric or the name of a partition type to an integer"""
+    try:
+        return TYPES[ptype]
+    except KeyError:
+        try:
+            return int(ptype, 0)
+        except TypeError:
+            return ptype
+
+
+# Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h
+SUBTYPES = {
+    BOOTLOADER_TYPE: {
+        "primary": 0x00,
+        "ota": 0x01,
+        "recovery": 0x02,
+    },
+    PARTITION_TABLE_TYPE: {
+        "primary": 0x00,
+        "ota": 0x01,
+    },
+    APP_TYPE: {
+        "factory": 0x00,
+        "test": 0x20,
+    },
+    DATA_TYPE: {
+        "ota": 0x00,
+        "phy": 0x01,
+        "nvs": 0x02,
+        "coredump": 0x03,
+        "nvs_keys": 0x04,
+        "efuse": 0x05,
+        "undefined": 0x06,
+        "esphttpd": 0x80,
+        "fat": 0x81,
+        "spiffs": 0x82,
+        "littlefs": 0x83,
+        "tee_ota": 0x90,
+    },
+}
+
+
+def get_subtype_as_int(ptype, subtype):
+    """Convert a string which might be numeric or the name of a partition subtype to an integer"""
+    try:
+        return SUBTYPES[get_ptype_as_int(ptype)][subtype]
+    except KeyError:
+        try:
+            return int(subtype, 0)
+        except TypeError:
+            return subtype
+
+
+ALIGNMENT = {
+    APP_TYPE: 0x10000,
+    DATA_TYPE: 0x1000,
+    BOOTLOADER_TYPE: 0x1000,
+    PARTITION_TABLE_TYPE: 0x1000,
+}
+
+
+def get_alignment_offset_for_type(ptype):
+    return ALIGNMENT.get(ptype, ALIGNMENT[DATA_TYPE])
+
+
+def get_alignment_size_for_type(ptype):
+    if ptype == APP_TYPE:
+        if secure == SECURE_V1:
+            # For secure boot v1 case, app partition must be 64K aligned
+            # signature block (68 bytes) lies at the very end of 64K block
+            return 0x10000
+        elif secure == SECURE_V2:
+            # For secure boot v2 case, app partition must be 4K aligned
+            # signature block (4K) is kept after padding the unsigned image to 64K boundary
+            return 0x1000
+        else:
+            # For no secure boot enabled case, app partition must be 4K aligned (min. flash erase size)
+            return 0x1000
+    # No specific size alignment requirement as such
+    return 0x1
+
+
+def get_partition_type(ptype):
+    if ptype == "app":
+        return APP_TYPE
+    if ptype == "data":
+        return DATA_TYPE
+    if ptype == "bootloader":
+        return BOOTLOADER_TYPE
+    if ptype == "partition_table":
+        return PARTITION_TABLE_TYPE
+    raise InputError("Invalid partition type")
+
+
+def add_extra_subtypes(csv):
+    for line_no in csv:
+        try:
+            fields = [line.strip() for line in line_no.split(",")]
+            for subtype, subtype_values in SUBTYPES.items():
+                if int(fields[2], 16) in subtype_values.values() and subtype == get_partition_type(fields[0]):
+                    raise ValueError("Found duplicate value in partition subtype")
+            SUBTYPES[TYPES[fields[0]]][fields[1]] = int(fields[2], 16)
+        except InputError as err:
+            raise InputError("Error parsing custom subtypes: %s" % err)
+
+
+quiet = False
+md5sum = True
+secure = SECURE_NONE
+offset_part_table = 0
+primary_bootloader_offset = None
+recovery_bootloader_offset = None
+
+
+def status(msg):
+    """Print status message to stderr"""
+    if not quiet:
+        critical(msg)
+
+
+def critical(msg):
+    """Print critical message to stderr"""
+    sys.stderr.write(msg)
+    sys.stderr.write("\n")
+
+
+class PartitionTable(list):
+    def __init__(self):
+        super(PartitionTable, self).__init__(self)
+
+    @classmethod
+    def from_file(cls, f):
+        data = f.read()
+        data_is_binary = data[0:2] == PartitionDefinition.MAGIC_BYTES
+        if data_is_binary:
+            status("Parsing binary partition input...")
+            return cls.from_binary(data), True
+
+        data = data.decode()
+        status("Parsing CSV input...")
+        return cls.from_csv(data), False
+
+    @classmethod
+    def from_csv(cls, csv_contents):
+        res = PartitionTable()
+        lines = csv_contents.splitlines()
+
+        def expand_vars(f):
+            f = os.path.expandvars(f)
+            m = re.match(r"(?<!\\)\$([A-Za-z_][A-Za-z0-9_]*)", f)
+            if m:
+                raise InputError("unknown variable '%s'" % m.group(1))
+            return f
+
+        for line_no in range(len(lines)):
+            line = expand_vars(lines[line_no]).strip()
+            if line.startswith("#") or len(line) == 0:
+                continue
+            try:
+                res.append(PartitionDefinition.from_csv(line, line_no + 1))
+            except InputError as err:
+                raise InputError(
+                    "Error at line %d: %s\nPlease check extra_partition_subtypes.inc file in build/config directory"
+                    % (line_no + 1, err)
+                )
+            except Exception:
+                critical("Unexpected error parsing CSV line %d: %s" % (line_no + 1, line))
+                raise
+
+        # fix up missing offsets & negative sizes
+        last_end = offset_part_table + PARTITION_TABLE_SIZE  # first offset after partition table
+        for e in res:
+            is_primary_bootloader = e.type == BOOTLOADER_TYPE and e.subtype == SUBTYPES[e.type]["primary"]
+            is_primary_partition_table = e.type == PARTITION_TABLE_TYPE and e.subtype == SUBTYPES[e.type]["primary"]
+            if is_primary_bootloader or is_primary_partition_table:
+                # They do not participate in the restoration of missing offsets
+                continue
+            if e.offset is not None and e.offset < last_end:
+                if e == res[0]:
+                    raise InputError(
+                        "CSV Error at line %d: Partitions overlap. Partition sets offset 0x%x. "
+                        "But partition table occupies the whole sector 0x%x. "
+                        "Use a free offset 0x%x or higher." % (e.line_no, e.offset, offset_part_table, last_end)
+                    )
+                else:
+                    raise InputError(
+                        "CSV Error at line %d: Partitions overlap. Partition sets offset 0x%x. "
+                        "Previous partition ends 0x%x" % (e.line_no, e.offset, last_end)
+                    )
+            if e.offset is None:
+                pad_to = get_alignment_offset_for_type(e.type)
+                if last_end % pad_to != 0:
+                    last_end += pad_to - (last_end % pad_to)
+                e.offset = last_end
+            if e.size < 0:
+                e.size = -e.size - e.offset
+            last_end = e.offset + e.size
+
+        return res
+
+    def __getitem__(self, item):
+        """Allow partition table access via name as well as by
+        numeric index."""
+        if isinstance(item, str):
+            for x in self:
+                if x.name == item:
+                    return x
+            raise ValueError("No partition entry named '%s'" % item)
+        else:
+            return super(PartitionTable, self).__getitem__(item)
+
+    def find_by_type(self, ptype, subtype):
+        """Return a partition by type & subtype, returns
+        None if not found"""
+        # convert ptype & subtypes names (if supplied this way) to integer values
+        ptype = get_ptype_as_int(ptype)
+        subtype = get_subtype_as_int(ptype, subtype)
+
+        for p in self:
+            if p.type == ptype and p.subtype == subtype:
+                yield p
+        return
+
+    def find_by_name(self, name):
+        for p in self:
+            if p.name == name:
+                return p
+        return None
+
+    def verify(self):
+        # verify each partition individually
+        for p in self:
+            p.verify()
+
+        # check on duplicate name
+        names = [p.name for p in self]
+        duplicates = {n for n in names if names.count(n) > 1}
+
+        # print sorted duplicate partitions by name
+        if len(duplicates) != 0:
+            critical("A list of partitions that have the same name:")
+            for p in sorted(self, key=lambda x: x.name):
+                if len(duplicates.intersection([p.name])) != 0:
+                    critical("%s" % (p.to_csv()))
+            raise InputError("Partition names must be unique")
+
+        # check for overlaps
+        last = None
+        for p in sorted(self, key=lambda x: x.offset):
+            if p.offset < offset_part_table + PARTITION_TABLE_SIZE:
+                is_primary_bootloader = p.type == BOOTLOADER_TYPE and p.subtype == SUBTYPES[p.type]["primary"]
+                is_primary_partition_table = p.type == PARTITION_TABLE_TYPE and p.subtype == SUBTYPES[p.type]["primary"]
+                if not (is_primary_bootloader or is_primary_partition_table):
+                    raise InputError(
+                        "Partition offset 0x%x is below 0x%x" % (p.offset, offset_part_table + PARTITION_TABLE_SIZE)
+                    )
+            if last is not None and p.offset < last.offset + last.size:
+                raise InputError(
+                    "Partition at 0x%x overlaps 0x%x-0x%x" % (p.offset, last.offset, last.offset + last.size - 1)
+                )
+            last = p
+
+        # check that otadata should be unique
+        otadata_duplicates = [p for p in self if p.type == TYPES["data"] and p.subtype == SUBTYPES[DATA_TYPE]["ota"]]
+        if len(otadata_duplicates) > 1:
+            for p in otadata_duplicates:
+                critical("%s" % (p.to_csv()))
+            raise InputError(
+                "Found multiple otadata partitions. Only one partition can be defined with "
+                'type="data"(1) and subtype="ota"(0).'
+            )
+
+        if len(otadata_duplicates) == 1 and otadata_duplicates[0].size != 0x2000:
+            p = otadata_duplicates[0]
+            critical("%s" % (p.to_csv()))
+            raise InputError("otadata partition must have size = 0x2000")
+
+        # Above checks but for TEE otadata
+        otadata_duplicates = [
+            p for p in self if p.type == TYPES["data"] and p.subtype == SUBTYPES[DATA_TYPE]["tee_ota"]
+        ]
+        if len(otadata_duplicates) > 1:
+            for p in otadata_duplicates:
+                critical("%s" % (p.to_csv()))
+            raise InputError(
+                "Found multiple TEE otadata partitions. Only one partition can be defined with "
+                'type="data"(1) and subtype="tee_ota"(0x90).'
+            )
+
+        if len(otadata_duplicates) == 1 and otadata_duplicates[0].size != 0x2000:
+            p = otadata_duplicates[0]
+            critical("%s" % (p.to_csv()))
+            raise InputError("TEE otadata partition must have size = 0x2000")
+
+    def flash_size(self):
+        """Return the size that partitions will occupy in flash
+        (ie the offset the last partition ends at)
+        """
+        try:
+            last = sorted(self, reverse=True)[0]
+        except IndexError:
+            return 0  # empty table!
+        return last.offset + last.size
+
+    def verify_size_fits(self, flash_size_bytes: int) -> None:
+        """Check that partition table fits into the given flash size.
+        Raises InputError otherwise.
+        """
+        table_size = self.flash_size()
+        if flash_size_bytes < table_size:
+            mb = 1024 * 1024
+            raise InputError(
+                "Partitions tables occupies %.1fMB of flash (%d bytes) which does not fit in configured "
+                "flash size %dMB. Change the flash size in menuconfig under the 'Serial Flasher Config' menu."
+                % (table_size / mb, table_size, flash_size_bytes / mb)
+            )
+
+    @classmethod
+    def from_binary(cls, b):
+        md5 = hashlib.md5()
+        result = cls()
+        for o in range(0, len(b), 32):
+            data = b[o : o + 32]
+            if len(data) != 32:
+                raise InputError("Partition table length must be a multiple of 32 bytes")
+            if data == b"\xff" * 32:
+                return result  # got end marker
+            if md5sum and data[:2] == MD5_PARTITION_BEGIN[:2]:  # check only the magic number part
+                if data[16:] == md5.digest():
+                    continue  # the next iteration will check for the end marker
+                else:
+                    raise InputError(
+                        "MD5 checksums don't match! (computed: 0x%s, parsed: 0x%s)"
+                        % (md5.hexdigest(), binascii.hexlify(data[16:]))
+                    )
+            else:
+                md5.update(data)
+            result.append(PartitionDefinition.from_binary(data))
+        raise InputError("Partition table is missing an end-of-table marker")
+
+    def to_binary(self):
+        result = b"".join(e.to_binary() for e in self)
+        if md5sum:
+            result += MD5_PARTITION_BEGIN + hashlib.md5(result).digest()
+        if len(result) >= MAX_PARTITION_LENGTH:
+            raise InputError("Binary partition table length (%d) longer than max" % len(result))
+        result += b"\xff" * (MAX_PARTITION_LENGTH - len(result))  # pad the sector, for signing
+        return result
+
+    def to_csv(self, simple_formatting=False):
+        rows = ["# ESP-IDF Partition Table", "# Name, Type, SubType, Offset, Size, Flags"]
+        rows += [x.to_csv(simple_formatting) for x in self]
+        return "\n".join(rows) + "\n"
+
+
+class PartitionDefinition(object):
+    MAGIC_BYTES = b"\xaa\x50"
+
+    # dictionary maps flag name (as used in CSV flags list, property name)
+    # to bit set in flags words in binary format
+    FLAGS = {"encrypted": 0, "readonly": 1}
+
+    # add subtypes for the 16 OTA slot values ("ota_XX, etc.")
+    for ota_slot in range(NUM_PARTITION_SUBTYPE_APP_OTA):
+        SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = MIN_PARTITION_SUBTYPE_APP_OTA + ota_slot
+
+    # add subtypes for the 2 TEE OTA slot values ("tee_XX, etc.")
+    for tee_slot in range(NUM_PARTITION_SUBTYPE_APP_TEE):
+        SUBTYPES[TYPES["app"]]["tee_%d" % tee_slot] = MIN_PARTITION_SUBTYPE_APP_TEE + tee_slot
+
+    def __init__(self):
+        self.name = ""
+        self.type = None
+        self.subtype = None
+        self.offset = None
+        self.size = None
+        self.encrypted = False
+        self.readonly = False
+
+    @classmethod
+    def from_csv(cls, line, line_no):
+        """Parse a line from the CSV"""
+        line_w_defaults = line + ",,,,"  # lazy way to support default fields
+        fields = [f.strip() for f in line_w_defaults.split(",")]
+
+        res = PartitionDefinition()
+        res.line_no = line_no
+        res.name = fields[0]
+        res.type = res.parse_type(fields[1])
+        res.subtype = res.parse_subtype(fields[2])
+        res.offset = res.parse_address(fields[3], res.type, res.subtype)
+        res.size = res.parse_size(fields[4], res.type)
+        if res.size is None:
+            raise InputError("Size field can't be empty")
+
+        flags = fields[5].split(":")
+        for flag in flags:
+            if flag in cls.FLAGS:
+                setattr(res, flag, True)
+            elif len(flag) > 0:
+                raise InputError("CSV flag column contains unknown flag '%s'" % (flag))
+
+        return res
+
+    def __eq__(self, other):
+        return (
+            self.name == other.name
+            and self.type == other.type
+            and self.subtype == other.subtype
+            and self.offset == other.offset
+            and self.size == other.size
+        )
+
+    def __repr__(self):
+        def maybe_hex(x):
+            return "0x%x" % x if x is not None else "None"
+
+        return "PartitionDefinition('%s', 0x%x, 0x%x, %s, %s)" % (
+            self.name,
+            self.type,
+            self.subtype or 0,
+            maybe_hex(self.offset),
+            maybe_hex(self.size),
+        )
+
+    def __str__(self):
+        return "Part '%s' %d/%d @ 0x%x size 0x%x" % (
+            self.name,
+            self.type,
+            self.subtype,
+            self.offset or -1,
+            self.size or -1,
+        )
+
+    def __cmp__(self, other):
+        return self.offset - other.offset
+
+    def __lt__(self, other):
+        return self.offset < other.offset
+
+    def __gt__(self, other):
+        return self.offset > other.offset
+
+    def __le__(self, other):
+        return self.offset <= other.offset
+
+    def __ge__(self, other):
+        return self.offset >= other.offset
+
+    def parse_type(self, strval):
+        if strval == "":
+            raise InputError("Field 'type' can't be left empty.")
+        return parse_int(strval, TYPES)
+
+    def parse_subtype(self, strval):
+        if strval == "":
+            if self.type == TYPES["app"]:
+                raise InputError("App partition cannot have an empty subtype")
+            return SUBTYPES[DATA_TYPE]["undefined"]
+        return parse_int(strval, SUBTYPES.get(self.type, {}))
+
+    def parse_size(self, strval, ptype):
+        if ptype == BOOTLOADER_TYPE:
+            if primary_bootloader_offset is None:
+                raise InputError("Primary bootloader offset is not defined. Please use --primary-bootloader-offset")
+            return offset_part_table - primary_bootloader_offset
+        if ptype == PARTITION_TABLE_TYPE:
+            return PARTITION_TABLE_SIZE
+        if strval == "":
+            return None  # PartitionTable will fill in default
+        return parse_int(strval)
+
+    def parse_address(self, strval, ptype, psubtype):
+        if ptype == BOOTLOADER_TYPE:
+            if psubtype == SUBTYPES[ptype]["primary"]:
+                if primary_bootloader_offset is None:
+                    raise InputError("Primary bootloader offset is not defined. Please use --primary-bootloader-offset")
+                return primary_bootloader_offset
+            if psubtype == SUBTYPES[ptype]["recovery"]:
+                if recovery_bootloader_offset is None:
+                    raise InputError(
+                        "Recovery bootloader offset is not defined. Please use --recovery-bootloader-offset"
+                    )
+                return recovery_bootloader_offset
+        if ptype == PARTITION_TABLE_TYPE and psubtype == SUBTYPES[ptype]["primary"]:
+            return offset_part_table
+        if strval == "":
+            return None  # PartitionTable will fill in default
+        return parse_int(strval)
+
+    def verify(self):
+        if self.type is None:
+            raise ValidationError(self, "Type field is not set")
+        if self.subtype is None:
+            raise ValidationError(self, "Subtype field is not set")
+        if self.offset is None:
+            raise ValidationError(self, "Offset field is not set")
+        if self.size is None:
+            raise ValidationError(self, "Size field is not set")
+        offset_align = get_alignment_offset_for_type(self.type)
+        if self.offset % offset_align:
+            raise ValidationError(self, "Offset 0x%x is not aligned to 0x%x" % (self.offset, offset_align))
+        if self.type == APP_TYPE:
+            size_align = get_alignment_size_for_type(self.type)
+            if self.size % size_align:
+                raise ValidationError(self, "Size 0x%x is not aligned to 0x%x" % (self.size, size_align))
+
+        if self.name in TYPES and TYPES.get(self.name, "") != self.type:
+            critical(
+                "WARNING: Partition has name '%s' which is a partition type, but does not match this partition's "
+                "type (0x%x). Mistake in partition table?" % (self.name, self.type)
+            )
+        all_subtype_names = []
+        for names in (t.keys() for t in SUBTYPES.values()):
+            all_subtype_names += names
+        if self.name in all_subtype_names and SUBTYPES.get(self.type, {}).get(self.name, "") != self.subtype:
+            critical(
+                "WARNING: Partition has name '%s' which is a partition subtype, but this partition has "
+                "non-matching type 0x%x and subtype 0x%x. Mistake in partition table?"
+                % (self.name, self.type, self.subtype)
+            )
+
+        always_rw_data_subtypes = [SUBTYPES[DATA_TYPE]["ota"], SUBTYPES[DATA_TYPE]["coredump"]]
+        if self.type == TYPES["data"] and self.subtype in always_rw_data_subtypes and self.readonly is True:
+            raise ValidationError(
+                self,
+                "'%s' partition of type %s and subtype %s is always read-write and cannot be read-only"
+                % (self.name, self.type, self.subtype),
+            )
+
+        if self.type == TYPES["data"] and self.subtype == SUBTYPES[DATA_TYPE]["nvs"]:
+            if self.size < NVS_RW_MIN_PARTITION_SIZE and self.readonly is False:
+                raise ValidationError(
+                    self,
+                    """'%s' partition of type %s and subtype %s of this size (0x%x) must be flagged as 'readonly' \
+(the size of read/write NVS has to be at least 0x%x)"""
+                    % (self.name, self.type, self.subtype, self.size, NVS_RW_MIN_PARTITION_SIZE),
+                )
+
+    STRUCT_FORMAT = b"<2sBBLL16sL"
+
+    @classmethod
+    def from_binary(cls, b):
+        if len(b) != 32:
+            raise InputError("Partition definition length must be exactly 32 bytes. Got %d bytes." % len(b))
+        res = cls()
+        (magic, res.type, res.subtype, res.offset, res.size, res.name, flags) = struct.unpack(cls.STRUCT_FORMAT, b)
+        if b"\x00" in res.name:  # strip null byte padding from name string
+            res.name = res.name[: res.name.index(b"\x00")]
+        res.name = res.name.decode()
+        if magic != cls.MAGIC_BYTES:
+            raise InputError("Invalid magic bytes (%r) for partition definition" % magic)
+        for flag, bit in cls.FLAGS.items():
+            if flags & (1 << bit):
+                setattr(res, flag, True)
+                flags &= ~(1 << bit)
+        if flags != 0:
+            critical("WARNING: Partition definition had unknown flag(s) 0x%08x. Newer binary format?" % flags)
+        return res
+
+    def get_flags_list(self):
+        return [flag for flag in self.FLAGS.keys() if getattr(self, flag)]
+
+    def to_binary(self):
+        flags = sum((1 << self.FLAGS[flag]) for flag in self.get_flags_list())
+        return struct.pack(
+            self.STRUCT_FORMAT,
+            self.MAGIC_BYTES,
+            self.type,
+            self.subtype,
+            self.offset,
+            self.size,
+            self.name.encode(),
+            flags,
+        )
+
+    def to_csv(self, simple_formatting=False):
+        def addr_format(a, include_sizes):
+            if not simple_formatting and include_sizes:
+                for val, suffix in [(0x100000, "M"), (0x400, "K")]:
+                    if a % val == 0:
+                        return "%d%s" % (a // val, suffix)
+            return "0x%x" % a
+
+        def lookup_keyword(t, keywords):
+            for k, v in keywords.items():
+                if simple_formatting is False and t == v:
+                    return k
+            return "%d" % t
+
+        def generate_text_flags():
+            """colon-delimited list of flags"""
+            return ":".join(self.get_flags_list())
+
+        return ",".join(
+            [
+                self.name,
+                lookup_keyword(self.type, TYPES),
+                lookup_keyword(self.subtype, SUBTYPES.get(self.type, {})),
+                addr_format(self.offset, False),
+                addr_format(self.size, True),
+                generate_text_flags(),
+            ]
+        )
+
+
+def parse_int(v, keywords={}):
+    """Generic parser for integer fields - int(x,0) with provision for
+    k/m/K/M suffixes and 'keyword' value lookup.
+    """
+    try:
+        for letter, multiplier in [("k", 1024), ("m", 1024 * 1024)]:
+            if v.lower().endswith(letter):
+                return parse_int(v[:-1], keywords) * multiplier
+        return int(v, 0)
+    except ValueError:
+        if len(keywords) == 0:
+            raise InputError("Invalid field value %s" % v)
+        try:
+            return keywords[v.lower()]
+        except KeyError:
+            raise InputError("Value '%s' is not valid. Known keywords: %s" % (v, ", ".join(keywords)))
+
+
+def main():
+    global quiet
+    global md5sum
+    global offset_part_table
+    global secure
+    global primary_bootloader_offset
+    global recovery_bootloader_offset
+    parser = argparse.ArgumentParser(description="ESP32 partition table utility")
+
+    parser.add_argument(
+        "--flash-size",
+        help="Optional flash size limit, checks partition table fits in flash",
+        nargs="?",
+        choices=["1MB", "2MB", "4MB", "8MB", "16MB", "32MB", "64MB", "128MB"],
+    )
+    parser.add_argument(
+        "--disable-md5sum", help="Disable md5 checksum for the partition table", default=False, action="store_true"
+    )
+    parser.add_argument("--no-verify", help="Don't verify partition table fields", action="store_true")
+    parser.add_argument(
+        "--verify",
+        "-v",
+        help="Verify partition table fields (deprecated, this behavior is "
+        "enabled by default and this flag does nothing.",
+        action="store_true",
+    )
+    parser.add_argument("--quiet", "-q", help="Don't print non-critical status messages to stderr", action="store_true")
+    parser.add_argument("--offset", "-o", help="Set offset partition table", default="0x8000")
+    parser.add_argument("--primary-bootloader-offset", help="Set primary bootloader offset", default=None)
+    parser.add_argument("--recovery-bootloader-offset", help="Set recovery bootloader offset", default=None)
+    parser.add_argument(
+        "--secure",
+        help="Require app partitions to be suitable for secure boot",
+        nargs="?",
+        const=SECURE_V1,
+        choices=[SECURE_V1, SECURE_V2],
+    )
+    parser.add_argument("--extra-partition-subtypes", help="Extra partition subtype entries", nargs="*")
+    parser.add_argument("input", help="Path to CSV or binary file to parse.", type=argparse.FileType("rb"))
+    parser.add_argument(
+        "output",
+        help="Path to output converted binary or CSV file. Will use stdout if omitted.",
+        nargs="?",
+        default="-",
+    )
+
+    args = parser.parse_args()
+
+    quiet = args.quiet
+    md5sum = not args.disable_md5sum
+    secure = args.secure
+    offset_part_table = int(args.offset, 0)
+    if args.primary_bootloader_offset is not None:
+        primary_bootloader_offset = int(args.primary_bootloader_offset, 0)
+        if primary_bootloader_offset >= offset_part_table:
+            raise InputError(
+                f"Unsupported configuration. Primary bootloader must be below partition table. "
+                f"Check --primary-bootloader-offset={primary_bootloader_offset:#x} and --offset={offset_part_table:#x}"
+            )
+    if args.recovery_bootloader_offset is not None:
+        recovery_bootloader_offset = int(args.recovery_bootloader_offset, 0)
+    if args.extra_partition_subtypes:
+        add_extra_subtypes(args.extra_partition_subtypes)
+
+    table, input_is_binary = PartitionTable.from_file(args.input)
+
+    if not args.no_verify:
+        status("Verifying table...")
+        table.verify()
+
+    if args.flash_size:
+        size_mb = int(args.flash_size.replace("MB", ""))
+        table.verify_size_fits(size_mb * 1024 * 1024)
+
+    # Make sure that the output directory is created
+    output_dir = os.path.abspath(os.path.dirname(args.output))
+
+    if not os.path.exists(output_dir):
+        try:
+            os.makedirs(output_dir)
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
+
+    if input_is_binary:
+        output = table.to_csv()
+        with sys.stdout if args.output == "-" else open(args.output, "w", encoding="utf-8") as f:
+            f.write(output)
+    else:
+        output = table.to_binary()
+        try:
+            stdout_binary = sys.stdout.buffer  # Python 3
+        except AttributeError:
+            stdout_binary = sys.stdout
+        with stdout_binary if args.output == "-" else open(args.output, "wb") as f:
+            f.write(output)
+
+
+class InputError(RuntimeError):
+    def __init__(self, e):
+        super(InputError, self).__init__(e)
+
+
+class ValidationError(InputError):
+    def __init__(self, partition, message):
+        super(ValidationError, self).__init__("Partition %s invalid: %s" % (partition.name, message))
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except InputError as e:
+        print(e, file=sys.stderr)
+        sys.exit(2)

tools/gen_insights_package.py

@@ -0,0 +1,53 @@
+import os
+import sys
+import shutil
+import json
+
+APP_HEADER_SIZE = 32
+VERSION_NAME_OFFSET = APP_HEADER_SIZE + 16
+VERSION_NAME_SIZE = 32
+PROJECT_NAME_OFFSET = VERSION_NAME_OFFSET + VERSION_NAME_SIZE
+PROJECT_NAME_SIZE = 32
+
+# Input path of temporary build directory created by Arduino
+BUILD_DIR = sys.argv[1]
+# Input project name
+PROJ_NAME = sys.argv[2]
+# Input path to create output package
+TARGET_PATH = sys.argv[3]
+
+
+def main():
+    print("Creating ESP Insights Firmware Package.")
+    archive_path = os.path.join(BUILD_DIR, PROJ_NAME)
+    out_path = os.path.join(TARGET_PATH, PROJ_NAME)
+
+    # Create target archive directories
+    os.makedirs(archive_path, exist_ok=True)
+    os.makedirs(os.path.join(archive_path, "partition_table"), exist_ok=True)
+    os.makedirs(os.path.join(archive_path, "bootloader"), exist_ok=True)
+
+    # Copy files from build directory to archive directory
+    shutil.copy2(os.path.join(BUILD_DIR, PROJ_NAME + ".bin"), archive_path)
+    shutil.copy2(os.path.join(BUILD_DIR, PROJ_NAME + ".elf"), archive_path)
+    shutil.copy2(os.path.join(BUILD_DIR, PROJ_NAME + ".map"), archive_path)
+    shutil.copy2(os.path.join(BUILD_DIR, "partitions.csv"), archive_path)
+    shutil.copy2(os.path.join(BUILD_DIR, PROJ_NAME + ".bootloader.bin"), os.path.join(archive_path, "bootloader"))
+    shutil.copy2(os.path.join(BUILD_DIR, PROJ_NAME + ".partitions.bin"), os.path.join(archive_path, "partition_table"))
+
+    with open(os.path.join(BUILD_DIR, PROJ_NAME + ".bin"), "rb") as bin_file:
+        bin_file.seek(VERSION_NAME_OFFSET)
+        version_name = (bin_file.read(VERSION_NAME_SIZE).decode("utf-8")).split("\x00", 1)[0]
+        bin_file.seek(PROJECT_NAME_OFFSET)
+        project_name = (bin_file.read(PROJECT_NAME_SIZE).decode("utf-8")).split("\x00", 1)[0]
+        project_build_config_obj = {"project": {"name": project_name, "version": version_name}}
+        with open(os.path.join(archive_path, "project_build_config.json"), "w") as json_file:
+            json_file.write(json.dumps(project_build_config_obj))
+
+    shutil.make_archive(out_path, "zip", BUILD_DIR, PROJ_NAME)
+    print("Archive created at {}".format(out_path + ".zip"))
+    return
+
+
+if __name__ == "__main__":
+    main()

tools/partitions/app3M_fat9M_16MB.csv

@@ -0,0 +1,8 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x300000,
+app1,     app,  ota_1,   0x310000,0x300000,
+ffat,     data, fat,     0x610000,0x9E0000,
+coredump, data, coredump,0xFF0000,0x10000,
+# to create/use ffat, see https://github.com/marcmerlin/esp32_fatfsimage

tools/partitions/app3M_fat9M_fact512k_16MB.csv

@@ -0,0 +1,9 @@
+# Name,   Type, SubType,  Offset,   Size,     Flags
+nvs,      data, nvs,        0x9000,   0x5000,
+otadata,  data, ota,        0xe000,   0x2000,
+app0,     app,  ota_0,     0x10000, 0x300000,
+app1,     app,  ota_1,    0x310000, 0x300000,
+ffat,     data, fat,      0x610000, 0x960000,
+factory,  app,  factory,  0xF70000,  0x80000,
+coredump, data, coredump, 0xFF0000,  0x10000,
+# to create/use ffat, see https://github.com/marcmerlin/esp32_fatfsimage

tools/partitions/app3M_spiffs9M_fact512k_16MB.csv

@@ -0,0 +1,8 @@
+# Name,   Type, SubType,  Offset,   Size,     Flags
+nvs,      data, nvs,        0x9000,   0x5000,
+otadata,  data, ota,        0xe000,   0x2000,
+app0,     app,  ota_0,     0x10000, 0x300000,
+app1,     app,  ota_1,    0x310000, 0x300000,
+spiffs,   data, spiffs,   0x610000, 0x960000,
+factory,  app,  factory,  0xF70000,  0x80000,
+coredump, data, coredump, 0xFF0000,  0x10000,

tools/partitions/bare_minimum_2MB.csv

@@ -0,0 +1,3 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     36K,     20K,
+factory,  app,  factory, 64K,     1900K,

tools/partitions/default.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x140000,
+app1,     app,  ota_1,   0x150000,0x140000,
+spiffs,   data, spiffs,  0x290000,0x160000,
+coredump, data, coredump,0x3F0000,0x10000,

tools/partitions/default_16MB.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x640000,
+app1,     app,  ota_1,   0x650000,0x640000,
+spiffs,   data, spiffs,  0xc90000,0x360000,
+coredump, data, coredump,0xFF0000,0x10000,

tools/partitions/default_32MB.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0xC80000,
+app1,     app,  ota_1,   0xC90000,0xC80000,
+spiffs,   data, spiffs,  0x1910000,0x6C0000,
+coredump, data, coredump,0x1FF0000,0x10000,

tools/partitions/default_8MB.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x330000,
+app1,     app,  ota_1,   0x340000,0x330000,
+spiffs,   data, spiffs,  0x670000,0x180000,
+coredump, data, coredump,0x7F0000,0x10000,

tools/partitions/default_ffat.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x140000,
+app1,     app,  ota_1,   0x150000,0x140000,
+ffat,     data, fat,     0x290000,0x160000,
+coredump, data, coredump,0x3F0000,0x10000,

tools/partitions/default_ffat_8MB.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x330000,
+app1,     app,  ota_1,   0x340000,0x330000,
+ffat,     data, fat,     0x670000,0x180000,
+coredump, data, coredump,0x7F0000,0x10000,

tools/partitions/esp_sr_16.csv

@@ -0,0 +1,8 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,       0x9000,   0x5000,
+otadata,  data, ota,       0xe000,   0x2000,
+app0,     app,  ota_0,    0x10000, 0x300000,
+app1,     app,  ota_1,   0x310000, 0x300000,
+spiffs,   data, spiffs,  0x610000, 0x700000,
+model,    data, spiffs,  0xD10000, 0x2E0000,
+coredump, data, coredump,0xFF0000,  0x10000,

tools/partitions/ffat.csv

@@ -0,0 +1,8 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x200000,
+app1,     app,  ota_1,   0x210000,0x200000,
+ffat,     data, fat,     0x410000,0xBE0000,
+coredump, data, coredump,0xFF0000,0x10000,
+# to create/use ffat, see https://github.com/marcmerlin/esp32_fatfsimage

tools/partitions/huge_app.csv

@@ -0,0 +1,6 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x300000,
+spiffs,   data, spiffs,  0x310000,0xE0000,
+coredump, data, coredump,0x3F0000,0x10000,

tools/partitions/large_fat_32MB.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x480000,
+app1,     app,  ota_1,   0x490000,0x480000,
+ffat,     data, fat,     0x910000,0x16E0000,
+coredump, data, coredump,0x1FF0000,0x10000,

tools/partitions/large_ffat_8MB.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x140000,
+app1,     app,  ota_1,   0x150000,0x140000,
+ffat,     data, fat,     0x290000,0x560000,
+coredump, data, coredump,0x7F0000,0x10000,

tools/partitions/large_littlefs_32MB.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x480000,
+app1,     app,  ota_1,   0x490000,0x480000,
+spiffs,   data, spiffs,  0x910000,0x16E0000,
+coredump, data, coredump,0x1FF0000,0x10000,

tools/partitions/large_spiffs_16MB.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x480000,
+app1,     app,  ota_1,   0x490000,0x480000,
+spiffs,   data, spiffs,  0x910000,0x6E0000,
+coredump, data, coredump,0xFF0000,0x10000,

tools/partitions/large_spiffs_8MB.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x140000,
+app1,     app,  ota_1,   0x150000,0x140000,
+spiffs,   data, spiffs,  0x290000,0x560000,
+coredump, data, coredump,0x7F0000,0x10000,

tools/partitions/m5stack_partitions_16MB_factory_4_apps.csv

@@ -0,0 +1,11 @@
+## 4 Apps + Factory
+## Name,   Type, SubType,    Offset,     Size
+nvs,      data, nvs,        0x9000,   0x5000
+otadata,  data, ota,        0xe000,   0x2000
+ota_0,    0,    ota_0,     0x10000, 0x300000
+ota_1,    0,    ota_1,    0x310000, 0x300000
+ota_2,    0,    ota_2,    0x610000, 0x300000
+ota_3,    0,    ota_3,    0x910000, 0x300000
+firmware, app,  factory,  0xC10000, 0x0F0000
+spiffs,   data, spiffs,   0xD00000, 0x2F0000
+coredump, data, coredump, 0xFF0000,  0x10000

tools/partitions/m5stack_partitions_16MB_factory_6_apps.csv

@@ -0,0 +1,13 @@
+# 6 Apps + Factory
+# Name,   Type, SubType,    Offset,     Size
+nvs,      data, nvs,        0x9000,   0x5000
+otadata,  data, ota,        0xe000,   0x2000
+ota_0,    0,    ota_0,     0x10000, 0x200000
+ota_1,    0,    ota_1,    0x210000, 0x200000
+ota_2,    0,    ota_2,    0x410000, 0x200000
+ota_3,    0,    ota_3,    0x610000, 0x200000
+ota_4,    0,    ota_4,    0x810000, 0x200000
+ota_5,    0,    ota_5,    0xA10000, 0x200000
+firmware, app,  factory,  0xC10000, 0x0F0000
+spiffs,   data, spiffs,   0xD00000, 0x2F0000
+coredump, data, coredump, 0xFF0000,  0x10000

tools/partitions/max_app_32MB.csv

@@ -0,0 +1,5 @@
+# Name,      Type, SubType,  Offset,     Size,      Flags
+nvs,         data, nvs,      0x9000,     0x5000,
+otadata,     data, ota,      0xE000,     0x2000,
+app0,        app,  factory,  0x10000,    0x1FE0000,
+coredump,    data, coredump, 0x1FF0000,  0x10000,

tools/partitions/max_app_4MB.csv

@@ -0,0 +1,5 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  factory, 0x10000, 0x3E0000,
+coredump, data, coredump,0x3F0000,0x10000,

tools/partitions/max_app_8MB.csv

@@ -0,0 +1,5 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  factory, 0x10000, 0x7E0000,
+coredump, data, coredump,0x7F0000,0x10000,

tools/partitions/min_spiffs.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x1E0000,
+app1,     app,  ota_1,   0x1F0000,0x1E0000,
+spiffs,   data, spiffs,  0x3D0000,0x20000,
+coredump, data, coredump,0x3F0000,0x10000,

tools/partitions/minimal.csv

@@ -0,0 +1,6 @@
+# Name,   Type, SubType, Offset,   Size, Flags
+nvs,      data, nvs,     0x9000,   0x5000,
+otadata,  data, ota,     0xe000,   0x2000,
+app0,     app,  ota_0,   0x10000,  0x140000,
+spiffs,   data, spiffs,  0x150000, 0xA0000,
+coredump, data, coredump,0x1F0000, 0x10000,

tools/partitions/no_fs.csv

@@ -0,0 +1,6 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x1F0000,
+app1,     app,  ota_1,   0x200000,0x1F0000,
+coredump, data, coredump,0x3F0000,0x10000,

tools/partitions/no_ota.csv

@@ -0,0 +1,6 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x200000,
+spiffs,   data, spiffs,  0x210000,0x1E0000,
+coredump, data, coredump,0x3F0000,0x10000,

tools/partitions/noota_3g.csv

@@ -0,0 +1,6 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x100000,
+spiffs,   data, spiffs,  0x110000,0x2E0000,
+coredump, data, coredump,0x3F0000,0x10000,

tools/partitions/noota_3gffat.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x100000,
+ffat,     data, fat,     0x110000,0x2E0000,
+coredump, data, coredump,0x3F0000,0x10000,
+# to create/use ffat, see https://github.com/marcmerlin/esp32_fatfsimage

tools/partitions/noota_ffat.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x200000,
+ffat,     data, fat,     0x210000,0x1E0000,
+coredump, data, coredump,0x3F0000,0x10000,
+# to create/use ffat, see https://github.com/marcmerlin/esp32_fatfsimage

tools/partitions/ota_nofs_4MB.csv

@@ -0,0 +1,6 @@
+#   Name, Type,  SubType,   Offset,      Size, Flags
+     nvs, data,      nvs,   0x9000,    0x5000,
+ otadata, data,      ota,   0xE000,    0x2000,
+    app0,  app,    ota_0,  0x10000,  0x1F0000,
+    app1,  app,    ota_1, 0x200000,  0x1F0000,
+coredump, data, coredump, 0x3F0000,   0x10000,

tools/partitions/rainmaker.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,       data,  nvs,     0x9000,   0x5000,
+otadata,   data,  ota,     0xe000,   0x2000,
+ota_0,     app,   ota_0,   0x10000,  0x1E0000,
+ota_1,     app,   ota_1,   0x1F0000, 0x1E0000,
+fctry,     data,  nvs,     0x3D0000, 0x6000,
+coredump,  data,  coredump,0x3F0000, 0x10000,

tools/partitions/rainmaker_4MB_no_ota.csv

@@ -0,0 +1,6 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,       data,  nvs,     0x9000,   0x5000,
+otadata,   data,  ota,     0xe000,   0x2000,
+ota_0,     app,   ota_0,   0x10000,  0x3DA000,
+fctry,     data,  nvs,     0x3EA000, 0x6000,
+coredump,  data,  coredump,0x3F0000, 0x10000,

tools/partitions/rainmaker_8MB.csv

@@ -0,0 +1,7 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,       data,  nvs,     0x9000,   0x5000,
+otadata,   data,  ota,     0xe000,   0x2000,
+ota_0,     app,   ota_0,   0x10000,  0x3EA000,
+ota_1,     app,   ota_1,   0x400000, 0x3EA000,
+fctry,     data,  nvs,     0x7EA000, 0x6000,
+coredump,  data,  coredump,0x7F0000, 0x10000,

tools/partitions/storage_4MB_noota.csv

@@ -0,0 +1,5 @@
+# Name,   Type, SubType, Offset,   Size,     Flags
+app0,     app,  factory, 0x10000,  0x200000,
+nvs,      data, nvs,     0x210000, 0x100000,
+spiffs,   data, spiffs,  0x310000, 0xE0000,
+coredump, data, coredump,0x3F0000, 0x10000,

tools/partitions/tinyuf2-partitions-16MB-noota.csv

@@ -0,0 +1,9 @@
+# ESP-IDF Partition Table
+# Name,   Type, SubType, Offset,  Size, Flags
+# bootloader.bin,,          0x1000, 32K
+# partition table,,         0x8000, 4K
+nvs,      data, nvs,      0x9000,  20K,
+otadata,  data, ota,      0xe000,  8K,
+ota_0,    app,  ota_0,   0x10000,  4096K,
+uf2,      app,  factory,0x410000,  256K,
+ffat,     data, fat,    0x450000,  11968K,

tools/partitions/tinyuf2-partitions-16MB.csv

@@ -0,0 +1,10 @@
+# ESP-IDF Partition Table
+# Name,   Type, SubType, Offset,  Size, Flags
+# bootloader.bin,,          0x1000, 32K
+# partition table,,         0x8000, 4K
+nvs,      data, nvs,      0x9000,  20K,
+otadata,  data, ota,      0xe000,  8K,
+ota_0,    app,  ota_0,   0x10000,  2048K,
+ota_1,    app,  ota_1,  0x210000,  2048K,
+uf2,      app,  factory,0x410000,  256K,
+ffat,     data, fat,    0x450000,  11968K,

tools/partitions/tinyuf2-partitions-4MB-noota.csv

@@ -0,0 +1,10 @@
+# ESP-IDF Partition Table
+# Name,   Type, SubType, Offset,  Size, Flags
+# bootloader.bin,,          0x1000, 32K
+# partition table,          0x8000, 4K
+
+nvs,      data, nvs,      0x9000,  20K,
+otadata,  data, ota,      0xe000,  8K,
+ota_0,    app,  ota_0,   0x10000,  2816K,
+uf2,      app,  factory,0x2d0000,  256K,
+ffat,     data, fat,    0x310000,  960K,

tools/partitions/tinyuf2-partitions-4MB.csv

@@ -0,0 +1,11 @@
+# ESP-IDF Partition Table
+# Name,   Type, SubType, Offset,  Size, Flags
+# bootloader.bin,,          0x1000, 32K
+# partition table,          0x8000, 4K
+
+nvs,      data, nvs,      0x9000,  20K,
+otadata,  data, ota,      0xe000,  8K,
+ota_0,    app,  ota_0,   0x10000,  1408K,
+ota_1,    app,  ota_1,  0x170000,  1408K,
+uf2,      app,  factory,0x2d0000,  256K,
+ffat,     data, fat,    0x310000,  960K,

tools/partitions/tinyuf2-partitions-8MB-noota.csv

@@ -0,0 +1,9 @@
+# ESP-IDF Partition Table
+# Name,   Type, SubType, Offset,  Size, Flags
+# bootloader.bin,,          0x1000, 32K
+# partition table,,         0x8000, 4K
+nvs,      data, nvs,      0x9000,  20K,
+otadata,  data, ota,      0xe000,  8K,
+ota_0,    app,    ota_0,   0x10000,  4096K,
+uf2,      app,  factory,0x410000,  256K,
+ffat,     data, fat,    0x450000,  3776K,

tools/partitions/tinyuf2-partitions-8MB.csv

@@ -0,0 +1,10 @@
+# ESP-IDF Partition Table
+# Name,   Type, SubType, Offset,  Size, Flags
+# bootloader.bin,,          0x1000, 32K
+# partition table,,         0x8000, 4K
+nvs,      data, nvs,      0x9000,  20K,
+otadata,  data, ota,      0xe000,  8K,
+ota_0,    app,    ota_0,   0x10000,  2048K,
+ota_1,    app,    ota_1,  0x210000,  2048K,
+uf2,      app,  factory,0x410000,  256K,
+ffat,     data, fat,    0x450000,  3776K,

tools/partitions/zigbee.csv

@@ -0,0 +1,9 @@
+# Name,     Type, SubType, Offset,  Size, Flags
+nvs,        data, nvs,     0x9000,  0x5000,
+otadata,    data, ota,     0xe000,  0x2000,
+app0,       app,  ota_0,   0x10000, 0x140000,
+app1,       app,  ota_1,   0x150000,0x140000,
+spiffs,     data, spiffs,  0x290000,0x15B000,
+zb_storage, data, fat,     0x3EB000,0x4000,
+zb_fct,     data, fat,     0x3EF000,0x1000,
+coredump,   data, coredump,0x3F0000,0x10000,

tools/partitions/zigbee_2MB.csv

@@ -0,0 +1,7 @@
+# Name,     Type, SubType, Offset,  Size, Flags
+nvs,        data, nvs,     0x9000,  0x5000,
+factory,    app,  factory, 0x10000, 0x140000,
+spiffs,     data, spiffs,  0x150000,0x9B000,
+zb_storage, data, fat,     0x1EB000,0x4000,
+zb_fct,     data, fat,     0x1EF000,0x1000,
+coredump,   data, coredump,0x1F0000,0x10000,

tools/partitions/zigbee_8MB.csv

@@ -0,0 +1,9 @@
+# Name,     Type, SubType, Offset,  Size, Flags
+nvs,        data, nvs,     0x9000,  0x5000,
+otadata,    data, ota,     0xe000,  0x2000,
+app0,       app,  ota_0,   0x10000, 0x340000,
+app1,       app,  ota_1,   0x350000,0x340000,
+spiffs,     data, spiffs,  0x690000,0x15B000,
+zb_storage, data, fat,     0x7EB000,0x4000,
+zb_fct,     data, fat,     0x7EF000,0x1000,
+coredump,   data, coredump,0x7F0000,0x10000,

tools/partitions/zigbee_zczr.csv

@@ -0,0 +1,10 @@
+# Name,     Type, SubType, Offset,  Size, Flags
+nvs,        data, nvs,     0x9000,  0x5000,
+otadata,    data, ota,     0xe000,  0x2000,
+app0,       app,  ota_0,   0x10000, 0x140000,
+app1,       app,  ota_1,   0x150000,0x140000,
+spiffs,     data, spiffs,  0x290000,0x15A000,
+zb_storage, data, fat,     0x3EA000,0x4000,
+zb_fct,     data, fat,     0x3EE000,0x1000,
+rcp_fw,     data, spiffs,  0x3EF000,0x1000,
+coredump,   data, coredump,0x3F0000,0x10000,

tools/partitions/zigbee_zczr_2MB.csv

@@ -0,0 +1,8 @@
+# Name,     Type, SubType, Offset,  Size, Flags
+nvs,        data, nvs,     0x9000,  0x5000,
+factory,    app,  factory, 0x10000, 0x140000,
+spiffs,     data, spiffs,  0x150000,0x9A000,
+zb_storage, data, fat,     0x1EA000,0x4000,
+zb_fct,     data, fat,     0x1EE000,0x1000,
+rcp_fw,     data, spiffs,  0x1EF000,0x1000,
+coredump,   data, coredump,0x1F0000,0x10000,

tools/partitions/zigbee_zczr_8MB.csv

@@ -0,0 +1,10 @@
+# Name,     Type, SubType, Offset,  Size, Flags
+nvs,        data, nvs,     0x9000,  0x5000,
+otadata,    data, ota,     0xe000,  0x2000,
+app0,       app,  ota_0,   0x10000, 0x340000,
+app1,       app,  ota_1,   0x350000,0x340000,
+spiffs,     data, spiffs,  0x690000,0x15A000,
+zb_storage, data, fat,     0x7EA000,0x4000,
+zb_fct,     data, fat,     0x7EE000,0x1000,
+rcp_fw,     data, spiffs,  0x7EF000,0x1000,
+coredump,   data, coredump,0x7F0000,0x10000,

tools/pioarduino-build.py

@@ -0,0 +1,263 @@
+# Copyright 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Arduino
+
+Arduino Wiring-based Framework allows writing cross-platform software to
+control devices attached to a wide range of Arduino boards to create all
+kinds of creative coding, interactive objects, spaces or physical experiences.
+
+http://arduino.cc/en/Reference/HomePage
+"""
+
+# Extends: https://github.com/pioarduino/platform-espressif32/blob/develop/builder/main.py
+
+from os.path import abspath, basename, isdir, isfile, join
+from copy import deepcopy
+from SCons.Script import DefaultEnvironment, SConscript
+
+env = DefaultEnvironment()
+platform = env.PioPlatform()
+board_config = env.BoardConfig()
+build_mcu = board_config.get("build.mcu", "").lower()
+chip_variant = board_config.get("build.chip_variant", "").lower()
+chip_variant = chip_variant if chip_variant else build_mcu
+partitions_name = board_config.get("build.partitions", board_config.get("build.arduino.partitions", ""))
+
+FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
+FRAMEWORK_LIBS_DIR = platform.get_package_dir("framework-arduinoespressif32-libs")
+assert isdir(FRAMEWORK_DIR)
+
+
+#
+# Helpers
+#
+
+
+def get_partition_table_csv(variants_dir):
+    fwpartitions_dir = join(FRAMEWORK_DIR, "tools", "partitions")
+    variant_partitions_dir = join(variants_dir, board_config.get("build.variant", ""))
+
+    if partitions_name:
+        # A custom partitions file is selected
+        if isfile(env.subst(join(variant_partitions_dir, partitions_name))):
+            return join(variant_partitions_dir, partitions_name)
+
+        return abspath(
+            join(fwpartitions_dir, partitions_name)
+            if isfile(env.subst(join(fwpartitions_dir, partitions_name)))
+            else partitions_name
+        )
+
+    variant_partitions = join(variant_partitions_dir, "partitions.csv")
+    return variant_partitions if isfile(env.subst(variant_partitions)) else join(fwpartitions_dir, "default.csv")
+
+
+def get_bootloader_image(variants_dir):
+    bootloader_image_file = "bootloader.bin"
+    if partitions_name.endswith("tinyuf2.csv"):
+        bootloader_image_file = "bootloader-tinyuf2.bin"
+
+    variant_bootloader = join(
+        variants_dir,
+        board_config.get("build.variant", ""),
+        board_config.get("build.arduino.custom_bootloader", bootloader_image_file),
+    )
+
+    return (
+        variant_bootloader
+        if isfile(env.subst(variant_bootloader))
+        else generate_bootloader_image(
+            join(
+                FRAMEWORK_LIBS_DIR,
+                chip_variant,
+                "bin",
+                "bootloader_${__get_board_boot_mode(__env__)}_${__get_board_f_boot(__env__)}.elf",
+            )
+        )
+    )
+
+
+def generate_bootloader_image(bootloader_elf):
+    bootloader_cmd = env.Command(
+        join("$BUILD_DIR", "bootloader.bin"),
+        bootloader_elf,
+        env.VerboseAction(
+            " ".join(
+                [
+                    "$OBJCOPY",
+                    "--chip",
+                    build_mcu,
+                    "elf2image",
+                    "--flash-mode",
+                    "${__get_board_flash_mode(__env__)}",
+                    "--flash-freq",
+                    "${__get_board_f_image(__env__)}",
+                    "--flash-size",
+                    board_config.get("upload.flash_size", "4MB"),
+                    "-o",
+                    "$TARGET",
+                    "$SOURCES",
+                ]
+            ),
+            "Building $TARGET",
+        ),
+    )
+
+    env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", bootloader_cmd)
+
+    # Because the Command always returns a NodeList, we have to
+    # access the first element in the list to get the Node object
+    # that actually represents the bootloader image.
+    # Also, this file is later used in generic Python code, so the
+    # Node object in converted to a generic string
+    return str(bootloader_cmd[0])
+
+
+def add_tinyuf2_extra_image():
+    tinuf2_image = board_config.get(
+        "upload.arduino.tinyuf2_image",
+        join(variants_dir, board_config.get("build.variant", ""), "tinyuf2.bin"),
+    )
+
+    # Add the UF2 image only if it exists and it's not already added
+    if not isfile(env.subst(tinuf2_image)):
+        print("Warning! The `%s` UF2 bootloader image doesn't exist" % env.subst(tinuf2_image))
+        return
+
+    if any("tinyuf2.bin" == basename(extra_image[1]) for extra_image in env.get("FLASH_EXTRA_IMAGES", [])):
+        print("Warning! An extra UF2 bootloader image is already added!")
+        return
+
+    env.Append(
+        FLASH_EXTRA_IMAGES=[
+            (
+                board_config.get(
+                    "upload.arduino.uf2_bootloader_offset",
+                    ("0x2d0000" if env.subst("$BOARD").startswith("adafruit") else "0x410000"),
+                ),
+                tinuf2_image,
+            ),
+        ]
+    )
+
+
+#
+# Run target-specific script to populate the environment with proper build flags
+#
+
+SConscript(
+    join(
+        FRAMEWORK_LIBS_DIR,
+        chip_variant,
+        "pioarduino-build.py",
+    )
+)
+
+#
+# Additional flags specific to Arduino core (not based on IDF)
+#
+
+env.Append(
+    CFLAGS=["-Werror=return-type"],
+    CXXFLAGS=["-Werror=return-type"],
+)
+
+#
+# Target: Build Core Library
+#
+
+# Set -DARDUINO_CORE_BUILD only for the core library
+corelib_env = env.Clone()
+corelib_env.Append(CPPDEFINES=["ARDUINO_CORE_BUILD"])
+
+libs = []
+
+variants_dir = join(FRAMEWORK_DIR, "variants")
+
+if "build.variants_dir" in board_config:
+    variants_dir = join("$PROJECT_DIR", board_config.get("build.variants_dir"))
+
+if "build.variant" in board_config:
+    env.Append(CPPPATH=[join(variants_dir, board_config.get("build.variant"))])
+    corelib_env.Append(CPPPATH=[join(variants_dir, board_config.get("build.variant"))])
+    corelib_env.BuildSources(
+        join("$BUILD_DIR", "FrameworkArduinoVariant"),
+        join(variants_dir, board_config.get("build.variant")),
+    )
+
+libs.append(
+    corelib_env.BuildLibrary(
+        join("$BUILD_DIR", "FrameworkArduino"),
+        join(FRAMEWORK_DIR, "cores", board_config.get("build.core")),
+    )
+)
+
+env.Prepend(LIBS=libs)
+
+#
+# Process framework extra images
+#
+
+env.Append(
+    LIBSOURCE_DIRS=[join(FRAMEWORK_DIR, "libraries")],
+    FLASH_EXTRA_IMAGES=[
+        (
+            (
+                "0x1000"
+                if build_mcu in ["esp32", "esp32s2"]
+                else ("0x2000" if build_mcu in ["esp32p4", "esp32c5"] else "0x0000")
+            ),
+            get_bootloader_image(variants_dir),
+        ),
+        (
+            board_config.get("upload.arduino.partitions_bin", "0x8000"),
+            join(env.subst("$BUILD_DIR"), "partitions.bin"),
+        ),
+        (
+            board_config.get("upload.arduino.boot_app0", "0xe000"),
+            join(FRAMEWORK_DIR, "tools", "partitions", "boot_app0.bin"),
+        ),
+    ]
+    + [(offset, join(FRAMEWORK_DIR, img)) for offset, img in board_config.get("upload.arduino.flash_extra_images", [])],
+)
+
+# Add an extra UF2 image if the 'TinyUF2' partition is selected
+if partitions_name.endswith("tinyuf2.csv") or board_config.get("upload.arduino.tinyuf2_image", ""):
+    add_tinyuf2_extra_image()
+
+#
+# Generate partition table
+#
+
+env.Replace(PARTITIONS_TABLE_CSV=get_partition_table_csv(variants_dir))
+
+partition_table = env.Command(
+    join("$BUILD_DIR", "partitions.bin"),
+    "$PARTITIONS_TABLE_CSV",
+    env.VerboseAction(
+        '"$PYTHONEXE" "%s" -q $SOURCE $TARGET' % join(FRAMEWORK_DIR, "tools", "gen_esp32part.py"),
+        "Generating partitions $TARGET",
+    ),
+)
+env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", partition_table)
+
+#
+#  Adjust the `esptoolpy` command in the `ElfToBin` builder with firmware checksum offset
+#
+
+action = deepcopy(env["BUILDERS"]["ElfToBin"].action)
+action.cmd_list = env["BUILDERS"]["ElfToBin"].action.cmd_list.replace("-o", "--elf-sha256-offset 0xb0 -o")
+env["BUILDERS"]["ElfToBin"].action = action