From aa6e97c91864ddb2901d2c1a05cd0d3178cb2483 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Thu, 22 Jan 2026 00:13:43 +0100 Subject: [PATCH] Arduino Core 3.3.6 --- .github/workflows/examples.yml | 1 + README.md | 29 +- boards/esp32-s3-devkitc1-n16r2.json | 2 +- boards/esp32-s3-devkitc1-n4r2.json | 2 +- boards/esp32-s3-devkitc1-n4r8.json | 2 +- boards/labplus_mpython.json | 2 +- boards/um_squixl.json | 2 +- builder/main.py | 898 +++++++++++++++++- builder/penv_setup.py | 2 + builder/spiffsgen.py | 762 +++++++++++++++ examples/arduino-fatfs/.gitignore | 5 + examples/arduino-fatfs/FATFS_INTEGRATION.md | 175 ++++ examples/arduino-fatfs/README.md | 224 +++++ examples/arduino-fatfs/TEST_GUIDE.md | 380 ++++++++ examples/arduino-fatfs/WEAR_LEVELING.md | 181 ++++ examples/arduino-fatfs/data/README.md | 224 +++++ examples/arduino-fatfs/data/partitions.csv | 6 + examples/arduino-fatfs/data/platformio.ini | 7 + examples/arduino-fatfs/data/test.txt | 1 + examples/arduino-fatfs/partitions.csv | 6 + examples/arduino-fatfs/platformio.ini | 6 + examples/arduino-fatfs/src/ffat.ino | 478 ++++++++++ examples/arduino-littlefs/.gitignore | 3 + examples/arduino-littlefs/data/README.md | 12 + examples/arduino-littlefs/data/partitions.csv | 6 + examples/arduino-littlefs/data/platformio.ini | 6 + examples/arduino-littlefs/data/test.txt | 2 + examples/arduino-littlefs/partitions.csv | 6 + examples/arduino-littlefs/platformio.ini | 6 + .../arduino-littlefs/src/littlefs_test.ino | 324 +++++++ examples/arduino-spiffs/.gitignore | 3 + examples/arduino-spiffs/README.md | 247 +++++ examples/arduino-spiffs/data/README.md | 12 + examples/arduino-spiffs/data/partitions.csv | 6 + examples/arduino-spiffs/data/platformio.ini | 6 + examples/arduino-spiffs/data/test.txt | 2 + examples/arduino-spiffs/partitions.csv | 6 + examples/arduino-spiffs/platformio.ini | 6 + examples/arduino-spiffs/src/spiffs_test.ino | 324 +++++++ platform.json | 46 +- platform.py | 80 +- 41 files changed, 4345 insertions(+), 153 deletions(-) create mode 100644 builder/spiffsgen.py create mode 100644 examples/arduino-fatfs/.gitignore create mode 100644 examples/arduino-fatfs/FATFS_INTEGRATION.md create mode 100644 examples/arduino-fatfs/README.md create mode 100644 examples/arduino-fatfs/TEST_GUIDE.md create mode 100644 examples/arduino-fatfs/WEAR_LEVELING.md create mode 100644 examples/arduino-fatfs/data/README.md create mode 100644 examples/arduino-fatfs/data/partitions.csv create mode 100644 examples/arduino-fatfs/data/platformio.ini create mode 100644 examples/arduino-fatfs/data/test.txt create mode 100644 examples/arduino-fatfs/partitions.csv create mode 100644 examples/arduino-fatfs/platformio.ini create mode 100644 examples/arduino-fatfs/src/ffat.ino create mode 100644 examples/arduino-littlefs/.gitignore create mode 100644 examples/arduino-littlefs/data/README.md create mode 100644 examples/arduino-littlefs/data/partitions.csv create mode 100644 examples/arduino-littlefs/data/platformio.ini create mode 100644 examples/arduino-littlefs/data/test.txt create mode 100644 examples/arduino-littlefs/partitions.csv create mode 100644 examples/arduino-littlefs/platformio.ini create mode 100644 examples/arduino-littlefs/src/littlefs_test.ino create mode 100644 examples/arduino-spiffs/.gitignore create mode 100644 examples/arduino-spiffs/README.md create mode 100644 examples/arduino-spiffs/data/README.md create mode 100644 examples/arduino-spiffs/data/partitions.csv create mode 100644 examples/arduino-spiffs/data/platformio.ini create mode 100644 examples/arduino-spiffs/data/test.txt create mode 100644 examples/arduino-spiffs/partitions.csv create mode 100644 examples/arduino-spiffs/platformio.ini create mode 100644 examples/arduino-spiffs/src/spiffs_test.ino diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index b8eb8bd..b306ae0 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -15,6 +15,7 @@ jobs: os: [ubuntu-latest, windows-latest, macos-15] example: - "examples/arduino-blink" + - "examples/arduino-fatfs" - "examples/arduino-rmt-blink" - "examples/arduino-usb-keyboard" - "examples/arduino-wifiscan" diff --git a/README.md b/README.md index 7bf100f..655b19e 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,35 @@ Prerequisites: [pioarduino Wiki](https://deepwiki.com/pioarduino/platform-espressif32) The Wiki is AI generated and insane detailed and accurate. +# Features + +## Filesystem Support + +pioarduino provides native support for multiple filesystem options, allowing you to choose the best solution for your project's needs: + +- **LittleFS** (default) - Modern wear-leveling filesystem designed specifically for flash memory. Offers excellent reliability and performance for ESP32 projects. +- **SPIFFS** - Simple legacy filesystem. While still functional, LittleFS is recommended for new projects due to better wear-leveling and reliability. +- **FatFS** - Industry-standard FAT filesystem with broad compatibility across platforms and operating systems. + +**Quick Start:** + +```ini +[env:myenv] +board_build.filesystem = fatfs +``` + +**Available Commands:** + +```bash +pio run -t buildfs # Build FatFS image +pio run -t uploadfs # Upload FatFS image +pio run -t download_fatfs # Download and extract FatFS from device +``` + +See the [arduino-fatfs example](examples/arduino-fatfs/) for a complete working example. + ### Stable Arduino -currently espressif Arduino 3.3.5 and IDF 5.5.1.251215 +currently espressif Arduino 3.3.6 and IDF 5.5.2+ ```ini [env:stable] diff --git a/boards/esp32-s3-devkitc1-n16r2.json b/boards/esp32-s3-devkitc1-n16r2.json index 8bd26bb..4cc0454 100644 --- a/boards/esp32-s3-devkitc1-n16r2.json +++ b/boards/esp32-s3-devkitc1-n16r2.json @@ -44,7 +44,7 @@ ], "name": "Espressif ESP32-S3-DevKitC-1-N16R2 (16 MB Flash Quad, 2 MB PSRAM Quad)", "upload": { - "flash_size": "8MB", + "flash_size": "16MB", "maximum_ram_size": 327680, "maximum_size": 16777216, "require_upload_port": true, diff --git a/boards/esp32-s3-devkitc1-n4r2.json b/boards/esp32-s3-devkitc1-n4r2.json index 2ef5ceb..a728dfc 100644 --- a/boards/esp32-s3-devkitc1-n4r2.json +++ b/boards/esp32-s3-devkitc1-n4r2.json @@ -44,7 +44,7 @@ ], "name": "Espressif ESP32-S3-DevKitC-1-N4R2 (4 MB Flash Quad, 2 MB PSRAM Quad)", "upload": { - "flash_size": "8MB", + "flash_size": "4MB", "maximum_ram_size": 327680, "maximum_size": 4194304, "require_upload_port": true, diff --git a/boards/esp32-s3-devkitc1-n4r8.json b/boards/esp32-s3-devkitc1-n4r8.json index e4f1ceb..695bcbb 100644 --- a/boards/esp32-s3-devkitc1-n4r8.json +++ b/boards/esp32-s3-devkitc1-n4r8.json @@ -44,7 +44,7 @@ ], "name": "Espressif ESP32-S3-DevKitC-1-N4R8 (4 MB Flash Quad, 8 MB PSRAM Octal)", "upload": { - "flash_size": "8MB", + "flash_size": "4MB", "maximum_ram_size": 327680, "maximum_size": 4194304, "require_upload_port": true, diff --git a/boards/labplus_mpython.json b/boards/labplus_mpython.json index 94b71ae..f89cd0d 100644 --- a/boards/labplus_mpython.json +++ b/boards/labplus_mpython.json @@ -23,7 +23,7 @@ ], "name": "Labplus mPython", "upload": { - "flash_size": "8MB", + "flash_size": "4MB", "maximum_ram_size": 327680, "maximum_size": 4194304, "require_upload_port": true, diff --git a/boards/um_squixl.json b/boards/um_squixl.json index 7d40f72..e0cba1b 100644 --- a/boards/um_squixl.json +++ b/boards/um_squixl.json @@ -40,7 +40,7 @@ "upload": { "flash_size": "16MB", "maximum_ram_size": 327680, - "maximum_size": 8388608, + "maximum_size": 16777216, "require_upload_port": true, "speed": 460800 }, diff --git a/builder/main.py b/builder/main.py index 69122b3..d543df0 100644 --- a/builder/main.py +++ b/builder/main.py @@ -12,15 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import importlib.util import locale import os import re import shlex +import shutil +import struct import subprocess import sys from os.path import isfile, join from pathlib import Path +from littlefs import LittleFS +from fatfs import Partition, RamDisk, create_extended_partition +from fatfs import create_esp32_wl_image +from fatfs import calculate_esp32_wl_overhead +from fatfs import is_esp32_wl_image, extract_fat_from_esp32_wl +from fatfs.partition_extended import PartitionExtended +from fatfs.wrapper import pyf_mkfs, PY_FR_OK as FR_OK +import importlib.util from SCons.Script import ( ARGUMENTS, @@ -48,6 +57,15 @@ build_dir = Path(projectconfig.get("platformio", "build_dir")) # Configure Python environment through centralized platform management PYTHON_EXE, esptool_binary_path = platform.setup_python_env(env) +# Load SPIFFS generator from local module +spiffsgen_path = platform_dir / "builder" / "spiffsgen.py" +spec = importlib.util.spec_from_file_location("spiffsgen", str(spiffsgen_path)) +spiffsgen = importlib.util.module_from_spec(spec) +sys.modules["spiffsgen"] = spiffsgen +spec.loader.exec_module(spiffsgen) +SpiffsFS = spiffsgen.SpiffsFS +SpiffsBuildConfig = spiffsgen.SpiffsBuildConfig + # Load board configuration and determine MCU architecture board = env.BoardConfig() board_id = env.subst("$BOARD") @@ -391,12 +409,6 @@ def fetch_fs_size(env): env["FS_PAGE"] = int("0x100", 16) env["FS_BLOCK"] = int("0x1000", 16) - # FFat specific offsets, see: - # https://github.com/lorol/arduino-esp32fatfs-plugin#notes-for-fatfs - if filesystem == "fatfs": - env["FS_START"] += 4096 - env["FS_SIZE"] -= 4096 - def __fetch_fs_size(target, source, env): """ @@ -414,6 +426,342 @@ def __fetch_fs_size(target, source, env): return (target, source) +def build_fs_image(target, source, env): + """ + Build filesystem image using littlefs-python. + + Args: + target: SCons target (output .bin file) + source: SCons source (directory with files) + env: SCons environment object + + Returns: + int: 0 on success, 1 on failure + """ + + # Get parameters + source_dir = str(source[0]) + target_file = str(target[0]) + fs_size = env["FS_SIZE"] + block_size = env.get("FS_BLOCK", 4096) + + # Calculate block count + block_count = fs_size // block_size + + # Get disk version from board config or project options + # Default to LittleFS version 2.1 (0x00020001) + disk_version_str = "2.1" + + # Try to read from project config (env-specific or common section) + for section in ["env:" + env["PIOENV"], "common"]: + if projectconfig.has_option(section, "board_build.littlefs_version"): + disk_version_str = projectconfig.get(section, "board_build.littlefs_version") + break + + # Parse version string and create proper version integer + # LittleFS version format: (major << 16) | (minor << 0) + try: + version_parts = str(disk_version_str).split(".") + major = int(version_parts[0]) + minor = int(version_parts[1]) if len(version_parts) > 1 else 0 + # Format: major in upper 16 bits, minor in lower 16 bits + disk_version = (major << 16) | minor + except (ValueError, IndexError): + print(f"Warning: Invalid littlefs version '{disk_version_str}', using default 2.1") + disk_version = (2 << 16) | 1 + + try: + # Create LittleFS instance with Arduino / IDF compatible parameters + fs = LittleFS( + block_size=block_size, + block_count=block_count, + read_size=1, # Minimum read size + prog_size=1, # Minimum program size + cache_size=block_size, # Cache size = block size + lookahead_size=32, # Default lookahead buffer + block_cycles=500, # Wear leveling cycles + name_max=64, # ESP-IDF default filename length + disk_version=disk_version, + mount=True + ) + + # Add all files from source directory + source_path = Path(source_dir) + if source_path.exists(): + for item in source_path.rglob("*"): + rel_path = item.relative_to(source_path) + fs_path = rel_path.as_posix() + + if item.is_dir(): + fs.makedirs(fs_path, exist_ok=True) + # Set directory mtime attribute + try: + mtime = int(item.stat().st_mtime) + fs.setattr(fs_path, 't', mtime.to_bytes(4, 'little')) + except Exception: + pass # Ignore timestamp errors + else: + # Ensure parent directories exist + if rel_path.parent != Path("."): + fs.makedirs(rel_path.parent.as_posix(), exist_ok=True) + # Copy file + with fs.open(fs_path, "wb") as dest: + dest.write(item.read_bytes()) + # Set file mtime attribute (ESP-IDF compatible) + try: + mtime = int(item.stat().st_mtime) + fs.setattr(fs_path, 't', mtime.to_bytes(4, 'little')) + except Exception: + pass # Ignore timestamp errors + + # Write filesystem image + with open(target_file, "wb") as f: + f.write(fs.context.buffer) + + return 0 + + except Exception as e: + print(f"Error building filesystem image: {e}") + return 1 + + +def build_spiffs_image(target, source, env): + """ + Build SPIFFS filesystem image using spiffsgen.py. + + Args: + target: SCons target (output .bin file) + source: SCons source (directory with files) + env: SCons environment object + + Returns: + int: 0 on success, 1 on failure + """ + + # Get parameters + source_dir = str(source[0]) + target_file = str(target[0]) + fs_size = env["FS_SIZE"] + page_size = env.get("FS_PAGE", 256) + block_size = env.get("FS_BLOCK", 4096) + + # Get SPIFFS configuration from project config or use defaults + obj_name_len = 32 + meta_len = 4 + use_magic = True + use_magic_len = True + aligned_obj_ix_tables = False + + # Check common section first, then env-specific (so env-specific takes precedence) + for section in ["common", "env:" + env["PIOENV"]]: + if projectconfig.has_option(section, "board_build.spiffs.obj_name_len"): + obj_name_len = int(projectconfig.get(section, "board_build.spiffs.obj_name_len")) + if projectconfig.has_option(section, "board_build.spiffs.meta_len"): + meta_len = int(projectconfig.get(section, "board_build.spiffs.meta_len")) + if projectconfig.has_option(section, "board_build.spiffs.use_magic"): + use_magic = projectconfig.getboolean(section, "board_build.spiffs.use_magic") + if projectconfig.has_option(section, "board_build.spiffs.use_magic_len"): + use_magic_len = projectconfig.getboolean(section, "board_build.spiffs.use_magic_len") + if projectconfig.has_option(section, "board_build.spiffs.aligned_obj_ix_tables"): + aligned_obj_ix_tables = projectconfig.getboolean(section, "board_build.spiffs.aligned_obj_ix_tables") + + try: + # Create SPIFFS build configuration + spiffs_build_config = SpiffsBuildConfig( + page_size=page_size, + page_ix_len=2, # SPIFFS_PAGE_IX_LEN + block_size=block_size, + block_ix_len=2, # SPIFFS_BLOCK_IX_LEN + meta_len=meta_len, + obj_name_len=obj_name_len, + obj_id_len=2, # SPIFFS_OBJ_ID_LEN + span_ix_len=2, # SPIFFS_SPAN_IX_LEN + packed=True, + aligned=True, + endianness='little', + use_magic=use_magic, + use_magic_len=use_magic_len, + aligned_obj_ix_tables=aligned_obj_ix_tables + ) + + # Create SPIFFS filesystem + spiffs = SpiffsFS(fs_size, spiffs_build_config) + + # Add all files from source directory + source_path = Path(source_dir) + if source_path.exists(): + for item in source_path.rglob("*"): + if item.is_file(): + rel_path = item.relative_to(source_path) + img_path = "/" + rel_path.as_posix() + spiffs.create_file(img_path, str(item)) + + # Generate binary image + image = spiffs.to_binary() + + # Write to file + with open(target_file, "wb") as f: + f.write(image) + + print(f"\nSuccessfully created SPIFFS image: {target_file}") + return 0 + + except Exception as e: + print(f"Error building SPIFFS image: {e}") + return 1 + + +def build_fatfs_image(target, source, env): + """ + Build FatFS filesystem image with ESP32 Wear Leveling support. + + Uses fatfs-ng module to create ESP-IDF compatible WL-wrapped FAT images. + + Args: + target: SCons target (output .bin file) + source: SCons source (directory with files) + env: SCons environment object + + Returns: + int: 0 on success, 1 on failure + """ + + # Get parameters + source_dir = str(source[0]) + target_file = str(target[0]) + fs_size = env["FS_SIZE"] + sector_size = env.get("FS_SECTOR", 4096) + + # ESP-IDF WL layout (following wl_fatfsgen.py): + # [dummy sector] [FAT data] [state1] [state2] [config] + # Total WL sectors: 1 dummy + 2 states + 1 config = 4 sectors + wl_info = calculate_esp32_wl_overhead(fs_size, sector_size) + + wl_reserved_sectors = wl_info['wl_overhead_sectors'] + fat_fs_size = wl_info['fat_size'] + sector_count = wl_info['fat_sectors'] + + try: + # Create RAM disk with the FAT filesystem size (without WL overhead) + storage = bytearray(fat_fs_size) + disk = RamDisk(storage, sector_size=sector_size, sector_count=sector_count) + + # Create partition, format, and mount + base_partition = Partition(disk) + + # Format the filesystem with proper workarea size for LFN support + # Workarea needs to be at least sector_size, use 2x for safety with LFN + workarea_size = sector_size * 2 + + # Create filesystem with parameters matching ESP-IDF expectations: + # - n_fat=2: Two FAT copies for redundancy + # - align=0: Auto-align (let FATFS decide) + # - n_root=512: Number of root directory entries (FAT12/16 only, 0 for FAT32) + # - au_size=0: Auto allocation unit size + ret = pyf_mkfs( + base_partition.pname, + n_fat=2, + align=0, + n_root=512, # Standard root entries for FAT16 + au_size=0, # Auto + workarea_size=workarea_size + ) + if ret != FR_OK: + raise Exception(f"Failed to format filesystem: error code {ret}") + + # Mount the filesystem + base_partition.mount() + + # Wrap with extended partition for directory support + partition = PartitionExtended(base_partition) + + # Track skipped files + skipped_files = [] + + # Add all files from source directory + source_path = Path(source_dir) + if source_path.exists(): + for item in source_path.rglob("*"): + rel_path = item.relative_to(source_path) + fs_path = "/" + rel_path.as_posix() + + if item.is_dir(): + try: + partition.mkdir(fs_path) + except Exception: + # Directory might already exist or be root + pass + else: + # Ensure parent directories exist + if rel_path.parent != Path("."): + parent_path = "/" + rel_path.parent.as_posix() + try: + partition.mkdir(parent_path) + except Exception: + pass # Directory might already exist + + # Copy file + try: + with partition.open(fs_path, "w") as dest: + dest.write(item.read_bytes()) + except Exception as e: + print(f"Warning: Failed to write file {rel_path}: {e}") + skipped_files.append(str(rel_path)) + + # Unmount filesystem + base_partition.unmount() + + # Read boot sector parameters for validation + bytes_per_sector = struct.unpack(' 10: + print(f" ... and {len(skipped_files) - 10} more") + + print(f"\nSuccessfully created ESP-IDF WL-wrapped FAT image: {target_file}") + + return 0 + + except Exception as e: + print(f"Error building FatFS image: {e}") + return 1 + + def check_lib_archive_exists(): """ Check if lib_archive is set in platformio.ini configuration. @@ -427,14 +775,29 @@ def check_lib_archive_exists(): return False +def build_fs_router(target, source, env): + """Route to appropriate filesystem builder based on filesystem type.""" + fs_type = board.get("build.filesystem", "littlefs") + if fs_type == "littlefs": + return build_fs_image(target, source, env) + elif fs_type == "fatfs": + return build_fatfs_image(target, source, env) + elif fs_type == "spiffs": + return build_spiffs_image(target, source, env) + else: + print(f"Error: Unknown filesystem type '{fs_type}'. Supported types: littlefs, fatfs, spiffs") + return 1 + + def switch_off_ldf(): """ - Disables LDF (Library Dependency Finder) for uploadfs, uploadfsota, and buildfs targets. + Disables LDF (Library Dependency Finder) for uploadfs, uploadfsota, buildfs, + download_fs, and erase targets. This optimization prevents unnecessary library dependency scanning and compilation when only filesystem operations are performed. """ - fs_targets = {"uploadfs", "uploadfsota", "buildfs", "erase"} + fs_targets = {"uploadfs", "uploadfsota", "buildfs", "erase", "download_fs"} if fs_targets & set(COMMAND_LINE_TARGETS): # Disable LDF by modifying project configuration directly env_section = "env:" + env["PIOENV"] @@ -494,22 +857,6 @@ env.Replace( ERASEFLAGS=["--chip", mcu, "--port", '"$UPLOAD_PORT"'], ERASETOOL=uploader_path, ERASECMD='$ERASETOOL $ERASEFLAGS erase-flash', - MKFSTOOL="mk%s" % filesystem - + ( - ( - "_${PIOPLATFORM}_" - + ( - "espidf" - if "espidf" in env.subst("$PIOFRAMEWORK") - else "${PIOFRAMEWORK}" - ) - ) - if filesystem == "spiffs" - else "" - ), - # Legacy `ESP32_SPIFFS_IMAGE_NAME` is used as the second fallback value - # for backward compatibility - ESP32_FS_IMAGE_NAME=env.get( "ESP32_FS_IMAGE_NAME", env.get("ESP32_SPIFFS_IMAGE_NAME", filesystem), @@ -559,15 +906,7 @@ env.Append( ), DataToBin=Builder( action=env.VerboseAction( - " ".join( - ['"$MKFSTOOL"', "-c", "$SOURCES", "-s", "$FS_SIZE"] - + ( - ["-p", "$FS_PAGE", "-b", "$FS_BLOCK"] - if filesystem in ("littlefs", "spiffs") - else [] - ) - + ["$TARGET"] - ), + build_fs_router, "Building FS image from '$SOURCES' directory to $TARGET", ), emitter=__fetch_fs_size, @@ -776,6 +1115,486 @@ def coredump_analysis(target, source, env): print(f"Error: Failed to run coredump analysis: {e}") print(f'Make sure esp-coredump is installed: uv pip install --python "{PYTHON_EXE}" esp-coredump') + +def _get_unpack_dir(env): + """ + Get the unpack directory from project configuration. + + Args: + env: SCons environment object + + Returns: + str: Unpack directory path + """ + unpack_dir = "unpacked_fs" + + # Read from project config (env-specific or common section) + for section in ["env:" + env["PIOENV"], "common"]: + if projectconfig.has_option(section, "board_build.unpack_dir"): + unpack_dir = projectconfig.get(section, "board_build.unpack_dir") + break + + return unpack_dir + + +def _prepare_unpack_dir(unpack_dir): + """ + Prepare the unpack directory by removing old content and creating fresh directory. + + Args: + unpack_dir: Directory path to prepare + + Returns: + Path: Path object for the unpack directory + """ + unpack_path = Path(get_project_dir()) / unpack_dir + if unpack_path.exists(): + shutil.rmtree(unpack_path) + unpack_path.mkdir(parents=True, exist_ok=True) + return unpack_path + + +def _download_partition_image(env, fs_type_filter=None): + """ + Common function to download partition table and filesystem image from device. + + Args: + env: SCons environment object + fs_type_filter: List of partition subtypes to look for (e.g., [0x82, 0x83] for LittleFS/SPIFFS) + or [0x81] for FAT. If None, accepts any data partition. + + Returns: + tuple: (fs_file_path, fs_start, fs_size, fs_subtype) or (None, None, None, None) on error + """ + # Ensure upload port is set + if not env.subst("$UPLOAD_PORT"): + env.AutodetectUploadPort() + + upload_port = env.subst("$UPLOAD_PORT") + download_speed = board.get("download.speed", "115200") + + # Download partition table from device + print(f"\nDownloading partition table from {upload_port}...\n") + + build_dir = Path(env.subst("$BUILD_DIR")) + build_dir.mkdir(parents=True, exist_ok=True) + partition_file = build_dir / "partition_table_from_flash.bin" + + esptool_cmd = [ + uploader_path.strip('"'), + "--port", upload_port, + "--baud", str(download_speed), + "--before", "default-reset", + "--after", "hard-reset", + "read-flash", + "0x8000", # Partition table offset + "0x1000", # Partition table size (4KB) + str(partition_file) + ] + + try: + result = subprocess.run(esptool_cmd, check=False) + if result.returncode != 0: + print("Error: Failed to download partition table") + return None, None, None, None + except Exception as e: + print(f"Error: {e}") + return None, None, None, None + + with open(partition_file, 'rb') as f: + partition_data = f.read() + + # Parse partition entries (format: 0xAA 0x50 followed by entry data) + entries = [e for e in partition_data.split(b'\xaaP') if len(e) > 0] + + fs_start = None + fs_size = None + fs_subtype = None + + for entry in entries: + if len(entry) < 32: + continue + + # Byte 0: Type (0x01 for data partitions) + # Byte 1: SubType (0x81=FAT, 0x82=SPIFFS, 0x83=LittleFS) + # Bytes 2-5: Offset (4 bytes, little-endian) + # Bytes 6-9: Size (4 bytes, little-endian) + + part_subtype = entry[1] + + # Check if this partition matches our filter + if fs_type_filter is None or part_subtype in fs_type_filter: + fs_start = int.from_bytes(entry[2:6], byteorder='little', signed=False) + fs_size = int.from_bytes(entry[6:10], byteorder='little', signed=False) + fs_subtype = part_subtype + break + + if fs_start is None or fs_size is None: + print("Error: No matching filesystem partition found in partition table") + return None, None, None, None + + print(f"\nFound filesystem partition (subtype {hex(fs_subtype)}):") + print(f" Start: {hex(fs_start)}") + print(f" Size: {hex(fs_size)} ({fs_size} bytes)") + + # Download filesystem image + fs_file = build_dir / f"downloaded_fs_{hex(fs_start)}_{hex(fs_size)}.bin" + + print("\nDownloading filesystem from device...\n") + + esptool_cmd = [ + uploader_path.strip('"'), + "--port", upload_port, + "--baud", str(download_speed), + "--before", "default-reset", + "--after", "hard-reset", + "read-flash", + hex(fs_start), + hex(fs_size), + str(fs_file) + ] + + try: + result = subprocess.run(esptool_cmd, check=False) + if result.returncode != 0: + print(f"Error: Download failed with code {result.returncode}") + return None, None, None, None + except Exception as e: + print(f"Error: {e}") + return None, None, None, None + + print(f"\nDownloaded to {fs_file}") + + return fs_file, fs_start, fs_size, fs_subtype + + +def _extract_littlefs(fs_file, fs_size, unpack_path, unpack_dir): + """Extract LittleFS filesystem.""" + # Read the downloaded filesystem image + with open(fs_file, 'rb') as f: + fs_data = f.read() + + # Use ESP-IDF defaults + block_size = 0x1000 # 4KB + block_count = fs_size // block_size + + # Create LittleFS instance and mount the image + fs = LittleFS( + block_size=block_size, + block_count=block_count, + mount=False + ) + fs.context.buffer = bytearray(fs_data) + fs.mount() + + # Extract all files + file_count = 0 + print("\nExtracted files:") + for root, dirs, files in fs.walk("/"): + if not root.endswith("/"): + root += "/" + + # Create directories + for dir_name in dirs: + src_path = root + dir_name + dst_path = unpack_path / src_path[1:] # Remove leading '/' + dst_path.mkdir(parents=True, exist_ok=True) + print(f" [DIR] {src_path}") + + # Extract files + for file_name in files: + src_path = root + file_name + dst_path = unpack_path / src_path[1:] # Remove leading '/' + dst_path.parent.mkdir(parents=True, exist_ok=True) + + with fs.open(src_path, "rb") as src: + file_data = src.read() + dst_path.write_bytes(file_data) + + print(f" [FILE] {src_path} ({len(file_data)} bytes)") + file_count += 1 + + fs.unmount() + print(f"\nSuccessfully extracted {file_count} file(s) to {unpack_dir}") + return 0 + + +def _parse_spiffs_config(fs_data, fs_size): + """ + Auto-detect SPIFFS configuration from the image. + Tries common configurations and validates against the image. + + Returns: + dict: SPIFFS configuration parameters or None + """ + # Common ESP32/ESP8266 SPIFFS configurations + common_configs = [ + # ESP32/ESP8266 defaults + {'page_size': 256, 'block_size': 4096, 'obj_name_len': 32}, + # Alternative configurations + {'page_size': 256, 'block_size': 8192, 'obj_name_len': 32}, + {'page_size': 512, 'block_size': 4096, 'obj_name_len': 32}, + {'page_size': 256, 'block_size': 4096, 'obj_name_len': 64}, + ] + + print("\nAuto-detecting SPIFFS configuration...") + + for config in common_configs: + try: + # Try to parse with this configuration + spiffs_build_config = SpiffsBuildConfig( + page_size=config['page_size'], + page_ix_len=2, + block_size=config['block_size'], + block_ix_len=2, + meta_len=4, + obj_name_len=config['obj_name_len'], + obj_id_len=2, + span_ix_len=2, + packed=True, + aligned=True, + endianness='little', + use_magic=True, + use_magic_len=True, + aligned_obj_ix_tables=False + ) + + # Try to create and parse the filesystem + spiffs = SpiffsFS(fs_size, spiffs_build_config) + spiffs.from_binary(fs_data) + + # If we got here without exception, this config works + print(" Detected SPIFFS configuration:") + print(f" Page size: {config['page_size']} bytes") + print(f" Block size: {config['block_size']} bytes") + print(f" Max filename length: {config['obj_name_len']}") + + return { + 'page_size': config['page_size'], + 'block_size': config['block_size'], + 'obj_name_len': config['obj_name_len'], + 'meta_len': 4, + 'use_magic': True, + 'use_magic_len': True, + 'aligned_obj_ix_tables': False + } + except Exception: + continue + + # If no config worked, return defaults + print(" Could not auto-detect configuration, using ESP32/ESP8266 defaults") + return { + 'page_size': 256, + 'block_size': 4096, + 'obj_name_len': 32, + 'meta_len': 4, + 'use_magic': True, + 'use_magic_len': True, + 'aligned_obj_ix_tables': False + } + + +def _extract_spiffs(fs_file, fs_size, unpack_path, unpack_dir): + """Extract SPIFFS filesystem with auto-detected configuration.""" + # Read the downloaded filesystem image + with open(fs_file, 'rb') as f: + fs_data = f.read() + + # Auto-detect SPIFFS configuration + config = _parse_spiffs_config(fs_data, fs_size) + + # Create SPIFFS build configuration + spiffs_build_config = SpiffsBuildConfig( + page_size=config['page_size'], + page_ix_len=2, + block_size=config['block_size'], + block_ix_len=2, + meta_len=config['meta_len'], + obj_name_len=config['obj_name_len'], + obj_id_len=2, + span_ix_len=2, + packed=True, + aligned=True, + endianness='little', + use_magic=config['use_magic'], + use_magic_len=config['use_magic_len'], + aligned_obj_ix_tables=config['aligned_obj_ix_tables'] + ) + + # Create SPIFFS filesystem and parse the image + spiffs = SpiffsFS(fs_size, spiffs_build_config) + spiffs.from_binary(fs_data) + + # Extract files + file_count = spiffs.extract_files(str(unpack_path)) + + if file_count == 0: + print("\nNo files were extracted.") + print("The filesystem may be empty, freshly formatted, or contain only deleted entries.") + else: + print(f"\nSuccessfully extracted {file_count} file(s) to {unpack_dir}") + + return 0 + + +def _extract_fatfs(fs_file, unpack_path, unpack_dir): + """Extract FatFS filesystem.""" + # Read the downloaded filesystem image + with open(fs_file, 'rb') as f: + fs_data = bytearray(f.read()) + + # Check if the image looks like a valid FAT filesystem + if len(fs_data) < 512: + print("Error: Downloaded image is too small to be a valid FAT filesystem") + return 1 + + # Try to detect and extract wear leveling layer + sector_size = 4096 # Default ESP32 sector size + + # Check if this is a wear-leveling wrapped image + if is_esp32_wl_image(fs_data, sector_size): + print("Detected Wear Leveling layer, extracting FAT data...") + fat_data = extract_fat_from_esp32_wl(fs_data, sector_size) + if fat_data is None: + print("Error: Failed to extract FAT data from wear-leveling image") + return 1 + fs_data = bytearray(fat_data) + print(f" Extracted FAT data: {len(fs_data)} bytes") + else: + print("No Wear Leveling layer detected, treating as raw FAT image...") + + # Read sector size from FAT boot sector (offset 0x0B, 2 bytes, little-endian) + sector_size = int.from_bytes(fs_data[0x0B:0x0D], byteorder='little') + + # Validate sector size + if sector_size not in [512, 1024, 2048, 4096]: + print(f"Error: Invalid sector size {sector_size}. Must be 512, 1024, 2048, or 4096") + return 1 + + # Mount with fatfs-python + fs_size_adjusted = len(fs_data) + sector_count = fs_size_adjusted // sector_size + disk = RamDisk(fs_data, sector_size=sector_size, sector_count=sector_count) + partition = create_extended_partition(disk) + partition.mount() + + # Extract all files using PartitionExtended.walk() and read_file() + print("Extracting files:\n") + extracted_count = 0 + for root, dirs, files in partition.walk("/"): + # Determine target directory + if root == "/": + abs_root = unpack_path + else: + rel_root = root[1:] if root.startswith("/") else root + abs_root = unpack_path / rel_root + abs_root.mkdir(parents=True, exist_ok=True) + + # Extract files in current directory + for filename in files: + # Construct source path + if root == "/": + src_file = "/" + filename + else: + src_file = root.rstrip("/") + "/" + filename + + dst_file = abs_root / filename + try: + data = partition.read_file(src_file) + dst_file.write_bytes(data) + print(f" FILE: {src_file} ({len(data)} bytes)") + extracted_count += 1 + except Exception as e: + print(f" Warning: Failed to extract {src_file}: {e}") + partition.unmount() + + # Summary + if extracted_count == 0: + print("\nNo files were extracted.") + print("The filesystem may be empty, freshly formatted, or contain only deleted entries.") + else: + print(f"\nSuccessfully extracted {extracted_count} file(s) to {unpack_dir}") + + return 0 + + +def download_fs_action(target, source, env): + """Download and extract filesystem from device.""" + # Get unpack directory (use global env, not the parameter) + unpack_dir = _get_unpack_dir(env) + + # Download partition image + fs_file, _fs_start, fs_size, fs_subtype = _download_partition_image(env, None) + + if fs_file is None: + return 1 + + # Read header for detailed filesystem detection + with open(fs_file, 'rb') as f: + header = f.read(16384) # Read more to check for offset FAT + + # Detect filesystem type with improved logic + fs_type = None + + # 1. Check for LittleFS magic at offset 8 of the superblock + if len(header) >= 16 and header[8:16] == b'littlefs': + fs_type = "littlefs" + + # 2. Check for FAT filesystem (with or without Wear Leveling) + if fs_type is None: + # Check multiple possible offsets for FAT boot sector + # ESP32 with WL often has FAT at offset 0x1000 (4096) + fat_offsets = [0, 4096, 8192] + + for offset in fat_offsets: + if len(header) >= offset + 512: + boot_sector = header[offset:offset+512] + + # Check for FAT boot signature at offset 510-511 + if boot_sector[510:512] == b'\x55\xAA': + # Additional validation: check for FAT filesystem markers + # Check for "FAT" string or "MSDOS" in boot sector + if (b'FAT' in boot_sector[0:90] or + b'MSDOS' in boot_sector[0:90] or + b'MSWIN' in boot_sector[0:90]): + # Verify bytes per sector + bytes_per_sector = int.from_bytes(boot_sector[11:13], byteorder='little') + if bytes_per_sector in [512, 1024, 2048, 4096]: + fs_type = "fatfs" + print(f" FAT boot sector found at offset 0x{offset:x}") + break + + # 3. Fall back to partition table subtype if no clear signature found + if fs_type is None: + if fs_subtype == 0x81: + fs_type = "fatfs" + elif fs_subtype == 0x82: + # Subtype 0x82 can be either SPIFFS or LittleFS, default to SPIFFS + fs_type = "spiffs" + elif fs_subtype == 0x83: + fs_type = "littlefs" + else: + print(f"Warning: Unknown partition subtype 0x{fs_subtype:02X}, defaulting to SPIFFS") + fs_type = "spiffs" + + print(f"\nDetected filesystem: {fs_type.upper()} (partition subtype: 0x{fs_subtype:02X})") + + # Prepare unpack directory + unpack_path = _prepare_unpack_dir(unpack_dir) + + # Extract filesystem + try: + if fs_type == "littlefs": + return _extract_littlefs(fs_file, fs_size, unpack_path, unpack_dir) + elif fs_type == "spiffs": + return _extract_spiffs(fs_file, fs_size, unpack_path, unpack_dir) + elif fs_type == "fatfs": + return _extract_fatfs(fs_file, unpack_path, unpack_dir) + except Exception as e: + print(f"Error: {e}") + return 1 + + # # Target: Build executable and linkable firmware or FS image # @@ -1013,6 +1832,17 @@ env.AddPlatformTarget( "Upload Filesystem Image OTA", ) +# Target: Download Filesystem (auto-detect type) +env.AddPlatformTarget( + "download_fs", + None, + [ + env.VerboseAction(BeforeUpload, "Looking for upload port..."), + env.VerboseAction(download_fs_action, "Downloading and extracting filesystem") + ], + "Download and extract filesystem from device", +) + # Target: Erase Flash and Upload env.AddPlatformTarget( "erase_upload", diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 5345bc3..5c3c23b 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -45,6 +45,8 @@ PLATFORMIO_URL_VERSION_RE = re.compile( # Python dependencies required for ESP32 platform builds python_deps = { "platformio": "https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip", + "littlefs-python": ">=0.16.0", + "fatfs-ng": ">=0.1.14", "pyyaml": ">=6.0.2", "rich-click": ">=1.8.6", "zopfli": ">=0.2.2", diff --git a/builder/spiffsgen.py b/builder/spiffsgen.py new file mode 100644 index 0000000..7d51e27 --- /dev/null +++ b/builder/spiffsgen.py @@ -0,0 +1,762 @@ +#!/usr/bin/env python +# +# spiffsgen is a tool used to generate a spiffs image from a directory +# +# SPDX-FileCopyrightText: 2019-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import argparse +import io +import math +import os +import struct + +try: + import typing + + TSP = typing.TypeVar('TSP', bound='SpiffsObjPageWithIdx') + ObjIdsItem = typing.Tuple[int, typing.Type[TSP]] +except ImportError: + pass + + +SPIFFS_PH_FLAG_USED_FINAL_INDEX = 0xF8 +SPIFFS_PH_FLAG_USED_FINAL = 0xFC + +SPIFFS_PH_FLAG_LEN = 1 +SPIFFS_PH_IX_SIZE_LEN = 4 +SPIFFS_PH_IX_OBJ_TYPE_LEN = 1 +SPIFFS_TYPE_FILE = 1 + +# Based on typedefs under spiffs_config.h +SPIFFS_OBJ_ID_LEN = 2 # spiffs_obj_id +SPIFFS_SPAN_IX_LEN = 2 # spiffs_span_ix +SPIFFS_PAGE_IX_LEN = 2 # spiffs_page_ix +SPIFFS_BLOCK_IX_LEN = 2 # spiffs_block_ix + + +class SpiffsBuildConfig(object): + def __init__(self, + page_size, # type: int + page_ix_len, # type: int + block_size, # type: int + block_ix_len, # type: int + meta_len, # type: int + obj_name_len, # type: int + obj_id_len, # type: int + span_ix_len, # type: int + packed, # type: bool + aligned, # type: bool + endianness, # type: str + use_magic, # type: bool + use_magic_len, # type: bool + aligned_obj_ix_tables # type: bool + ): + if block_size % page_size != 0: + raise RuntimeError('block size should be a multiple of page size') + + self.page_size = page_size + self.block_size = block_size + self.obj_id_len = obj_id_len + self.span_ix_len = span_ix_len + self.packed = packed + self.aligned = aligned + self.obj_name_len = obj_name_len + self.meta_len = meta_len + self.page_ix_len = page_ix_len + self.block_ix_len = block_ix_len + self.endianness = endianness + self.use_magic = use_magic + self.use_magic_len = use_magic_len + self.aligned_obj_ix_tables = aligned_obj_ix_tables + + self.PAGES_PER_BLOCK = self.block_size // self.page_size + self.OBJ_LU_PAGES_PER_BLOCK = int(math.ceil(self.block_size / self.page_size * self.obj_id_len / self.page_size)) + self.OBJ_USABLE_PAGES_PER_BLOCK = self.PAGES_PER_BLOCK - self.OBJ_LU_PAGES_PER_BLOCK + + self.OBJ_LU_PAGES_OBJ_IDS_LIM = self.page_size // self.obj_id_len + + self.OBJ_DATA_PAGE_HEADER_LEN = self.obj_id_len + self.span_ix_len + SPIFFS_PH_FLAG_LEN + + pad = 4 - (4 if self.OBJ_DATA_PAGE_HEADER_LEN % 4 == 0 else self.OBJ_DATA_PAGE_HEADER_LEN % 4) + + self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED = self.OBJ_DATA_PAGE_HEADER_LEN + pad + self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED_PAD = pad + self.OBJ_DATA_PAGE_CONTENT_LEN = self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN + + self.OBJ_INDEX_PAGES_HEADER_LEN = (self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED + SPIFFS_PH_IX_SIZE_LEN + + SPIFFS_PH_IX_OBJ_TYPE_LEN + self.obj_name_len + self.meta_len) + if aligned_obj_ix_tables: + self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED = (self.OBJ_INDEX_PAGES_HEADER_LEN + SPIFFS_PAGE_IX_LEN - 1) & ~(SPIFFS_PAGE_IX_LEN - 1) + self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD = self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED - self.OBJ_INDEX_PAGES_HEADER_LEN + else: + self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED = self.OBJ_INDEX_PAGES_HEADER_LEN + self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD = 0 + + self.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM = (self.page_size - self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED) // self.block_ix_len + self.OBJ_INDEX_PAGES_OBJ_IDS_LIM = (self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED) // self.block_ix_len + + +class SpiffsFullError(RuntimeError): + pass + + +class SpiffsPage(object): + _endianness_dict = { + 'little': '<', + 'big': '>' + } + + _len_dict = { + 1: 'B', + 2: 'H', + 4: 'I', + 8: 'Q' + } + + def __init__(self, bix, build_config): # type: (int, SpiffsBuildConfig) -> None + self.build_config = build_config + self.bix = bix + + def to_binary(self): # type: () -> bytes + raise NotImplementedError() + + +class SpiffsObjPageWithIdx(SpiffsPage): + def __init__(self, obj_id, build_config): # type: (int, SpiffsBuildConfig) -> None + super(SpiffsObjPageWithIdx, self).__init__(0, build_config) + self.obj_id = obj_id + + def to_binary(self): # type: () -> bytes + raise NotImplementedError() + + +class SpiffsObjLuPage(SpiffsPage): + def __init__(self, bix, build_config): # type: (int, SpiffsBuildConfig) -> None + SpiffsPage.__init__(self, bix, build_config) + + self.obj_ids_limit = self.build_config.OBJ_LU_PAGES_OBJ_IDS_LIM + self.obj_ids = list() # type: typing.List[ObjIdsItem] + + def _calc_magic(self, blocks_lim): # type: (int) -> int + # Calculate the magic value mirroring computation done by the macro SPIFFS_MAGIC defined in + # spiffs_nucleus.h + magic = 0x20140529 ^ self.build_config.page_size + if self.build_config.use_magic_len: + magic = magic ^ (blocks_lim - self.bix) + # narrow the result to build_config.obj_id_len bytes + mask = (2 << (8 * self.build_config.obj_id_len)) - 1 + return magic & mask + + def register_page(self, page): # type: (TSP) -> None + if not self.obj_ids_limit > 0: + raise SpiffsFullError() + + obj_id = (page.obj_id, page.__class__) + self.obj_ids.append(obj_id) + self.obj_ids_limit -= 1 + + def to_binary(self): # type: () -> bytes + img = b'' + + for (obj_id, page_type) in self.obj_ids: + if page_type == SpiffsObjIndexPage: + obj_id ^= (1 << ((self.build_config.obj_id_len * 8) - 1)) + img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.obj_id_len], obj_id) + + assert len(img) <= self.build_config.page_size + + img += b'\xFF' * (self.build_config.page_size - len(img)) + + return img + + def magicfy(self, blocks_lim): # type: (int) -> None + # Only use magic value if no valid obj id has been written to the spot, which is the + # spot taken up by the last obj id on last lookup page. The parent is responsible + # for determining which is the last lookup page and calling this function. + remaining = self.obj_ids_limit + empty_obj_id_dict = { + 1: 0xFF, + 2: 0xFFFF, + 4: 0xFFFFFFFF, + 8: 0xFFFFFFFFFFFFFFFF + } + if remaining >= 2: + for i in range(remaining): + if i == remaining - 2: + self.obj_ids.append((self._calc_magic(blocks_lim), SpiffsObjDataPage)) + break + else: + self.obj_ids.append((empty_obj_id_dict[self.build_config.obj_id_len], SpiffsObjDataPage)) + self.obj_ids_limit -= 1 + + +class SpiffsObjIndexPage(SpiffsObjPageWithIdx): + def __init__(self, obj_id, span_ix, size, name, build_config + ): # type: (int, int, int, str, SpiffsBuildConfig) -> None + super(SpiffsObjIndexPage, self).__init__(obj_id, build_config) + self.span_ix = span_ix + self.name = name + self.size = size + + if self.span_ix == 0: + self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM + else: + self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_LIM + + self.pages = list() # type: typing.List[int] + + def register_page(self, page): # type: (SpiffsObjDataPage) -> None + if not self.pages_lim > 0: + raise SpiffsFullError() + + self.pages.append(page.offset) + self.pages_lim -= 1 + + def to_binary(self): # type: () -> bytes + obj_id = self.obj_id ^ (1 << ((self.build_config.obj_id_len * 8) - 1)) + img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.obj_id_len] + + SpiffsPage._len_dict[self.build_config.span_ix_len] + + SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], + obj_id, + self.span_ix, + SPIFFS_PH_FLAG_USED_FINAL_INDEX) + + # Add padding before the object index page specific information + img += b'\xFF' * self.build_config.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED_PAD + + # If this is the first object index page for the object, add filename, type + # and size information + if self.span_ix == 0: + img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[SPIFFS_PH_IX_SIZE_LEN] + + SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], + self.size, + SPIFFS_TYPE_FILE) + + img += self.name.encode() + (b'\x00' * ( + (self.build_config.obj_name_len - len(self.name)) + + self.build_config.meta_len + + self.build_config.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD)) + + # Finally, add the page index of data pages + for page in self.pages: + page = page >> int(math.log(self.build_config.page_size, 2)) + img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.page_ix_len], page) + + assert len(img) <= self.build_config.page_size + + img += b'\xFF' * (self.build_config.page_size - len(img)) + + return img + + +class SpiffsObjDataPage(SpiffsObjPageWithIdx): + def __init__(self, offset, obj_id, span_ix, contents, build_config + ): # type: (int, int, int, bytes, SpiffsBuildConfig) -> None + super(SpiffsObjDataPage, self).__init__(obj_id, build_config) + self.span_ix = span_ix + self.contents = contents + self.offset = offset + + def to_binary(self): # type: () -> bytes + img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.obj_id_len] + + SpiffsPage._len_dict[self.build_config.span_ix_len] + + SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], + self.obj_id, + self.span_ix, + SPIFFS_PH_FLAG_USED_FINAL) + + img += self.contents + + assert len(img) <= self.build_config.page_size + + img += b'\xFF' * (self.build_config.page_size - len(img)) + + return img + + +class SpiffsBlock(object): + def _reset(self): # type: () -> None + self.cur_obj_index_span_ix = 0 + self.cur_obj_data_span_ix = 0 + self.cur_obj_id = 0 + self.cur_obj_idx_page = None # type: typing.Optional[SpiffsObjIndexPage] + + def __init__(self, bix, build_config): # type: (int, SpiffsBuildConfig) -> None + self.build_config = build_config + self.offset = bix * self.build_config.block_size + self.remaining_pages = self.build_config.OBJ_USABLE_PAGES_PER_BLOCK + self.pages = list() # type: typing.List[SpiffsPage] + self.bix = bix + + lu_pages = list() + for i in range(self.build_config.OBJ_LU_PAGES_PER_BLOCK): + page = SpiffsObjLuPage(self.bix, self.build_config) + lu_pages.append(page) + + self.pages.extend(lu_pages) + + self.lu_page_iter = iter(lu_pages) + self.lu_page = next(self.lu_page_iter) + + self._reset() + + def _register_page(self, page): # type: (TSP) -> None + if isinstance(page, SpiffsObjDataPage): + assert self.cur_obj_idx_page is not None + self.cur_obj_idx_page.register_page(page) # can raise SpiffsFullError + + try: + self.lu_page.register_page(page) + except SpiffsFullError: + self.lu_page = next(self.lu_page_iter) + try: + self.lu_page.register_page(page) + except AttributeError: # no next lookup page + # Since the amount of lookup pages is pre-computed at every block instance, + # this should never occur + raise RuntimeError('invalid attempt to add page to a block when there is no more space in lookup') + + self.pages.append(page) + + def begin_obj(self, obj_id, size, name, obj_index_span_ix=0, obj_data_span_ix=0 + ): # type: (int, int, str, int, int) -> None + if not self.remaining_pages > 0: + raise SpiffsFullError() + self._reset() + + self.cur_obj_id = obj_id + self.cur_obj_index_span_ix = obj_index_span_ix + self.cur_obj_data_span_ix = obj_data_span_ix + + page = SpiffsObjIndexPage(obj_id, self.cur_obj_index_span_ix, size, name, self.build_config) + self._register_page(page) + + self.cur_obj_idx_page = page + + self.remaining_pages -= 1 + self.cur_obj_index_span_ix += 1 + + def update_obj(self, contents): # type: (bytes) -> None + if not self.remaining_pages > 0: + raise SpiffsFullError() + page = SpiffsObjDataPage(self.offset + (len(self.pages) * self.build_config.page_size), + self.cur_obj_id, self.cur_obj_data_span_ix, contents, self.build_config) + + self._register_page(page) + + self.cur_obj_data_span_ix += 1 + self.remaining_pages -= 1 + + def end_obj(self): # type: () -> None + self._reset() + + def is_full(self): # type: () -> bool + return self.remaining_pages <= 0 + + def to_binary(self, blocks_lim): # type: (int) -> bytes + img = b'' + + if self.build_config.use_magic: + for (idx, page) in enumerate(self.pages): + if idx == self.build_config.OBJ_LU_PAGES_PER_BLOCK - 1: + assert isinstance(page, SpiffsObjLuPage) + page.magicfy(blocks_lim) + img += page.to_binary() + else: + for page in self.pages: + img += page.to_binary() + + assert len(img) <= self.build_config.block_size + + img += b'\xFF' * (self.build_config.block_size - len(img)) + return img + + def _parse_from_binary(self, block_data): # type: (bytes) -> None + """Parse block data from binary image. + + Args: + block_data: Raw block bytes + """ + self._raw_data = block_data + + +class SpiffsFS(object): + def __init__(self, img_size, build_config): # type: (int, SpiffsBuildConfig) -> None + if img_size % build_config.block_size != 0: + raise RuntimeError('image size should be a multiple of block size') + + self.img_size = img_size + self.build_config = build_config + + self.blocks = list() # type: typing.List[SpiffsBlock] + self.blocks_lim = self.img_size // self.build_config.block_size + self.remaining_blocks = self.blocks_lim + self.cur_obj_id = 1 # starting object id + + def _create_block(self): # type: () -> SpiffsBlock + if self.is_full(): + raise SpiffsFullError('the image size has been exceeded') + + block = SpiffsBlock(len(self.blocks), self.build_config) + self.blocks.append(block) + self.remaining_blocks -= 1 + return block + + def is_full(self): # type: () -> bool + return self.remaining_blocks <= 0 + + def create_file(self, img_path, file_path): # type: (str, str) -> None + if len(img_path) > self.build_config.obj_name_len: + raise RuntimeError("object name '%s' too long" % img_path) + + name = img_path + + with open(file_path, 'rb') as obj: + contents = obj.read() + + stream = io.BytesIO(contents) + + try: + block = self.blocks[-1] + block.begin_obj(self.cur_obj_id, len(contents), name) + except (IndexError, SpiffsFullError): + block = self._create_block() + block.begin_obj(self.cur_obj_id, len(contents), name) + + contents_chunk = stream.read(self.build_config.OBJ_DATA_PAGE_CONTENT_LEN) + + while contents_chunk: + try: + block = self.blocks[-1] + try: + # This can fail because either (1) all the pages in block have been + # used or (2) object index has been exhausted. + block.update_obj(contents_chunk) + except SpiffsFullError: + # If its (1), use the outer exception handler + if block.is_full(): + raise SpiffsFullError + # If its (2), write another object index page + block.begin_obj(self.cur_obj_id, len(contents), name, + obj_index_span_ix=block.cur_obj_index_span_ix, + obj_data_span_ix=block.cur_obj_data_span_ix) + continue + except (IndexError, SpiffsFullError): + # All pages in the block have been exhausted. Create a new block, copying + # the previous state of the block to a new one for the continuation of the + # current object + prev_block = block + block = self._create_block() + block.cur_obj_id = prev_block.cur_obj_id + block.cur_obj_idx_page = prev_block.cur_obj_idx_page + block.cur_obj_data_span_ix = prev_block.cur_obj_data_span_ix + block.cur_obj_index_span_ix = prev_block.cur_obj_index_span_ix + continue + + contents_chunk = stream.read(self.build_config.OBJ_DATA_PAGE_CONTENT_LEN) + + block.end_obj() + + self.cur_obj_id += 1 + + def to_binary(self): # type: () -> bytes + img = b'' + all_blocks = [] + for block in self.blocks: + all_blocks.append(block.to_binary(self.blocks_lim)) + bix = len(self.blocks) + if self.build_config.use_magic: + # Create empty blocks with magic numbers + while self.remaining_blocks > 0: + block = SpiffsBlock(bix, self.build_config) + all_blocks.append(block.to_binary(self.blocks_lim)) + self.remaining_blocks -= 1 + bix += 1 + else: + # Just fill remaining spaces FF's + all_blocks.append(b'\xFF' * (self.img_size - len(all_blocks) * self.build_config.block_size)) + img += b''.join([blk for blk in all_blocks]) + return img + + def from_binary(self, image_data): # type: (bytes) -> None + """Parse a SPIFFS binary image and populate the filesystem structure. + + Args: + image_data: Raw SPIFFS image bytes + """ + if len(image_data) != self.img_size: + raise RuntimeError(f'image size mismatch: expected {self.img_size}, got {len(image_data)}') + + # Parse blocks from the image + blocks_count = self.img_size // self.build_config.block_size + for bix in range(blocks_count): + block_offset = bix * self.build_config.block_size + block_data = image_data[block_offset:block_offset + self.build_config.block_size] + block = SpiffsBlock(bix, self.build_config) + block._parse_from_binary(block_data) + self.blocks.append(block) + + def extract_files(self, output_dir): # type: (str) -> int + """Extract all files from the SPIFFS filesystem to a directory. + + Args: + output_dir: Directory path where files will be extracted + + Returns: + int: Number of files extracted + """ + + # Build a map of object_id -> file info + files_map = {} # obj_id -> {'name': str, 'size': int, 'data_pages': [(span_ix, page_data)]} + + for block in self.blocks: + # Parse lookup pages to find valid objects + for page_idx in range(self.build_config.OBJ_LU_PAGES_PER_BLOCK): + lu_page_offset = page_idx * self.build_config.page_size + lu_page_data = block._raw_data[lu_page_offset:lu_page_offset + self.build_config.page_size] + + # Parse object IDs from lookup page + for i in range(0, len(lu_page_data), self.build_config.obj_id_len): + if i + self.build_config.obj_id_len > len(lu_page_data): + break + + obj_id_bytes = lu_page_data[i:i + self.build_config.obj_id_len] + obj_id = struct.unpack( + SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.obj_id_len], + obj_id_bytes + )[0] + + # Check if it's a valid object (not erased/empty) + empty_values = {1: 0xFF, 2: 0xFFFF, 4: 0xFFFFFFFF, 8: 0xFFFFFFFFFFFFFFFF} + if obj_id == empty_values[self.build_config.obj_id_len]: + continue + + # Check if it's an index page (MSB set) + is_index = obj_id & (1 << ((self.build_config.obj_id_len * 8) - 1)) + real_obj_id = obj_id & ~(1 << ((self.build_config.obj_id_len * 8) - 1)) + + if is_index and real_obj_id not in files_map: + files_map[real_obj_id] = {'name': None, 'size': 0, 'data_pages': []} + + # Parse actual pages to get file metadata and content + for page_idx in range(self.build_config.OBJ_LU_PAGES_PER_BLOCK, self.build_config.PAGES_PER_BLOCK): + page_offset = page_idx * self.build_config.page_size + page_data = block._raw_data[page_offset:page_offset + self.build_config.page_size] + + # Parse page header + header_fmt = ( + SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.obj_id_len] + + SpiffsPage._len_dict[self.build_config.span_ix_len] + + SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN] + ) + header_size = struct.calcsize(header_fmt) + + if len(page_data) < header_size: + continue + + obj_id, span_ix, flags = struct.unpack(header_fmt, page_data[:header_size]) + + # Check for valid page + empty_id = {1: 0xFF, 2: 0xFFFF, 4: 0xFFFFFFFF, 8: 0xFFFFFFFFFFFFFFFF}[self.build_config.obj_id_len] + if obj_id == empty_id: + continue + + is_index = obj_id & (1 << ((self.build_config.obj_id_len * 8) - 1)) + real_obj_id = obj_id & ~(1 << ((self.build_config.obj_id_len * 8) - 1)) + + if is_index and flags == SPIFFS_PH_FLAG_USED_FINAL_INDEX: + # Index page - contains file metadata + if real_obj_id not in files_map: + files_map[real_obj_id] = {'name': None, 'size': 0, 'data_pages': []} + + # Only first index page (span_ix == 0) has filename and size + if span_ix == 0: + # Skip to size and type fields + offset = header_size + self.build_config.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED_PAD + size_type_fmt = ( + SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[SPIFFS_PH_IX_SIZE_LEN] + + SpiffsPage._len_dict[SPIFFS_PH_IX_OBJ_TYPE_LEN] + ) + size_type_size = struct.calcsize(size_type_fmt) + + if offset + size_type_size <= len(page_data): + file_size, obj_type = struct.unpack(size_type_fmt, page_data[offset:offset + size_type_size]) + offset += size_type_size + + # Read filename + name_end = offset + self.build_config.obj_name_len + if name_end <= len(page_data): + name_bytes = page_data[offset:name_end] + # Find null terminator + null_pos = name_bytes.find(b'\x00') + if null_pos != -1: + name_bytes = name_bytes[:null_pos] + filename = name_bytes.decode('utf-8', errors='ignore') + files_map[real_obj_id]['name'] = filename + files_map[real_obj_id]['size'] = file_size + + elif not is_index and flags == SPIFFS_PH_FLAG_USED_FINAL: + # Data page - contains file content + if real_obj_id in files_map: + # Extract content (skip header, no padding on data pages) + content_start = header_size + content = page_data[content_start:content_start + self.build_config.OBJ_DATA_PAGE_CONTENT_LEN] + files_map[real_obj_id]['data_pages'].append((span_ix, content)) + + # Extract files to output directory + file_count = 0 + for obj_id, file_info in files_map.items(): + if file_info['name'] is None: + continue + + # Remove leading slash if present + rel_path = file_info['name'].lstrip('/') + file_path = os.path.join(output_dir, rel_path) + if not rel_path: + print(f" Warning: Skipping file with empty path (obj_id={obj_id})") + continue + + # Create parent directories + os.makedirs(os.path.dirname(file_path), exist_ok=True) + + # Sort data pages by span index + file_info['data_pages'].sort(key=lambda x: x[0]) + + # Write file content + with open(file_path, 'wb') as f: + total_written = 0 + for span_ix, content in file_info['data_pages']: + # Write only up to the file size + remaining = file_info['size'] - total_written + if remaining <= 0: + break + to_write = min(len(content), remaining) + f.write(content[:to_write]) + total_written += to_write + + file_count += 1 + print(f" Extracted: {file_info['name']} ({file_info['size']} bytes)") + + return file_count + + +class CustomHelpFormatter(argparse.HelpFormatter): + """ + Similar to argparse.ArgumentDefaultsHelpFormatter, except it + doesn't add the default value if "(default:" is already present. + This helps in the case of options with action="store_false", like + --no-magic or --no-magic-len. + """ + def _get_help_string(self, action): # type: (argparse.Action) -> str + if action.help is None: + return '' + if '%(default)' not in action.help and '(default:' not in action.help: + if action.default is not argparse.SUPPRESS: + defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE] + if action.option_strings or action.nargs in defaulting_nargs: + return action.help + ' (default: %(default)s)' + return action.help + + +def main(): # type: () -> None + parser = argparse.ArgumentParser(description='SPIFFS Image Generator', + formatter_class=CustomHelpFormatter) + + parser.add_argument('image_size', + help='Size of the created image') + + parser.add_argument('base_dir', + help='Path to directory from which the image will be created') + + parser.add_argument('output_file', + help='Created image output file path') + + parser.add_argument('--page-size', + help='Logical page size. Set to value same as CONFIG_SPIFFS_PAGE_SIZE.', + type=int, + default=256) + + parser.add_argument('--block-size', + help="Logical block size. Set to the same value as the flash chip's sector size (g_rom_flashchip.sector_size).", + type=int, + default=4096) + + parser.add_argument('--obj-name-len', + help='File full path maximum length. Set to value same as CONFIG_SPIFFS_OBJ_NAME_LEN.', + type=int, + default=32) + + parser.add_argument('--meta-len', + help='File metadata length. Set to value same as CONFIG_SPIFFS_META_LENGTH.', + type=int, + default=4) + + parser.add_argument('--use-magic', + dest='use_magic', + help='Use magic number to create an identifiable SPIFFS image. Specify if CONFIG_SPIFFS_USE_MAGIC.', + action='store_true') + + parser.add_argument('--no-magic', + dest='use_magic', + help='Inverse of --use-magic (default: --use-magic is enabled)', + action='store_false') + + parser.add_argument('--use-magic-len', + dest='use_magic_len', + help='Use position in memory to create different magic numbers for each block. Specify if CONFIG_SPIFFS_USE_MAGIC_LENGTH.', + action='store_true') + + parser.add_argument('--no-magic-len', + dest='use_magic_len', + help='Inverse of --use-magic-len (default: --use-magic-len is enabled)', + action='store_false') + + parser.add_argument('--follow-symlinks', + help='Take into account symbolic links during partition image creation.', + action='store_true') + + parser.add_argument('--big-endian', + help='Specify if the target architecture is big-endian. If not specified, little-endian is assumed.', + action='store_true') + + parser.add_argument('--aligned-obj-ix-tables', + action='store_true', + help='Use aligned object index tables. Specify if SPIFFS_ALIGNED_OBJECT_INDEX_TABLES is set.') + + parser.set_defaults(use_magic=True, use_magic_len=True) + + args = parser.parse_args() + + if not os.path.exists(args.base_dir): + raise RuntimeError('given base directory %s does not exist' % args.base_dir) + + with open(args.output_file, 'wb') as image_file: + image_size = int(args.image_size, 0) + spiffs_build_default = SpiffsBuildConfig(args.page_size, SPIFFS_PAGE_IX_LEN, + args.block_size, SPIFFS_BLOCK_IX_LEN, args.meta_len, + args.obj_name_len, SPIFFS_OBJ_ID_LEN, SPIFFS_SPAN_IX_LEN, + True, True, 'big' if args.big_endian else 'little', + args.use_magic, args.use_magic_len, args.aligned_obj_ix_tables) + + spiffs = SpiffsFS(image_size, spiffs_build_default) + + for root, dirs, files in os.walk(args.base_dir, followlinks=args.follow_symlinks): + for f in files: + full_path = os.path.join(root, f) + spiffs.create_file('/' + os.path.relpath(full_path, args.base_dir).replace('\\', '/'), full_path) + + image = spiffs.to_binary() + + image_file.write(image) + + +if __name__ == '__main__': + main() diff --git a/examples/arduino-fatfs/.gitignore b/examples/arduino-fatfs/.gitignore new file mode 100644 index 0000000..89cc49c --- /dev/null +++ b/examples/arduino-fatfs/.gitignore @@ -0,0 +1,5 @@ +.pio +.vscode/.browse.c_cpp.db* +.vscode/c_cpp_properties.json +.vscode/launch.json +.vscode/ipch diff --git a/examples/arduino-fatfs/FATFS_INTEGRATION.md b/examples/arduino-fatfs/FATFS_INTEGRATION.md new file mode 100644 index 0000000..8be1ddb --- /dev/null +++ b/examples/arduino-fatfs/FATFS_INTEGRATION.md @@ -0,0 +1,175 @@ +# FatFS Integration for Platform-Espressif32 + +This platform now supports FatFS as a filesystem option, analogous to the existing LittleFS integration. + +## Features + +- **Build FatFS Image**: Creates a FatFS filesystem image from a directory +- **Upload FatFS Image**: Uploads the FatFS image to the ESP32 device +- **Download FatFS Image**: Downloads the FatFS image from the device and extracts it + +## Configuration + +### platformio.ini + +```ini +[env:myenv] +platform = espressif32 +board = esp32dev +framework = arduino + +; Select FatFS as filesystem +board_build.filesystem = fatfs + +; Optional: Directory for extracted files (default: unpacked_fs) +board_build.unpack_dir = unpacked_fs +``` + +### Partition Table + +The partition table must contain a FAT partition (Subtype 0x81): + +```csv +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +ffat, data, fat, 0x290000,0x170000, +``` + +## Usage + +### Build FatFS Image + +```bash +# Place files in data/ directory +mkdir -p data +echo "Hello FatFS" > data/test.txt + +# Build image +pio run -t buildfs +``` + +### Upload FatFS Image + +```bash +pio run -t uploadfs +``` + +### Download FatFS Image from Device + +```bash +pio run -t download_fatfs +``` + +Files will be extracted to the configured directory (default: `unpacked_fs`). + +## Technical Details + +### Python Dependencies + +The integration uses the `fatfs-ng` package, which is automatically installed. + +### Build Process + +1. A RAM disk is created with the configured FAT data size (partition size minus WL overhead) +2. The FatFS is formatted with proper parameters (2 FATs, LFN support) +3. All files from the `data/` directory are copied +4. The FAT image is wrapped with ESP32 Wear Leveling layer +5. The final image is saved as a `.bin` file + +**Important**: The build process automatically adds the ESP32 Wear Leveling layer, which is required by the Arduino FFat library. See [WEAR_LEVELING.md](WEAR_LEVELING.md) for details. + +### Wear Leveling Layer + +ESP32's FFat library requires a wear leveling layer around the FAT filesystem. The build process automatically: +- Reserves sectors for wear leveling metadata +- Wraps the FAT filesystem with WL_State structures +- Calculates proper CRC32 checksums + +### Download Process + +1. The partition table is downloaded from the device +2. The FAT partition is identified (Subtype 0x81) +3. The filesystem image is downloaded +4. The wear leveling layer is automatically detected and removed +5. The FAT data is mounted and extracted + +## Extended Features + +The `pyfatfs` package includes extended features for complete directory traversal: + +- **Complete Directory Traversal**: `walk()`, `listdir()`, `stat()` +- **Path Operations**: `exists()`, `isfile()`, `isdir()` +- **File Operations**: `remove()`, `rmdir()`, `rename()`, `makedirs()` +- **Convenience Methods**: `read_file()`, `write_file()` +- **Bulk Operations**: `copy_tree_from()`, `copy_tree_to()` + +These features enable full filesystem extraction and manipulation. + +## Comparison: LittleFS vs FatFS + +| Feature | LittleFS | FatFS | +|---------|----------|-------| +| Wear Leveling | Yes | Yes | +| Power-Loss Protection | Yes | Limited | +| Compatibility | ESP-IDF specific | Standard FAT | +| Sector Size | 4096 | 4096 | +| Filesystem Size | Flexible | Larger | + +## Example Code (Arduino) + +```cpp +#include + +void setup() { + Serial.begin(115200); + + // Mount FatFS + if (!FFat.begin(true)) { + Serial.println("FFat Mount Failed"); + return; + } + + // Read file + File file = FFat.open("/test.txt", "r"); + if (file) { + Serial.println(file.readString()); + file.close(); + } + + // Write file + file = FFat.open("/output.txt", "w"); + if (file) { + file.println("Hello from ESP32!"); + file.close(); + } +} + +void loop() { + // ... +} +``` + +## Troubleshooting + +### "No FAT filesystem partition found" + +- Check the partition table +- Ensure a partition with subtype `fat` (0x81) exists + +### Build Errors + +```bash +# Recreate Python environment +rm -rf ~/.platformio/penv +pio run +``` + +## Further Information + +- [FatFS Documentation](http://elm-chan.org/fsw/ff/00index_e.html) +- [ESP-IDF FFat Documentation](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/fatfs.html) +- [fatfs-ng Repository](https://github.com/Jason2866/pyfatfs) +- [Original fatfs-python](https://github.com/krakonos/fatfs-python) diff --git a/examples/arduino-fatfs/README.md b/examples/arduino-fatfs/README.md new file mode 100644 index 0000000..62fde9f --- /dev/null +++ b/examples/arduino-fatfs/README.md @@ -0,0 +1,224 @@ +# ESP32 FAT Filesystem Test Project + +This project tests the FAT filesystem implementation with ESP32 Wear Leveling support. + +## Overview + +This project demonstrates: +- Building FAT filesystem images with wear leveling layer +- Uploading FAT images to ESP32 +- Mounting and reading FAT filesystem on ESP32 +- Downloading and extracting FAT images from ESP32 + +## Requirements + +- PlatformIO +- ESP32 development board +- USB cable + +## Project Structure + +``` +arduino-fatfs/ +├── data/ # Files to be included in FAT image +│ ├── test.txt +│ ├── README.md +│ └── ... +├── src/ +│ └── ffat.ino # Main Arduino sketch +├── partitions.csv # Partition table with FAT partition +├── platformio.ini # PlatformIO configuration +└── unpacked_fs/ # Downloaded files (created by download_fatfs) +``` + +## Usage + +### 1. Build Firmware + +```bash +pio run +``` + +### 2. Build FAT Filesystem Image + +Place your files in the `data/` directory, then: + +```bash +pio run -t buildfs +``` + +This creates a FAT filesystem image with ESP32 wear leveling layer at: +`.pio/build/esp32dev/fatfs.bin` + +### 3. Upload Firmware and Filesystem + +```bash +# Upload firmware +pio run -t upload + +# Upload filesystem +pio run -t uploadfs +``` + +### 4. Monitor Serial Output + +```bash +pio run -t monitor +``` + +Expected output: +``` +FFat mounted successfully +Test begin +Total space: 1486848 +Free space: 1482752 +Listing directory: / + FILE: test.txt SIZE: 12 + FILE: README.md SIZE: 1234 +Test complete +``` + +### 5. Download Filesystem from Device + +To download and extract the filesystem from the device: + +```bash +pio run -t download_fatfs +``` + +Files will be extracted to `unpacked_fs/` directory. + +## Partition Table + +The `partitions.csv` defines the flash layout: + +```csv +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +ffat, data, fat, 0x290000,0x170000, +``` + +The `ffat` partition: +- **Type**: data +- **SubType**: fat (0x81) +- **Offset**: 0x290000 (2,686,976 bytes) +- **Size**: 0x170000 (1,507,328 bytes = ~1.44 MB) + +## Wear Leveling + +The FAT filesystem is automatically wrapped with ESP32's wear leveling layer: + +- **Total partition**: 1,507,328 bytes (368 sectors × 4096 bytes) +- **WL overhead**: 20,480 bytes (5 sectors) +- **FAT data**: 1,486,848 bytes (363 sectors) + +Structure: +``` +[WL State 1][WL State 2][FAT Data][Temp][WL State 3][WL State 4] +``` + +See [WEAR_LEVELING.md](../../WEAR_LEVELING.md) for details. + +## Troubleshooting + +### "FFat Mount Failed" + +**Possible causes:** +1. Filesystem not uploaded +2. Wrong partition table +3. Corrupted filesystem + +**Solutions:** +```bash +# Rebuild and upload filesystem +pio run -t buildfs +pio run -t uploadfs + +# Or erase flash and start fresh +pio run -t erase +pio run -t upload +pio run -t uploadfs +``` + +### "No FAT filesystem partition found" + +**Cause:** Partition table doesn't have a FAT partition + +**Solution:** Check `partitions.csv` has a partition with `SubType: fat` + +### Files not appearing + +**Cause:** Files not in `data/` directory when building + +**Solution:** +1. Add files to `data/` directory +2. Rebuild filesystem: `pio run -t buildfs` +3. Upload: `pio run -t uploadfs` + +## Code Example + +```cpp +#include "FFat.h" + +void setup() { + Serial.begin(115200); + + // Mount FAT filesystem + if (!FFat.begin(false)) { + Serial.println("FFat Mount Failed"); + return; + } + + // List files + File root = FFat.open("/"); + File file = root.openNextFile(); + while (file) { + Serial.printf("File: %s, Size: %d\n", + file.name(), file.size()); + file = root.openNextFile(); + } + + // Read file + File f = FFat.open("/test.txt", "r"); + if (f) { + String content = f.readString(); + Serial.println(content); + f.close(); + } + + // Write file + f = FFat.open("/output.txt", "w"); + if (f) { + f.println("Hello from ESP32!"); + f.close(); + } +} + +void loop() { + // Your code here +} +``` + +## Platform Configuration + +This project uses a custom platform with FAT filesystem support: + +```ini +[env:esp32dev] +platform = espressif32 +framework = arduino +board = esp32dev +board_build.filesystem = fatfs +board_build.partitions = partitions.csv +``` + +## References + +- [ESP32 FFat Library](https://github.com/espressif/arduino-esp32/tree/master/libraries/FFat) +- [ESP-IDF FAT Filesystem](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/fatfs.html) +- [ESP-IDF Wear Levelling](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/wear-levelling.html) +- [Platform-Espressif32 FAT Integration](../../FATFS_INTEGRATION.md) +- [Wear Leveling Implementation](../../platform-espressif32/WEAR_LEVELING.md) diff --git a/examples/arduino-fatfs/TEST_GUIDE.md b/examples/arduino-fatfs/TEST_GUIDE.md new file mode 100644 index 0000000..33e681f --- /dev/null +++ b/examples/arduino-fatfs/TEST_GUIDE.md @@ -0,0 +1,380 @@ +# FFat Filesystem Test Guide + +This guide explains how to test the ESP32 FFat filesystem with pre-flashed images using Wear Leveling. + +## Quick Start + +### 1. Prepare Test Files + +Add files to the `data/` directory that you want to include in the filesystem: + +```bash +# Example files already included: +data/ +├── test.txt +├── README.md +├── platformio.ini +└── partitions.csv +``` + +### 2. Build Filesystem Image + +Build the FAT filesystem image with Wear Leveling layer: + +```bash +pio run -t buildfs +``` + +This creates `.pio/build/esp32dev/fatfs.bin` with: +- Your files from `data/` directory +- ESP32 Wear Leveling layer +- Proper FAT filesystem structure + +### 3. Upload Firmware and Filesystem + +```bash +# Upload firmware +pio run -t upload + +# Upload filesystem +pio run -t uploadfs +``` + +### 4. Monitor Serial Output + +```bash +pio run -t monitor +``` + +## Test Configuration + +Edit `src/ffat.ino` to configure tests: + +```cpp +// Set to true to format the partition (erases all data!) +#define FORMAT_FFAT false + +// Test settings +#define TEST_READ_EXISTING true // Test reading pre-flashed files +#define TEST_WRITE_NEW true // Test writing new files +#define TEST_FILE_IO true // Test I/O performance +``` + +## Expected Output + +### Successful Mount + +``` +============================================================ +ESP32 FFat Filesystem Test +Testing pre-flashed image with Wear Leveling +============================================================ + +Mounting FFat filesystem... +✓ FFat mounted successfully! + +=== Filesystem Information === +Total space: 1486848 bytes (1.42 MB) +Used space: 12288 bytes (0.01 MB) +Free space: 1474560 bytes (1.41 MB) +Usage: 0.8% +``` + +### Reading Pre-Flashed Files + +``` +============================================================ + +=== Testing Pre-Flashed Files === + +Files in root directory: +Listing directory: / + FILE: test.txt SIZE: 12 + FILE: README.md SIZE: 1234 + FILE: platformio.ini SIZE: 456 + FILE: partitions.csv SIZE: 234 + +Reading test files: + +--- File: /test.txt --- +Reading file: /test.txt +- read from file: +Hello World! + +--- File: /README.md --- +Reading file: /README.md +- read from file: +[README content...] +``` + +### Write Operations + +``` +============================================================ + +=== Testing Write Operations === + +1. Creating new file... +Writing file: /test_write.txt +- file written + +2. Appending to file... +Appending to file: /test_write.txt +- message appended +Appending to file: /test_write.txt +- message appended + +3. Reading back written file: +Reading file: /test_write.txt +- read from file: +Hello from ESP32! +This line was appended. +And another line. + +4. Testing rename... +Renaming file /test_write.txt to /renamed.txt +- file renamed +Reading file: /renamed.txt +[content...] + +5. Testing delete... +Deleting file: /renamed.txt +- file deleted +File successfully deleted +``` + +### Performance Test + +``` +============================================================ + +Testing file I/O with /benchmark.bin +- writing................................ + - 1048576 bytes written in 2345 ms +- reading................................ + - 1048576 bytes read in 1234 ms +``` + +## Troubleshooting + +### "FFat Mount Failed" + +**Possible causes:** + +1. **No FFat partition in partition table** + - Check `partitions.csv` has a `fat` partition + - Verify partition is flashed + +2. **Filesystem not flashed** + ```bash + pio run -t buildfs + pio run -t uploadfs + ``` + +3. **Missing Wear Leveling layer** + - Ensure you're using the updated platform with WL support + - Rebuild filesystem image + +4. **Corrupted filesystem** + - Set `FORMAT_FFAT true` to reformat + - Or erase flash: `pio run -t erase` + +### "File not found" + +If pre-flashed files are not found: + +1. Check files exist in `data/` directory +2. Rebuild filesystem: `pio run -t buildfs` +3. Upload filesystem: `pio run -t uploadfs` +4. Reset ESP32 + +### "Write failed" + +If write operations fail: + +1. Check filesystem is not full +2. Verify partition has write permissions +3. Check for filesystem corruption + +## Advanced Testing + +### Download and Verify Filesystem + +After running tests, download the filesystem to verify changes: + +```bash +pio run -t download_fatfs +``` + +Files will be extracted to `unpacked_fs/` directory. + +### Compare Original and Downloaded + +```bash +# Compare original files +diff data/test.txt unpacked_fs/test.txt + +# Check for new files created by tests +ls -la unpacked_fs/ +``` + +### Test Wear Leveling + +To verify wear leveling is working: + +1. Write many files +2. Download filesystem +3. Check WL state is valid: + +```python +from fatfs import is_esp32_wl_image + +with open('.pio/build/esp32dev/downloaded_fs_*.bin', 'rb') as f: + data = f.read() + +if is_esp32_wl_image(data): + print("✓ Wear Leveling layer is intact") +else: + print("✗ Wear Leveling layer is missing or corrupted") +``` + +## Test Scenarios + +### Scenario 1: Fresh Filesystem + +```cpp +#define FORMAT_FFAT true +#define TEST_READ_EXISTING false +#define TEST_WRITE_NEW true +#define TEST_FILE_IO true +``` + +Tests creating a new filesystem from scratch. + +### Scenario 2: Pre-Flashed Image (Default) + +```cpp +#define FORMAT_FFAT false +#define TEST_READ_EXISTING true +#define TEST_WRITE_NEW true +#define TEST_FILE_IO true +``` + +Tests reading pre-flashed files and writing new ones. + +### Scenario 3: Read-Only Test + +```cpp +#define FORMAT_FFAT false +#define TEST_READ_EXISTING true +#define TEST_WRITE_NEW false +#define TEST_FILE_IO false +``` + +Only tests reading pre-flashed files without modifications. + +### Scenario 4: Performance Only + +```cpp +#define FORMAT_FFAT false +#define TEST_READ_EXISTING false +#define TEST_WRITE_NEW false +#define TEST_FILE_IO true +``` + +Only tests I/O performance. + +## Continuous Integration + +For automated testing: + +```bash +#!/bin/bash +# test_fatfs.sh + +# Build and upload +pio run -t buildfs +pio run -t upload +pio run -t uploadfs + +# Wait for ESP32 to boot +sleep 2 + +# Monitor output and check for success +pio run -t monitor | tee test_output.log + +# Verify output +if grep -q "✓ All tests completed!" test_output.log; then + echo "Tests PASSED" + exit 0 +else + echo "Tests FAILED" + exit 1 +fi +``` + +## Debugging + +### Enable Debug Output + +```cpp +void setup() { + Serial.begin(115200); + Serial.setDebugOutput(true); // Enable ESP32 debug output + // ... +} +``` + +### Check Partition Table + +```bash +# Read partition table from device +pio run -t monitor + +# In another terminal +esptool.py --port /dev/ttyUSB0 read_flash 0x8000 0x1000 partition_table.bin + +# Parse partition table +python -c " +import struct +with open('partition_table.bin', 'rb') as f: + data = f.read() + for i in range(0, len(data), 32): + entry = data[i:i+32] + if entry[:2] == b'\xAA\x50': + print(f'Partition at {i}: {entry.hex()}') +" +``` + +### Verify Wear Leveling + +```bash +# Download filesystem +pio run -t download_fatfs + +# Check WL structure +python -c " +import glob +from fatfs import is_esp32_wl_image, ESP32WearLeveling +import struct + +files = glob.glob('.pio/build/esp32dev/downloaded_fs_*.bin') +with open(files[0], 'rb') as f: + data = f.read() + +if is_esp32_wl_image(data): + wl = ESP32WearLeveling() + state = data[:48] + fields = struct.unpack('32MB) + +## Troubleshooting + +### "FFat Mount Failed" + +**Cause**: Image doesn't have wear leveling layer + +**Solution**: Rebuild with updated `build_fatfs_image()`: +```bash +pio run -t buildfs +pio run -t uploadfs +``` + +### "Invalid sector size" + +**Cause**: Sector size mismatch between build and ESP32 config + +**Solution**: Ensure `CONFIG_WL_SECTOR_SIZE=4096` in sdkconfig + +### "Partition too small" + +**Cause**: FAT data + WL overhead exceeds partition size + +**Solution**: Increase partition size in `partitions.csv` or reduce data + +## References + +- [ESP-IDF Wear Levelling Component](https://github.com/espressif/esp-idf/tree/master/components/wear_levelling) +- [ESP-IDF FAT Filesystem](https://github.com/espressif/esp-idf/tree/master/components/fatfs) +- [Arduino-ESP32 FFat Library](https://github.com/espressif/arduino-esp32/tree/master/libraries/FFat) +- [mk_esp32fat Tool](https://github.com/TobleMiner/mk_esp32fat) (alternative C implementation) + +## License + +Same as platform-espressif32 (Apache 2.0) diff --git a/examples/arduino-fatfs/data/README.md b/examples/arduino-fatfs/data/README.md new file mode 100644 index 0000000..3828402 --- /dev/null +++ b/examples/arduino-fatfs/data/README.md @@ -0,0 +1,224 @@ +# ESP32 FAT Filesystem Test Project + +This project tests the FAT filesystem implementation with ESP32 Wear Leveling support. + +## Overview + +This project demonstrates: +- Building FAT filesystem images with wear leveling layer +- Uploading FAT images to ESP32 +- Mounting and reading FAT filesystem on ESP32 +- Downloading and extracting FAT images from ESP32 + +## Requirements + +- PlatformIO +- ESP32 development board +- USB cable + +## Project Structure + +``` +arduino-fatfs/ +├── data/ # Files to be included in FAT image +│ ├── test.txt +│ ├── README.md +│ └── ... +├── src/ +│ └── ffat.ino # Main Arduino sketch +├── partitions.csv # Partition table with FAT partition +├── platformio.ini # PlatformIO configuration +└── unpacked_fs/ # Downloaded files (created by download_fatfs) +``` + +## Usage + +### 1. Build Firmware + +```bash +pio run +``` + +### 2. Build FAT Filesystem Image + +Place your files in the `data/` directory, then: + +```bash +pio run -t buildfs +``` + +This creates a FAT filesystem image with ESP32 wear leveling layer at: +`.pio/build/esp32dev/fatfs.bin` + +### 3. Upload Firmware and Filesystem + +```bash +# Upload firmware +pio run -t upload + +# Upload filesystem +pio run -t uploadfs +``` + +### 4. Monitor Serial Output + +```bash +pio run -t monitor +``` + +Expected output: +``` +FFat mounted successfully +Test begin +Total space: 1486848 +Free space: 1482752 +Listing directory: / + FILE: test.txt SIZE: 12 + FILE: README.md SIZE: 1234 +Test complete +``` + +### 5. Download Filesystem from Device + +To download and extract the filesystem from the device: + +```bash +pio run -t download_fatfs +``` + +Files will be extracted to `unpacked_fs/` directory. + +## Partition Table + +The `partitions.csv` defines the flash layout: + +```csv +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +ffat, data, fat, 0x290000,0x170000, +``` + +The `ffat` partition: +- **Type**: data +- **SubType**: fat (0x81) +- **Offset**: 0x290000 (2,686,976 bytes) +- **Size**: 0x170000 (1,507,328 bytes = ~1.44 MB) + +## Wear Leveling + +The FAT filesystem is automatically wrapped with ESP32's wear leveling layer: + +- **Total partition**: 1,507,328 bytes (368 sectors × 4096 bytes) +- **WL overhead**: 20,480 bytes (5 sectors) +- **FAT data**: 1,486,848 bytes (363 sectors) + +Structure: +``` +[WL State 1][WL State 2][FAT Data][Temp][WL State 3][WL State 4] +``` + +See [WEAR_LEVELING.md](../../WEAR_LEVELING.md) for details. + +## Troubleshooting + +### "FFat Mount Failed" + +**Possible causes:** +1. Filesystem not uploaded +2. Wrong partition table +3. Corrupted filesystem + +**Solutions:** +```bash +# Rebuild and upload filesystem +pio run -t buildfs +pio run -t uploadfs + +# Or erase flash and start fresh +pio run -t erase +pio run -t upload +pio run -t uploadfs +``` + +### "No FAT filesystem partition found" + +**Cause:** Partition table doesn't have a FAT partition + +**Solution:** Check `partitions.csv` has a partition with `SubType: fat` + +### Files not appearing + +**Cause:** Files not in `data/` directory when building + +**Solution:** +1. Add files to `data/` directory +2. Rebuild filesystem: `pio run -t buildfs` +3. Upload: `pio run -t uploadfs` + +## Code Example + +```cpp +#include "FFat.h" + +void setup() { + Serial.begin(115200); + + // Mount FAT filesystem + if (!FFat.begin(false)) { + Serial.println("FFat Mount Failed"); + return; + } + + // List files + File root = FFat.open("/"); + File file = root.openNextFile(); + while (file) { + Serial.printf("File: %s, Size: %d\n", + file.name(), file.size()); + file = root.openNextFile(); + } + + // Read file + File f = FFat.open("/test.txt", "r"); + if (f) { + String content = f.readString(); + Serial.println(content); + f.close(); + } + + // Write file + f = FFat.open("/output.txt", "w"); + if (f) { + f.println("Hello from ESP32!"); + f.close(); + } +} + +void loop() { + // Your code here +} +``` + +## Platform Configuration + +This project uses a custom platform with FAT filesystem support: + +```ini +[env:esp32dev] +platform = espressif32 +framework = arduino +board = esp32dev +board_build.filesystem = fatfs +board_build.partitions = partitions.csv +``` + +## References + +- [ESP32 FFat Library](https://github.com/espressif/arduino-esp32/tree/master/libraries/FFat) +- [ESP-IDF FAT Filesystem](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/fatfs.html) +- [ESP-IDF Wear Levelling](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/wear-levelling.html) +- [Platform-Espressif32 FAT Integration](../platform-espressif32/FATFS_INTEGRATION.md) +- [Wear Leveling Implementation](../platform-espressif32/WEAR_LEVELING.md) diff --git a/examples/arduino-fatfs/data/partitions.csv b/examples/arduino-fatfs/data/partitions.csv new file mode 100644 index 0000000..d921c9f --- /dev/null +++ b/examples/arduino-fatfs/data/partitions.csv @@ -0,0 +1,6 @@ +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +ffat, data, fat, 0x290000,0x170000, diff --git a/examples/arduino-fatfs/data/platformio.ini b/examples/arduino-fatfs/data/platformio.ini new file mode 100644 index 0000000..c3c6387 --- /dev/null +++ b/examples/arduino-fatfs/data/platformio.ini @@ -0,0 +1,7 @@ +[env:esp32dev] +platform = espressif32 +framework = arduino +board = esp32dev +board_build.filesystem = fatfs +board_build.partitions = partitions.csv +monitor_speed = 115200 diff --git a/examples/arduino-fatfs/data/test.txt b/examples/arduino-fatfs/data/test.txt new file mode 100644 index 0000000..524acff --- /dev/null +++ b/examples/arduino-fatfs/data/test.txt @@ -0,0 +1 @@ +Test file diff --git a/examples/arduino-fatfs/partitions.csv b/examples/arduino-fatfs/partitions.csv new file mode 100644 index 0000000..d921c9f --- /dev/null +++ b/examples/arduino-fatfs/partitions.csv @@ -0,0 +1,6 @@ +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +ffat, data, fat, 0x290000,0x170000, diff --git a/examples/arduino-fatfs/platformio.ini b/examples/arduino-fatfs/platformio.ini new file mode 100644 index 0000000..5ebaa83 --- /dev/null +++ b/examples/arduino-fatfs/platformio.ini @@ -0,0 +1,6 @@ +[env:esp32dev] +platform = espressif32 +framework = arduino +board = esp32dev +board_build.filesystem = fatfs +board_build.partitions = partitions.csv diff --git a/examples/arduino-fatfs/src/ffat.ino b/examples/arduino-fatfs/src/ffat.ino new file mode 100644 index 0000000..b1a1385 --- /dev/null +++ b/examples/arduino-fatfs/src/ffat.ino @@ -0,0 +1,478 @@ +#include "FS.h" +#include "FFat.h" +#include "esp_partition.h" + +// Test configuration +// Set to true to format the partition (will erase all data!) +// Set to false to test the pre-flashed image from data/ directory +#define FORMAT_FFAT false // Test pre-flashed image + +// Test settings +#define TEST_READ_EXISTING true // Test reading files from flashed image +#define TEST_WRITE_NEW true // Test writing new files +#define TEST_FILE_IO true // Test file I/O performance + +void listDir(fs::FS &fs, const char *dirname, uint8_t levels) { + Serial.printf("Listing directory: %s\r\n", dirname); + + File root = fs.open(dirname); + if (!root) { + Serial.println("- failed to open directory"); + return; + } + if (!root.isDirectory()) { + Serial.println(" - not a directory"); + return; + } + + File file = root.openNextFile(); + while (file) { + if (file.isDirectory()) { + Serial.print(" DIR : "); + Serial.println(file.name()); + if (levels) { + listDir(fs, file.path(), levels - 1); + } + } else { + Serial.print(" FILE: "); + Serial.print(file.name()); + Serial.print("\tSIZE: "); + Serial.println(file.size()); + } + file = root.openNextFile(); + } +} + +void readFile(fs::FS &fs, const char *path) { + Serial.printf("Reading file: %s\r\n", path); + + File file = fs.open(path); + if (!file || file.isDirectory()) { + Serial.println("- failed to open file for reading"); + return; + } + + Serial.println("- read from file:"); + while (file.available()) { + Serial.write(file.read()); + } + file.close(); +} + +void writeFile(fs::FS &fs, const char *path, const char *message) { + Serial.printf("Writing file: %s\r\n", path); + + File file = fs.open(path, FILE_WRITE); + if (!file) { + Serial.println("- failed to open file for writing"); + return; + } + if (file.print(message)) { + Serial.println("- file written"); + } else { + Serial.println("- write failed"); + } + file.close(); +} + +void appendFile(fs::FS &fs, const char *path, const char *message) { + Serial.printf("Appending to file: %s\r\n", path); + + File file = fs.open(path, FILE_APPEND); + if (!file) { + Serial.println("- failed to open file for appending"); + return; + } + if (file.print(message)) { + Serial.println("- message appended"); + } else { + Serial.println("- append failed"); + } + file.close(); +} + +void renameFile(fs::FS &fs, const char *path1, const char *path2) { + Serial.printf("Renaming file %s to %s\r\n", path1, path2); + if (fs.rename(path1, path2)) { + Serial.println("- file renamed"); + } else { + Serial.println("- rename failed"); + } +} + +void deleteFile(fs::FS &fs, const char *path) { + Serial.printf("Deleting file: %s\r\n", path); + if (fs.remove(path)) { + Serial.println("- file deleted"); + } else { + Serial.println("- delete failed"); + } +} + +void testFileIO(fs::FS &fs, const char *path) { + Serial.printf("Testing file I/O with %s\r\n", path); + + static uint8_t buf[512]; + size_t len = 0; + File file = fs.open(path, FILE_WRITE); + if (!file) { + Serial.println("- failed to open file for writing"); + return; + } + + size_t i; + Serial.print("- writing"); + uint32_t start = millis(); + for (i = 0; i < 2048; i++) { + if ((i & 0x001F) == 0x001F) { + Serial.print("."); + } + file.write(buf, 512); + } + Serial.println(""); + uint32_t end = millis() - start; + Serial.printf(" - %u bytes written in %lu ms\r\n", 2048 * 512, end); + file.close(); + + file = fs.open(path); + start = millis(); + end = start; + i = 0; + if (file && !file.isDirectory()) { + len = file.size(); + size_t flen = len; + start = millis(); + Serial.print("- reading"); + while (len) { + size_t toRead = len; + if (toRead > 512) { + toRead = 512; + } + file.read(buf, toRead); + if ((i++ & 0x001F) == 0x001F) { + Serial.print("."); + } + len -= toRead; + } + Serial.println(""); + end = millis() - start; + Serial.printf("- %u bytes read in %lu ms\r\n", flen, end); + file.close(); + } else { + Serial.println("- failed to open file for reading"); + } +} + +void testExistingFiles(fs::FS &fs) { + Serial.println("\n=== Testing Pre-Flashed Files ==="); + + // List all files in root + Serial.println("\nFiles in root directory:"); + listDir(fs, "/", 2); + + // Test reading specific files that should exist + const char* testFiles[] = { + "/test.txt", + "/README.md", + "/platformio.ini", + "/partitions.csv" + }; + + Serial.println("\nReading test files:"); + for (int i = 0; i < 4; i++) { + if (fs.exists(testFiles[i])) { + Serial.printf("\n--- File: %s ---\n", testFiles[i]); + readFile(fs, testFiles[i]); + } else { + Serial.printf("File not found: %s\n", testFiles[i]); + } + } +} + +void testWriteOperations(fs::FS &fs) { + Serial.println("\n=== Testing Write Operations ==="); + + // Test creating new file + Serial.println("\n1. Creating new file..."); + writeFile(fs, "/test_write.txt", "Hello from ESP32!\n"); + + // Test appending + Serial.println("\n2. Appending to file..."); + appendFile(fs, "/test_write.txt", "This line was appended.\n"); + appendFile(fs, "/test_write.txt", "And another line.\n"); + + // Read back + Serial.println("\n3. Reading back written file:"); + readFile(fs, "/test_write.txt"); + + // Test rename + Serial.println("\n4. Testing rename..."); + renameFile(fs, "/test_write.txt", "/renamed.txt"); + readFile(fs, "/renamed.txt"); + + // Test delete + Serial.println("\n5. Testing delete..."); + deleteFile(fs, "/renamed.txt"); + + // Verify deletion + if (!fs.exists("/renamed.txt")) { + Serial.println("File successfully deleted"); + } else { + Serial.println("ERROR: File still exists!"); + } +} + +void testFileSystem(fs::FS &fs) { + Serial.println("\n=== Filesystem Information ==="); + Serial.printf("Total space: %10u bytes (%.2f MB)\n", + FFat.totalBytes(), FFat.totalBytes() / 1024.0 / 1024.0); + Serial.printf("Used space: %10u bytes (%.2f MB)\n", + FFat.usedBytes(), FFat.usedBytes() / 1024.0 / 1024.0); + Serial.printf("Free space: %10u bytes (%.2f MB)\n", + FFat.freeBytes(), FFat.freeBytes() / 1024.0 / 1024.0); + + float usage = (FFat.usedBytes() * 100.0) / FFat.totalBytes(); + Serial.printf("Usage: %.1f%%\n", usage); +} + +void printSeparator() { + Serial.println("\n============================================================"); +} + +void setup() { + Serial.begin(115200); + delay(1000); // Wait for serial monitor + + // Enable ESP-IDF debug logging + esp_log_level_set("*", ESP_LOG_DEBUG); + esp_log_level_set("vfs_fat", ESP_LOG_VERBOSE); + esp_log_level_set("wear_levelling", ESP_LOG_VERBOSE); + + Serial.println("\n\n"); + printSeparator(); + Serial.println("ESP32 FFat Filesystem Test"); + Serial.println("Testing pre-flashed image with Wear Leveling"); + printSeparator(); + + // Format if requested + if (FORMAT_FFAT) { + Serial.println("\n============================================================"); + Serial.println("WARNING: Formatting FFat partition..."); + Serial.println("This will erase all data!"); + Serial.println("============================================================"); + delay(2000); + + // First try to mount the WL layer without formatting + Serial.println("\nStep 1: Checking if WL layer can be mounted..."); + if (FFat.begin(false, "/ffat", 10, "ffat")) { + Serial.println("✓ WL layer mounted successfully!"); + Serial.println(" Now formatting filesystem..."); + FFat.end(); + } else { + Serial.println("✗ WL layer mount failed - will try to format anyway"); + } + + Serial.println("\nStep 2: Formatting partition..."); + if (!FFat.format(false, (char*)"ffat")) { + Serial.println("ERROR: FFat Format Failed"); + Serial.println("\nThis means the Wear Leveling layer itself is broken."); + Serial.println("The WL structure on flash is not compatible with ESP-IDF."); + return; + } + Serial.println("✓ FFat formatted successfully"); + + Serial.println("\nStep 3: Mounting formatted partition..."); + if (!FFat.begin(false, "/ffat", 10, "ffat")) { + Serial.println("ERROR: Cannot mount even after format!"); + return; + } + Serial.println("✓ Mounted successfully after format"); + + // Write some test files so we have data to compare + Serial.println("\nWriting test files..."); + writeFile(FFat, "/test.txt", "This is a test file created by ESP32\n"); + writeFile(FFat, "/README.md", "# ESP32 Formatted Filesystem\n\nThis was formatted on device.\n"); + + Serial.println("\n*** IMPORTANT: Now download this filesystem ***"); + Serial.println("Run: pio run -t download_fatfs"); + Serial.println("This will save the ESP32-formatted image for comparison"); + + delay(5000); // Give time to read the message + } + + // Mount the filesystem + Serial.println("\nMounting FFat filesystem..."); + Serial.println("Partition label: 'ffat'"); + Serial.println("Format on fail: false"); + Serial.println("Max open files: 10"); + + if (!FFat.begin(false, "/ffat", 10, "ffat")) { // formatOnFail = false + Serial.println("\nERROR: FFat Mount Failed"); + Serial.println("\nDiagnostics:"); + Serial.println("1. Checking partition..."); + + // Try to get partition info + const esp_partition_t* partition = esp_partition_find_first( + ESP_PARTITION_TYPE_DATA, + ESP_PARTITION_SUBTYPE_DATA_FAT, + "ffat" + ); + + if (partition) { + Serial.printf(" ✓ Partition found: %s\n", partition->label); + Serial.printf(" Address: 0x%06X\n", partition->address); + Serial.printf(" Size: %u bytes (%.2f MB)\n", + partition->size, partition->size / 1024.0 / 1024.0); + + // Check FAT boot sector at offset 0 (RAW FAT, no WL layer in flash) + Serial.println("\n2. Checking FAT boot sector..."); + uint8_t buffer[512]; + if (esp_partition_read(partition, 0, buffer, 512) == ESP_OK) { + Serial.println(" First 64 bytes at offset 0 (RAW FAT):"); + for (int i = 0; i < 64; i++) { + Serial.printf("%02X ", buffer[i]); + if ((i + 1) % 16 == 0) Serial.println(); + } + Serial.println(); + + // Check boot signature + if (buffer[510] == 0x55 && buffer[511] == 0xAA) { + Serial.println(" ✓ Boot signature found (0x55AA)"); + + // Parse boot sector + char oem[9]; + memcpy(oem, buffer + 3, 8); + oem[8] = 0; + uint16_t bytes_per_sector = buffer[11] | (buffer[12] << 8); + uint16_t total_sectors = buffer[19] | (buffer[20] << 8); + + Serial.printf(" OEM Name: '%s'\n", oem); + Serial.printf(" Bytes per sector: %u\n", bytes_per_sector); + Serial.printf(" Total sectors: %u\n", total_sectors); + + if (bytes_per_sector == 4096 && total_sectors == 362) { + Serial.println(" ✓ Correct format for ESP32 FFat!"); + } + } else { + Serial.printf(" ✗ Invalid boot signature: 0x%02X%02X (expected 0x55AA)\n", + buffer[511], buffer[510]); + } + } + + // Check FAT table + Serial.println("\n3. Checking FAT table..."); + if (esp_partition_read(partition, 4096, buffer, 32) == ESP_OK) { + Serial.println(" FAT1 (first 32 bytes at offset 4096):"); + for (int i = 0; i < 32; i++) { + Serial.printf("%02X ", buffer[i]); + if ((i + 1) % 16 == 0) Serial.println(); + } + Serial.println(); + + if (buffer[0] == 0xF8 && buffer[1] == 0xFF && buffer[2] == 0xFF) { + Serial.println(" ✓ Media descriptor correct (F8 FF FF)"); + + // Check if rest is 00 + bool clean = true; + for (int i = 3; i < 32; i++) { + if (buffer[i] != 0x00) { + clean = false; + break; + } + } + + if (clean) { + Serial.println(" ✓ FAT table clean (all zeros after media descriptor)"); + } else { + Serial.println(" ✗ FAT table has non-zero bytes after media descriptor"); + } + } + } + + // Check WL metadata at end of partition + Serial.println("\n4. Checking WL metadata at end..."); + // WL state1 should be at partition_size - 3 * sector_size + uint32_t wl_state1_offset = partition->size - (3 * 4096); + if (esp_partition_read(partition, wl_state1_offset, buffer, 48) == ESP_OK) { + Serial.printf(" WL State at offset 0x%06X (first 48 bytes):\n", wl_state1_offset); + for (int i = 0; i < 48; i++) { + Serial.printf("%02X ", buffer[i]); + if ((i + 1) % 16 == 0) Serial.println(); + } + Serial.println(); + + // Parse WL_State + uint32_t* words = (uint32_t*)buffer; + Serial.printf(" pos: %u\n", words[0]); + Serial.printf(" max_pos: %u\n", words[1]); + Serial.printf(" block_size: %u\n", words[5]); + Serial.printf(" version: %u\n", words[6]); + + if (words[5] == 4096 && words[6] == 2) { + Serial.println(" ✓ WL metadata looks valid"); + } else { + Serial.println(" ✗ WL metadata invalid"); + } + } + } else { + Serial.println(" ✗ Partition 'ffat' not found!"); + Serial.println(" Check partition table in platformio.ini"); + } + + Serial.println("\nPossible causes:"); + Serial.println("- Corrupted FAT filesystem"); + Serial.println("- Wrong sector size (should be 4096)"); + Serial.println("- Wrong sector count (should be 362)"); + Serial.println("\nTry: pio run -t erase && pio run -t upload && pio run -t uploadfs"); + return; + } + + Serial.println("✓ FFat mounted successfully!"); + + // Show filesystem info + testFileSystem(FFat); + + // Test reading existing files + if (TEST_READ_EXISTING) { + printSeparator(); + testExistingFiles(FFat); + } + + // Test write operations + if (TEST_WRITE_NEW) { + printSeparator(); + testWriteOperations(FFat); + + // Show updated filesystem info + printSeparator(); + Serial.println("\nFilesystem after write tests:"); + testFileSystem(FFat); + } + + // Test file I/O performance + if (TEST_FILE_IO) { + printSeparator(); + testFileIO(FFat, "/benchmark.bin"); + + // Clean up benchmark file + deleteFile(FFat, "/benchmark.bin"); + } + + // Final directory listing + printSeparator(); + Serial.println("\nFinal directory listing:"); + listDir(FFat, "/", 2); + + printSeparator(); + Serial.println("\n✓ All tests completed!"); + Serial.println("\nFilesystem remains mounted for further testing."); + Serial.println("You can now:"); + Serial.println("- Download filesystem: pio run -t download_fatfs"); + Serial.println("- Reset to re-run tests"); + printSeparator(); +} + +void loop() { + // Keep the filesystem mounted + // You can add interactive commands here if needed +} diff --git a/examples/arduino-littlefs/.gitignore b/examples/arduino-littlefs/.gitignore new file mode 100644 index 0000000..7ea31ea --- /dev/null +++ b/examples/arduino-littlefs/.gitignore @@ -0,0 +1,3 @@ +.pio +.vscode +unpacked_fs/ diff --git a/examples/arduino-littlefs/data/README.md b/examples/arduino-littlefs/data/README.md new file mode 100644 index 0000000..746d930 --- /dev/null +++ b/examples/arduino-littlefs/data/README.md @@ -0,0 +1,12 @@ +# LittleFS Test Data + +This directory contains test files that will be included in the LittleFS filesystem image. + +## Files + +- `test.txt` - Simple text file +- `README.md` - This file +- `platformio.ini` - Copy of project configuration +- `partitions.csv` - Copy of partition table + +These files will be flashed to the ESP32's LittleFS partition and can be read by the firmware. diff --git a/examples/arduino-littlefs/data/partitions.csv b/examples/arduino-littlefs/data/partitions.csv new file mode 100644 index 0000000..e9772b6 --- /dev/null +++ b/examples/arduino-littlefs/data/partitions.csv @@ -0,0 +1,6 @@ +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +spiffs, data, spiffs, 0x290000,0x170000, diff --git a/examples/arduino-littlefs/data/platformio.ini b/examples/arduino-littlefs/data/platformio.ini new file mode 100644 index 0000000..a233137 --- /dev/null +++ b/examples/arduino-littlefs/data/platformio.ini @@ -0,0 +1,6 @@ +[env:esp32dev] +platform = espressif32 +framework = arduino +board = esp32dev +board_build.filesystem = littlefs +board_build.partitions = partitions.csv diff --git a/examples/arduino-littlefs/data/test.txt b/examples/arduino-littlefs/data/test.txt new file mode 100644 index 0000000..9293d7c --- /dev/null +++ b/examples/arduino-littlefs/data/test.txt @@ -0,0 +1,2 @@ +Hello from LittleFS! +This is a test file. diff --git a/examples/arduino-littlefs/partitions.csv b/examples/arduino-littlefs/partitions.csv new file mode 100644 index 0000000..e9772b6 --- /dev/null +++ b/examples/arduino-littlefs/partitions.csv @@ -0,0 +1,6 @@ +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +spiffs, data, spiffs, 0x290000,0x170000, diff --git a/examples/arduino-littlefs/platformio.ini b/examples/arduino-littlefs/platformio.ini new file mode 100644 index 0000000..a233137 --- /dev/null +++ b/examples/arduino-littlefs/platformio.ini @@ -0,0 +1,6 @@ +[env:esp32dev] +platform = espressif32 +framework = arduino +board = esp32dev +board_build.filesystem = littlefs +board_build.partitions = partitions.csv diff --git a/examples/arduino-littlefs/src/littlefs_test.ino b/examples/arduino-littlefs/src/littlefs_test.ino new file mode 100644 index 0000000..99a7de0 --- /dev/null +++ b/examples/arduino-littlefs/src/littlefs_test.ino @@ -0,0 +1,324 @@ +#include "FS.h" +#include "LittleFS.h" + +// Test configuration +#define FORMAT_LITTLEFS false // Set to true to format (will erase all data!) + +// Test settings +#define TEST_READ_EXISTING true // Test reading files from flashed image +#define TEST_WRITE_NEW true // Test writing new files +#define TEST_FILE_IO true // Test file I/O performance + +void listDir(fs::FS &fs, const char *dirname, uint8_t levels) { + Serial.printf("Listing directory: %s\r\n", dirname); + + File root = fs.open(dirname); + if (!root) { + Serial.println("- failed to open directory"); + return; + } + if (!root.isDirectory()) { + Serial.println(" - not a directory"); + return; + } + + File file = root.openNextFile(); + while (file) { + if (file.isDirectory()) { + Serial.print(" DIR : "); + Serial.println(file.name()); + if (levels) { + listDir(fs, file.path(), levels - 1); + } + } else { + Serial.print(" FILE: "); + Serial.print(file.name()); + Serial.print("\tSIZE: "); + Serial.println(file.size()); + } + file = root.openNextFile(); + } +} + +void readFile(fs::FS &fs, const char *path) { + Serial.printf("Reading file: %s\r\n", path); + + File file = fs.open(path); + if (!file || file.isDirectory()) { + Serial.println("- failed to open file for reading"); + return; + } + + Serial.println("- read from file:"); + while (file.available()) { + Serial.write(file.read()); + } + file.close(); +} + +void writeFile(fs::FS &fs, const char *path, const char *message) { + Serial.printf("Writing file: %s\r\n", path); + + File file = fs.open(path, FILE_WRITE); + if (!file) { + Serial.println("- failed to open file for writing"); + return; + } + if (file.print(message)) { + Serial.println("- file written"); + } else { + Serial.println("- write failed"); + } + file.close(); +} + +void appendFile(fs::FS &fs, const char *path, const char *message) { + Serial.printf("Appending to file: %s\r\n", path); + + File file = fs.open(path, FILE_APPEND); + if (!file) { + Serial.println("- failed to open file for appending"); + return; + } + if (file.print(message)) { + Serial.println("- message appended"); + } else { + Serial.println("- append failed"); + } + file.close(); +} + +void renameFile(fs::FS &fs, const char *path1, const char *path2) { + Serial.printf("Renaming file %s to %s\r\n", path1, path2); + if (fs.rename(path1, path2)) { + Serial.println("- file renamed"); + } else { + Serial.println("- rename failed"); + } +} + +void deleteFile(fs::FS &fs, const char *path) { + Serial.printf("Deleting file: %s\r\n", path); + if (fs.remove(path)) { + Serial.println("- file deleted"); + } else { + Serial.println("- delete failed"); + } +} + +void testFileIO(fs::FS &fs, const char *path) { + Serial.printf("Testing file I/O with %s\r\n", path); + + static uint8_t buf[512]; + size_t len = 0; + File file = fs.open(path, FILE_WRITE); + if (!file) { + Serial.println("- failed to open file for writing"); + return; + } + + size_t i; + Serial.print("- writing"); + uint32_t start = millis(); + for (i = 0; i < 2048; i++) { + if ((i & 0x001F) == 0x001F) { + Serial.print("."); + } + file.write(buf, 512); + } + Serial.println(""); + uint32_t end = millis() - start; + Serial.printf(" - %u bytes written in %lu ms\r\n", 2048 * 512, end); + file.close(); + + file = fs.open(path); + start = millis(); + end = start; + i = 0; + if (file && !file.isDirectory()) { + len = file.size(); + size_t flen = len; + start = millis(); + Serial.print("- reading"); + while (len) { + size_t toRead = len; + if (toRead > 512) { + toRead = 512; + } + file.read(buf, toRead); + if ((i++ & 0x001F) == 0x001F) { + Serial.print("."); + } + len -= toRead; + } + Serial.println(""); + end = millis() - start; + Serial.printf("- %u bytes read in %lu ms\r\n", flen, end); + file.close(); + } else { + Serial.println("- failed to open file for reading"); + } +} + +void testExistingFiles(fs::FS &fs) { + Serial.println("\n=== Testing Pre-Flashed Files ==="); + + // List all files in root + Serial.println("\nFiles in root directory:"); + listDir(fs, "/", 2); + + // Test reading specific files that should exist + const char* testFiles[] = { + "/test.txt", + "/README.md", + "/platformio.ini", + "/partitions.csv" + }; + + Serial.println("\nReading test files:"); + for (int i = 0; i < 4; i++) { + if (fs.exists(testFiles[i])) { + Serial.printf("\n--- File: %s ---\n", testFiles[i]); + readFile(fs, testFiles[i]); + } else { + Serial.printf("File not found: %s\n", testFiles[i]); + } + } +} + +void testWriteOperations(fs::FS &fs) { + Serial.println("\n=== Testing Write Operations ==="); + + // Test creating new file + Serial.println("\n1. Creating new file..."); + writeFile(fs, "/test_write.txt", "Hello from ESP32!\n"); + + // Test appending + Serial.println("\n2. Appending to file..."); + appendFile(fs, "/test_write.txt", "This line was appended.\n"); + appendFile(fs, "/test_write.txt", "And another line.\n"); + + // Read back + Serial.println("\n3. Reading back written file:"); + readFile(fs, "/test_write.txt"); + + // Test rename + Serial.println("\n4. Testing rename..."); + renameFile(fs, "/test_write.txt", "/renamed.txt"); + readFile(fs, "/renamed.txt"); + + // Test delete + Serial.println("\n5. Testing delete..."); + deleteFile(fs, "/renamed.txt"); + + // Verify deletion + if (!fs.exists("/renamed.txt")) { + Serial.println("File successfully deleted"); + } else { + Serial.println("ERROR: File still exists!"); + } +} + +void testFileSystem(fs::FS &fs) { + Serial.println("\n=== Filesystem Information ==="); + Serial.printf("Total space: %10u bytes (%.2f MB)\n", + LittleFS.totalBytes(), LittleFS.totalBytes() / 1024.0 / 1024.0); + Serial.printf("Used space: %10u bytes (%.2f MB)\n", + LittleFS.usedBytes(), LittleFS.usedBytes() / 1024.0 / 1024.0); + + float usage = (LittleFS.usedBytes() * 100.0) / LittleFS.totalBytes(); + Serial.printf("Usage: %.1f%%\n", usage); +} + +void printSeparator() { + Serial.println("\n============================================================"); +} + +void setup() { + Serial.begin(115200); + delay(1000); // Wait for serial monitor + + Serial.println("\n\n"); + printSeparator(); + Serial.println("ESP32 LittleFS Filesystem Test"); + Serial.println("Testing pre-flashed LittleFS image"); + printSeparator(); + + // Format if requested + if (FORMAT_LITTLEFS) { + Serial.println("\n============================================================"); + Serial.println("WARNING: Formatting LittleFS partition..."); + Serial.println("This will erase all data!"); + Serial.println("============================================================"); + delay(2000); + + if (!LittleFS.format()) { + Serial.println("ERROR: LittleFS Format Failed"); + return; + } + Serial.println(" LittleFS formatted successfully"); + } + + // Mount the filesystem + Serial.println("\nMounting LittleFS filesystem..."); + Serial.println("Format on fail: false"); + + if (!LittleFS.begin(false)) { // formatOnFail = false + Serial.println("\nERROR: LittleFS Mount Failed"); + Serial.println("\nPossible causes:"); + Serial.println("- Filesystem not uploaded"); + Serial.println("- Corrupted LittleFS filesystem"); + Serial.println("- Wrong partition table"); + Serial.println("\nTry: pio run -t uploadfs"); + return; + } + + Serial.println(" LittleFS mounted successfully!"); + + // Show filesystem info + testFileSystem(LittleFS); + + // Test reading existing files + if (TEST_READ_EXISTING) { + printSeparator(); + testExistingFiles(LittleFS); + } + + // Test write operations + if (TEST_WRITE_NEW) { + printSeparator(); + testWriteOperations(LittleFS); + + // Show updated filesystem info + printSeparator(); + Serial.println("\nFilesystem after write tests:"); + testFileSystem(LittleFS); + } + + // Test file I/O performance + if (TEST_FILE_IO) { + printSeparator(); + testFileIO(LittleFS, "/benchmark.bin"); + + // Clean up benchmark file + deleteFile(LittleFS, "/benchmark.bin"); + } + + // Final directory listing + printSeparator(); + Serial.println("\nFinal directory listing:"); + listDir(LittleFS, "/", 2); + + printSeparator(); + Serial.println("\n All tests completed!"); + Serial.println("\nFilesystem remains mounted for further testing."); + Serial.println("You can now:"); + Serial.println("- Download filesystem: pio run -t download_fs"); + Serial.println("- Reset to re-run tests"); + printSeparator(); +} + +void loop() { + // Keep the filesystem mounted + // You can add interactive commands here if needed +} diff --git a/examples/arduino-spiffs/.gitignore b/examples/arduino-spiffs/.gitignore new file mode 100644 index 0000000..7ea31ea --- /dev/null +++ b/examples/arduino-spiffs/.gitignore @@ -0,0 +1,3 @@ +.pio +.vscode +unpacked_fs/ diff --git a/examples/arduino-spiffs/README.md b/examples/arduino-spiffs/README.md new file mode 100644 index 0000000..490faca --- /dev/null +++ b/examples/arduino-spiffs/README.md @@ -0,0 +1,247 @@ +# ESP32 SPIFFS Filesystem Test Project + +This project tests the SPIFFS filesystem implementation on ESP32. + +## Overview + +This project demonstrates: +- Building SPIFFS filesystem images +- Uploading SPIFFS images to ESP32 +- Mounting and reading SPIFFS filesystem on ESP32 +- Downloading and extracting SPIFFS images from ESP32 + +## Requirements + +- PlatformIO +- ESP32 development board +- USB cable + +## Project Structure + +``` +arduino-spiffs/ +├── data/ # Files to be included in SPIFFS image +│ ├── test.txt +│ ├── README.md +│ └── ... +├── src/ +│ └── spiffs_test.ino # Main Arduino sketch +├── partitions.csv # Partition table with SPIFFS partition +├── platformio.ini # PlatformIO configuration +└── unpacked_fs/ # Downloaded files (created by download_spiffs) +``` + +## Usage + +### 1. Build Firmware + +```bash +pio run +``` + +### 2. Build SPIFFS Filesystem Image + +Place your files in the `data/` directory, then: + +```bash +pio run -t buildfs +``` + +This creates a SPIFFS filesystem image at: +`.pio/build/esp32dev/spiffs.bin` + +### 3. Upload Firmware and Filesystem + +```bash +# Upload firmware +pio run -t upload + +# Upload filesystem +pio run -t uploadfs +``` + +### 4. Monitor Serial Output + +```bash +pio run -t monitor +``` + +Expected output: +``` +SPIFFS mounted successfully +Total space: 1507328 +Used space: 12345 +Listing directory: / + FILE: test.txt SIZE: 12 + FILE: README.md SIZE: 456 +Test complete +``` + +### 5. Download Filesystem from Device + +To download and extract the filesystem from the device: + +```bash +pio run -t download_spiffs +``` + +Files will be extracted to `unpacked_fs/` directory. + +## Partition Table + +The `partitions.csv` defines the flash layout: + +```csv +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +spiffs, data, spiffs, 0x290000,0x170000, +``` + +The `spiffs` partition: +- **Type**: data +- **SubType**: spiffs (0x82) +- **Offset**: 0x290000 (2,686,976 bytes) +- **Size**: 0x170000 (1,507,328 bytes = ~1.44 MB) + +## SPIFFS Configuration + +Default SPIFFS configuration (can be customized in `platformio.ini`): + +```ini +board_build.spiffs.page_size = 256 +board_build.spiffs.block_size = 4096 +board_build.spiffs.obj_name_len = 32 +board_build.spiffs.meta_len = 4 +board_build.spiffs.use_magic = true +board_build.spiffs.use_magic_len = true +board_build.spiffs.aligned_obj_ix_tables = false +``` + +These match ESP-IDF defaults: +- `CONFIG_SPIFFS_PAGE_SIZE = 256` +- `CONFIG_SPIFFS_OBJ_NAME_LEN = 32` +- `CONFIG_SPIFFS_META_LENGTH = 4` +- `CONFIG_SPIFFS_USE_MAGIC = true` +- `CONFIG_SPIFFS_USE_MAGIC_LENGTH = true` + +## Troubleshooting + +### "SPIFFS Mount Failed" + +**Possible causes:** +1. Filesystem not uploaded +2. Wrong partition table +3. Corrupted filesystem + +**Solutions:** +```bash +# Rebuild and upload filesystem +pio run -t buildfs +pio run -t uploadfs + +# Or erase flash and start fresh +pio run -t erase +pio run -t upload +pio run -t uploadfs +``` + +### "No SPIFFS filesystem partition found" + +**Cause:** Partition table doesn't have a SPIFFS partition + +**Solution:** Check `partitions.csv` has a partition with `SubType: spiffs` + +### Files not appearing + +**Cause:** Files not in `data/` directory when building + +**Solution:** +1. Add files to `data/` directory +2. Rebuild filesystem: `pio run -t buildfs` +3. Upload: `pio run -t uploadfs` + +## Code Example + +```cpp +#include "FS.h" +#include "SPIFFS.h" + +void setup() { + Serial.begin(115200); + + // Mount SPIFFS filesystem + if (!SPIFFS.begin(false)) { + Serial.println("SPIFFS Mount Failed"); + return; + } + + // List files + File root = SPIFFS.open("/"); + File file = root.openNextFile(); + while (file) { + Serial.printf("File: %s, Size: %d\n", + file.name(), file.size()); + file = root.openNextFile(); + } + + // Read file + File f = SPIFFS.open("/test.txt", "r"); + if (f) { + String content = f.readString(); + Serial.println(content); + f.close(); + } + + // Write file + f = SPIFFS.open("/output.txt", "w"); + if (f) { + f.println("Hello from ESP32!"); + f.close(); + } +} + +void loop() { + // Your code here +} +``` + +## Platform Configuration + +This project uses a custom platform with SPIFFS filesystem support: + +```ini +[env:esp32dev] +platform = espressif32 +framework = arduino +board = esp32dev +board_build.filesystem = spiffs +board_build.partitions = partitions.csv +``` + +## Comparison with Other Filesystems + +| Feature | SPIFFS | LittleFS | FatFS | +|---------|--------|----------|-------| +| Wear Leveling | Built-in | Built-in | Requires WL layer | +| Max File Size | ~1MB | Limited by partition | 4GB | +| Directories | No | Yes | Yes | +| Performance | Medium | Fast | Fast | +| RAM Usage | Low | Low | Medium | +| Reliability | Good | Excellent | Good | + +## References + +- [ESP32 SPIFFS Library](https://github.com/espressif/arduino-esp32/tree/master/libraries/SPIFFS) +- [ESP-IDF SPIFFS](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/storage/spiffs.html) +- [SPIFFS Specification](https://github.com/pellepl/spiffs) + +## Notes + +- SPIFFS does not support directories (flat filesystem) +- File names are limited to 32 characters by default +- SPIFFS has built-in wear leveling +- Maximum file size depends on available RAM +- SPIFFS is being deprecated in favor of LittleFS in newer ESP-IDF versions diff --git a/examples/arduino-spiffs/data/README.md b/examples/arduino-spiffs/data/README.md new file mode 100644 index 0000000..64a1d35 --- /dev/null +++ b/examples/arduino-spiffs/data/README.md @@ -0,0 +1,12 @@ +# SPIFFS Test Data + +This directory contains test files that will be included in the SPIFFS filesystem image. + +## Files + +- `test.txt` - Simple text file +- `README.md` - This file +- `platformio.ini` - Copy of project configuration +- `partitions.csv` - Copy of partition table + +These files will be flashed to the ESP32's SPIFFS partition and can be read by the firmware. diff --git a/examples/arduino-spiffs/data/partitions.csv b/examples/arduino-spiffs/data/partitions.csv new file mode 100644 index 0000000..e9772b6 --- /dev/null +++ b/examples/arduino-spiffs/data/partitions.csv @@ -0,0 +1,6 @@ +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +spiffs, data, spiffs, 0x290000,0x170000, diff --git a/examples/arduino-spiffs/data/platformio.ini b/examples/arduino-spiffs/data/platformio.ini new file mode 100644 index 0000000..9f0961a --- /dev/null +++ b/examples/arduino-spiffs/data/platformio.ini @@ -0,0 +1,6 @@ +[env:esp32dev] +platform = espressif32 +framework = arduino +board = esp32dev +board_build.filesystem = spiffs +board_build.partitions = partitions.csv diff --git a/examples/arduino-spiffs/data/test.txt b/examples/arduino-spiffs/data/test.txt new file mode 100644 index 0000000..8fda702 --- /dev/null +++ b/examples/arduino-spiffs/data/test.txt @@ -0,0 +1,2 @@ +Hello from SPIFFS! +This is a test file. diff --git a/examples/arduino-spiffs/partitions.csv b/examples/arduino-spiffs/partitions.csv new file mode 100644 index 0000000..e9772b6 --- /dev/null +++ b/examples/arduino-spiffs/partitions.csv @@ -0,0 +1,6 @@ +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +spiffs, data, spiffs, 0x290000,0x170000, diff --git a/examples/arduino-spiffs/platformio.ini b/examples/arduino-spiffs/platformio.ini new file mode 100644 index 0000000..9f0961a --- /dev/null +++ b/examples/arduino-spiffs/platformio.ini @@ -0,0 +1,6 @@ +[env:esp32dev] +platform = espressif32 +framework = arduino +board = esp32dev +board_build.filesystem = spiffs +board_build.partitions = partitions.csv diff --git a/examples/arduino-spiffs/src/spiffs_test.ino b/examples/arduino-spiffs/src/spiffs_test.ino new file mode 100644 index 0000000..579cc2d --- /dev/null +++ b/examples/arduino-spiffs/src/spiffs_test.ino @@ -0,0 +1,324 @@ +#include "FS.h" +#include "SPIFFS.h" + +// Test configuration +#define FORMAT_SPIFFS false // Set to true to format (will erase all data!) + +// Test settings +#define TEST_READ_EXISTING true // Test reading files from flashed image +#define TEST_WRITE_NEW true // Test writing new files +#define TEST_FILE_IO true // Test file I/O performance + +void listDir(fs::FS &fs, const char *dirname, uint8_t levels) { + Serial.printf("Listing directory: %s\r\n", dirname); + + File root = fs.open(dirname); + if (!root) { + Serial.println("- failed to open directory"); + return; + } + if (!root.isDirectory()) { + Serial.println(" - not a directory"); + return; + } + + File file = root.openNextFile(); + while (file) { + if (file.isDirectory()) { + Serial.print(" DIR : "); + Serial.println(file.name()); + if (levels) { + listDir(fs, file.path(), levels - 1); + } + } else { + Serial.print(" FILE: "); + Serial.print(file.name()); + Serial.print("\tSIZE: "); + Serial.println(file.size()); + } + file = root.openNextFile(); + } +} + +void readFile(fs::FS &fs, const char *path) { + Serial.printf("Reading file: %s\r\n", path); + + File file = fs.open(path); + if (!file || file.isDirectory()) { + Serial.println("- failed to open file for reading"); + return; + } + + Serial.println("- read from file:"); + while (file.available()) { + Serial.write(file.read()); + } + file.close(); +} + +void writeFile(fs::FS &fs, const char *path, const char *message) { + Serial.printf("Writing file: %s\r\n", path); + + File file = fs.open(path, FILE_WRITE); + if (!file) { + Serial.println("- failed to open file for writing"); + return; + } + if (file.print(message)) { + Serial.println("- file written"); + } else { + Serial.println("- write failed"); + } + file.close(); +} + +void appendFile(fs::FS &fs, const char *path, const char *message) { + Serial.printf("Appending to file: %s\r\n", path); + + File file = fs.open(path, FILE_APPEND); + if (!file) { + Serial.println("- failed to open file for appending"); + return; + } + if (file.print(message)) { + Serial.println("- message appended"); + } else { + Serial.println("- append failed"); + } + file.close(); +} + +void renameFile(fs::FS &fs, const char *path1, const char *path2) { + Serial.printf("Renaming file %s to %s\r\n", path1, path2); + if (fs.rename(path1, path2)) { + Serial.println("- file renamed"); + } else { + Serial.println("- rename failed"); + } +} + +void deleteFile(fs::FS &fs, const char *path) { + Serial.printf("Deleting file: %s\r\n", path); + if (fs.remove(path)) { + Serial.println("- file deleted"); + } else { + Serial.println("- delete failed"); + } +} + +void testFileIO(fs::FS &fs, const char *path) { + Serial.printf("Testing file I/O with %s\r\n", path); + + static uint8_t buf[512]; + size_t len = 0; + File file = fs.open(path, FILE_WRITE); + if (!file) { + Serial.println("- failed to open file for writing"); + return; + } + + size_t i; + Serial.print("- writing"); + uint32_t start = millis(); + for (i = 0; i < 2048; i++) { + if ((i & 0x001F) == 0x001F) { + Serial.print("."); + } + file.write(buf, 512); + } + Serial.println(""); + uint32_t end = millis() - start; + Serial.printf(" - %u bytes written in %lu ms\r\n", 2048 * 512, end); + file.close(); + + file = fs.open(path); + start = millis(); + end = start; + i = 0; + if (file && !file.isDirectory()) { + len = file.size(); + size_t flen = len; + start = millis(); + Serial.print("- reading"); + while (len) { + size_t toRead = len; + if (toRead > 512) { + toRead = 512; + } + file.read(buf, toRead); + if ((i++ & 0x001F) == 0x001F) { + Serial.print("."); + } + len -= toRead; + } + Serial.println(""); + end = millis() - start; + Serial.printf("- %u bytes read in %lu ms\r\n", flen, end); + file.close(); + } else { + Serial.println("- failed to open file for reading"); + } +} + +void testExistingFiles(fs::FS &fs) { + Serial.println("\n=== Testing Pre-Flashed Files ==="); + + // List all files in root + Serial.println("\nFiles in root directory:"); + listDir(fs, "/", 2); + + // Test reading specific files that should exist + const char* testFiles[] = { + "/test.txt", + "/README.md", + "/platformio.ini", + "/partitions.csv" + }; + + Serial.println("\nReading test files:"); + for (int i = 0; i < 4; i++) { + if (fs.exists(testFiles[i])) { + Serial.printf("\n--- File: %s ---\n", testFiles[i]); + readFile(fs, testFiles[i]); + } else { + Serial.printf("File not found: %s\n", testFiles[i]); + } + } +} + +void testWriteOperations(fs::FS &fs) { + Serial.println("\n=== Testing Write Operations ==="); + + // Test creating new file + Serial.println("\n1. Creating new file..."); + writeFile(fs, "/test_write.txt", "Hello from ESP32!\n"); + + // Test appending + Serial.println("\n2. Appending to file..."); + appendFile(fs, "/test_write.txt", "This line was appended.\n"); + appendFile(fs, "/test_write.txt", "And another line.\n"); + + // Read back + Serial.println("\n3. Reading back written file:"); + readFile(fs, "/test_write.txt"); + + // Test rename + Serial.println("\n4. Testing rename..."); + renameFile(fs, "/test_write.txt", "/renamed.txt"); + readFile(fs, "/renamed.txt"); + + // Test delete + Serial.println("\n5. Testing delete..."); + deleteFile(fs, "/renamed.txt"); + + // Verify deletion + if (!fs.exists("/renamed.txt")) { + Serial.println("File successfully deleted"); + } else { + Serial.println("ERROR: File still exists!"); + } +} + +void testFileSystem(fs::FS &fs) { + Serial.println("\n=== Filesystem Information ==="); + Serial.printf("Total space: %10u bytes (%.2f MB)\n", + SPIFFS.totalBytes(), SPIFFS.totalBytes() / 1024.0 / 1024.0); + Serial.printf("Used space: %10u bytes (%.2f MB)\n", + SPIFFS.usedBytes(), SPIFFS.usedBytes() / 1024.0 / 1024.0); + + float usage = (SPIFFS.usedBytes() * 100.0) / SPIFFS.totalBytes(); + Serial.printf("Usage: %.1f%%\n", usage); +} + +void printSeparator() { + Serial.println("\n============================================================"); +} + +void setup() { + Serial.begin(115200); + delay(1000); // Wait for serial monitor + + Serial.println("\n\n"); + printSeparator(); + Serial.println("ESP32 SPIFFS Filesystem Test"); + Serial.println("Testing pre-flashed SPIFFS image"); + printSeparator(); + + // Format if requested + if (FORMAT_SPIFFS) { + Serial.println("\n============================================================"); + Serial.println("WARNING: Formatting SPIFFS partition..."); + Serial.println("This will erase all data!"); + Serial.println("============================================================"); + delay(2000); + + if (!SPIFFS.format()) { + Serial.println("ERROR: SPIFFS Format Failed"); + return; + } + Serial.println("✓ SPIFFS formatted successfully"); + } + + // Mount the filesystem + Serial.println("\nMounting SPIFFS filesystem..."); + Serial.println("Format on fail: false"); + + if (!SPIFFS.begin(false)) { // formatOnFail = false + Serial.println("\nERROR: SPIFFS Mount Failed"); + Serial.println("\nPossible causes:"); + Serial.println("- Filesystem not uploaded"); + Serial.println("- Corrupted SPIFFS filesystem"); + Serial.println("- Wrong partition table"); + Serial.println("\nTry: pio run -t uploadfs"); + return; + } + + Serial.println("✓ SPIFFS mounted successfully!"); + + // Show filesystem info + testFileSystem(SPIFFS); + + // Test reading existing files + if (TEST_READ_EXISTING) { + printSeparator(); + testExistingFiles(SPIFFS); + } + + // Test write operations + if (TEST_WRITE_NEW) { + printSeparator(); + testWriteOperations(SPIFFS); + + // Show updated filesystem info + printSeparator(); + Serial.println("\nFilesystem after write tests:"); + testFileSystem(SPIFFS); + } + + // Test file I/O performance + if (TEST_FILE_IO) { + printSeparator(); + testFileIO(SPIFFS, "/benchmark.bin"); + + // Clean up benchmark file + deleteFile(SPIFFS, "/benchmark.bin"); + } + + // Final directory listing + printSeparator(); + Serial.println("\nFinal directory listing:"); + listDir(SPIFFS, "/", 2); + + printSeparator(); + Serial.println("\n✓ All tests completed!"); + Serial.println("\nFilesystem remains mounted for further testing."); + Serial.println("You can now:"); + Serial.println("- Download filesystem: pio run -t download_spiffs"); + Serial.println("- Reset to re-run tests"); + printSeparator(); +} + +void loop() { + // Keep the filesystem mounted + // You can add interactive commands here if needed +} diff --git a/platform.json b/platform.json index 02a997f..2429f77 100644 --- a/platform.json +++ b/platform.json @@ -18,7 +18,7 @@ "type": "git", "url": "https://github.com/pioarduino/platform-espressif32.git" }, - "version": "55.03.35", + "version": "55.03.36", "frameworks": { "arduino": { "script": "builder/frameworks/arduino.py" @@ -33,31 +33,31 @@ "type": "framework", "optional": true, "owner": "espressif", - "version": "https://github.com/espressif/arduino-esp32/releases/download/3.3.5/esp32-3.3.5.tar.xz" + "version": "https://github.com/espressif/arduino-esp32/releases/download/3.3.6/esp32-core-3.3.6.tar.xz" }, "framework-arduinoespressif32-libs": { "type": "framework", "optional": true, "owner": "espressif", - "version": "https://github.com/espressif/arduino-esp32/releases/download/3.3.5/esp32-3.3.5-libs.tar.xz" + "version": "https://github.com/espressif/arduino-esp32/releases/download/3.3.6/esp32-core-3.3.6-libs.tar.xz" }, "framework-arduino-c2-skeleton-lib": { "type": "framework", "optional": true, "owner": "espressif", - "version": "https://github.com/pioarduino/platform-espressif32/releases/download/c2-skeleton/c2_8410210c9a_compile_skeleton.zip" + "version": "https://github.com/pioarduino/platform-espressif32/releases/download/c2-skeleton/c2_366dadf354_compile_skeleton.zip" }, "framework-arduino-c61-skeleton-lib": { "type": "framework", "optional": true, "owner": "espressif", - "version": "https://github.com/pioarduino/platform-espressif32/releases/download/c61-skeleton/c61_d1e854920a_compile_skeleton.zip" + "version": "https://github.com/pioarduino/platform-espressif32/releases/download/c61-skeleton/c61_366dadf354_compile_skeleton.zip" }, "framework-espidf": { "type": "framework", "optional": true, "owner": "pioarduino", - "version": "https://github.com/pioarduino/esp-idf/releases/download/v5.5.1.251215/esp-idf-v5.5.1.tar.xz" + "version": "https://github.com/pioarduino/esp-idf/releases/download/v5.5.2.260116/esp-idf-v5.5.2.tar.xz" }, "toolchain-xtensa-esp-elf": { "type": "toolchain", @@ -115,7 +115,7 @@ "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/contrib-piohome-3.4.4.tar.gz" }, "tool-dfuutil-arduino": { - "type": "uploader", + "type": "tool", "optional": true, "owner": "pioarduino", "package-version": "1.11.0", @@ -125,36 +125,8 @@ "type": "debugger", "optional": true, "owner": "pioarduino", - "package-version": "2.1200.20250707", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/openocd-v0.12.0-esp32-20250707.zip" - }, - "tool-mklittlefs": { - "type": "uploader", - "optional": true, - "owner": "pioarduino", - "package-version": "3.2.0", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/mklittlefs-3.2.0-new.zip" - }, - "tool-mklittlefs4": { - "type": "uploader", - "optional": true, - "owner": "pioarduino", - "package-version": "4.0.2", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/mklittlefs-4.0.2.zip" - }, - "tool-mkspiffs": { - "type": "uploader", - "optional": true, - "owner": "pioarduino", - "package-version": "2.230.0", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/mkspiffs-v2.230.0.zip" - }, - "tool-mkfatfs": { - "type": "uploader", - "optional": true, - "owner": "pioarduino", - "package-version": "2.0.1", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/mkfatfs-v2.0.1.zip" + "package-version": "2.1200.20251215", + "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/openocd-v0.12.0-esp32-20251215.zip" }, "tool-cppcheck": { "type": "tool", diff --git a/platform.py b/platform.py index a780adf..f018f5a 100644 --- a/platform.py +++ b/platform.py @@ -707,83 +707,13 @@ class Espressif32Platform(PlatformBase): if any(tool in package for tool in check_tools): self.install_tool(package) - def _ensure_mklittlefs_version(self) -> None: - """Ensure correct mklittlefs version is installed.""" - piopm_path = Path(self.packages_dir) / "tool-mklittlefs" / ".piopm" - - if piopm_path.exists(): - try: - with open(piopm_path, 'r', encoding='utf-8') as f: - package_data = json.load(f) - version = package_data.get('version', '') - if not version.startswith("3."): - safe_remove_file(piopm_path) - logger.info(f"Incompatible mklittlefs version {version} removed (required: 3.x)") - except (json.JSONDecodeError, KeyError): - logger.exception("Error reading mklittlefs package metadata") - - def _setup_mklittlefs_for_download(self) -> None: - """Setup mklittlefs for download functionality with version 4.x.""" - mklittlefs_dir = Path(self.packages_dir) / "tool-mklittlefs" - mklittlefs4_dir = Path(self.packages_dir) / "tool-mklittlefs4" - - # Ensure mklittlefs 3.x is installed - if not mklittlefs_dir.exists(): - self.install_tool("tool-mklittlefs") - if (mklittlefs_dir / "tools.json").exists(): - self.install_tool("tool-mklittlefs") - - # Install mklittlefs 4.x - if not mklittlefs4_dir.exists(): - self.install_tool("tool-mklittlefs4") - if (mklittlefs4_dir / "tools.json").exists(): - self.install_tool("tool-mklittlefs4") - - # Copy mklittlefs 4.x over 3.x - if mklittlefs4_dir.exists(): - # Copy 3.x package.json into 4.x before mirroring 4.x -> 3.x, - # so 3.x dir ends up with 4.x binaries and 3.x metadata. - package_src = mklittlefs_dir / "package.json" - package_dst = mklittlefs4_dir / "package.json" - safe_copy_file(package_src, package_dst) - shutil.copytree(mklittlefs4_dir, mklittlefs_dir, dirs_exist_ok=True) - self.packages.pop("tool-mkfatfs", None) - - def _handle_littlefs_tool(self, for_download: bool) -> None: - """Handle LittleFS tool installation with special download configuration.""" - if for_download: - self._setup_mklittlefs_for_download() - else: - self._ensure_mklittlefs_version() - self.install_tool("tool-mklittlefs") - - def _install_filesystem_tool(self, filesystem: str, for_download: bool = False) -> None: - """Install filesystem-specific tools based on the filesystem type.""" - tool_mapping = { - "default": lambda: self._handle_littlefs_tool(for_download), - "fatfs": lambda: self.install_tool("tool-mkfatfs"), - "spiffs": lambda: self.install_tool("tool-mkspiffs") - } - - handler = tool_mapping.get(filesystem, tool_mapping["default"]) - handler() - - def _handle_dfuutil_tool(self, variables: Dict, for_download: bool = False) -> None: + def _handle_dfuutil_tool(self, variables: Dict) -> None: """Install dfuutil tool for Arduino Nano ESP32 board.""" - # Currently only Arduino Nano ESP32 uses the dfuutil tool as uploader - if variables.get("board") == "arduino_nano_esp32": + board_config = self.board_config(variables.get("board")) + uploader = variables.get("board_upload.protocol", board_config.get("upload.protocol", "esptool")) + if uploader == "dfu": self.install_tool("tool-dfuutil-arduino") - def _configure_filesystem_tools(self, variables: Dict, targets: List[str]) -> None: - """Configure filesystem tools based on build targets and filesystem type.""" - filesystem = variables.get("board_build.filesystem", "littlefs") - - if any(target in targets for target in ["buildfs", "uploadfs"]): - self._install_filesystem_tool(filesystem, for_download=False) - - if "downloadfs" in targets: - self._install_filesystem_tool(filesystem, for_download=True) - def setup_python_env(self, env): """Configure SCons environment with centrally managed Python executable paths.""" # Python environment is centrally managed in configure_default_packages @@ -822,14 +752,12 @@ class Espressif32Platform(PlatformBase): self._configure_arduino_framework(frameworks, mcu) self._configure_espidf_framework(frameworks, variables, board_config, mcu) self._configure_mcu_toolchains(mcu, variables, targets) - self._handle_littlefs_tool(for_download=False) # Ensure mklittlefs is installed if "espidf" in frameworks: self._install_common_idf_packages() self._configure_rom_elfs_for_exception_decoder(variables) self._configure_check_tools(variables) - self._configure_filesystem_tools(variables, targets) self._handle_dfuutil_tool(variables) logger.info("Package configuration completed successfully")