pioduino/platform-espressif32
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

penv setup moved in platform (#296)

Browse Source
This commit is contained in:
Jason2866
2025-10-08 18:57:41 +02:00
committed by GitHub
parent 85062ff9e3
commit dabbde41f9
9 changed files with 933 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -29,7 +29,7 @@ Prerequisites:
The Wiki is AI generated and insane detailed and accurate. The Wiki is AI generated and insane detailed and accurate.
### Stable Arduino ### Stable Arduino
currently espressif Arduino 3.3.1 and IDF 5.5.1 currently espressif Arduino 3.3.2 and IDF 5.5.1.250929
```ini ```ini
[env:stable] [env:stable]
builder/frameworks/arduino.py
+4 -4
View File
@@ -22,10 +22,10 @@ kinds of creative coding, interactive objects, spaces or physical experiences.
http://arduino.cc/en/Reference/HomePage http://arduino.cc/en/Reference/HomePage
""" """
import os
import sys
import shutil
import hashlib import hashlib
import os
import shutil
import sys
import threading import threading
from contextlib import suppress from contextlib import suppress
from os.path import join, exists, isabs, splitdrive, commonpath, relpath from os.path import join, exists, isabs, splitdrive, commonpath, relpath
@@ -886,7 +886,7 @@ if check_reinstall_frwrk():
if flag_custom_sdkconfig and not flag_any_custom_sdkconfig: if flag_custom_sdkconfig and not flag_any_custom_sdkconfig:
call_compile_libs() call_compile_libs()
# Main logic for Arduino Framework # Arduino framework configuration and build logic
pioframework = env.subst("$PIOFRAMEWORK") pioframework = env.subst("$PIOFRAMEWORK")
arduino_lib_compile_flag = env.subst("$ARDUINO_LIB_COMPILE_FLAG") arduino_lib_compile_flag = env.subst("$ARDUINO_LIB_COMPILE_FLAG")
builder/frameworks/component_manager.py
+2 -2
View File
@@ -12,9 +12,9 @@ import os
import shutil import shutil
import re import re
import yaml import yaml
from yaml import SafeLoader
from pathlib import Path from pathlib import Path
from typing import Set, Optional, Dict, Any, List, Tuple, Pattern from typing import Set, Optional, Dict, Any, List, Tuple, Pattern
from yaml import SafeLoader
class ComponentManagerConfig: class ComponentManagerConfig:
@@ -252,7 +252,7 @@ class ComponentHandler:
Returns: Returns:
Absolute path to the component YAML file Absolute path to the component YAML file
""" """
# Try Arduino framework first # Check Arduino framework directory first
afd = self.config.arduino_framework_dir afd = self.config.arduino_framework_dir
framework_yml = str(Path(afd) / "idf_component.yml") if afd else "" framework_yml = str(Path(afd) / "idf_component.yml") if afd else ""
if framework_yml and os.path.exists(framework_yml): if framework_yml and os.path.exists(framework_yml):
builder/frameworks/espidf.py
+539 -91
View File
@@ -23,14 +23,13 @@ https://github.com/espressif/esp-idf
import copy import copy
import importlib.util import importlib.util
import json import json
import subprocess
import sys
import shutil
import os import os
from os.path import join import platform as sys_platform
import re import re
import requests import requests
import platform as sys_platform import shutil
import subprocess
import sys
from pathlib import Path from pathlib import Path
from urllib.parse import urlsplit, unquote from urllib.parse import urlsplit, unquote
@@ -79,7 +78,7 @@ config = env.GetProjectConfig()
board = env.BoardConfig() board = env.BoardConfig()
mcu = board.get("build.mcu", "esp32") mcu = board.get("build.mcu", "esp32")
flash_speed = board.get("build.f_flash", "40000000L") flash_speed = board.get("build.f_flash", "40000000L")
flash_frequency = str(flash_speed.replace("000000L", "m")) flash_frequency = str(flash_speed.replace("000000L", ""))
flash_mode = board.get("build.flash_mode", "dio") flash_mode = board.get("build.flash_mode", "dio")
idf_variant = mcu.lower() idf_variant = mcu.lower()
flag_custom_sdkonfig = False flag_custom_sdkonfig = False
@@ -105,6 +104,47 @@ if not TOOLCHAIN_DIR or not os.path.isdir(TOOLCHAIN_DIR):
env.Exit(1) env.Exit(1)
def get_framework_version():
def _extract_from_cmake_version_file():
version_cmake_file = str(Path(FRAMEWORK_DIR) / "tools" / "cmake" / "version.cmake")
if not os.path.isfile(version_cmake_file):
return
with open(version_cmake_file, encoding="utf8") as fp:
pattern = r"set\(IDF_VERSION_(MAJOR|MINOR|PATCH) (\d+)\)"
matches = re.findall(pattern, fp.read())
if len(matches) != 3:
return
# If found all three parts of the version
return ".".join([match[1] for match in matches])
pkg = platform.get_package("framework-espidf")
version = get_original_version(str(pkg.metadata.version.truncate()))
if not version:
# Fallback value extracted directly from the cmake version file
version = _extract_from_cmake_version_file()
if not version:
version = "0.0.0"
# Normalize to semver (handles "6.0.0-rc1", VCS metadata, etc.)
try:
coerced = semantic_version.Version.coerce(version, partial=True)
major = coerced.major or 0
minor = coerced.minor or 0
patch = coerced.patch or 0
return f"{major}.{minor}.{patch}"
except (ValueError, TypeError):
m = re.match(r"(\d+)\.(\d+)\.(\d+)", str(version))
return ".".join(m.groups()) if m else "0.0.0"
# Configure ESP-IDF version environment variables
framework_version = get_framework_version()
_mv = framework_version.split(".")
major_version = f"{_mv[0]}.{_mv[1] if len(_mv) > 1 else '0'}"
os.environ["ESP_IDF_VERSION"] = major_version
def create_silent_action(action_func): def create_silent_action(action_func):
"""Create a silent SCons action that suppresses output""" """Create a silent SCons action that suppresses output"""
silent_action = env.Action(action_func) silent_action = env.Action(action_func)
@@ -178,6 +218,29 @@ if config.has_option("env:"+env["PIOENV"], "custom_sdkconfig"):
if "espidf.custom_sdkconfig" in board: if "espidf.custom_sdkconfig" in board:
flag_custom_sdkonfig = True flag_custom_sdkonfig = True
# Check for board-specific configurations that require sdkconfig generation
def has_board_specific_config():
"""Check if board has configuration that needs to be applied to sdkconfig."""
# Check for PSRAM support
extra_flags = board.get("build.extra_flags", [])
has_psram = any("-DBOARD_HAS_PSRAM" in flag for flag in extra_flags)
# Check for special memory types
memory_type = None
build_section = board.get("build", {})
arduino_section = build_section.get("arduino", {})
if "memory_type" in arduino_section:
memory_type = arduino_section["memory_type"]
elif "memory_type" in build_section:
memory_type = build_section["memory_type"]
has_special_memory = memory_type and ("opi" in memory_type.lower())
return has_psram or has_special_memory
if has_board_specific_config():
flag_custom_sdkonfig = True
def HandleArduinoIDFsettings(env): def HandleArduinoIDFsettings(env):
""" """
Handles Arduino IDF settings configuration with custom sdkconfig support. Handles Arduino IDF settings configuration with custom sdkconfig support.
@@ -244,48 +307,322 @@ def HandleArduinoIDFsettings(env):
return line.split("=")[0] return line.split("=")[0]
return None return None
def generate_board_specific_config():
"""Generate board-specific sdkconfig settings from board.json manifest."""
board_config_flags = []
# Handle memory type configuration with platformio.ini override support
# Priority: platformio.ini > board.json manifest
memory_type = None
# Check for memory_type override in platformio.ini
if hasattr(env, 'GetProjectOption'):
try:
memory_type = env.GetProjectOption("board_build.memory_type", None)
except:
pass
# Fallback to board.json manifest
if not memory_type:
build_section = board.get("build", {})
arduino_section = build_section.get("arduino", {})
if "memory_type" in arduino_section:
memory_type = arduino_section["memory_type"]
elif "memory_type" in build_section:
memory_type = build_section["memory_type"]
flash_memory_type = None
psram_memory_type = None
if memory_type:
parts = memory_type.split("_")
if len(parts) == 2:
flash_memory_type, psram_memory_type = parts
else:
flash_memory_type = memory_type
# Check for additional flash configuration indicators
boot_mode = board.get("build", {}).get("boot", None)
flash_mode = board.get("build", {}).get("flash_mode", None)
# Override flash_memory_type if boot mode indicates OPI
if boot_mode == "opi" or flash_mode in ["dout", "opi"]:
if not flash_memory_type or flash_memory_type.lower() != "opi":
flash_memory_type = "opi"
print(f"Info: Detected OPI Flash via boot_mode='{boot_mode}' or flash_mode='{flash_mode}'")
# Set CPU frequency with platformio.ini override support
# Priority: platformio.ini > board.json manifest
f_cpu = None
if hasattr(env, 'GetProjectOption'):
# Check for board_build.f_cpu override in platformio.ini
try:
f_cpu = env.GetProjectOption("board_build.f_cpu", None)
except:
pass
# Fallback to board.json manifest
if not f_cpu:
f_cpu = board.get("build.f_cpu", None)
if f_cpu:
cpu_freq = str(f_cpu).replace("000000L", "")
board_config_flags.append(f"CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ={cpu_freq}")
# Disable other CPU frequency options and enable the specific one
common_cpu_freqs = ["80", "160", "240"]
for freq in common_cpu_freqs:
if freq != cpu_freq:
if mcu == "esp32":
board_config_flags.append(f"# CONFIG_ESP32_DEFAULT_CPU_FREQ_{freq} is not set")
elif mcu in ["esp32s2", "esp32s3"]:
board_config_flags.append(f"# CONFIG_ESP32S2_DEFAULT_CPU_FREQ_{freq} is not set" if mcu == "esp32s2" else f"# CONFIG_ESP32S3_DEFAULT_CPU_FREQ_{freq} is not set")
elif mcu in ["esp32c2", "esp32c3", "esp32c6"]:
board_config_flags.append(f"# CONFIG_ESP32C3_DEFAULT_CPU_FREQ_{freq} is not set")
# Enable the specific CPU frequency
if mcu == "esp32":
board_config_flags.append(f"CONFIG_ESP32_DEFAULT_CPU_FREQ_{cpu_freq}=y")
elif mcu == "esp32s2":
board_config_flags.append(f"CONFIG_ESP32S2_DEFAULT_CPU_FREQ_{cpu_freq}=y")
elif mcu == "esp32s3":
board_config_flags.append(f"CONFIG_ESP32S3_DEFAULT_CPU_FREQ_{cpu_freq}=y")
elif mcu in ["esp32c2", "esp32c3", "esp32c6"]:
board_config_flags.append(f"CONFIG_ESP32C3_DEFAULT_CPU_FREQ_{cpu_freq}=y")
# Set flash size with platformio.ini override support
# Priority: platformio.ini > board.json manifest
flash_size = None
if hasattr(env, 'GetProjectOption'):
# Check for board_upload.flash_size override in platformio.ini
try:
flash_size = env.GetProjectOption("board_upload.flash_size", None)
except:
pass
# Fallback to board.json manifest
if not flash_size:
flash_size = board.get("upload", {}).get("flash_size", None)
if flash_size:
# Configure both string and boolean flash size formats
# Disable other flash size options first
flash_sizes = ["4MB", "8MB", "16MB", "32MB", "64MB", "128MB"]
for size in flash_sizes:
if size != flash_size:
board_config_flags.append(f"# CONFIG_ESPTOOLPY_FLASHSIZE_{size} is not set")
# Set the specific flash size configs
board_config_flags.append(f"CONFIG_ESPTOOLPY_FLASHSIZE=\"{flash_size}\"")
board_config_flags.append(f"CONFIG_ESPTOOLPY_FLASHSIZE_{flash_size}=y")
# Handle Flash and PSRAM frequency configuration with platformio.ini override support
# Priority: platformio.ini > board.json manifest
# From 80MHz onwards, Flash and PSRAM frequencies must be identical
# Get f_flash with override support
f_flash = None
if hasattr(env, 'GetProjectOption'):
try:
f_flash = env.GetProjectOption("board_build.f_flash", None)
except:
pass
if not f_flash:
f_flash = board.get("build.f_flash", None)
# Get f_boot with override support
f_boot = None
if hasattr(env, 'GetProjectOption'):
try:
f_boot = env.GetProjectOption("board_build.f_boot", None)
except:
pass
if not f_boot:
f_boot = board.get("build.f_boot", None)
# Determine the frequencies to use
esptool_flash_freq = f_flash # Always use f_flash for esptool compatibility
compile_freq = f_boot if f_boot else f_flash # Use f_boot for compile-time if available
if f_flash and compile_freq:
# Ensure frequency compatibility (>= 80MHz must be identical for Flash and PSRAM)
compile_freq_val = int(str(compile_freq).replace("000000L", ""))
if compile_freq_val >= 80:
# Above 80MHz, both Flash and PSRAM must use same frequency
unified_freq = compile_freq_val
flash_freq_str = f"{unified_freq}m"
psram_freq_str = str(unified_freq)
print(f"Info: Unified frequency mode (>= 80MHz): {unified_freq}MHz for both Flash and PSRAM")
else:
# Below 80MHz, frequencies can differ
flash_freq_str = str(compile_freq).replace("000000L", "m")
psram_freq_str = str(compile_freq).replace("000000L", "")
print(f"Info: Independent frequency mode (< 80MHz): Flash={flash_freq_str}, PSRAM={psram_freq_str}")
# Configure Flash frequency
# Disable other flash frequency options first
flash_freqs = ["20m", "26m", "40m", "80m", "120m"]
for freq in flash_freqs:
if freq != flash_freq_str:
board_config_flags.append(f"# CONFIG_ESPTOOLPY_FLASHFREQ_{freq.upper()} is not set")
# Then set the specific frequency configs
board_config_flags.append(f"CONFIG_ESPTOOLPY_FLASHFREQ=\"{flash_freq_str}\"")
board_config_flags.append(f"CONFIG_ESPTOOLPY_FLASHFREQ_{flash_freq_str.upper()}=y")
# Configure PSRAM frequency (same as Flash for >= 80MHz)
# Disable other SPIRAM speed options first
psram_freqs = ["40", "80", "120"]
for freq in psram_freqs:
if freq != psram_freq_str:
board_config_flags.append(f"# CONFIG_SPIRAM_SPEED_{freq}M is not set")
# Then set the specific SPIRAM configs
board_config_flags.append(f"CONFIG_SPIRAM_SPEED={psram_freq_str}")
board_config_flags.append(f"CONFIG_SPIRAM_SPEED_{psram_freq_str}M=y")
# Enable experimental features for frequencies > 80MHz
if compile_freq_val > 80:
board_config_flags.append("CONFIG_IDF_EXPERIMENTAL_FEATURES=y")
board_config_flags.append("CONFIG_SPI_FLASH_HPM_ENABLE=y")
board_config_flags.append("CONFIG_SPI_FLASH_HPM_AUTO=y")
# Check for PSRAM support based on board flags
extra_flags = board.get("build.extra_flags", [])
has_psram = any("-DBOARD_HAS_PSRAM" in flag for flag in extra_flags)
# Additional PSRAM detection methods
if not has_psram:
# Check if memory_type contains psram indicators
if memory_type and ("opi" in memory_type.lower() or "psram" in memory_type.lower()):
has_psram = True
# Check build.psram_type
elif "psram_type" in board.get("build", {}):
has_psram = True
# Check for SPIRAM mentions in extra_flags
elif any("SPIRAM" in str(flag) for flag in extra_flags):
has_psram = True
# For ESP32-S3, assume PSRAM capability (can be disabled later if not present)
elif mcu == "esp32s3":
has_psram = True
if has_psram:
# Enable basic SPIRAM support
board_config_flags.append("CONFIG_SPIRAM=y")
# Determine PSRAM type with platformio.ini override support
# Priority: platformio.ini > memory_type > build.psram_type > default
psram_type = None
# Priority 1: Check for platformio.ini override
if hasattr(env, 'GetProjectOption'):
try:
psram_type = env.GetProjectOption("board_build.psram_type", None)
if psram_type:
psram_type = psram_type.lower()
except:
pass
# Priority 2: Check psram_memory_type from memory_type field (e.g., "qio_opi")
if not psram_type and psram_memory_type:
psram_type = psram_memory_type.lower()
# Priority 3: Check build.psram_type field as fallback
elif not psram_type and "psram_type" in board.get("build", {}):
psram_type = board.get("build.psram_type", "qio").lower()
# Priority 4: Default to qio
elif not psram_type:
psram_type = "qio"
# Configure PSRAM mode based on detected type
if psram_type == "opi":
# Octal PSRAM configuration (for ESP32-S3 only)
if mcu == "esp32s3":
board_config_flags.extend([
"CONFIG_IDF_EXPERIMENTAL_FEATURES=y",
"# CONFIG_SPIRAM_MODE_QUAD is not set",
"CONFIG_SPIRAM_MODE_OCT=y",
"CONFIG_SPIRAM_TYPE_AUTO=y"
])
else:
# Fallback to QUAD for non-S3 chips
board_config_flags.extend([
"# CONFIG_SPIRAM_MODE_OCT is not set",
"CONFIG_SPIRAM_MODE_QUAD=y"
])
elif psram_type in ["qio", "qspi"]:
# Quad PSRAM configuration
if mcu in ["esp32s2", "esp32s3"]:
board_config_flags.extend([
"# CONFIG_SPIRAM_MODE_OCT is not set",
"CONFIG_SPIRAM_MODE_QUAD=y"
])
elif mcu == "esp32":
board_config_flags.extend([
"# CONFIG_SPIRAM_MODE_OCT is not set",
"# CONFIG_SPIRAM_MODE_QUAD is not set"
])
# Use flash_memory_type for flash config
if flash_memory_type and "opi" in flash_memory_type.lower():
# OPI Flash configurations require specific settings
board_config_flags.extend([
"# CONFIG_ESPTOOLPY_FLASHMODE_QIO is not set",
"# CONFIG_ESPTOOLPY_FLASHMODE_QOUT is not set",
"# CONFIG_ESPTOOLPY_FLASHMODE_DIO is not set",
"CONFIG_ESPTOOLPY_FLASHMODE_DOUT=y",
"CONFIG_ESPTOOLPY_OCT_FLASH=y",
"# CONFIG_ESPTOOLPY_FLASH_SAMPLE_MODE_STR is not set",
"CONFIG_ESPTOOLPY_FLASH_SAMPLE_MODE_DTR=y"
])
return board_config_flags
def build_idf_config_flags(): def build_idf_config_flags():
"""Build complete IDF configuration flags from all sources.""" """Build complete IDF configuration flags from all sources."""
flags = [] flags = []
# Add board-specific flags first # FIRST: Add board-specific flags derived from board.json manifest
if "espidf.custom_sdkconfig" in board: board_flags = generate_board_specific_config()
board_flags = board.get("espidf.custom_sdkconfig", []) if board_flags:
if board_flags: flags.extend(board_flags)
flags.extend(board_flags)
# Add custom sdkconfig file content # SECOND: Add board-specific flags from board manifest (espidf.custom_sdkconfig)
if "espidf.custom_sdkconfig" in board:
board_manifest_flags = board.get("espidf.custom_sdkconfig", [])
if board_manifest_flags:
flags.extend(board_manifest_flags)
# THIRD: Add custom sdkconfig file content
custom_file_content = load_custom_sdkconfig_file() custom_file_content = load_custom_sdkconfig_file()
if custom_file_content: if custom_file_content:
flags.append(custom_file_content) flags.append(custom_file_content)
# Add project-level custom sdkconfig # FOURTH: Add project-level custom sdkconfig (highest precedence for user overrides)
if config.has_option("env:" + env["PIOENV"], "custom_sdkconfig"): if config.has_option("env:" + env["PIOENV"], "custom_sdkconfig"):
custom_flags = env.GetProjectOption("custom_sdkconfig").rstrip("\n") custom_flags = env.GetProjectOption("custom_sdkconfig").rstrip("\n")
if custom_flags: if custom_flags:
flags.append(custom_flags) flags.append(custom_flags)
# FIFTH: Apply ESP32-specific compatibility fixes
all_flags_str = "\n".join(flags) + "\n" if flags else ""
esp32_compatibility_flags = apply_esp32_compatibility_fixes(all_flags_str)
if esp32_compatibility_flags:
flags.extend(esp32_compatibility_flags)
return "\n".join(flags) + "\n" if flags else "" return "\n".join(flags) + "\n" if flags else ""
def add_flash_configuration(config_flags): def apply_esp32_compatibility_fixes(config_flags_str):
"""Add flash frequency and mode configuration.""" """Apply ESP32-specific compatibility fixes based on final configuration."""
if flash_frequency != "80m": compatibility_flags = []
config_flags += "# CONFIG_ESPTOOLPY_FLASHFREQ_80M is not set\n"
config_flags += f"CONFIG_ESPTOOLPY_FLASHFREQ_{flash_frequency.upper()}=y\n"
config_flags += f"CONFIG_ESPTOOLPY_FLASHFREQ=\"{flash_frequency}\"\n"
if flash_mode != "qio":
config_flags += "# CONFIG_ESPTOOLPY_FLASHMODE_QIO is not set\n"
flash_mode_flag = f"CONFIG_ESPTOOLPY_FLASHMODE_{flash_mode.upper()}=y\n"
if flash_mode_flag not in config_flags:
config_flags += flash_mode_flag
# ESP32 specific SPIRAM configuration # ESP32 specific SPIRAM configuration
if mcu == "esp32" and "CONFIG_FREERTOS_UNICORE=y" in config_flags: # On ESP32, SPIRAM is not used with UNICORE mode
config_flags += "# CONFIG_SPIRAM is not set\n" if mcu == "esp32" and "CONFIG_FREERTOS_UNICORE=y" in config_flags_str:
if "CONFIG_SPIRAM=y" in config_flags_str:
compatibility_flags.append("# CONFIG_SPIRAM is not set")
print("Info: ESP32 SPIRAM disabled since solo1 core mode is enabled")
return config_flags return compatibility_flags
def write_sdkconfig_file(idf_config_flags, checksum_source): def write_sdkconfig_file(idf_config_flags, checksum_source):
if "arduino" not in env.subst("$PIOFRAMEWORK"): if "arduino" not in env.subst("$PIOFRAMEWORK"):
@@ -306,7 +643,9 @@ def HandleArduinoIDFsettings(env):
dst.write(f"# TASMOTA__{checksum}\n") dst.write(f"# TASMOTA__{checksum}\n")
# Process each line from source sdkconfig # Process each line from source sdkconfig
for line in src: src_lines = src.readlines()
for line in src_lines:
flag_name = extract_flag_name(line) flag_name = extract_flag_name(line)
if flag_name is None: if flag_name is None:
@@ -335,20 +674,25 @@ def HandleArduinoIDFsettings(env):
print(f"Add: {cleaned_flag}") print(f"Add: {cleaned_flag}")
dst.write(cleaned_flag + "\n") dst.write(cleaned_flag + "\n")
# Main execution logic # Main execution logic
has_custom_config = ( has_custom_config = (
config.has_option("env:" + env["PIOENV"], "custom_sdkconfig") or config.has_option("env:" + env["PIOENV"], "custom_sdkconfig") or
"espidf.custom_sdkconfig" in board "espidf.custom_sdkconfig" in board
) )
if not has_custom_config: has_board_config = has_board_specific_config()
if not has_custom_config and not has_board_config:
return return
print("*** Add \"custom_sdkconfig\" settings to IDF sdkconfig.defaults ***") if has_board_config and not has_custom_config:
print("*** Apply board-specific settings to IDF sdkconfig.defaults ***")
else:
print("*** Add \"custom_sdkconfig\" settings to IDF sdkconfig.defaults ***")
# Build complete configuration # Build complete configuration
idf_config_flags = build_idf_config_flags() idf_config_flags = build_idf_config_flags()
idf_config_flags = add_flash_configuration(idf_config_flags)
# Convert to list for processing # Convert to list for processing
idf_config_list = [line for line in idf_config_flags.splitlines() if line.strip()] idf_config_list = [line for line in idf_config_flags.splitlines() if line.strip()]
@@ -946,8 +1290,8 @@ def generate_project_ld_script(sdk_config, ignore_targets=None):
initial_ld_script = str(Path(FRAMEWORK_DIR) / "components" / "esp_system" / "ld" / idf_variant / "sections.ld.in") initial_ld_script = str(Path(FRAMEWORK_DIR) / "components" / "esp_system" / "ld" / idf_variant / "sections.ld.in")
framework_version = [int(v) for v in get_framework_version().split(".")] framework_version_list = [int(v) for v in get_framework_version().split(".")]
if framework_version[:2] > [5, 2]: if framework_version_list[:2] > [5, 2]:
initial_ld_script = preprocess_linker_file( initial_ld_script = preprocess_linker_file(
initial_ld_script, initial_ld_script,
str(Path(BUILD_DIR) / "esp-idf" / "esp_system" / "ld" / "sections.ld.in"), str(Path(BUILD_DIR) / "esp-idf" / "esp_system" / "ld" / "sections.ld.in"),
@@ -1023,11 +1367,14 @@ def compile_source_files(
# Canonical, symlink-resolved absolute path of the components directory # Canonical, symlink-resolved absolute path of the components directory
components_dir_path = (Path(FRAMEWORK_DIR) / "components").resolve() components_dir_path = (Path(FRAMEWORK_DIR) / "components").resolve()
for source in config.get("sources", []): for source in config.get("sources", []):
if source["path"].endswith(".rule"): src_path = source["path"]
if src_path.endswith(".rule"):
continue
# Always skip dummy_src.c to avoid duplicate build actions
if os.path.basename(src_path) == "dummy_src.c":
continue continue
compile_group_idx = source.get("compileGroupIndex") compile_group_idx = source.get("compileGroupIndex")
if compile_group_idx is not None: if compile_group_idx is not None:
src_path = source.get("path")
if not os.path.isabs(src_path): if not os.path.isabs(src_path):
# For cases when sources are located near CMakeLists.txt # For cases when sources are located near CMakeLists.txt
src_path = str(Path(project_src_dir) / src_path) src_path = str(Path(project_src_dir) / src_path)
@@ -1130,7 +1477,10 @@ def get_lib_ignore_components():
lib_handler = _component_manager.LibraryIgnoreHandler(config, logger) lib_handler = _component_manager.LibraryIgnoreHandler(config, logger)
# Get the processed lib_ignore entries (already converted to component names) # Get the processed lib_ignore entries (already converted to component names)
lib_ignore_entries = lib_handler._get_lib_ignore_entries() get_entries = getattr(lib_handler, "get_lib_ignore_entries", None)
lib_ignore_entries = (
get_entries() if callable(get_entries) else lib_handler._get_lib_ignore_entries()
)
return lib_ignore_entries return lib_ignore_entries
except (OSError, ValueError, RuntimeError, KeyError) as e: except (OSError, ValueError, RuntimeError, KeyError) as e:
@@ -1182,6 +1532,9 @@ def build_bootloader(sdk_config):
"-DPROJECT_SOURCE_DIR=" + PROJECT_DIR, "-DPROJECT_SOURCE_DIR=" + PROJECT_DIR,
"-DLEGACY_INCLUDE_COMMON_HEADERS=", "-DLEGACY_INCLUDE_COMMON_HEADERS=",
"-DEXTRA_COMPONENT_DIRS=" + str(Path(FRAMEWORK_DIR) / "components" / "bootloader"), "-DEXTRA_COMPONENT_DIRS=" + str(Path(FRAMEWORK_DIR) / "components" / "bootloader"),
f"-DESP_IDF_VERSION={major_version}",
f"-DESP_IDF_VERSION_MAJOR={framework_version.split('.')[0]}",
f"-DESP_IDF_VERSION_MINOR={framework_version.split('.')[1]}",
], ],
) )
@@ -1222,7 +1575,84 @@ def build_bootloader(sdk_config):
) )
bootloader_env.MergeFlags(link_args) bootloader_env.MergeFlags(link_args)
bootloader_env.Append(LINKFLAGS=extra_flags)
# Handle ESP-IDF 6.0 linker script preprocessing for .ld.in files
# In bootloader context, only .ld.in templates exist and need preprocessing
processed_extra_flags = []
# Bootloader preprocessing configuration
bootloader_config_dir = str(Path(BUILD_DIR) / "bootloader" / "config")
bootloader_extra_includes = [
str(Path(FRAMEWORK_DIR) / "components" / "bootloader" / "subproject" / "main" / "ld" / idf_variant)
]
i = 0
while i < len(extra_flags):
if extra_flags[i] == "-T" and i + 1 < len(extra_flags):
linker_script = extra_flags[i + 1]
# Process .ld.in templates directly
if linker_script.endswith(".ld.in"):
script_name = os.path.basename(linker_script).replace(".ld.in", ".ld")
target_script = str(Path(BUILD_DIR) / "bootloader" / script_name)
preprocessed_script = preprocess_linker_file(
linker_script,
target_script,
config_dir=bootloader_config_dir,
extra_include_dirs=bootloader_extra_includes
)
bootloader_env.Depends("$BUILD_DIR/bootloader.elf", preprocessed_script)
processed_extra_flags.extend(["-T", target_script])
# Handle .ld files - prioritize using original scripts when available
elif linker_script.endswith(".ld"):
script_basename = os.path.basename(linker_script)
# Check if the original .ld file exists in framework and use it directly
original_script_path = str(Path(FRAMEWORK_DIR) / "components" / "bootloader" / "subproject" / "main" / "ld" / idf_variant / script_basename)
if os.path.isfile(original_script_path):
# Use the original script directly - no preprocessing needed
processed_extra_flags.extend(["-T", original_script_path])
else:
# Only generate from template if no original .ld file exists
script_name_in = script_basename.replace(".ld", ".ld.in")
bootloader_script_in_path = str(Path(FRAMEWORK_DIR) / "components" / "bootloader" / "subproject" / "main" / "ld" / idf_variant / script_name_in)
# ESP32-P4 specific: Check for bootloader.rev3.ld.in
if idf_variant == "esp32p4" and script_basename == "bootloader.ld":
sdk_config = get_sdk_configuration()
if sdk_config.get("ESP32P4_REV_MIN_300", False):
bootloader_rev3_path = str(Path(FRAMEWORK_DIR) / "components" / "bootloader" / "subproject" / "main" / "ld" / idf_variant / "bootloader.rev3.ld.in")
if os.path.isfile(bootloader_rev3_path):
bootloader_script_in_path = bootloader_rev3_path
# Preprocess the .ld.in template to generate the .ld file
if os.path.isfile(bootloader_script_in_path):
target_script = str(Path(BUILD_DIR) / "bootloader" / script_basename)
preprocessed_script = preprocess_linker_file(
bootloader_script_in_path,
target_script,
config_dir=bootloader_config_dir,
extra_include_dirs=bootloader_extra_includes
)
bootloader_env.Depends("$BUILD_DIR/bootloader.elf", preprocessed_script)
processed_extra_flags.extend(["-T", target_script])
else:
# Pass through if neither original nor template found (e.g., ROM scripts)
processed_extra_flags.extend(["-T", linker_script])
else:
# Pass through any other linker flags unchanged
processed_extra_flags.extend(["-T", linker_script])
i += 2
else:
processed_extra_flags.append(extra_flags[i])
i += 1
bootloader_env.Append(LINKFLAGS=processed_extra_flags)
bootloader_libs = find_lib_deps(components_map, elf_config, link_args) bootloader_libs = find_lib_deps(components_map, elf_config, link_args)
bootloader_env.Prepend(__RPATH="-Wl,--start-group ") bootloader_env.Prepend(__RPATH="-Wl,--start-group ")
@@ -1318,31 +1748,6 @@ def find_default_component(target_configs):
env.Exit(1) env.Exit(1)
def get_framework_version():
def _extract_from_cmake_version_file():
version_cmake_file = str(Path(FRAMEWORK_DIR) / "tools" / "cmake" / "version.cmake")
if not os.path.isfile(version_cmake_file):
return
with open(version_cmake_file, encoding="utf8") as fp:
pattern = r"set\(IDF_VERSION_(MAJOR|MINOR|PATCH) (\d+)\)"
matches = re.findall(pattern, fp.read())
if len(matches) != 3:
return
# If found all three parts of the version
return ".".join([match[1] for match in matches])
pkg = platform.get_package("framework-espidf")
version = get_original_version(str(pkg.metadata.version.truncate()))
if not version:
# Fallback value extracted directly from the cmake version file
version = _extract_from_cmake_version_file()
if not version:
version = "0.0.0"
return version
def create_version_file(): def create_version_file():
version_file = str(Path(FRAMEWORK_DIR) / "version.txt") version_file = str(Path(FRAMEWORK_DIR) / "version.txt")
if not os.path.isfile(version_file): if not os.path.isfile(version_file):
@@ -1427,26 +1832,73 @@ def get_app_partition_offset(pt_table, pt_offset):
return factory_app_params.get("offset", "0x10000") return factory_app_params.get("offset", "0x10000")
def preprocess_linker_file(src_ld_script, target_ld_script): def preprocess_linker_file(src_ld_script, target_ld_script, config_dir=None, extra_include_dirs=None):
return env.Command( """
target_ld_script, Preprocess a linker script file (.ld.in) to generate the final .ld file.
src_ld_script, Supports both IDF 5.x (linker_script_generator.cmake) and IDF 6.x (linker_script_preprocessor.cmake).
env.VerboseAction(
" ".join( Args:
[ src_ld_script: Source .ld.in file path
target_ld_script: Target .ld file path
config_dir: Configuration directory (defaults to BUILD_DIR/config for main app)
extra_include_dirs: Additional include directories (list)
"""
if config_dir is None:
config_dir = str(Path(BUILD_DIR) / "config")
# Convert all paths to forward slashes for CMake compatibility on Windows
config_dir = fs.to_unix_path(config_dir)
src_ld_script = fs.to_unix_path(src_ld_script)
target_ld_script = fs.to_unix_path(target_ld_script)
# Check IDF version to determine which CMake script to use
framework_version_list = [int(v) for v in get_framework_version().split(".")]
# IDF 6.0+ uses linker_script_preprocessor.cmake with CFLAGS approach
if framework_version_list[0] >= 6:
include_dirs = [f'"{config_dir}"']
include_dirs.append(f'"{fs.to_unix_path(str(Path(FRAMEWORK_DIR) / "components" / "esp_system" / "ld"))}"')
if extra_include_dirs:
include_dirs.extend(f'"{fs.to_unix_path(dir_path)}"' for dir_path in extra_include_dirs)
cflags_value = "-I" + " -I".join(include_dirs)
return env.Command(
target_ld_script,
src_ld_script,
env.VerboseAction(
" ".join([
f'"{CMAKE_DIR}"',
f'-DCC="{fs.to_unix_path(str(Path(TOOLCHAIN_DIR) / "bin" / "$CC"))}"',
f'-DSOURCE="{src_ld_script}"',
f'-DTARGET="{target_ld_script}"',
f'-DCFLAGS="{cflags_value}"',
"-P",
f'"{fs.to_unix_path(str(Path(FRAMEWORK_DIR) / "tools" / "cmake" / "linker_script_preprocessor.cmake"))}"',
]),
"Generating LD script $TARGET",
),
)
else:
# IDF 5.x: Use legacy linker_script_generator.cmake method
return env.Command(
target_ld_script,
src_ld_script,
env.VerboseAction(
" ".join([
f'"{CMAKE_DIR}"', f'"{CMAKE_DIR}"',
f'-DCC="{str(Path(TOOLCHAIN_DIR) / "bin" / "$CC")}"', f'-DCC="{str(Path(TOOLCHAIN_DIR) / "bin" / "$CC")}"',
"-DSOURCE=$SOURCE", "-DSOURCE=$SOURCE",
"-DTARGET=$TARGET", "-DTARGET=$TARGET",
f'-DCONFIG_DIR="{str(Path(BUILD_DIR) / "config")}"', f'-DCONFIG_DIR="{config_dir}"',
f'-DLD_DIR="{str(Path(FRAMEWORK_DIR) / "components" / "esp_system" / "ld")}"', f'-DLD_DIR="{str(Path(FRAMEWORK_DIR) / "components" / "esp_system" / "ld")}"',
"-P", "-P",
f'"{str(Path("$BUILD_DIR") / "esp-idf" / "esp_system" / "ld" / "linker_script_generator.cmake")}"', f'"{str(Path("$BUILD_DIR") / "esp-idf" / "esp_system" / "ld" / "linker_script_generator.cmake")}"',
] ]),
"Generating LD script $TARGET",
), ),
"Generating LD script $TARGET", )
),
)
def generate_mbedtls_bundle(sdk_config): def generate_mbedtls_bundle(sdk_config):
@@ -1530,6 +1982,7 @@ def install_python_deps():
# https://github.com/platformio/platform-espressif32/issues/635 # https://github.com/platformio/platform-espressif32/issues/635
"cryptography": "~=44.0.0", "cryptography": "~=44.0.0",
"pyparsing": ">=3.1.0,<4", "pyparsing": ">=3.1.0,<4",
"pydantic": "~=2.11.10",
"idf-component-manager": "~=2.2", "idf-component-manager": "~=2.2",
"esp-idf-kconfig": "~=2.5.0" "esp-idf-kconfig": "~=2.5.0"
} }
@@ -1672,8 +2125,8 @@ def get_python_exe():
ensure_python_venv_available() ensure_python_venv_available()
# ESP-IDF package doesn't contain .git folder, instead package version is specified # ESP-IDF package version is determined from version.h file
# in a special file "version.h" in the root folder of the package # since the package distribution doesn't include .git metadata
create_version_file() create_version_file()
@@ -1690,8 +2143,8 @@ generate_default_component()
if not board.get("build.ldscript", ""): if not board.get("build.ldscript", ""):
initial_ld_script = board.get("build.esp-idf.ldscript", str(Path(FRAMEWORK_DIR) / "components" / "esp_system" / "ld" / idf_variant / "memory.ld.in")) initial_ld_script = board.get("build.esp-idf.ldscript", str(Path(FRAMEWORK_DIR) / "components" / "esp_system" / "ld" / idf_variant / "memory.ld.in"))
framework_version = [int(v) for v in get_framework_version().split(".")] framework_version_list = [int(v) for v in get_framework_version().split(".")]
if framework_version[:2] > [5, 2]: if framework_version_list[:2] > [5, 2]:
initial_ld_script = preprocess_linker_file( initial_ld_script = preprocess_linker_file(
initial_ld_script, initial_ld_script,
str(Path(BUILD_DIR) / "esp-idf" / "esp_system" / "ld" / "memory.ld.in") str(Path(BUILD_DIR) / "esp-idf" / "esp_system" / "ld" / "memory.ld.in")
@@ -1712,7 +2165,7 @@ if not board.get("build.ldscript", ""):
# #
# Current build script limitations # Known build system limitations
# #
if any(" " in p for p in (FRAMEWORK_DIR, BUILD_DIR)): if any(" " in p for p in (FRAMEWORK_DIR, BUILD_DIR)):
@@ -1751,12 +2204,7 @@ if "arduino" in env.subst("$PIOFRAMEWORK"):
LIBSOURCE_DIRS=[str(Path(ARDUINO_FRAMEWORK_DIR) / "libraries")] LIBSOURCE_DIRS=[str(Path(ARDUINO_FRAMEWORK_DIR) / "libraries")]
) )
# Set ESP-IDF version environment variables (needed for proper Kconfig processing) # Setup CMake configuration arguments
framework_version = get_framework_version()
major_version = framework_version.split('.')[0] + '.' + framework_version.split('.')[1]
os.environ["ESP_IDF_VERSION"] = major_version
# Configure CMake arguments with ESP-IDF version
extra_cmake_args = [ extra_cmake_args = [
"-DIDF_TARGET=" + idf_variant, "-DIDF_TARGET=" + idf_variant,
"-DPYTHON_DEPS_CHECKED=1", "-DPYTHON_DEPS_CHECKED=1",
@@ -1850,7 +2298,7 @@ if flag_custom_sdkonfig == False:
env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", build_bootloader(sdk_config)) env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", build_bootloader(sdk_config))
# #
# Target: ESP-IDF menuconfig # ESP-IDF menuconfig target implementation
# #
env.AddPlatformTarget( env.AddPlatformTarget(
@@ -1995,8 +2443,8 @@ if "__test" not in COMMAND_LINE_TARGETS or env.GetProjectOption(
): ):
project_env = env.Clone() project_env = env.Clone()
if project_target_name != "__idf_main": if project_target_name != "__idf_main":
# Manually add dependencies to CPPPATH since ESP-IDF build system doesn't generate # Add dependencies to CPPPATH for non-main source directories
# this info if the folder with sources is not named 'main' # ESP-IDF build system requires manual dependency handling for custom source folders
# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html#rename-main # https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html#rename-main
project_env.AppendUnique(CPPPATH=app_includes["plain_includes"]) project_env.AppendUnique(CPPPATH=app_includes["plain_includes"])
@@ -2044,7 +2492,7 @@ if board_flash_size != idf_flash_size:
# #
extra_elf2bin_flags = "--elf-sha256-offset 0xb0" extra_elf2bin_flags = "--elf-sha256-offset 0xb0"
# https://github.com/espressif/esp-idf/blob/master/components/esptool_py/project_include.cmake#L58 # Reference: ESP-IDF esptool_py component configuration
# For chips that support configurable MMU page size feature # For chips that support configurable MMU page size feature
# If page size is configured to values other than the default "64KB" in menuconfig, # If page size is configured to values other than the default "64KB" in menuconfig,
mmu_page_size = "64KB" mmu_page_size = "64KB"
builder/main.py
+9 -10
View File
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import importlib.util
import locale import locale
import os import os
import re import re
@@ -20,7 +21,6 @@ import subprocess
import sys import sys
from os.path import isfile, join from os.path import isfile, join
from pathlib import Path from pathlib import Path
import importlib.util
from SCons.Script import ( from SCons.Script import (
ARGUMENTS, ARGUMENTS,
@@ -34,9 +34,8 @@ from SCons.Script import (
from platformio.project.helpers import get_project_dir from platformio.project.helpers import get_project_dir
from platformio.util import get_serial_ports from platformio.util import get_serial_ports
from platformio.compat import IS_WINDOWS from platformio.compat import IS_WINDOWS
from penv_setup import setup_python_environment
# Initialize environment and configuration # Initialize SCons environment and project configuration
env = DefaultEnvironment() env = DefaultEnvironment()
platform = env.PioPlatform() platform = env.PioPlatform()
projectconfig = env.GetProjectConfig() projectconfig = env.GetProjectConfig()
@@ -46,10 +45,10 @@ framework_dir = platform.get_package_dir("framework-arduinoespressif32")
core_dir = projectconfig.get("platformio", "core_dir") core_dir = projectconfig.get("platformio", "core_dir")
build_dir = Path(projectconfig.get("platformio", "build_dir")) build_dir = Path(projectconfig.get("platformio", "build_dir"))
# Setup Python virtual environment and get executable paths # Configure Python environment through centralized platform management
PYTHON_EXE, esptool_binary_path = setup_python_environment(env, platform, core_dir) PYTHON_EXE, esptool_binary_path = platform.setup_python_env(env)
# Initialize board configuration and MCU settings # Load board configuration and determine MCU architecture
board = env.BoardConfig() board = env.BoardConfig()
board_id = env.subst("$BOARD") board_id = env.subst("$BOARD")
mcu = board.get("build.mcu", "esp32") mcu = board.get("build.mcu", "esp32")
@@ -451,7 +450,7 @@ load_board_script(env)
if not is_xtensa: if not is_xtensa:
toolchain_arch = "riscv32-esp" toolchain_arch = "riscv32-esp"
# Initialize integration extra data if not present # Ensure integration extra data structure exists
if "INTEGRATION_EXTRA_DATA" not in env: if "INTEGRATION_EXTRA_DATA" not in env:
env["INTEGRATION_EXTRA_DATA"] = {} env["INTEGRATION_EXTRA_DATA"] = {}
@@ -461,7 +460,7 @@ uploader_path = (
if ' ' in esptool_binary_path if ' ' in esptool_binary_path
else esptool_binary_path else esptool_binary_path
) )
# Configure build tools and environment variables # Configure SCons build tools and compiler settings
env.Replace( env.Replace(
__get_board_boot_mode=_get_board_boot_mode, __get_board_boot_mode=_get_board_boot_mode,
__get_board_f_flash=_get_board_f_flash, __get_board_f_flash=_get_board_f_flash,
@@ -612,7 +611,7 @@ def firmware_metrics(target, source, env):
return return
try: try:
cmd = [PYTHON_EXE, "-m", "esp_idf_size", "--ng"] cmd = [PYTHON_EXE, "-m", "esp_idf_size"]
# Parameters from platformio.ini # Parameters from platformio.ini
extra_args = env.GetProjectOption("custom_esp_idf_size_args", "") extra_args = env.GetProjectOption("custom_esp_idf_size_args", "")
@@ -637,7 +636,7 @@ def firmware_metrics(target, source, env):
if env.GetProjectOption("custom_esp_idf_size_verbose", False): if env.GetProjectOption("custom_esp_idf_size_verbose", False):
print(f"Running command: {' '.join(cmd)}") print(f"Running command: {' '.join(cmd)}")
# Call esp-idf-size with modified environment # Execute esp-idf-size with current environment
result = subprocess.run(cmd, check=False, capture_output=False, env=os.environ) result = subprocess.run(cmd, check=False, capture_output=False, env=os.environ)
if result.returncode != 0: if result.returncode != 0:
builder/penv_setup.py
+297 -117
View File
@@ -15,11 +15,11 @@
import json import json
import os import os
import re import re
import site
import semantic_version import semantic_version
import site
import socket
import subprocess import subprocess
import sys import sys
import socket
from pathlib import Path from pathlib import Path
from platformio.package.version import pepver_to_semver from platformio.package.version import pepver_to_semver
@@ -34,14 +34,14 @@ if sys.version_info < (3, 10):
) )
sys.exit(1) sys.exit(1)
github_actions = os.getenv('GITHUB_ACTIONS') github_actions = bool(os.getenv("GITHUB_ACTIONS"))
PLATFORMIO_URL_VERSION_RE = re.compile( PLATFORMIO_URL_VERSION_RE = re.compile(
r'/v?(\d+\.\d+\.\d+(?:[.-]\w+)?(?:\.\d+)?)(?:\.(?:zip|tar\.gz|tar\.bz2))?$', r'/v?(\d+\.\d+\.\d+(?:[.-]\w+)?(?:\.\d+)?)(?:\.(?:zip|tar\.gz|tar\.bz2))?$',
re.IGNORECASE, re.IGNORECASE,
) )
# Python dependencies required for the build process # Python dependencies required for ESP32 platform builds
python_deps = { python_deps = {
"platformio": "https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip", "platformio": "https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip",
"pyyaml": ">=6.0.2", "pyyaml": ">=6.0.2",
@@ -49,12 +49,13 @@ python_deps = {
"zopfli": ">=0.2.2", "zopfli": ">=0.2.2",
"intelhex": ">=2.3.0", "intelhex": ">=2.3.0",
"rich": ">=14.0.0", "rich": ">=14.0.0",
"urllib3": "<2",
"cryptography": ">=45.0.3", "cryptography": ">=45.0.3",
"certifi": ">=2025.8.3", "certifi": ">=2025.8.3",
"ecdsa": ">=0.19.1", "ecdsa": ">=0.19.1",
"bitstring": ">=4.3.1", "bitstring": ">=4.3.1",
"reedsolo": ">=1.5.3,<1.8", "reedsolo": ">=1.5.3,<1.8",
"esp-idf-size": ">=1.6.1" "esp-idf-size": ">=2.0.0"
} }
@@ -64,10 +65,9 @@ def has_internet_connection(host="1.1.1.1", port=53, timeout=2):
Returns True if a connection is possible, otherwise False. Returns True if a connection is possible, otherwise False.
""" """
try: try:
socket.setdefaulttimeout(timeout) with socket.create_connection((host, port), timeout=timeout):
socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) return True
return True except OSError:
except Exception:
return False return False
@@ -89,8 +89,8 @@ def setup_pipenv_in_package(env, penv_dir):
Returns: Returns:
str or None: Path to uv executable if uv was used, None if python -m venv was used str or None: Path to uv executable if uv was used, None if python -m venv was used
""" """
if not os.path.exists(penv_dir): if not os.path.isfile(get_executable_path(penv_dir, "python")):
# First try to create virtual environment with uv # Attempt virtual environment creation using uv package manager
uv_success = False uv_success = False
uv_cmd = None uv_cmd = None
try: try:
@@ -126,12 +126,16 @@ def setup_pipenv_in_package(env, penv_dir):
) )
) )
# Verify that the virtual environment was created properly # Validate virtual environment creation
# Check for python executable # Ensure Python executable is available
assert os.path.isfile( penv_python = get_executable_path(penv_dir, "python")
get_executable_path(penv_dir, "python") if not os.path.isfile(penv_python):
), f"Error: Failed to create a proper virtual environment. Missing the `python` binary! Created with uv: {uv_success}" sys.stderr.write(
f"Error: Failed to create a proper virtual environment. "
f"Missing the `python` binary at {penv_python}! Created with uv: {uv_success}\n"
)
sys.exit(1)
return uv_cmd if uv_success else None return uv_cmd if uv_success else None
return None return None
@@ -220,7 +224,7 @@ def install_python_deps(python_exe, external_uv_executable):
[external_uv_executable, "pip", "install", "uv>=0.1.0", f"--python={python_exe}", "--quiet"], [external_uv_executable, "pip", "install", "uv>=0.1.0", f"--python={python_exe}", "--quiet"],
stdout=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
timeout=120 timeout=300
) )
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print(f"Error: uv installation failed with exit code {e.returncode}") print(f"Error: uv installation failed with exit code {e.returncode}")
@@ -241,7 +245,7 @@ def install_python_deps(python_exe, external_uv_executable):
[python_exe, "-m", "pip", "install", "uv>=0.1.0", "--quiet"], [python_exe, "-m", "pip", "install", "uv>=0.1.0", "--quiet"],
stdout=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
timeout=120 timeout=300
) )
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print(f"Error: uv installation via pip failed with exit code {e.returncode}") print(f"Error: uv installation via pip failed with exit code {e.returncode}")
@@ -272,7 +276,7 @@ def install_python_deps(python_exe, external_uv_executable):
capture_output=True, capture_output=True,
text=True, text=True,
encoding='utf-8', encoding='utf-8',
timeout=120 timeout=300
) )
if result_obj.returncode == 0: if result_obj.returncode == 0:
@@ -282,18 +286,18 @@ def install_python_deps(python_exe, external_uv_executable):
for p in packages: for p in packages:
result[p["name"].lower()] = pepver_to_semver(p["version"]) result[p["name"].lower()] = pepver_to_semver(p["version"])
else: else:
print(f"Warning: uv pip list failed with exit code {result_obj.returncode}") print(f"Error: uv pip list failed with exit code {result_obj.returncode}")
if result_obj.stderr: if result_obj.stderr:
print(f"Error output: {result_obj.stderr.strip()}") print(f"Error output: {result_obj.stderr.strip()}")
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
print("Warning: uv pip list command timed out") print("Error: uv pip list command timed out")
except (json.JSONDecodeError, KeyError) as e: except (json.JSONDecodeError, KeyError) as e:
print(f"Warning: Could not parse package list: {e}") print(f"Error: Could not parse package list: {e}")
except FileNotFoundError: except FileNotFoundError:
print("Warning: uv command not found") print("Error: uv command not found")
except Exception as e: except Exception as e:
print(f"Warning! Couldn't extract the list of installed Python packages: {e}") print(f"Error! Couldn't extract the list of installed Python packages: {e}")
return result return result
@@ -302,39 +306,39 @@ def install_python_deps(python_exe, external_uv_executable):
if packages_to_install: if packages_to_install:
packages_list = [] packages_list = []
package_map = {}
for p in packages_to_install: for p in packages_to_install:
spec = python_deps[p] spec = python_deps[p]
if spec.startswith(('http://', 'https://', 'git+', 'file://')): if spec.startswith(('http://', 'https://', 'git+', 'file://')):
packages_list.append(spec) packages_list.append(spec)
package_map[spec] = p
else: else:
packages_list.append(f"{p}{spec}") full_spec = f"{p}{spec}"
packages_list.append(full_spec)
package_map[full_spec] = p
cmd = [ for package_spec in packages_list:
penv_uv_executable, "pip", "install", cmd = [
f"--python={python_exe}", penv_uv_executable, "pip", "install",
"--quiet", "--upgrade" f"--python={python_exe}",
] + packages_list "--quiet", "--upgrade",
package_spec
try: ]
subprocess.check_call( try:
cmd, subprocess.check_call(
stdout=subprocess.DEVNULL, cmd,
stderr=subprocess.STDOUT, stdout=subprocess.DEVNULL,
timeout=120 stderr=subprocess.STDOUT,
) timeout=300
)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print(f"Error: Failed to install Python dependencies (exit code: {e.returncode})") print(f"Error: Installing package '{package_map.get(package_spec, package_spec)}' failed (exit code {e.returncode}).")
return False except subprocess.TimeoutExpired:
except subprocess.TimeoutExpired: print(f"Error: Installing package '{package_map.get(package_spec, package_spec)}' timed out.")
print("Error: Python dependencies installation timed out") except FileNotFoundError:
return False print("Error: uv command not found")
except FileNotFoundError: except Exception as e:
print("Error: uv command not found") print(f"Error: Installing package '{package_map.get(package_spec, package_spec)}': {e}.")
return False
except Exception as e:
print(f"Error installing Python dependencies: {e}")
return False
return True return True
@@ -353,7 +357,7 @@ def install_esptool(env, platform, python_exe, uv_executable):
Raises: Raises:
SystemExit: If esptool installation fails or package directory not found SystemExit: If esptool installation fails or package directory not found
""" """
esptool_repo_path = env.subst(platform.get_package_dir("tool-esptoolpy") or "") esptool_repo_path = platform.get_package_dir("tool-esptoolpy") or ""
if not esptool_repo_path or not os.path.isdir(esptool_repo_path): if not esptool_repo_path or not os.path.isdir(esptool_repo_path):
sys.stderr.write( sys.stderr.write(
f"Error: 'tool-esptoolpy' package directory not found: {esptool_repo_path!r}\n" f"Error: 'tool-esptoolpy' package directory not found: {esptool_repo_path!r}\n"
@@ -400,6 +404,245 @@ def install_esptool(env, platform, python_exe, uv_executable):
sys.exit(1) sys.exit(1)
def setup_penv_minimal(platform, platformio_dir: str, install_esptool: bool = True):
"""
Minimal Python virtual environment setup without SCons dependencies.
Args:
platform: PlatformIO platform object
platformio_dir (str): Path to PlatformIO core directory
install_esptool (bool): Whether to install esptool (default: True)
Returns:
tuple[str, str]: (Path to penv Python executable, Path to esptool script)
Raises:
SystemExit: If Python version < 3.10 or dependency installation fails
"""
return _setup_python_environment_core(None, platform, platformio_dir, should_install_esptool=install_esptool)
def _setup_python_environment_core(env, platform, platformio_dir, should_install_esptool=True):
"""
Core Python environment setup logic shared by both SCons and minimal versions.
Args:
env: SCons environment object (None for minimal setup)
platform: PlatformIO platform object
platformio_dir (str): Path to PlatformIO core directory
should_install_esptool (bool): Whether to install esptool (default: True)
Returns:
tuple[str, str]: (Path to penv Python executable, Path to esptool script)
"""
penv_dir = str(Path(platformio_dir) / "penv")
# Create virtual environment if not present
if env is not None:
# SCons version
used_uv_executable = setup_pipenv_in_package(env, penv_dir)
else:
# Minimal version
used_uv_executable = _setup_pipenv_minimal(penv_dir)
# Set Python executable path
penv_python = get_executable_path(penv_dir, "python")
# Update SCons environment if available
if env is not None:
env.Replace(PYTHONEXE=penv_python)
# check for python binary, exit with error when not found
if not os.path.isfile(penv_python):
sys.stderr.write(f"Error: Python executable not found: {penv_python}\n")
sys.exit(1)
# Setup Python module search paths
setup_python_paths(penv_dir)
# Set executable paths from tools
esptool_binary_path = get_executable_path(penv_dir, "esptool")
uv_executable = get_executable_path(penv_dir, "uv")
# Install required Python dependencies for ESP32 platform
if has_internet_connection() or github_actions:
if not install_python_deps(penv_python, used_uv_executable):
sys.stderr.write("Error: Failed to install Python dependencies into penv\n")
sys.exit(1)
else:
print("Warning: No internet connection detected, Python dependency check will be skipped.")
# Install esptool package if required
if should_install_esptool:
if env is not None:
# SCons version
install_esptool(env, platform, penv_python, uv_executable)
else:
# Minimal setup - install esptool from tool package
_install_esptool_from_tl_install(platform, penv_python, uv_executable)
# Setup certifi environment variables
_setup_certifi_env(env, penv_python)
return penv_python, esptool_binary_path
def _setup_pipenv_minimal(penv_dir):
"""
Setup virtual environment without SCons dependencies.
Args:
penv_dir (str): Path to virtual environment directory
Returns:
str or None: Path to uv executable if uv was used, None if python -m venv was used
"""
if not os.path.isfile(get_executable_path(penv_dir, "python")):
# Attempt virtual environment creation using uv package manager
uv_success = False
uv_cmd = None
try:
# Derive uv path from current Python path
python_dir = os.path.dirname(sys.executable)
uv_exe_suffix = ".exe" if IS_WINDOWS else ""
uv_cmd = str(Path(python_dir) / f"uv{uv_exe_suffix}")
# Fall back to system uv if derived path doesn't exist
if not os.path.isfile(uv_cmd):
uv_cmd = "uv"
subprocess.check_call(
[uv_cmd, "venv", "--clear", f"--python={sys.executable}", penv_dir],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=90
)
uv_success = True
print(f"Created pioarduino Python virtual environment using uv: {penv_dir}")
except Exception:
pass
# Fallback to python -m venv if uv failed or is not available
if not uv_success:
uv_cmd = None
try:
subprocess.check_call([
sys.executable, "-m", "venv", "--clear", penv_dir
])
print(f"Created pioarduino Python virtual environment: {penv_dir}")
except subprocess.CalledProcessError as e:
sys.stderr.write(f"Error: Failed to create virtual environment: {e}\n")
sys.exit(1)
# Validate virtual environment creation
# Ensure Python executable is available
penv_python = get_executable_path(penv_dir, "python")
if not os.path.isfile(penv_python):
sys.stderr.write(
f"Error: Failed to create a proper virtual environment. "
f"Missing the `python` binary at {penv_python}! Created with uv: {uv_success}\n"
)
sys.exit(1)
return uv_cmd if uv_success else None
return None
def _install_esptool_from_tl_install(platform, python_exe, uv_executable):
"""
Install esptool from tl-install provided path into penv.
Args:
platform: PlatformIO platform object
python_exe (str): Path to Python executable in virtual environment
uv_executable (str): Path to uv executable
Raises:
SystemExit: If esptool installation fails or package directory not found
"""
# Get esptool path from tool-esptoolpy package (provided by tl-install)
esptool_repo_path = platform.get_package_dir("tool-esptoolpy") or ""
if not esptool_repo_path or not os.path.isdir(esptool_repo_path):
return (None, None)
# Check if esptool is already installed from the correct path
try:
result = subprocess.run(
[
python_exe,
"-c",
(
"import esptool, os, sys; "
"expected_path = os.path.normcase(os.path.realpath(sys.argv[1])); "
"actual_path = os.path.normcase(os.path.realpath(os.path.dirname(esptool.__file__))); "
"print('MATCH' if actual_path.startswith(expected_path) else 'MISMATCH')"
),
esptool_repo_path,
],
capture_output=True,
check=True,
text=True,
timeout=5
)
if result.stdout.strip() == "MATCH":
return
except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError):
pass
try:
subprocess.check_call([
uv_executable, "pip", "install", "--quiet", "--force-reinstall",
f"--python={python_exe}",
"-e", esptool_repo_path
], timeout=60)
print(f"Installed esptool from tl-install path: {esptool_repo_path}")
except subprocess.CalledProcessError as e:
print(f"Warning: Failed to install esptool from {esptool_repo_path} (exit {e.returncode})")
# Don't exit - esptool installation is not critical for penv setup
def _setup_certifi_env(env, python_exe):
"""
Setup certifi environment variables from the given python_exe virtual environment.
Uses a subprocess call to extract certifi path from that environment to guarantee penv usage.
"""
try:
# Run python executable from penv to get certifi path
out = subprocess.check_output(
[python_exe, "-c", "import certifi; print(certifi.where())"],
text=True,
timeout=5
)
cert_path = out.strip()
except Exception as e:
print(f"Error: Failed to obtain certifi path from the virtual environment: {e}")
return
# Set environment variables for certificate bundles
os.environ["CERTIFI_PATH"] = cert_path
os.environ["SSL_CERT_FILE"] = cert_path
os.environ["REQUESTS_CA_BUNDLE"] = cert_path
os.environ["CURL_CA_BUNDLE"] = cert_path
os.environ["GIT_SSL_CAINFO"] = cert_path
# Also propagate to SCons environment if available
if env is not None:
env_vars = dict(env.get("ENV", {}))
env_vars.update({
"CERTIFI_PATH": cert_path,
"SSL_CERT_FILE": cert_path,
"REQUESTS_CA_BUNDLE": cert_path,
"CURL_CA_BUNDLE": cert_path,
"GIT_SSL_CAINFO": cert_path,
})
env.Replace(ENV=env_vars)
def setup_python_environment(env, platform, platformio_dir): def setup_python_environment(env, platform, platformio_dir):
""" """
Main function to setup the Python virtual environment and dependencies. Main function to setup the Python virtual environment and dependencies.
@@ -415,67 +658,4 @@ def setup_python_environment(env, platform, platformio_dir):
Raises: Raises:
SystemExit: If Python version < 3.10 or dependency installation fails SystemExit: If Python version < 3.10 or dependency installation fails
""" """
# Check Python version requirement return _setup_python_environment_core(env, platform, platformio_dir, should_install_esptool=True)
if sys.version_info < (3, 10):
sys.stderr.write(
f"Error: Python 3.10 or higher is required. "
f"Current version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\n"
f"Please update your Python installation.\n"
)
sys.exit(1)
penv_dir = str(Path(platformio_dir) / "penv")
# Setup virtual environment if needed
used_uv_executable = setup_pipenv_in_package(env, penv_dir)
# Set Python Scons Var to env Python
penv_python = get_executable_path(penv_dir, "python")
env.Replace(PYTHONEXE=penv_python)
# check for python binary, exit with error when not found
assert os.path.isfile(penv_python), f"Python executable not found: {penv_python}"
# Setup Python module search paths
setup_python_paths(penv_dir)
# Set executable paths from tools
esptool_binary_path = get_executable_path(penv_dir, "esptool")
uv_executable = get_executable_path(penv_dir, "uv")
# Install espressif32 Python dependencies
if has_internet_connection() or github_actions:
if not install_python_deps(penv_python, used_uv_executable):
sys.stderr.write("Error: Failed to install Python dependencies into penv\n")
sys.exit(1)
else:
print("Warning: No internet connection detected, Python dependency check will be skipped.")
# Install esptool after dependencies
install_esptool(env, platform, penv_python, uv_executable)
# Setup certifi environment variables
def setup_certifi_env():
try:
import certifi
except ImportError:
print("Info: certifi not available; skipping CA environment setup.")
return
cert_path = certifi.where()
os.environ["CERTIFI_PATH"] = cert_path
os.environ["SSL_CERT_FILE"] = cert_path
os.environ["REQUESTS_CA_BUNDLE"] = cert_path
os.environ["CURL_CA_BUNDLE"] = cert_path
# Also propagate to SCons environment for future env.Execute calls
env_vars = dict(env.get("ENV", {}))
env_vars.update({
"CERTIFI_PATH": cert_path,
"SSL_CERT_FILE": cert_path,
"REQUESTS_CA_BUNDLE": cert_path,
"CURL_CA_BUNDLE": cert_path,
})
env.Replace(ENV=env_vars)
setup_certifi_env()
return penv_python, esptool_binary_path
examples/arduino-blink/platformio.ini
+1 -4
View File
@@ -114,16 +114,13 @@ lib_ignore = wifi
Matter Matter
Zigbee Zigbee
ESP RainMaker ESP RainMaker
custom_sdkconfig = CONFIG_SPIRAM_MODE_OCT=y custom_sdkconfig = CONFIG_LCD_RGB_ISR_IRAM_SAFE=y
CONFIG_SPIRAM_SPEED_120M=y
CONFIG_LCD_RGB_ISR_IRAM_SAFE=y
CONFIG_GDMA_CTRL_FUNC_IN_IRAM=y CONFIG_GDMA_CTRL_FUNC_IN_IRAM=y
CONFIG_I2S_ISR_IRAM_SAFE=y CONFIG_I2S_ISR_IRAM_SAFE=y
CONFIG_GDMA_ISR_IRAM_SAFE=y CONFIG_GDMA_ISR_IRAM_SAFE=y
CONFIG_SPIRAM_XIP_FROM_PSRAM=y CONFIG_SPIRAM_XIP_FROM_PSRAM=y
CONFIG_SPIRAM_FETCH_INSTRUCTIONS=y CONFIG_SPIRAM_FETCH_INSTRUCTIONS=y
CONFIG_SPIRAM_RODATA=y CONFIG_SPIRAM_RODATA=y
CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y
CONFIG_ESP32S3_DATA_CACHE_64KB=y CONFIG_ESP32S3_DATA_CACHE_64KB=y
CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y
custom_component_remove = espressif/esp_hosted custom_component_remove = espressif/esp_hosted
platform.json
+12 -12
View File
@@ -18,7 +18,7 @@
"type": "git", "type": "git",
"url": "https://github.com/pioarduino/platform-espressif32.git" "url": "https://github.com/pioarduino/platform-espressif32.git"
}, },
"version": "55.03.31", "version": "55.03.32",
"frameworks": { "frameworks": {
"arduino": { "arduino": {
"script": "builder/frameworks/arduino.py" "script": "builder/frameworks/arduino.py"
@@ -33,13 +33,13 @@
"type": "framework", "type": "framework",
"optional": true, "optional": true,
"owner": "espressif", "owner": "espressif",
"version": "https://github.com/espressif/arduino-esp32/releases/download/3.3.1/esp32-3.3.1.zip" "version": "https://github.com/espressif/arduino-esp32/releases/download/3.3.2/esp32-3.3.2.tar.xz"
}, },
"framework-arduinoespressif32-libs": { "framework-arduinoespressif32-libs": {
"type": "framework", "type": "framework",
"optional": true, "optional": true,
"owner": "espressif", "owner": "espressif",
"version": "https://github.com/espressif/esp32-arduino-lib-builder/releases/download/idf-release_v5.5/esp32-arduino-libs-idf-release_v5.5-129cd0d2-v4.zip" "version": "https://github.com/espressif/arduino-esp32/releases/download/3.3.2/esp32-3.3.2-libs.tar.xz"
}, },
"framework-arduino-c2-skeleton-lib": { "framework-arduino-c2-skeleton-lib": {
"type": "framework", "type": "framework",
@@ -51,21 +51,21 @@
"type": "framework", "type": "framework",
"optional": true, "optional": true,
"owner": "pioarduino", "owner": "pioarduino",
"version": "https://github.com/pioarduino/esp-idf/releases/download/v5.5.1/esp-idf-v5.5.1.tar.xz" "version": "https://github.com/pioarduino/esp-idf/releases/download/v5.5.1.250929/esp-idf-v5.5.1.tar.xz"
}, },
"toolchain-xtensa-esp-elf": { "toolchain-xtensa-esp-elf": {
"type": "toolchain", "type": "toolchain",
"optional": true, "optional": true,
"owner": "pioarduino", "owner": "pioarduino",
"package-version": "14.2.0+20241119", "package-version": "14.2.0+20250730",
"version": "https://github.com/pioarduino/registry/releases/download/0.0.1/xtensa-esp-elf-14.2.0_20241119.zip" "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/xtensa-esp-elf-14.2.0_20250730.zip"
}, },
"toolchain-riscv32-esp": { "toolchain-riscv32-esp": {
"type": "toolchain", "type": "toolchain",
"optional": true, "optional": true,
"owner": "pioarduino", "owner": "pioarduino",
"package-version": "14.2.0+20241119", "package-version": "14.2.0+20250730",
"version": "https://github.com/pioarduino/registry/releases/download/0.0.1/riscv32-esp-elf-14.2.0_20241119.zip" "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/riscv32-esp-elf-14.2.0_20250730.zip"
}, },
"toolchain-esp32ulp": { "toolchain-esp32ulp": {
"type": "toolchain", "type": "toolchain",
@@ -78,15 +78,15 @@
"type": "debugger", "type": "debugger",
"optional": true, "optional": true,
"owner": "pioarduino", "owner": "pioarduino",
"package-version": "16.2.0+20250324", "package-version": "16.3.0+20250913",
"version": "https://github.com/pioarduino/registry/releases/download/0.0.1/xtensa-esp-gdb-v16.2_20250324.zip" "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/xtensa-esp-gdb-16.3_20250913.zip"
}, },
"tool-riscv32-esp-elf-gdb": { "tool-riscv32-esp-elf-gdb": {
"type": "debugger", "type": "debugger",
"optional": true, "optional": true,
"owner": "pioarduino", "owner": "pioarduino",
"package-version": "16.2.0+20250324", "package-version": "16.3.0+20250913",
"version": "https://github.com/pioarduino/registry/releases/download/0.0.1/riscv32-esp-gdb-v16.2_20250324.zip" "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/riscv32-esp-gdb-16.3_20250913.zip"
}, },
"tool-esptoolpy": { "tool-esptoolpy": {
"type": "uploader", "type": "uploader",
platform.py
+68 -28
View File
@@ -26,14 +26,15 @@ else:
del _lzma del _lzma
import fnmatch import fnmatch
import os import importlib.util
import json import json
import logging
import os
import requests import requests
import shutil
import socket import socket
import subprocess import subprocess
import sys import sys
import shutil
import logging
from pathlib import Path from pathlib import Path
from typing import Optional, Dict, List, Any, Union from typing import Optional, Dict, List, Any, Union
@@ -43,6 +44,17 @@ from platformio.proc import get_pythonexe_path
from platformio.project.config import ProjectConfig from platformio.project.config import ProjectConfig
from platformio.package.manager.tool import ToolPackageManager from platformio.package.manager.tool import ToolPackageManager
# Import penv_setup functionality using explicit module loading for centralized Python environment management
penv_setup_path = Path(__file__).parent / "builder" / "penv_setup.py"
spec = importlib.util.spec_from_file_location("penv_setup", str(penv_setup_path))
penv_setup_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(penv_setup_module)
setup_penv_minimal = penv_setup_module.setup_penv_minimal
get_executable_path = penv_setup_module.get_executable_path
# Constants # Constants
DEFAULT_DEBUG_SPEED = "5000" DEFAULT_DEBUG_SPEED = "5000"
DEFAULT_APP_OFFSET = "0x10000" DEFAULT_APP_OFFSET = "0x10000"
@@ -214,7 +226,7 @@ class Espressif32Platform(PlatformBase):
logger.debug(f"No version check required for {tl_install_name}") logger.debug(f"No version check required for {tl_install_name}")
return True return True
# Check if tool is already installed # Check current installation status
tl_install_path = self.packages_dir / tl_install_name tl_install_path = self.packages_dir / tl_install_name
package_json_path = tl_install_path / "package.json" package_json_path = tl_install_path / "package.json"
@@ -232,10 +244,10 @@ class Espressif32Platform(PlatformBase):
logger.warning(f"Installed version for {tl_install_name} unknown, installing {required_version}") logger.warning(f"Installed version for {tl_install_name} unknown, installing {required_version}")
return self._install_tl_install(required_version) return self._install_tl_install(required_version)
# IMPORTANT: Compare versions correctly # Compare versions to avoid unnecessary reinstallation
if self._compare_tl_install_versions(installed_version, required_version): if self._compare_tl_install_versions(installed_version, required_version):
logger.debug(f"{tl_install_name} version {installed_version} is already correctly installed") logger.debug(f"{tl_install_name} version {installed_version} is already correctly installed")
# IMPORTANT: Set package as available, but do NOT reinstall # Mark package as available without reinstalling
self.packages[tl_install_name]["optional"] = True self.packages[tl_install_name]["optional"] = True
return True return True
else: else:
@@ -293,8 +305,7 @@ class Espressif32Platform(PlatformBase):
def _install_tl_install(self, version: str) -> bool: def _install_tl_install(self, version: str) -> bool:
""" """
Install tool-esp_install ONLY when necessary Install tool-esp_install with version validation and legacy compatibility.
and handles backwards compatibility for tl-install.
Args: Args:
version: Version string or URL to install version: Version string or URL to install
@@ -308,7 +319,7 @@ class Espressif32Platform(PlatformBase):
try: try:
old_tl_install_exists = old_tl_install_path.exists() old_tl_install_exists = old_tl_install_path.exists()
if old_tl_install_exists: if old_tl_install_exists:
# remove outdated tl-install # Remove legacy tl-install directory
safe_remove_directory(old_tl_install_path) safe_remove_directory(old_tl_install_path)
if tl_install_path.exists(): if tl_install_path.exists():
@@ -319,7 +330,7 @@ class Espressif32Platform(PlatformBase):
self.packages[tl_install_name]["optional"] = False self.packages[tl_install_name]["optional"] = False
self.packages[tl_install_name]["version"] = version self.packages[tl_install_name]["version"] = version
pm.install(version) pm.install(version)
# Ensure backward compatibility by removing pio install status indicator # Remove PlatformIO install marker to prevent version conflicts
tl_piopm_path = tl_install_path / ".piopm" tl_piopm_path = tl_install_path / ".piopm"
safe_remove_file(tl_piopm_path) safe_remove_file(tl_piopm_path)
@@ -327,9 +338,9 @@ class Espressif32Platform(PlatformBase):
logger.info(f"{tl_install_name} successfully installed and verified") logger.info(f"{tl_install_name} successfully installed and verified")
self.packages[tl_install_name]["optional"] = True self.packages[tl_install_name]["optional"] = True
# Handle old tl-install to keep backwards compatibility # Maintain backwards compatibility with legacy tl-install references
if old_tl_install_exists: if old_tl_install_exists:
# Copy tool-esp_install content to tl-install location # Copy tool-esp_install content to legacy tl-install location
if safe_copy_directory(tl_install_path, old_tl_install_path): if safe_copy_directory(tl_install_path, old_tl_install_path):
logger.info(f"Content copied from {tl_install_name} to old tl-install location") logger.info(f"Content copied from {tl_install_name} to old tl-install location")
else: else:
@@ -395,14 +406,17 @@ class Espressif32Platform(PlatformBase):
'tool_exists': Path(paths['tool_path']).exists() 'tool_exists': Path(paths['tool_path']).exists()
} }
def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str) -> bool: def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str, penv_python: Optional[str] = None) -> bool:
""" """
Execute idf_tools.py install command. Execute idf_tools.py install command.
Note: No timeout is set to allow installations to complete on slow networks. Note: No timeout is set to allow installations to complete on slow networks.
The tool-esp_install handles the retry logic. The tool-esp_install handles the retry logic.
""" """
# Use penv Python if available, fallback to system Python
python_executable = penv_python or python_exe
cmd = [ cmd = [
python_exe, python_executable,
idf_tools_path, idf_tools_path,
"--quiet", "--quiet",
"--non-interactive", "--non-interactive",
@@ -415,13 +429,15 @@ class Espressif32Platform(PlatformBase):
logger.info(f"Installing tools via idf_tools.py (this may take several minutes)...") logger.info(f"Installing tools via idf_tools.py (this may take several minutes)...")
result = subprocess.run( result = subprocess.run(
cmd, cmd,
stdout=subprocess.DEVNULL, stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, stderr=subprocess.PIPE,
text=True,
check=False check=False
) )
if result.returncode != 0: if result.returncode != 0:
logger.error("idf_tools.py installation failed") tail = (result.stderr or result.stdout or "").strip()[-1000:]
logger.error("idf_tools.py installation failed (rc=%s). Tail:\n%s", result.returncode, tail)
return False return False
logger.debug("idf_tools.py executed successfully") logger.debug("idf_tools.py executed successfully")
@@ -433,7 +449,7 @@ class Espressif32Platform(PlatformBase):
def _check_tool_version(self, tool_name: str) -> bool: def _check_tool_version(self, tool_name: str) -> bool:
"""Check if the installed tool version matches the required version.""" """Check if the installed tool version matches the required version."""
# Clean up versioned directories FIRST, before any version checks # Clean up versioned directories before version checks to prevent conflicts
self._cleanup_versioned_tool_directories(tool_name) self._cleanup_versioned_tool_directories(tool_name)
paths = self._get_tool_paths(tool_name) paths = self._get_tool_paths(tool_name)
@@ -472,11 +488,14 @@ class Espressif32Platform(PlatformBase):
paths = self._get_tool_paths(tool_name) paths = self._get_tool_paths(tool_name)
status = self._check_tool_status(tool_name) status = self._check_tool_status(tool_name)
# Case 1: New installation with idf_tools # Use centrally configured Python executable if available
if status['has_idf_tools'] and status['has_tools_json']: penv_python = getattr(self, '_penv_python', None)
return self._install_with_idf_tools(tool_name, paths)
# Case 2: Tool already installed, version check # Case 1: Fresh installation using idf_tools.py
if status['has_idf_tools'] and status['has_tools_json']:
return self._install_with_idf_tools(tool_name, paths, penv_python)
# Case 2: Tool already installed, perform version validation
if (status['has_idf_tools'] and status['has_piopm'] and if (status['has_idf_tools'] and status['has_piopm'] and
not status['has_tools_json']): not status['has_tools_json']):
return self._handle_existing_tool(tool_name, paths) return self._handle_existing_tool(tool_name, paths)
@@ -484,14 +503,14 @@ class Espressif32Platform(PlatformBase):
logger.debug(f"Tool {tool_name} already configured") logger.debug(f"Tool {tool_name} already configured")
return True return True
def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str]) -> bool: def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str], penv_python: Optional[str] = None) -> bool:
"""Install tool using idf_tools.py installation method.""" """Install tool using idf_tools.py installation method."""
if not self._run_idf_tools_install( if not self._run_idf_tools_install(
paths['tools_json_path'], paths['idf_tools_path'] paths['tools_json_path'], paths['idf_tools_path'], penv_python
): ):
return False return False
# Copy tool files # Copy tool metadata to IDF tools directory
target_package_path = Path(IDF_TOOLS_PATH) / "tools" / tool_name / "package.json" target_package_path = Path(IDF_TOOLS_PATH) / "tools" / tool_name / "package.json"
if not safe_copy_file(paths['package_path'], target_package_path): if not safe_copy_file(paths['package_path'], target_package_path):
@@ -514,7 +533,7 @@ class Espressif32Platform(PlatformBase):
logger.debug(f"Tool {tool_name} found with correct version") logger.debug(f"Tool {tool_name} found with correct version")
return True return True
# Wrong version, reinstall - cleanup is already done in _check_tool_version # Version mismatch detected, reinstall tool (cleanup already performed)
logger.info(f"Reinstalling {tool_name} due to version mismatch") logger.info(f"Reinstalling {tool_name} due to version mismatch")
# Remove the main tool directory (if it still exists after cleanup) # Remove the main tool directory (if it still exists after cleanup)
@@ -603,7 +622,7 @@ class Espressif32Platform(PlatformBase):
logger.error("Error during tool-esp_install version check / installation") logger.error("Error during tool-esp_install version check / installation")
return return
# Remove pio install marker to avoid issues when switching versions # Remove legacy PlatformIO install marker to prevent version conflicts
old_tl_piopm_path = Path(self.packages_dir) / "tl-install" / ".piopm" old_tl_piopm_path = Path(self.packages_dir) / "tl-install" / ".piopm"
if old_tl_piopm_path.exists(): if old_tl_piopm_path.exists():
safe_remove_file(old_tl_piopm_path) safe_remove_file(old_tl_piopm_path)
@@ -714,6 +733,14 @@ class Espressif32Platform(PlatformBase):
if "downloadfs" in targets: if "downloadfs" in targets:
self._install_filesystem_tool(filesystem, for_download=True) self._install_filesystem_tool(filesystem, for_download=True)
def setup_python_env(self, env):
"""Configure SCons environment with centrally managed Python executable paths."""
# Python environment is centrally managed in configure_default_packages
if hasattr(self, '_penv_python') and hasattr(self, '_esptool_path'):
# Update SCons environment with centrally configured Python executable
env.Replace(PYTHONEXE=self._penv_python)
return self._penv_python, self._esptool_path
def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any: def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any:
"""Main configuration method with optimized package management.""" """Main configuration method with optimized package management."""
if not variables.get("board"): if not variables.get("board"):
@@ -725,9 +752,22 @@ class Espressif32Platform(PlatformBase):
frameworks = list(variables.get("pioframework", [])) # Create copy frameworks = list(variables.get("pioframework", [])) # Create copy
try: try:
# Configuration steps # FIRST: Install required packages
self._configure_installer() self._configure_installer()
self._install_esptool_package() self._install_esptool_package()
# Complete Python virtual environment setup
config = ProjectConfig.get_instance()
core_dir = config.get("platformio", "core_dir")
# Setup penv using minimal function (no SCons dependencies, esptool from tl-install)
penv_python, esptool_path = setup_penv_minimal(self, core_dir, install_esptool=True)
# Store both for later use
self._penv_python = penv_python
self._esptool_path = esptool_path
# Configuration steps (now with penv available)
self._configure_arduino_framework(frameworks) self._configure_arduino_framework(frameworks)
self._configure_espidf_framework(frameworks, variables, board_config, mcu) self._configure_espidf_framework(frameworks, variables, board_config, mcu)
self._configure_mcu_toolchains(mcu, variables, targets) self._configure_mcu_toolchains(mcu, variables, targets)