pioduino/platform-espressif32
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix merge error

Browse Source
This commit is contained in:
Jason2866
2025-07-23 17:46:36 +02:00
committed by GitHub
parent df9b181bf0
commit afd9b73025
1 changed files with 473 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
builder/main.py
+472 -27
View File
@@ -13,8 +13,10 @@
# limitations under the License.
import locale
import json
import os
import re
import semantic_version
import shlex
import subprocess
import sys
@@ -30,22 +32,354 @@ from SCons.Script import (
)
from platformio.project.helpers import get_project_dir
from platformio.package.version import pepver_to_semver
from platformio.util import get_serial_ports
# Python dependencies required for the build process
python_deps = {
"uv": ">=0.1.0",
"pyyaml": ">=6.0.2",
"rich-click": ">=1.8.6",
"zopfli": ">=0.2.2",
"intelhex": ">=2.3.0",
"rich": ">=14.0.0",
"esp-idf-size": ">=1.6.1"
}
# Initialize environment and configuration
env = DefaultEnvironment()
platform = env.PioPlatform()
projectconfig = env.GetProjectConfig()
terminal_cp = locale.getpreferredencoding().lower()
PYTHON_EXE = env.subst("$PYTHONEXE") # Global Python executable path
# Framework directory path
FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
def add_to_pythonpath(path):
"""
Add a path to the PYTHONPATH environment variable (cross-platform).
Args:
path (str): The path to add to PYTHONPATH
"""
# Normalize the path for the current OS
normalized_path = os.path.normpath(path)
# Add to PYTHONPATH environment variable
if "PYTHONPATH" in os.environ:
current_paths = os.environ["PYTHONPATH"].split(os.pathsep)
normalized_current_paths = [os.path.normpath(p) for p in current_paths]
if normalized_path not in normalized_current_paths:
os.environ["PYTHONPATH"] = normalized_path + os.pathsep + os.environ.get("PYTHONPATH", "")
else:
os.environ["PYTHONPATH"] = normalized_path
# Also add to sys.path for immediate availability
if normalized_path not in sys.path:
sys.path.insert(0, normalized_path)
def setup_python_paths():
"""
Setup Python paths based on the actual Python executable being used.
"""
if not PYTHON_EXE or not os.path.isfile(PYTHON_EXE):
return
# Get the directory containing the Python executable
python_dir = os.path.dirname(PYTHON_EXE)
add_to_pythonpath(python_dir)
# Try to find site-packages directory using the actual Python executable
result = subprocess.run(
[PYTHON_EXE, "-c", "import site; print(site.getsitepackages()[0])"],
capture_output=True,
text=True,
timeout=5
)
if result.returncode == 0:
site_packages = result.stdout.strip()
if os.path.isdir(site_packages):
add_to_pythonpath(site_packages)
# Setup Python paths based on the actual Python executable
setup_python_paths()
def _get_executable_path(python_exe, executable_name):
"""
Get the path to an executable binary (esptool, uv, etc.) based on the Python executable path.
Args:
python_exe (str): Path to Python executable
executable_name (str): Name of the executable to find (e.g., 'esptool', 'uv')
Returns:
str: Path to executable or fallback to executable name
"""
if not python_exe or not os.path.isfile(python_exe):
return executable_name # Fallback to command name
python_dir = os.path.dirname(python_exe)
if sys.platform == "win32":
scripts_dir = os.path.join(python_dir, "Scripts")
executable_path = os.path.join(scripts_dir, f"{executable_name}.exe")
else:
# For Unix-like systems, executables are typically in the same directory as python
# or in a bin subdirectory
executable_path = os.path.join(python_dir, executable_name)
# If not found in python directory, try bin subdirectory
if not os.path.isfile(executable_path):
bin_dir = os.path.join(python_dir, "bin")
executable_path = os.path.join(bin_dir, executable_name)
if os.path.isfile(executable_path):
return executable_path
return executable_name # Fallback to command name
def _get_esptool_executable_path(python_exe):
"""
Get the path to the esptool executable binary.
Args:
python_exe (str): Path to Python executable
Returns:
str: Path to esptool executable
"""
return _get_executable_path(python_exe, "esptool")
def _get_uv_executable_path(python_exe):
"""
Get the path to the uv executable binary.
Args:
python_exe (str): Path to Python executable
Returns:
str: Path to uv executable
"""
return _get_executable_path(python_exe, "uv")
def get_packages_to_install(deps, installed_packages):
"""
Generator for Python packages that need to be installed.
Args:
deps (dict): Dictionary of package names and version specifications
installed_packages (dict): Dictionary of currently installed packages
Yields:
str: Package name that needs to be installed
"""
for package, spec in deps.items():
if package not in installed_packages:
yield package
else:
version_spec = semantic_version.Spec(spec)
if not version_spec.match(installed_packages[package]):
yield package
def install_python_deps():
"""
Ensure uv package manager is available and install required Python dependencies.
Returns:
bool: True if successful, False otherwise
"""
# Get uv executable path
uv_executable = _get_uv_executable_path(PYTHON_EXE)
try:
result = subprocess.run(
[uv_executable, "--version"],
capture_output=True,
text=True,
timeout=3
)
uv_available = result.returncode == 0
except (FileNotFoundError, subprocess.TimeoutExpired):
uv_available = False
if not uv_available:
try:
result = subprocess.run(
[PYTHON_EXE, "-m", "pip", "install", "uv>=0.1.0", "-q", "-q", "-q"],
capture_output=True,
text=True,
timeout=30, # 30 second timeout
env=os.environ # Use modified environment with custom PYTHONPATH
)
if result.returncode != 0:
if result.stderr:
print(f"Error output: {result.stderr.strip()}")
return False
# Update uv executable path after installation
uv_executable = _get_uv_executable_path(PYTHON_EXE)
# Add Scripts directory to PATH for Windows
if sys.platform == "win32":
python_dir = os.path.dirname(PYTHON_EXE)
scripts_dir = os.path.join(python_dir, "Scripts")
if os.path.isdir(scripts_dir):
os.environ["PATH"] = scripts_dir + os.pathsep + os.environ.get("PATH", "")
except subprocess.TimeoutExpired:
print("Error: uv installation timed out")
return False
except FileNotFoundError:
print("Error: Python executable not found")
return False
except Exception as e:
print(f"Error installing uv package manager: {e}")
return False
def _get_installed_uv_packages():
"""
Get list of installed packages using uv.
Returns:
dict: Dictionary of installed packages with versions
"""
result = {}
try:
cmd = [uv_executable, "pip", "list", "--format=json"]
result_obj = subprocess.run(
cmd,
capture_output=True,
text=True,
encoding='utf-8',
timeout=30, # 30 second timeout
env=os.environ # Use modified environment with custom PYTHONPATH
)
if result_obj.returncode == 0:
content = result_obj.stdout.strip()
if content:
packages = json.loads(content)
for p in packages:
result[p["name"]] = pepver_to_semver(p["version"])
else:
print(f"Warning: pip list failed with exit code {result_obj.returncode}")
if result_obj.stderr:
print(f"Error output: {result_obj.stderr.strip()}")
except subprocess.TimeoutExpired:
print("Warning: uv pip list command timed out")
except (json.JSONDecodeError, KeyError) as e:
print(f"Warning: Could not parse package list: {e}")
except FileNotFoundError:
print("Warning: uv command not found")
except Exception as e:
print(f"Warning! Couldn't extract the list of installed Python packages: {e}")
return result
installed_packages = _get_installed_uv_packages()
packages_to_install = list(get_packages_to_install(python_deps, installed_packages))
if packages_to_install:
packages_list = [f"{p}{python_deps[p]}" for p in packages_to_install]
cmd = [
uv_executable, "pip", "install",
f"--python={PYTHON_EXE}",
"--quiet", "--upgrade"
] + packages_list
try:
result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=30, # 30 second timeout for package installation
env=os.environ # Use modified environment with custom PYTHONPATH
)
if result.returncode != 0:
print(f"Error: Failed to install Python dependencies (exit code: {result.returncode})")
if result.stderr:
print(f"Error output: {result.stderr.strip()}")
return False
except subprocess.TimeoutExpired:
print("Error: Python dependencies installation timed out")
return False
except FileNotFoundError:
print("Error: uv command not found")
return False
except Exception as e:
print(f"Error installing Python dependencies: {e}")
return False
return True
def install_esptool():
"""
Install esptool from package folder "tool-esptoolpy" using uv package manager.
Also determines the path to the esptool executable binary.
Returns:
str: Path to esptool executable, or 'esptool' as fallback
"""
try:
subprocess.check_call(
[PYTHON_EXE, "-c", "import esptool"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
env=os.environ
)
esptool_binary_path = _get_esptool_executable_path(PYTHON_EXE)
return esptool_binary_path
except (subprocess.CalledProcessError, FileNotFoundError):
pass
esptool_repo_path = env.subst(platform.get_package_dir("tool-esptoolpy") or "")
if esptool_repo_path and os.path.isdir(esptool_repo_path):
uv_executable = _get_uv_executable_path(PYTHON_EXE)
try:
subprocess.check_call([
uv_executable, "pip", "install", "--quiet",
f"--python={PYTHON_EXE}",
"-e", esptool_repo_path
], env=os.environ)
esptool_binary_path = _get_esptool_executable_path(PYTHON_EXE)
return esptool_binary_path
except subprocess.CalledProcessError as e:
print(f"Warning: Failed to install esptool: {e}")
return 'esptool' # Fallback
return 'esptool' # Fallback
# Install Python dependencies and esptool
install_python_deps()
esptool_binary_path = install_esptool()
def BeforeUpload(target, source, env):
"""
Prepare the environment before uploading firmware.
Handles port detection and special upload configurations.
Args:
target: SCons target
source: SCons source
env: SCons environment object
"""
upload_options = {}
if "BOARD" in env:
@@ -65,7 +399,12 @@ def BeforeUpload(target, source, env):
def _get_board_memory_type(env):
"""
Determine the memory type configuration for the board.
Returns the appropriate memory type string based on board configuration.
Args:
env: SCons environment object
Returns:
str: The appropriate memory type string based on board configuration
"""
board_config = env.BoardConfig()
default_type = "%s_%s" % (
@@ -86,20 +425,41 @@ def _get_board_memory_type(env):
def _normalize_frequency(frequency):
"""
Convert frequency value to normalized string format (e.g., "40m").
Removes 'L' suffix and converts to MHz format.
Args:
frequency: Frequency value to normalize
Returns:
str: Normalized frequency string with 'm' suffix
"""
frequency = str(frequency).replace("L", "")
return str(int(int(frequency) / 1000000)) + "m"
def _get_board_f_flash(env):
"""Get the flash frequency for the board."""
"""
Get the flash frequency for the board.
Args:
env: SCons environment object
Returns:
str: Flash frequency string
"""
frequency = env.subst("$BOARD_F_FLASH")
return _normalize_frequency(frequency)
def _get_board_f_image(env):
"""Get the image frequency for the board, fallback to flash frequency."""
"""
Get the image frequency for the board, fallback to flash frequency.
Args:
env: SCons environment object
Returns:
str: Image frequency string
"""
board_config = env.BoardConfig()
if "build.f_image" in board_config:
return _normalize_frequency(board_config.get("build.f_image"))
@@ -108,7 +468,15 @@ def _get_board_f_image(env):
def _get_board_f_boot(env):
"""Get the boot frequency for the board, fallback to flash frequency."""
"""
Get the boot frequency for the board, fallback to flash frequency.
Args:
env: SCons environment object
Returns:
str: Boot frequency string
"""
board_config = env.BoardConfig()
if "build.f_boot" in board_config:
return _normalize_frequency(board_config.get("build.f_boot"))
@@ -120,6 +488,12 @@ def _get_board_flash_mode(env):
"""
Determine the appropriate flash mode for the board.
Handles special cases for OPI memory types.
Args:
env: SCons environment object
Returns:
str: Flash mode string
"""
if _get_board_memory_type(env) in ("opi_opi", "opi_qspi"):
return "dout"
@@ -134,6 +508,12 @@ def _get_board_boot_mode(env):
"""
Determine the boot mode for the board.
Handles special cases for OPI memory types.
Args:
env: SCons environment object
Returns:
str: Boot mode string
"""
memory_type = env.BoardConfig().get("build.arduino.memory_type", "")
build_boot = env.BoardConfig().get("build.boot", "$BOARD_FLASH_MODE")
@@ -145,7 +525,12 @@ def _get_board_boot_mode(env):
def _parse_size(value):
"""
Parse size values from various formats (int, hex, K/M suffixes).
Returns the size in bytes as an integer.
Args:
value: Size value to parse
Returns:
int: Size in bytes as an integer
"""
if isinstance(value, int):
return value
@@ -163,6 +548,12 @@ def _parse_partitions(env):
"""
Parse the partition table CSV file and return partition information.
Also sets the application offset for the environment.
Args:
env: SCons environment object
Returns:
list: List of partition dictionaries
"""
partitions_csv = env.subst("$PARTITIONS_TABLE_CSV")
if not isfile(partitions_csv):
@@ -214,6 +605,9 @@ def _update_max_upload_size(env):
"""
Update the maximum upload size based on partition table configuration.
Prioritizes user-specified partition names.
Args:
env: SCons environment object
"""
if not env.get("PARTITIONS_TABLE_CSV"):
return
@@ -249,7 +643,15 @@ def _update_max_upload_size(env):
def _to_unix_slashes(path):
"""Convert Windows-style backslashes to Unix-style forward slashes."""
"""
Convert Windows-style backslashes to Unix-style forward slashes.
Args:
path (str): Path to convert
Returns:
str: Path with Unix-style slashes
"""
return path.replace("\\", "/")
@@ -257,6 +659,9 @@ def fetch_fs_size(env):
"""
Extract filesystem size and offset information from partition table.
Sets FS_START, FS_SIZE, FS_PAGE, and FS_BLOCK environment variables.
Args:
env: SCons environment object
"""
fs = None
for p in _parse_partitions(env):
@@ -287,7 +692,17 @@ def fetch_fs_size(env):
def __fetch_fs_size(target, source, env):
"""Wrapper function for fetch_fs_size to be used as SCons emitter."""
"""
Wrapper function for fetch_fs_size to be used as SCons emitter.
Args:
target: SCons target
source: SCons source
env: SCons environment object
Returns:
tuple: (target, source) tuple
"""
fetch_fs_size(env)
return (target, source)
@@ -295,7 +710,9 @@ def __fetch_fs_size(target, source, env):
def check_lib_archive_exists():
"""
Check if lib_archive is set in platformio.ini configuration.
Returns True if found, False otherwise.
Returns:
bool: True if found, False otherwise
"""
for section in projectconfig.sections():
if "lib_archive" in projectconfig.options(section):
@@ -303,6 +720,22 @@ def check_lib_archive_exists():
return False
def switch_off_ldf():
"""
Disables LDF (Library Dependency Finder) for uploadfs, uploadfsota, and buildfs targets.
This optimization prevents unnecessary library dependency scanning and compilation
when only filesystem operations are performed.
"""
fs_targets = {"uploadfs", "uploadfsota", "buildfs", "erase"}
if fs_targets & set(COMMAND_LINE_TARGETS):
# Disable LDF by modifying project configuration directly
env_section = "env:" + env["PIOENV"]
if not projectconfig.has_section(env_section):
projectconfig.add_section(env_section)
projectconfig.set(env_section, "lib_ldf_mode", "off")
# Initialize board configuration and MCU settings
board = env.BoardConfig()
mcu = board.get("build.mcu", "esp32")
@@ -317,6 +750,13 @@ if mcu in ("esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32h2", "esp32p4"):
if "INTEGRATION_EXTRA_DATA" not in env:
env["INTEGRATION_EXTRA_DATA"] = {}
# Take care of possible whitespaces in path
objcopy_value = (
f'"{esptool_binary_path}"'
if ' ' in esptool_binary_path
else esptool_binary_path
)
# Configure build tools and environment variables
env.Replace(
__get_board_boot_mode=_get_board_boot_mode,
@@ -346,7 +786,7 @@ env.Replace(
"bin",
"%s-elf-gdb" % toolchain_arch,
),
OBJCOPY=join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"),
OBJCOPY=objcopy_value,
RANLIB="%s-elf-gcc-ranlib" % toolchain_arch,
SIZETOOL="%s-elf-size" % toolchain_arch,
ARFLAGS=["rc"],
@@ -356,7 +796,7 @@ env.Replace(
SIZECHECKCMD="$SIZETOOL -A -d $SOURCES",
SIZEPRINTCMD="$SIZETOOL -B -d $SOURCES",
ERASEFLAGS=["--chip", mcu, "--port", '"$UPLOAD_PORT"'],
ERASECMD='"$PYTHONEXE" "$OBJCOPY" $ERASEFLAGS erase-flash',
ERASECMD='"$OBJCOPY" $ERASEFLAGS erase-flash',
# mkspiffs package contains two different binaries for IDF and Arduino
MKFSTOOL="mk%s" % filesystem
+ (
@@ -373,6 +813,7 @@ env.Replace(
),
# Legacy `ESP32_SPIFFS_IMAGE_NAME` is used as the second fallback value
# for backward compatibility
ESP32_FS_IMAGE_NAME=env.get(
"ESP32_FS_IMAGE_NAME",
env.get("ESP32_SPIFFS_IMAGE_NAME", filesystem),
@@ -401,7 +842,7 @@ env.Append(
action=env.VerboseAction(
" ".join(
[
'"$PYTHONEXE" "$OBJCOPY"',
"$OBJCOPY",
"--chip",
mcu,
"elf2image",
@@ -444,10 +885,20 @@ env.Append(
if not env.get("PIOFRAMEWORK"):
env.SConscript("frameworks/_bare.py", exports="env")
# Disable LDF for filesystem operations
switch_off_ldf()
def firmware_metrics(target, source, env):
"""
Custom target to run esp-idf-size with support for command line parameters
Custom target to run esp-idf-size with support for command line parameters.
Usage: pio run -t metrics -- [esp-idf-size arguments]
Args:
target: SCons target
source: SCons source
env: SCons environment object
"""
if terminal_cp != "utf-8":
print("Firmware metrics can not be shown. Set the terminal codepage to \"utf-8\"")
@@ -464,11 +915,7 @@ def firmware_metrics(target, source, env):
return
try:
import subprocess
import sys
import shlex
cmd = [env.subst("$PYTHONEXE"), "-m", "esp_idf_size", "--ng"]
cmd = [PYTHON_EXE, "-m", "esp_idf_size", "--ng"]
# Parameters from platformio.ini
extra_args = env.GetProjectOption("custom_esp_idf_size_args", "")
@@ -494,8 +941,8 @@ def firmware_metrics(target, source, env):
if env.GetProjectOption("custom_esp_idf_size_verbose", False):
print(f"Running command: {' '.join(cmd)}")
# Call esp-idf-size
result = subprocess.run(cmd, check=False, capture_output=False)
# Call esp-idf-size with modified environment
result = subprocess.run(cmd, check=False, capture_output=False, env=os.environ)
if result.returncode != 0:
print(f"Warning: esp-idf-size exited with code {result.returncode}")
@@ -510,6 +957,7 @@ def firmware_metrics(target, source, env):
print(f"Error: Failed to run firmware metrics: {e}")
print("Make sure esp-idf-size is installed: pip install esp-idf-size")
#
# Target: Build executable and linkable firmware or FS image
#
@@ -595,7 +1043,7 @@ if upload_protocol == "espota":
env.Replace(
UPLOADER=join(FRAMEWORK_DIR, "tools", "espota.py"),
UPLOADERFLAGS=["--debug", "--progress", "-i", "$UPLOAD_PORT"],
UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS -f $SOURCE',
UPLOADCMD=f'"{PYTHON_EXE}" "$UPLOADER" $UPLOADERFLAGS -f $SOURCE',
)
if set(["uploadfs", "uploadfsota"]) & set(COMMAND_LINE_TARGETS):
env.Append(UPLOADERFLAGS=["--spiffs"])
@@ -604,9 +1052,7 @@ if upload_protocol == "espota":
# Configure upload protocol: esptool
elif upload_protocol == "esptool":
env.Replace(
UPLOADER=join(
platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"
),
UPLOADER=objcopy_value,
UPLOADERFLAGS=[
"--chip",
mcu,
@@ -627,8 +1073,7 @@ elif upload_protocol == "esptool":
"--flash-size",
"detect",
],
UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS '
"$ESP32_APP_OFFSET $SOURCE",
UPLOADCMD='$UPLOADER $UPLOADERFLAGS $ESP32_APP_OFFSET $SOURCE'
)
for image in env.get("FLASH_EXTRA_IMAGES", []):
env.Append(UPLOADERFLAGS=[image[0], env.subst(image[1])])
@@ -656,7 +1101,7 @@ elif upload_protocol == "esptool":
"detect",
"$FS_START",
],
UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS $SOURCE',
UPLOADCMD='"$UPLOADER" $UPLOADERFLAGS $SOURCE',
)
upload_actions = [