fix merge error

builder/main.py

@@ -13,8 +13,10 @@
 # limitations under the License.
 
 import locale
+import json
 import os
 import re
+import semantic_version
 import shlex
 import subprocess
 import sys
@@ -30,22 +32,354 @@ from SCons.Script import (
 )
 
 from platformio.project.helpers import get_project_dir
+from platformio.package.version import pepver_to_semver
 from platformio.util import get_serial_ports
 
+# Python dependencies required for the build process
+python_deps = {
+    "uv": ">=0.1.0",
+    "pyyaml": ">=6.0.2",
+    "rich-click": ">=1.8.6",
+    "zopfli": ">=0.2.2",
+    "intelhex": ">=2.3.0",
+    "rich": ">=14.0.0",
+    "esp-idf-size": ">=1.6.1"
+}
+
 # Initialize environment and configuration
 env = DefaultEnvironment()
 platform = env.PioPlatform()
 projectconfig = env.GetProjectConfig()
 terminal_cp = locale.getpreferredencoding().lower()
+PYTHON_EXE = env.subst("$PYTHONEXE")  # Global Python executable path
 
 # Framework directory path
 FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
 
 
+def add_to_pythonpath(path):
+    """
+    Add a path to the PYTHONPATH environment variable (cross-platform).
+    
+    Args:
+        path (str): The path to add to PYTHONPATH
+    """
+    # Normalize the path for the current OS
+    normalized_path = os.path.normpath(path)
+    
+    # Add to PYTHONPATH environment variable
+    if "PYTHONPATH" in os.environ:
+        current_paths = os.environ["PYTHONPATH"].split(os.pathsep)
+        normalized_current_paths = [os.path.normpath(p) for p in current_paths]
+        if normalized_path not in normalized_current_paths:
+            os.environ["PYTHONPATH"] = normalized_path + os.pathsep + os.environ.get("PYTHONPATH", "")
+    else:
+        os.environ["PYTHONPATH"] = normalized_path
+    
+    # Also add to sys.path for immediate availability
+    if normalized_path not in sys.path:
+        sys.path.insert(0, normalized_path)
+
+
+def setup_python_paths():
+    """
+    Setup Python paths based on the actual Python executable being used.
+    """
+    if not PYTHON_EXE or not os.path.isfile(PYTHON_EXE):
+        return
+    
+    # Get the directory containing the Python executable
+    python_dir = os.path.dirname(PYTHON_EXE)
+    add_to_pythonpath(python_dir)
+    
+    # Try to find site-packages directory using the actual Python executable
+    result = subprocess.run(
+        [PYTHON_EXE, "-c", "import site; print(site.getsitepackages()[0])"],
+        capture_output=True,
+        text=True,
+        timeout=5
+    )
+    if result.returncode == 0:
+        site_packages = result.stdout.strip()
+        if os.path.isdir(site_packages):
+            add_to_pythonpath(site_packages)
+
+# Setup Python paths based on the actual Python executable
+setup_python_paths()
+
+
+def _get_executable_path(python_exe, executable_name):
+    """
+    Get the path to an executable binary (esptool, uv, etc.) based on the Python executable path.
+    
+    Args:
+        python_exe (str): Path to Python executable
+        executable_name (str): Name of the executable to find (e.g., 'esptool', 'uv')
+        
+    Returns:
+        str: Path to executable or fallback to executable name
+    """
+    if not python_exe or not os.path.isfile(python_exe):
+        return executable_name  # Fallback to command name
+    
+    python_dir = os.path.dirname(python_exe)
+    
+    if sys.platform == "win32":
+        scripts_dir = os.path.join(python_dir, "Scripts")
+        executable_path = os.path.join(scripts_dir, f"{executable_name}.exe")
+    else:
+        # For Unix-like systems, executables are typically in the same directory as python
+        # or in a bin subdirectory
+        executable_path = os.path.join(python_dir, executable_name)
+        
+        # If not found in python directory, try bin subdirectory
+        if not os.path.isfile(executable_path):
+            bin_dir = os.path.join(python_dir, "bin")
+            executable_path = os.path.join(bin_dir, executable_name)
+    
+    if os.path.isfile(executable_path):
+        return executable_path
+    
+    return executable_name  # Fallback to command name
+
+
+def _get_esptool_executable_path(python_exe):
+    """
+    Get the path to the esptool executable binary.
+    
+    Args:
+        python_exe (str): Path to Python executable
+        
+    Returns:
+        str: Path to esptool executable
+    """
+    return _get_executable_path(python_exe, "esptool")
+
+
+def _get_uv_executable_path(python_exe):
+    """
+    Get the path to the uv executable binary.
+    
+    Args:
+        python_exe (str): Path to Python executable
+        
+    Returns:
+        str: Path to uv executable
+    """
+    return _get_executable_path(python_exe, "uv")
+
+
+def get_packages_to_install(deps, installed_packages):
+    """
+    Generator for Python packages that need to be installed.
+    
+    Args:
+        deps (dict): Dictionary of package names and version specifications
+        installed_packages (dict): Dictionary of currently installed packages
+        
+    Yields:
+        str: Package name that needs to be installed
+    """
+    for package, spec in deps.items():
+        if package not in installed_packages:
+            yield package
+        else:
+            version_spec = semantic_version.Spec(spec)
+            if not version_spec.match(installed_packages[package]):
+                yield package
+
+
+def install_python_deps():
+    """
+    Ensure uv package manager is available and install required Python dependencies.
+    
+    Returns:
+        bool: True if successful, False otherwise
+    """
+    # Get uv executable path
+    uv_executable = _get_uv_executable_path(PYTHON_EXE)
+    
+    try:
+        result = subprocess.run(
+            [uv_executable, "--version"],
+            capture_output=True,
+            text=True,
+            timeout=3
+        )
+        uv_available = result.returncode == 0
+    except (FileNotFoundError, subprocess.TimeoutExpired):
+        uv_available = False
+    
+    if not uv_available:
+        try:
+            result = subprocess.run(
+                [PYTHON_EXE, "-m", "pip", "install", "uv>=0.1.0", "-q", "-q", "-q"],
+                capture_output=True,
+                text=True,
+                timeout=30,  # 30 second timeout
+                env=os.environ  # Use modified environment with custom PYTHONPATH
+            )
+            if result.returncode != 0:
+                if result.stderr:
+                    print(f"Error output: {result.stderr.strip()}")
+                return False
+            
+            # Update uv executable path after installation
+            uv_executable = _get_uv_executable_path(PYTHON_EXE)
+            
+            # Add Scripts directory to PATH for Windows
+            if sys.platform == "win32":
+                python_dir = os.path.dirname(PYTHON_EXE)
+                scripts_dir = os.path.join(python_dir, "Scripts")
+                if os.path.isdir(scripts_dir):
+                    os.environ["PATH"] = scripts_dir + os.pathsep + os.environ.get("PATH", "")
+                    
+        except subprocess.TimeoutExpired:
+            print("Error: uv installation timed out")
+            return False
+        except FileNotFoundError:
+            print("Error: Python executable not found")
+            return False
+        except Exception as e:
+            print(f"Error installing uv package manager: {e}")
+            return False
+
+    
+    def _get_installed_uv_packages():
+        """
+        Get list of installed packages using uv.
+        
+        Returns:
+            dict: Dictionary of installed packages with versions
+        """
+        result = {}
+        try:
+            cmd = [uv_executable, "pip", "list", "--format=json"]
+            result_obj = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                encoding='utf-8',
+                timeout=30,  # 30 second timeout
+                env=os.environ  # Use modified environment with custom PYTHONPATH
+            )
+            
+            if result_obj.returncode == 0:
+                content = result_obj.stdout.strip()
+                if content:
+                    packages = json.loads(content)
+                    for p in packages:
+                        result[p["name"]] = pepver_to_semver(p["version"])
+            else:
+                print(f"Warning: pip list failed with exit code {result_obj.returncode}")
+                if result_obj.stderr:
+                    print(f"Error output: {result_obj.stderr.strip()}")
+                
+        except subprocess.TimeoutExpired:
+            print("Warning: uv pip list command timed out")
+        except (json.JSONDecodeError, KeyError) as e:
+            print(f"Warning: Could not parse package list: {e}")
+        except FileNotFoundError:
+            print("Warning: uv command not found")
+        except Exception as e:
+            print(f"Warning! Couldn't extract the list of installed Python packages: {e}")
+
+        return result
+
+    installed_packages = _get_installed_uv_packages()
+    packages_to_install = list(get_packages_to_install(python_deps, installed_packages))
+    
+    if packages_to_install:
+        packages_list = [f"{p}{python_deps[p]}" for p in packages_to_install]
+        
+        cmd = [
+            uv_executable, "pip", "install",
+            f"--python={PYTHON_EXE}",
+            "--quiet", "--upgrade"
+        ] + packages_list
+        
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=30,  # 30 second timeout for package installation
+                env=os.environ  # Use modified environment with custom PYTHONPATH
+            )
+            
+            if result.returncode != 0:
+                print(f"Error: Failed to install Python dependencies (exit code: {result.returncode})")
+                if result.stderr:
+                    print(f"Error output: {result.stderr.strip()}")
+                return False
+                
+        except subprocess.TimeoutExpired:
+            print("Error: Python dependencies installation timed out")
+            return False
+        except FileNotFoundError:
+            print("Error: uv command not found")
+            return False
+        except Exception as e:
+            print(f"Error installing Python dependencies: {e}")
+            return False
+    
+    return True
+
+
+def install_esptool():
+    """
+    Install esptool from package folder "tool-esptoolpy" using uv package manager.
+    Also determines the path to the esptool executable binary.
+    
+    Returns:
+        str: Path to esptool executable, or 'esptool' as fallback
+    """
+    try:
+        subprocess.check_call(
+            [PYTHON_EXE, "-c", "import esptool"], 
+            stdout=subprocess.DEVNULL, 
+            stderr=subprocess.DEVNULL,
+            env=os.environ
+        )
+        esptool_binary_path = _get_esptool_executable_path(PYTHON_EXE)
+        return esptool_binary_path
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        pass
+
+    esptool_repo_path = env.subst(platform.get_package_dir("tool-esptoolpy") or "")
+    if esptool_repo_path and os.path.isdir(esptool_repo_path):
+        uv_executable = _get_uv_executable_path(PYTHON_EXE)
+        try:
+            subprocess.check_call([
+                uv_executable, "pip", "install", "--quiet",
+                f"--python={PYTHON_EXE}",
+                "-e", esptool_repo_path
+            ], env=os.environ)
+
+            esptool_binary_path = _get_esptool_executable_path(PYTHON_EXE)
+            return esptool_binary_path
+            
+        except subprocess.CalledProcessError as e:
+            print(f"Warning: Failed to install esptool: {e}")
+            return 'esptool'  # Fallback
+    
+    return 'esptool'  # Fallback
+
+
+# Install Python dependencies and esptool
+install_python_deps()
+esptool_binary_path = install_esptool()
+
+
 def BeforeUpload(target, source, env):
     """
     Prepare the environment before uploading firmware.
     Handles port detection and special upload configurations.
+    
+    Args:
+        target: SCons target
+        source: SCons source
+        env: SCons environment object
     """
     upload_options = {}
     if "BOARD" in env:
@@ -65,7 +399,12 @@ def BeforeUpload(target, source, env):
 def _get_board_memory_type(env):
     """
     Determine the memory type configuration for the board.
-    Returns the appropriate memory type string based on board configuration.
+    
+    Args:
+        env: SCons environment object
+        
+    Returns:
+        str: The appropriate memory type string based on board configuration
     """
     board_config = env.BoardConfig()
     default_type = "%s_%s" % (
@@ -86,20 +425,41 @@ def _get_board_memory_type(env):
 def _normalize_frequency(frequency):
     """
     Convert frequency value to normalized string format (e.g., "40m").
-    Removes 'L' suffix and converts to MHz format.
+    
+    Args:
+        frequency: Frequency value to normalize
+        
+    Returns:
+        str: Normalized frequency string with 'm' suffix
     """
     frequency = str(frequency).replace("L", "")
     return str(int(int(frequency) / 1000000)) + "m"
 
 
 def _get_board_f_flash(env):
-    """Get the flash frequency for the board."""
+    """
+    Get the flash frequency for the board.
+    
+    Args:
+        env: SCons environment object
+        
+    Returns:
+        str: Flash frequency string
+    """
     frequency = env.subst("$BOARD_F_FLASH")
     return _normalize_frequency(frequency)
 
 
 def _get_board_f_image(env):
-    """Get the image frequency for the board, fallback to flash frequency."""
+    """
+    Get the image frequency for the board, fallback to flash frequency.
+    
+    Args:
+        env: SCons environment object
+        
+    Returns:
+        str: Image frequency string
+    """
     board_config = env.BoardConfig()
     if "build.f_image" in board_config:
         return _normalize_frequency(board_config.get("build.f_image"))
@@ -108,7 +468,15 @@ def _get_board_f_image(env):
 
 
 def _get_board_f_boot(env):
-    """Get the boot frequency for the board, fallback to flash frequency."""
+    """
+    Get the boot frequency for the board, fallback to flash frequency.
+    
+    Args:
+        env: SCons environment object
+        
+    Returns:
+        str: Boot frequency string
+    """
     board_config = env.BoardConfig()
     if "build.f_boot" in board_config:
         return _normalize_frequency(board_config.get("build.f_boot"))
@@ -120,6 +488,12 @@ def _get_board_flash_mode(env):
     """
     Determine the appropriate flash mode for the board.
     Handles special cases for OPI memory types.
+    
+    Args:
+        env: SCons environment object
+        
+    Returns:
+        str: Flash mode string
     """
     if _get_board_memory_type(env) in ("opi_opi", "opi_qspi"):
         return "dout"
@@ -134,6 +508,12 @@ def _get_board_boot_mode(env):
     """
     Determine the boot mode for the board.
     Handles special cases for OPI memory types.
+    
+    Args:
+        env: SCons environment object
+        
+    Returns:
+        str: Boot mode string
     """
     memory_type = env.BoardConfig().get("build.arduino.memory_type", "")
     build_boot = env.BoardConfig().get("build.boot", "$BOARD_FLASH_MODE")
@@ -145,7 +525,12 @@ def _get_board_boot_mode(env):
 def _parse_size(value):
     """
     Parse size values from various formats (int, hex, K/M suffixes).
-    Returns the size in bytes as an integer.
+    
+    Args:
+        value: Size value to parse
+        
+    Returns:
+        int: Size in bytes as an integer
     """
     if isinstance(value, int):
         return value
@@ -163,6 +548,12 @@ def _parse_partitions(env):
     """
     Parse the partition table CSV file and return partition information.
     Also sets the application offset for the environment.
+    
+    Args:
+        env: SCons environment object
+        
+    Returns:
+        list: List of partition dictionaries
     """
     partitions_csv = env.subst("$PARTITIONS_TABLE_CSV")
     if not isfile(partitions_csv):
@@ -214,6 +605,9 @@ def _update_max_upload_size(env):
     """
     Update the maximum upload size based on partition table configuration.
     Prioritizes user-specified partition names.
+    
+    Args:
+        env: SCons environment object
     """
     if not env.get("PARTITIONS_TABLE_CSV"):
         return
@@ -249,7 +643,15 @@ def _update_max_upload_size(env):
 
 
 def _to_unix_slashes(path):
-    """Convert Windows-style backslashes to Unix-style forward slashes."""
+    """
+    Convert Windows-style backslashes to Unix-style forward slashes.
+    
+    Args:
+        path (str): Path to convert
+        
+    Returns:
+        str: Path with Unix-style slashes
+    """
     return path.replace("\\", "/")
 
 
@@ -257,6 +659,9 @@ def fetch_fs_size(env):
     """
     Extract filesystem size and offset information from partition table.
     Sets FS_START, FS_SIZE, FS_PAGE, and FS_BLOCK environment variables.
+    
+    Args:
+        env: SCons environment object
     """
     fs = None
     for p in _parse_partitions(env):
@@ -287,7 +692,17 @@ def fetch_fs_size(env):
 
 
 def __fetch_fs_size(target, source, env):
-    """Wrapper function for fetch_fs_size to be used as SCons emitter."""
+    """
+    Wrapper function for fetch_fs_size to be used as SCons emitter.
+    
+    Args:
+        target: SCons target
+        source: SCons source
+        env: SCons environment object
+        
+    Returns:
+        tuple: (target, source) tuple
+    """
     fetch_fs_size(env)
     return (target, source)
 
@@ -295,7 +710,9 @@ def __fetch_fs_size(target, source, env):
 def check_lib_archive_exists():
     """
     Check if lib_archive is set in platformio.ini configuration.
-    Returns True if found, False otherwise.
+    
+    Returns:
+        bool: True if found, False otherwise
     """
     for section in projectconfig.sections():
         if "lib_archive" in projectconfig.options(section):
@@ -303,6 +720,22 @@ def check_lib_archive_exists():
     return False
 
 
+def switch_off_ldf():
+    """
+    Disables LDF (Library Dependency Finder) for uploadfs, uploadfsota, and buildfs targets.
+
+    This optimization prevents unnecessary library dependency scanning and compilation
+    when only filesystem operations are performed.
+    """
+    fs_targets = {"uploadfs", "uploadfsota", "buildfs", "erase"}
+    if fs_targets & set(COMMAND_LINE_TARGETS):
+        # Disable LDF by modifying project configuration directly
+        env_section = "env:" + env["PIOENV"]
+        if not projectconfig.has_section(env_section):
+            projectconfig.add_section(env_section)
+        projectconfig.set(env_section, "lib_ldf_mode", "off")
+
+
 # Initialize board configuration and MCU settings
 board = env.BoardConfig()
 mcu = board.get("build.mcu", "esp32")
@@ -317,6 +750,13 @@ if mcu in ("esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32h2", "esp32p4"):
 if "INTEGRATION_EXTRA_DATA" not in env:
     env["INTEGRATION_EXTRA_DATA"] = {}
 
+# Take care of possible whitespaces in path
+objcopy_value = (
+    f'"{esptool_binary_path}"' 
+    if ' ' in esptool_binary_path 
+    else esptool_binary_path
+)
+
 # Configure build tools and environment variables
 env.Replace(
     __get_board_boot_mode=_get_board_boot_mode,
@@ -346,7 +786,7 @@ env.Replace(
         "bin",
         "%s-elf-gdb" % toolchain_arch,
     ),
-    OBJCOPY=join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"),
+    OBJCOPY=objcopy_value,
     RANLIB="%s-elf-gcc-ranlib" % toolchain_arch,
     SIZETOOL="%s-elf-size" % toolchain_arch,
     ARFLAGS=["rc"],
@@ -356,7 +796,7 @@ env.Replace(
     SIZECHECKCMD="$SIZETOOL -A -d $SOURCES",
     SIZEPRINTCMD="$SIZETOOL -B -d $SOURCES",
     ERASEFLAGS=["--chip", mcu, "--port", '"$UPLOAD_PORT"'],
-    ERASECMD='"$PYTHONEXE" "$OBJCOPY" $ERASEFLAGS erase-flash',
+    ERASECMD='"$OBJCOPY" $ERASEFLAGS erase-flash',
     # mkspiffs package contains two different binaries for IDF and Arduino
     MKFSTOOL="mk%s" % filesystem
     + (
@@ -373,6 +813,7 @@ env.Replace(
     ),
     # Legacy `ESP32_SPIFFS_IMAGE_NAME` is used as the second fallback value
     # for backward compatibility
+
     ESP32_FS_IMAGE_NAME=env.get(
         "ESP32_FS_IMAGE_NAME",
         env.get("ESP32_SPIFFS_IMAGE_NAME", filesystem),
@@ -401,7 +842,7 @@ env.Append(
             action=env.VerboseAction(
                 " ".join(
                     [
-                        '"$PYTHONEXE" "$OBJCOPY"',
+                        "$OBJCOPY",
                         "--chip",
                         mcu,
                         "elf2image",
@@ -444,10 +885,20 @@ env.Append(
 if not env.get("PIOFRAMEWORK"):
     env.SConscript("frameworks/_bare.py", exports="env")
 
+
+# Disable LDF for filesystem operations
+switch_off_ldf()
+
+
 def firmware_metrics(target, source, env):
     """
-    Custom target to run esp-idf-size with support for command line parameters
+    Custom target to run esp-idf-size with support for command line parameters.
     Usage: pio run -t metrics -- [esp-idf-size arguments]
+    
+    Args:
+        target: SCons target
+        source: SCons source
+        env: SCons environment object
     """
     if terminal_cp != "utf-8":
         print("Firmware metrics can not be shown. Set the terminal codepage to \"utf-8\"")
@@ -463,12 +914,8 @@ def firmware_metrics(target, source, env):
         print("Make sure the project is built first with 'pio run'")
         return
 
-    try:
+    try:        
-        import subprocess
+        cmd = [PYTHON_EXE, "-m", "esp_idf_size", "--ng"]
-        import sys
-        import shlex
-        
-        cmd = [env.subst("$PYTHONEXE"), "-m", "esp_idf_size", "--ng"]
         
         # Parameters from platformio.ini
         extra_args = env.GetProjectOption("custom_esp_idf_size_args", "")
@@ -494,8 +941,8 @@ def firmware_metrics(target, source, env):
         if env.GetProjectOption("custom_esp_idf_size_verbose", False):
             print(f"Running command: {' '.join(cmd)}")
         
-        # Call esp-idf-size
+        # Call esp-idf-size with modified environment
-        result = subprocess.run(cmd, check=False, capture_output=False)
+        result = subprocess.run(cmd, check=False, capture_output=False, env=os.environ)
         
         if result.returncode != 0:
             print(f"Warning: esp-idf-size exited with code {result.returncode}")
@@ -510,6 +957,7 @@ def firmware_metrics(target, source, env):
         print(f"Error: Failed to run firmware metrics: {e}")
         print("Make sure esp-idf-size is installed: pip install esp-idf-size")
 
+
 #
 # Target: Build executable and linkable firmware or FS image
 #
@@ -595,7 +1043,7 @@ if upload_protocol == "espota":
     env.Replace(
         UPLOADER=join(FRAMEWORK_DIR, "tools", "espota.py"),
         UPLOADERFLAGS=["--debug", "--progress", "-i", "$UPLOAD_PORT"],
-        UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS -f $SOURCE',
+        UPLOADCMD=f'"{PYTHON_EXE}" "$UPLOADER" $UPLOADERFLAGS -f $SOURCE',
     )
     if set(["uploadfs", "uploadfsota"]) & set(COMMAND_LINE_TARGETS):
         env.Append(UPLOADERFLAGS=["--spiffs"])
@@ -604,9 +1052,7 @@ if upload_protocol == "espota":
 # Configure upload protocol: esptool
 elif upload_protocol == "esptool":
     env.Replace(
-        UPLOADER=join(
+        UPLOADER=objcopy_value,
-            platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"
-        ),
         UPLOADERFLAGS=[
             "--chip",
             mcu,
@@ -627,8 +1073,7 @@ elif upload_protocol == "esptool":
             "--flash-size",
             "detect",
         ],
-        UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS '
+        UPLOADCMD='$UPLOADER $UPLOADERFLAGS $ESP32_APP_OFFSET $SOURCE'
-        "$ESP32_APP_OFFSET $SOURCE",
     )
     for image in env.get("FLASH_EXTRA_IMAGES", []):
         env.Append(UPLOADERFLAGS=[image[0], env.subst(image[1])])
@@ -656,7 +1101,7 @@ elif upload_protocol == "esptool":
                 "detect",
                 "$FS_START",
             ],
-            UPLOADCMD='"$PYTHONEXE" "$UPLOADER" $UPLOADERFLAGS $SOURCE',
+            UPLOADCMD='"$UPLOADER" $UPLOADERFLAGS $SOURCE',
         )
 
     upload_actions = [