Release 55.03.30-2

builder/main.py

@@ -34,6 +34,16 @@ from SCons.Script import (
 from platformio.project.helpers import get_project_dir
 from platformio.package.version import pepver_to_semver
 from platformio.util import get_serial_ports
+from platformio.compat import IS_WINDOWS
+
+# Check Python version requirement
+if sys.version_info < (3, 10):
+    sys.stderr.write(
+        f"Error: Python 3.10 or higher is required. "
+        f"Current version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\n"
+        f"Please update your Python installation.\n"
+    )
+    sys.exit(1)
 
 # Python dependencies required for the build process
 python_deps = {
@@ -56,6 +66,68 @@ PYTHON_EXE = env.subst("$PYTHONEXE")  # Global Python executable path
 # Framework directory path
 FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
 
+platformio_dir = projectconfig.get("platformio", "core_dir")
+penv_dir = os.path.join(platformio_dir, "penv")
+
+pip_path = os.path.join(
+    penv_dir,
+    "Scripts" if IS_WINDOWS else "bin",
+    "pip" + (".exe" if IS_WINDOWS else ""),
+)
+
+def setup_pipenv_in_package():
+    """
+    Checks if 'penv' folder exists in platformio dir and creates virtual environment if not.
+    """
+    if not os.path.exists(penv_dir):
+        env.Execute(
+            env.VerboseAction(
+                '"$PYTHONEXE" -m venv --clear "%s"' % penv_dir,
+                "Creating a new virtual environment for Python dependencies",
+            )
+        )
+
+        assert os.path.isfile(
+            pip_path
+        ), "Error: Failed to create a proper virtual environment. Missing the `pip` binary!"
+
+    penv_python = os.path.join(penv_dir, "Scripts", "python.exe") if IS_WINDOWS else os.path.join(penv_dir, "bin", "python")
+    env.Replace(PYTHONEXE=penv_python)
+    print(f"PYTHONEXE updated to penv environment: {penv_python}")
+
+setup_pipenv_in_package()
+# Update global PYTHON_EXE variable after potential pipenv setup
+PYTHON_EXE = env.subst("$PYTHONEXE")
+python_exe = PYTHON_EXE
+
+# Ensure penv Python directory is in PATH for subprocess calls
+python_dir = os.path.dirname(PYTHON_EXE)
+current_path = os.environ.get("PATH", "")
+if python_dir not in current_path:
+    os.environ["PATH"] = python_dir + os.pathsep + current_path
+
+# Verify the Python executable exists
+assert os.path.isfile(PYTHON_EXE), f"Python executable not found: {PYTHON_EXE}"
+
+if os.path.isfile(python_exe):
+    # Update sys.path to include penv site-packages
+    if IS_WINDOWS:
+        penv_site_packages = os.path.join(penv_dir, "Lib", "site-packages")
+    else:
+        # Find the actual site-packages directory in the venv
+        penv_lib_dir = os.path.join(penv_dir, "lib")
+        if os.path.isdir(penv_lib_dir):
+            for python_dir in os.listdir(penv_lib_dir):
+                if python_dir.startswith("python"):
+                    penv_site_packages = os.path.join(penv_lib_dir, python_dir, "site-packages")
+                    break
+            else:
+                penv_site_packages = None
+        else:
+            penv_site_packages = None
+
+    if penv_site_packages and os.path.isdir(penv_site_packages) and penv_site_packages not in sys.path:
+        sys.path.insert(0, penv_site_packages)
 
 def add_to_pythonpath(path):
     """
@@ -80,14 +152,10 @@ def add_to_pythonpath(path):
     if normalized_path not in sys.path:
         sys.path.insert(0, normalized_path)
 
-
 def setup_python_paths():
     """
     Setup Python paths based on the actual Python executable being used.
-    """
+    """    
-    if not PYTHON_EXE or not os.path.isfile(PYTHON_EXE):
-        return
-    
     # Get the directory containing the Python executable
     python_dir = os.path.dirname(PYTHON_EXE)
     add_to_pythonpath(python_dir)
@@ -107,7 +175,6 @@ def setup_python_paths():
 # Setup Python paths based on the actual Python executable
 setup_python_paths()
 
-
 def _get_executable_path(python_exe, executable_name):
     """
     Get the path to an executable binary (esptool, uv, etc.) based on the Python executable path.
@@ -119,14 +186,11 @@ def _get_executable_path(python_exe, executable_name):
     Returns:
         str: Path to executable or fallback to executable name
     """
-    if not python_exe or not os.path.isfile(python_exe):
-        return executable_name  # Fallback to command name
     
     python_dir = os.path.dirname(python_exe)
     
-    if sys.platform == "win32":
+    if IS_WINDOWS:
-        scripts_dir = os.path.join(python_dir, "Scripts")
+        executable_path = os.path.join(python_dir, f"{executable_name}.exe")
-        executable_path = os.path.join(scripts_dir, f"{executable_name}.exe")
     else:
         # For Unix-like systems, executables are typically in the same directory as python
         # or in a bin subdirectory
@@ -228,7 +292,7 @@ def install_python_deps():
             uv_executable = _get_uv_executable_path(PYTHON_EXE)
             
             # Add Scripts directory to PATH for Windows
-            if sys.platform == "win32":
+            if IS_WINDOWS:
                 python_dir = os.path.dirname(PYTHON_EXE)
                 scripts_dir = os.path.join(python_dir, "Scripts")
                 if os.path.isdir(scripts_dir):
@@ -366,8 +430,10 @@ def install_esptool():
     return 'esptool'  # Fallback
 
 
-# Install Python dependencies and esptool
+# Install Python dependencies
 install_python_deps()
+
+# Install esptool after dependencies
 esptool_binary_path = install_esptool()
 
 
@@ -756,7 +822,6 @@ objcopy_value = (
     if ' ' in esptool_binary_path 
     else esptool_binary_path
 )
-
 # Configure build tools and environment variables
 env.Replace(
     __get_board_boot_mode=_get_board_boot_mode,

platform.json

@@ -92,13 +92,14 @@
       "type": "uploader",
       "optional": true,
       "owner": "pioarduino",
-      "package-version": "5.0.1",
+      "package-version": "5.0.2",
-      "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/esptoolpy-v5.0.1.zip"
+      "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/esptoolpy-v5.0.2.zip"
     },
-    "tl-install": {
+    "tool-esp_install": {
       "type": "tool",
       "optional": false,
       "owner": "pioarduino",
+      "package-version": "5.1.0",
       "version": "https://github.com/pioarduino/esp_install/releases/download/v5.1.0/esp_install-v5.1.0.zip"
     },
     "contrib-piohome": {

platform.py

@@ -31,6 +31,7 @@ RETRY_LIMIT = 3
 SUBPROCESS_TIMEOUT = 300
 DEFAULT_DEBUG_SPEED = "5000"
 DEFAULT_APP_OFFSET = "0x10000"
+tl_install_name = "tool-esp_install"
 
 # MCUs that support ESP-builtin debug
 ESP_BUILTIN_DEBUG_MCUS = frozenset([
@@ -98,6 +99,15 @@ def safe_file_operation(operation_func):
     return wrapper
 
 
+@safe_file_operation
+def safe_remove_file(path: str) -> bool:
+    """Safely remove a file with error handling."""
+    if os.path.exists(path) and os.path.isfile(path):
+        os.remove(path)
+        logger.debug(f"File removed: {path}")
+    return True
+
+
 @safe_file_operation
 def safe_remove_directory(path: str) -> bool:
     """Safely remove directories with error handling."""
@@ -130,6 +140,15 @@ def safe_copy_file(src: str, dst: str) -> bool:
     return True
 
 
+@safe_file_operation
+def safe_copy_directory(src: str, dst: str) -> bool:
+    """Safely copy directories with error handling."""
+    os.makedirs(os.path.dirname(dst), exist_ok=True)
+    shutil.copytree(src, dst, dirs_exist_ok=True)
+    logger.debug(f"Directory copied: {src} -> {dst}")
+    return True
+
+
 class Espressif32Platform(PlatformBase):
     """ESP32 platform implementation for PlatformIO with optimized toolchain management."""
 
@@ -148,22 +167,183 @@ class Espressif32Platform(PlatformBase):
             self._packages_dir = config.get("platformio", "packages_dir")
         return self._packages_dir
 
+    def _check_tl_install_version(self) -> bool:
+        """
+        Check if tool-esp_install is installed in the correct version.
+        Install the correct version only if version differs.
+        
+        Returns:
+            bool: True if correct version is available, False on error
+        """
+        
+        # Get required version from platform.json
+        required_version = self.packages.get(tl_install_name, {}).get("version")
+        if not required_version:
+            logger.debug(f"No version check required for {tl_install_name}")
+            return True
+        
+        # Check if tool is already installed
+        tl_install_path = os.path.join(self.packages_dir, tl_install_name)
+        package_json_path = os.path.join(tl_install_path, "package.json")
+        
+        if not os.path.exists(package_json_path):
+            logger.info(f"{tl_install_name} not installed, installing version {required_version}")
+            return self._install_tl_install(required_version)
+        
+        # Read installed version
+        try:
+            with open(package_json_path, 'r', encoding='utf-8') as f:
+                package_data = json.load(f)
+            
+            installed_version = package_data.get("version")
+            if not installed_version:
+                logger.warning(f"Installed version for {tl_install_name} unknown, installing {required_version}")
+                return self._install_tl_install(required_version)
+            
+            # IMPORTANT: Compare versions correctly
+            if self._compare_tl_install_versions(installed_version, required_version):
+                logger.debug(f"{tl_install_name} version {installed_version} is already correctly installed")
+                # IMPORTANT: Set package as available, but do NOT reinstall
+                self.packages[tl_install_name]["optional"] = True
+                return True
+            else:
+                logger.info(
+                    f"Version mismatch for {tl_install_name}: "
+                    f"installed={installed_version}, required={required_version}, installing correct version"
+                )
+                return self._install_tl_install(required_version)
+            
+        except (json.JSONDecodeError, FileNotFoundError) as e:
+            logger.error(f"Error reading package data for {tl_install_name}: {e}")
+            return self._install_tl_install(required_version)
+
+    def _compare_tl_install_versions(self, installed: str, required: str) -> bool:
+        """
+        Compare installed and required version of tool-esp_install.
+        
+        Args:
+            installed: Currently installed version string
+            required: Required version string from platform.json
+            
+        Returns:
+            bool: True if versions match, False otherwise
+        """
+        # For URL-based versions: Extract version string from URL
+        installed_clean = self._extract_version_from_url(installed)
+        required_clean = self._extract_version_from_url(required)
+        
+        logger.debug(f"Version comparison: installed='{installed_clean}' vs required='{required_clean}'")
+        
+        return installed_clean == required_clean
+
+    def _extract_version_from_url(self, version_string: str) -> str:
+        """
+        Extract version information from URL or return version directly.
+        
+        Args:
+            version_string: Version string or URL containing version
+            
+        Returns:
+            str: Extracted version string
+        """
+        if version_string.startswith(('http://', 'https://')):
+            # Extract version from URL like: .../v5.1.0/esp_install-v5.1.0.zip
+            import re
+            version_match = re.search(r'v(\d+\.\d+\.\d+)', version_string)
+            if version_match:
+                return version_match.group(1)  # Returns "5.1.0"
+            else:
+                # Fallback: Use entire URL
+                return version_string
+        else:
+            # Direct version number
+            return version_string.strip()
+
+    def _install_tl_install(self, version: str) -> bool:
+        """
+        Install tool-esp_install ONLY when necessary
+        and handles backwards compatibility for tl-install.
+
+        Args:
+            version: Version string or URL to install
+   
+        Returns:
+            bool: True if installation successful, False otherwise
+        """
+        tl_install_path = os.path.join(self.packages_dir, tl_install_name)
+        old_tl_install_path = os.path.join(self.packages_dir, "tl-install")
+
+        try:
+            old_tl_install_exists = os.path.exists(old_tl_install_path)
+            if old_tl_install_exists:
+                # remove outdated tl-install
+                safe_remove_directory(old_tl_install_path)
+
+            if os.path.exists(tl_install_path):
+                logger.info(f"Removing old {tl_install_name} installation")
+                safe_remove_directory(tl_install_path)
+
+            logger.info(f"Installing {tl_install_name} version {version}")
+            self.packages[tl_install_name]["optional"] = False
+            self.packages[tl_install_name]["version"] = version
+            pm.install(version)
+            # Ensure backward compatibility by removing pio install status indicator
+            tl_piopm_path = os.path.join(tl_install_path, ".piopm")
+            safe_remove_file(tl_piopm_path)
+
+            if os.path.exists(os.path.join(tl_install_path, "package.json")):
+                logger.info(f"{tl_install_name} successfully installed and verified")
+                self.packages[tl_install_name]["optional"] = True
+            
+                # Handle old tl-install to keep backwards compatibility
+                if old_tl_install_exists:
+                    # Copy tool-esp_install content to tl-install location
+                    if safe_copy_directory(tl_install_path, old_tl_install_path):
+                        logger.info(f"Content copied from {tl_install_name} to old tl-install location")
+                    else:
+                        logger.warning("Failed to copy content to old tl-install location")
+                return True
+            else:
+                logger.error(f"{tl_install_name} installation failed - package.json not found")
+                return False
+        
+        except Exception as e:
+            logger.error(f"Error installing {tl_install_name}: {e}")
+            return False
+
+    def _cleanup_versioned_tool_directories(self, tool_name: str) -> None:
+        """
+        Clean up versioned tool directories containing '@' or version suffixes.
+        This function should be called during every tool version check.
+        
+        Args:
+            tool_name: Name of the tool to clean up
+        """
+        if not os.path.exists(self.packages_dir) or not os.path.isdir(self.packages_dir):
+            return
+            
+        try:
+            # Remove directories with '@' in their name (e.g., tool-name@version, tool-name@src)
+            safe_remove_directory_pattern(self.packages_dir, f"{tool_name}@*")
+            
+            # Remove directories with version suffixes (e.g., tool-name.12345)
+            safe_remove_directory_pattern(self.packages_dir, f"{tool_name}.*")
+            
+            # Also check for any directory that starts with tool_name and contains '@'
+            for item in os.listdir(self.packages_dir):
+                if item.startswith(tool_name) and '@' in item:
+                    item_path = os.path.join(self.packages_dir, item)
+                    if os.path.isdir(item_path):
+                        safe_remove_directory(item_path)
+                        logger.debug(f"Removed versioned directory: {item_path}")
+                        
+        except OSError as e:
+            logger.error(f"Error cleaning up versioned directories for {tool_name}: {e}")
+
     def _get_tool_paths(self, tool_name: str) -> Dict[str, str]:
         """Get centralized path calculation for tools with caching."""
         if tool_name not in self._tools_cache:
             tool_path = os.path.join(self.packages_dir, tool_name)
-            # Remove all directories containing '@' in their name
-            try:
-                if os.path.exists(self.packages_dir) and os.path.isdir(self.packages_dir):
-                    for item in os.listdir(self.packages_dir):
-                        if '@' in item and item.startswith(tool_name):
-                            item_path = os.path.join(self.packages_dir, item)
-                            if os.path.isdir(item_path):
-                                safe_remove_directory(item_path)
-                                logger.debug(f"Removed directory with '@' in name: {item_path}")
-
-            except OSError as e:
-                logger.error(f"Error scanning packages directory for '@' directories: {e}")
             
             self._tools_cache[tool_name] = {
                 'tool_path': tool_path,
@@ -171,7 +351,7 @@ class Espressif32Platform(PlatformBase):
                 'tools_json_path': os.path.join(tool_path, "tools.json"),
                 'piopm_path': os.path.join(tool_path, ".piopm"),
                 'idf_tools_path': os.path.join(
-                    self.packages_dir, "tl-install", "tools", "idf_tools.py"
+                    self.packages_dir, tl_install_name, "tools", "idf_tools.py"
                 )
             }
         return self._tools_cache[tool_name]
@@ -223,6 +403,9 @@ class Espressif32Platform(PlatformBase):
 
     def _check_tool_version(self, tool_name: str) -> bool:
         """Check if the installed tool version matches the required version."""
+        # Clean up versioned directories FIRST, before any version checks
+        self._cleanup_versioned_tool_directories(tool_name)
+        
         paths = self._get_tool_paths(tool_name)
 
         try:
@@ -314,23 +497,16 @@ class Espressif32Platform(PlatformBase):
             logger.debug(f"Tool {tool_name} found with correct version")
             return True
 
-        # Wrong version, reinstall - remove similar paths too
+        # Wrong version, reinstall - cleanup is already done in _check_tool_version
         logger.info(f"Reinstalling {tool_name} due to version mismatch")
-    
+
-        tool_base_name = os.path.basename(paths['tool_path'])
+        # Remove the main tool directory (if it still exists after cleanup)
-        packages_dir = os.path.dirname(paths['tool_path'])
-    
-        # Remove similar directories with version suffixes FIRST (e.g., xtensa@src, xtensa.12232)
-        safe_remove_directory_pattern(packages_dir, f"{tool_base_name}@*")
-        safe_remove_directory_pattern(packages_dir, f"{tool_base_name}.*")
-    
-        # Then remove the main tool directory (if it still exists)
         safe_remove_directory(paths['tool_path'])
 
         return self.install_tool(tool_name, retry_count + 1)
 
     def _configure_arduino_framework(self, frameworks: List[str]) -> None:
-        """Configure Arduino framework"""
+        """Configure Arduino framework dependencies."""
         if "arduino" not in frameworks:
             return
 
@@ -403,12 +579,28 @@ class Espressif32Platform(PlatformBase):
             self.install_tool("tool-openocd-esp32")
 
     def _configure_installer(self) -> None:
-        """Configure the ESP-IDF tools installer."""
+        """Configure the ESP-IDF tools installer with proper version checking."""
+        
+        # Check version - installs only when needed
+        if not self._check_tl_install_version():
+            logger.error("Error during tool-esp_install version check / installation")
+            return
+
+        # Remove pio install marker to avoid issues when switching versions
+        old_tl_piopm_path = os.path.join(self.packages_dir, "tl-install", ".piopm")
+        if os.path.exists(old_tl_piopm_path):
+            safe_remove_file(old_tl_piopm_path)
+        
+        # Check if idf_tools.py is available
         installer_path = os.path.join(
-            self.packages_dir, "tl-install", "tools", "idf_tools.py"
+            self.packages_dir, tl_install_name, "tools", "idf_tools.py"
         )
+        
         if os.path.exists(installer_path):
-            self.packages["tl-install"]["optional"] = True
+            logger.debug(f"{tl_install_name} is available and ready")
+            self.packages[tl_install_name]["optional"] = True
+        else:
+            logger.warning(f"idf_tools.py not found in {installer_path}")
 
     def _install_esptool_package(self) -> None:
         """Install esptool package required for all builds."""
@@ -443,7 +635,7 @@ class Espressif32Platform(PlatformBase):
                     os.remove(piopm_path)
                     logger.info(f"Incompatible mklittlefs version {version} removed (required: 3.x)")
             except (json.JSONDecodeError, KeyError) as e:
-                logger.error(f"Error reading mklittlefs package data: {e}")
+                logger.error(f"Error reading mklittlefs package  {e}")
 
     def _setup_mklittlefs_for_download(self) -> None:
         """Setup mklittlefs for download functionality with version 4.x."""