add Hybrid Compile (#73)

.github/workflows/examples.yml

@@ -20,6 +20,7 @@ jobs:
           - "examples/arduino-wifiscan"
           - "examples/arduino-zigbee-light"
           - "examples/arduino-zigbee-switch"
+          - "examples/tasmota"
           - "examples/espidf-arduino-h2zero-BLE_scan"
           - "examples/espidf-arduino-matter-light"
           - "examples/espidf-arduino-blink"
@@ -49,7 +50,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install wheel
           pip install -U https://github.com/platformio/platformio/archive/develop.zip
-          pio pkg install --global --platform symlink://.
+          pio pkg install --global --platform file://.
         if: "matrix.os == 'windows-2022'"
         env:
           PLATFORMIO_CORE_DIR: C:\plat
@@ -57,7 +58,7 @@ jobs:
           PLATFORMIO_PLATFORMS_DIR: C:\plat\plat
       - name: Build Windows examples
         run: pio run -d ${{ matrix.example }}
-        if: "matrix.os == 'windows-2022'"
+        if: "matrix.example != 'examples/tasmota' && matrix.os == 'windows-2022'"
         env:
           PLATFORMIO_CORE_DIR: C:\plat
           PLATFORMIO_PACKAGES_DIR: C:\plat\pack
@@ -67,8 +68,13 @@ jobs:
           python -m pip install --upgrade pip
           pip install wheel
           pip install -U https://github.com/platformio/platformio/archive/develop.zip
-          pio pkg install --global --platform symlink://.
+          pio pkg install --global --platform file://.
         if: "matrix.os != 'windows-2022'"
+      - name: git clone Tasmota and add to examples
+        run: |
+          git clone -b development --depth 1 https://github.com/arendst/Tasmota.git examples/tasmota
+          cp examples/tasmota_platformio_override.ini examples/tasmota/platformio_override.ini
+        if: "matrix.example == 'examples/tasmota' && matrix.os != 'windows-2022'"
       - name: Build Ubuntu Mac examples
         run: pio run -d ${{ matrix.example }}
         if: "matrix.os != 'windows-2022'"

README.md

@@ -50,7 +50,7 @@ based on devel branch Arduino 3.1.0 and IDF 5.3.1+
 
 ```ini
 [env:development]
-platform = https://github.com/pioarduino/platform-espressif32.git#Arduino/IDF53
+platform = https://github.com/pioarduino/platform-espressif32.git#Arduino/IDF53_work_hybrid
 board = ...
 ...
 ```

boards/esp32-solo1.json

@@ -1,11 +1,8 @@
 {
   "build": {
-    "arduino":{
-      "ldscript": "esp32_out.ld"
-    },
     "core": "esp32",
     "extra_flags": "-DARDUINO_ESP32_DEV -DARDUINO_USB_CDC_ON_BOOT=0 -DCORE32SOLO1",
-    "f_cpu": "80000000L",
+    "f_cpu": "160000000L",
     "f_flash": "40000000L",
     "flash_mode": "dio",
     "mcu": "esp32",
@@ -32,6 +29,11 @@
     "require_upload_port": true,
     "speed": 460800
   },
+  "espidf": {
+    "custom_sdkconfig": [
+      "CONFIG_FREERTOS_UNICORE=y" 
+    ]
+  },
   "url": "https://en.wikipedia.org/wiki/ESP32",
   "vendor": "Espressif"
 }

boards/esp32s3_120_16_8-qio_opi.json

@@ -0,0 +1,50 @@
+{
+  "build": {
+    "arduino": {
+      "memory_type": "qio_opi",
+      "partitions": "default_16MB.csv"
+    },
+    "core": "esp32",
+    "f_cpu": "240000000L",
+    "f_flash": "80000000L",
+    "f_boot": "120000000L",
+    "boot": "qio",
+    "flash_mode": "qio",
+    "extra_flags": [
+      "-DBOARD_HAS_PSRAM",
+      "-DARDUINO_USB_MODE=1",
+      "-DARDUINO_USB_CDC_ON_BOOT=1",
+      "-DARDUINO_RUNNING_CORE=1",
+      "-DARDUINO_EVENT_RUNNING_CORE=1"
+    ],
+    "hwids": [
+      [
+        "0x303A",
+        "0x1001"
+      ]
+    ],
+    "mcu": "esp32s3",
+    "variant": "esp32s3"
+  },
+  "connectivity": [
+    "bluetooth",
+    "wifi"
+  ],
+  "debug": {
+    "openocd_target": "esp32s3.cfg"
+  },
+  "frameworks": [
+    "arduino",
+    "espidf"
+  ],
+  "name": "ESP32-S3 16MB QIO, 8MB OPI PSRAM",
+  "upload": {
+    "flash_size": "16MB",
+    "maximum_ram_size": 327680,
+    "maximum_size": 16777216,
+    "require_upload_port": true,
+    "speed": 460800
+  },
+  "url": "https://www.espressif.com/sites/default/files/documentation/esp32-s3-wroom-1_wroom-1u_datasheet_en.pdf",
+  "vendor": "espressif"
+}

builder/build_lib/CMakeLists.txt

@@ -0,0 +1 @@
+idf_component_register(SRCS "sketch.cpp" "arduino-lib-builder-gcc.c" "arduino-lib-builder-cpp.cpp" "arduino-lib-builder-as.S" INCLUDE_DIRS ".")

builder/build_lib/idf_component.yml

@@ -0,0 +1,12 @@
+dependencies:
+  # Required IDF version
+  idf: ">=5.1"
+  espressif/cmake_utilities:
+    version: "0.*"
+  espressif/fb_gfx:
+    version: "master"
+    path: components/fb_gfx
+    git: https://github.com/espressif/esp32-arduino-lib-builder.git
+    require: public
+    rules:
+      - if: "target in [esp32, esp32s2, esp32s3, esp32p4]"

builder/build_lib/sketch.cpp

@@ -0,0 +1,10 @@
+#include "Arduino.h"
+
+void setup() {
+  Serial.begin(115200);
+}
+
+void loop() {
+  Serial.println("Hello World!");
+  delay(1000);
+}

builder/frameworks/arduino.py

@@ -22,23 +22,179 @@ kinds of creative coding, interactive objects, spaces or physical experiences.
 http://arduino.cc/en/Reference/HomePage
 """
 
+import subprocess
+import json
+import semantic_version
 import os
+import shutil
 from os.path import join
 
-from SCons.Script import DefaultEnvironment, SConscript
+from SCons.Script import COMMAND_LINE_TARGETS, DefaultEnvironment, SConscript
+from platformio.package.version import pepver_to_semver
+from platformio.project.config import ProjectConfig
+from platformio.package.manager.tool import ToolPackageManager
 
 env = DefaultEnvironment()
+pm = ToolPackageManager()
+platform = env.PioPlatform()
+config = env.GetProjectConfig()
 board = env.BoardConfig()
-build_core = board.get("build.core", "").lower()
+mcu = board.get("build.mcu", "esp32")
+board_sdkconfig = board.get("espidf.custom_sdkconfig", "")
+entry_custom_sdkconfig = "\n"
+flag_custom_sdkconfig = False
+
+if config.has_option("env:"+env["PIOENV"], "custom_sdkconfig"):
+    entry_custom_sdkconfig = env.GetProjectOption("custom_sdkconfig")
+    flag_custom_sdkconfig = True
+
+if len(str(board_sdkconfig)) > 2:
+    flag_custom_sdkconfig = True
+
+extra_flags = (''.join([element for element in board.get("build.extra_flags", "")])).replace("-D", " ")
+framework_reinstall = False
+flag_any_custom_sdkconfig = False
 
 SConscript("_embed_files.py", exports="env")
 
-if "espidf" not in env.subst("$PIOFRAMEWORK"):
+flag_any_custom_sdkconfig = os.path.exists(join(platform.get_package_dir("framework-arduinoespressif32-libs"),"sdkconfig"))
-    if os.path.exists(join(DefaultEnvironment().PioPlatform().get_package_dir(
+
+# Esp32-solo1 libs needs adopted settings
+if flag_custom_sdkconfig == True and ("CORE32SOLO1" in extra_flags or "CONFIG_FREERTOS_UNICORE=y" in entry_custom_sdkconfig or "CONFIG_FREERTOS_UNICORE=y" in board_sdkconfig):
+    if len(str(env.GetProjectOption("build_unflags"))) == 2: # No valid env, needs init
+        env['BUILD_UNFLAGS'] = {}
+    build_unflags = " ".join(env['BUILD_UNFLAGS'])
+    build_unflags = build_unflags + " -mdisable-hardware-atomics -ustart_app_other_cores"
+    new_build_unflags = build_unflags.split()
+    env.Replace(
+      BUILD_UNFLAGS=new_build_unflags
+    )
+
+def install_python_deps():
+    def _get_installed_pip_packages():
+        result = {}
+        packages = {}
+        pip_output = subprocess.check_output(
+            [
+                env.subst("$PYTHONEXE"),
+                "-m",
+                "pip",
+                "list",
+                "--format=json",
+                "--disable-pip-version-check",
+            ]
+        )
+        try:
+            packages = json.loads(pip_output)
+        except:
+            print("Warning! Couldn't extract the list of installed Python packages.")
+            return {}
+        for p in packages:
+            result[p["name"]] = pepver_to_semver(p["version"])
+
+        return result
+
+    deps = {
+        "wheel": ">=0.35.1",
+        "PyYAML": ">=6.0.2",
+        "intelhex": ">=2.3.0"
+    }
+
+    installed_packages = _get_installed_pip_packages()
+    packages_to_install = []
+    for package, spec in deps.items():
+        if package not in installed_packages:
+            packages_to_install.append(package)
+        else:
+            version_spec = semantic_version.Spec(spec)
+            if not version_spec.match(installed_packages[package]):
+                packages_to_install.append(package)
+
+    if packages_to_install:
+        env.Execute(
+            env.VerboseAction(
+                (
+                    '"$PYTHONEXE" -m pip install -U '
+                    + " ".join(
+                        [
+                            '"%s%s"' % (p, deps[p])
+                            for p in packages_to_install
+                        ]
+                    )
+                ),
+                "Installing Arduino Python dependencies",
+            )
+        )
+    return
+
+install_python_deps()
+
+def get_MD5_hash(phrase):
+    import hashlib
+    return hashlib.md5((phrase).encode('utf-8')).hexdigest()[:16]
+
+
+def matching_custom_sdkconfig():
+    # check if current env is matching to existing sdkconfig
+    cust_sdk_is_present = False
+    matching_sdkconfig = False
+    last_sdkconfig_path = join(env.subst("$PROJECT_DIR"),"sdkconfig.defaults")
+    if flag_any_custom_sdkconfig == False:
+        matching_sdkconfig = True
+        return matching_sdkconfig, cust_sdk_is_present
+    if os.path.exists(last_sdkconfig_path) == False:
+        return matching_sdkconfig, cust_sdk_is_present
+    if flag_custom_sdkconfig == False:
+        matching_sdkconfig = False
+        return matching_sdkconfig, cust_sdk_is_present
+    with open(last_sdkconfig_path) as src:
+        line = src.readline()
+        if line.startswith("# TASMOTA__"):
+            cust_sdk_is_present = True;
+            costum_options = entry_custom_sdkconfig
+            if (line.split("__")[1]).strip() == get_MD5_hash((costum_options).strip() + mcu):
+                matching_sdkconfig = True
+
+    return matching_sdkconfig, cust_sdk_is_present
+
+def check_reinstall_frwrk():
+    framework_reinstall = False
+    cust_sdk_is_present = False
+    matching_sdkconfig = False
+    if flag_custom_sdkconfig == True:
+        matching_sdkconfig, cust_sdk_is_present = matching_custom_sdkconfig()
+    if flag_custom_sdkconfig == False and flag_any_custom_sdkconfig == True:
+        # case custom sdkconfig exists and a env without "custom_sdkconfig"
+        framework_reinstall = True
+    if flag_custom_sdkconfig == True  and matching_sdkconfig == False:
+        # check if current custom sdkconfig is different from existing
+        framework_reinstall = True
+    return framework_reinstall
+
+def call_compile_libs():
+    if mcu == "esp32c2":
+        ARDUINO_FRMWRK_C2_LIB_DIR = join(platform.get_package_dir("framework-arduinoespressif32-libs"),mcu)
+        ARDUINO_C2_DIR = join(platform.get_package_dir("framework-arduino-c2-skeleton-lib"),mcu)
+        shutil.copytree(ARDUINO_C2_DIR, ARDUINO_FRMWRK_C2_LIB_DIR, dirs_exist_ok=True)
+    print("*** Compile Arduino IDF libs for %s ***" % env["PIOENV"])
+    SConscript("espidf.py")
+
+if check_reinstall_frwrk() == True:
+    print("*** Reinstall Arduino framework libs ***")
+    shutil.rmtree(platform.get_package_dir("framework-arduinoespressif32-libs"))
+    ARDUINO_FRMWRK_LIB_URL = str(platform.get_package_spec("framework-arduinoespressif32-libs")).split("uri=",1)[1][:-1]
+    pm.install(ARDUINO_FRMWRK_LIB_URL)
+    if flag_custom_sdkconfig == True:
+        call_compile_libs()
+        flag_custom_sdkconfig = False
+    
+if flag_custom_sdkconfig == True and flag_any_custom_sdkconfig == False:
+    call_compile_libs()
+
+if "arduino" in env.subst("$PIOFRAMEWORK") and "espidf" not in env.subst("$PIOFRAMEWORK") and env.subst("$ARDUINO_LIB_COMPILE_FLAG") in ("Inactive", "True"):
+    if os.path.exists(join(platform.get_package_dir(
             "framework-arduinoespressif32"), "tools", "platformio-build.py")):
         PIO_BUILD = "platformio-build.py"
     else:
         PIO_BUILD = "pioarduino-build.py"
-    SConscript(
+    SConscript(join(platform.get_package_dir("framework-arduinoespressif32"), "tools", PIO_BUILD))
-        join(DefaultEnvironment().PioPlatform().get_package_dir(
-            "framework-arduinoespressif32"), "tools", PIO_BUILD))

builder/frameworks/espidf.py

@@ -26,7 +26,9 @@ import subprocess
 import sys
 import shutil
 import os
+from os.path import join
 import re
+import requests
 import platform as sys_platform
 
 import click
@@ -42,6 +44,7 @@ from platformio import fs, __version__
 from platformio.compat import IS_WINDOWS
 from platformio.proc import exec_command
 from platformio.builder.tools.piolib import ProjectAsLibBuilder
+from platformio.project.config import ProjectConfig
 from platformio.package.version import get_original_version, pepver_to_semver
 
 # Added to avoid conflicts between installed Python packages from
@@ -53,13 +56,78 @@ if os.environ.get("PYTHONPATH"):
 env = DefaultEnvironment()
 env.SConscript("_embed_files.py", exports="env")
 
+def install_standard_python_deps():
+    def _get_installed_standard_pip_packages():
+        result = {}
+        packages = {}
+        pip_output = subprocess.check_output(
+            [
+                env.subst("$PYTHONEXE"),
+                "-m",
+                "pip",
+                "list",
+                "--format=json",
+                "--disable-pip-version-check",
+            ]
+        )
+        try:
+            packages = json.loads(pip_output)
+        except:
+            print("Warning! Couldn't extract the list of installed Python packages.")
+            return {}
+        for p in packages:
+            result[p["name"]] = pepver_to_semver(p["version"])
+
+        return result
+
+    deps = {
+        "wheel": ">=0.35.1",
+        "PyYAML": ">=6.0.2"
+    }
+
+    installed_packages = _get_installed_standard_pip_packages()
+    packages_to_install = []
+    for package, spec in deps.items():
+        if package not in installed_packages:
+            packages_to_install.append(package)
+        else:
+            version_spec = semantic_version.Spec(spec)
+            if not version_spec.match(installed_packages[package]):
+                packages_to_install.append(package)
+
+    if packages_to_install:
+        env.Execute(
+            env.VerboseAction(
+                (
+                    '"$PYTHONEXE" -m pip install -U '
+                    + " ".join(
+                        [
+                            '"%s%s"' % (p, deps[p])
+                            for p in packages_to_install
+                        ]
+                    )
+                ),
+                "Installing standard Python dependencies",
+            )
+        )
+    return
+
+install_standard_python_deps()
+
 # Allow changes in folders of managed components
 os.environ["IDF_COMPONENT_OVERWRITE_MANAGED_COMPONENTS"] = "1"
 
 platform = env.PioPlatform()
+config = env.GetProjectConfig()
 board = env.BoardConfig()
 mcu = board.get("build.mcu", "esp32")
+flash_speed = board.get("build.f_flash", "40000000L")
+flash_frequency = str(flash_speed.replace("000000L", "m"))
+flash_mode = board.get("build.flash_mode", "dio")
 idf_variant = mcu.lower()
+flag_custom_sdkonfig = False
+flag_custom_component_add = False
+flag_custom_component_remove = False
 
 IDF5 = (
     platform.get_package_version("framework-espidf")
@@ -78,7 +146,6 @@ TOOLCHAIN_DIR = platform.get_package_dir(
 assert os.path.isdir(FRAMEWORK_DIR)
 assert os.path.isdir(TOOLCHAIN_DIR)
 
-# The latest IDF uses a standalone GDB package which requires at least PlatformIO 6.1.11
 if (
     ["espidf"] == env.get("PIOFRAMEWORK")
     and semantic_version.Version.coerce(__version__)
@@ -87,9 +154,10 @@ if (
 ):
     print("Warning! Debugging an IDF project requires PlatformIO Core >= 6.1.11!")
 
-# Arduino framework as a component is not compatible with ESP-IDF >5.2
+# Arduino framework as a component is not compatible with ESP-IDF >5.3
 if "arduino" in env.subst("$PIOFRAMEWORK"):
     ARDUINO_FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
+    ARDUINO_FRMWRK_LIB_DIR = platform.get_package_dir("framework-arduinoespressif32-libs")
     # Possible package names in 'package@version' format is not compatible with CMake
     if "@" in os.path.basename(ARDUINO_FRAMEWORK_DIR):
         new_path = os.path.join(
@@ -109,6 +177,208 @@ SDKCONFIG_PATH = os.path.expandvars(board.get(
         os.path.join(PROJECT_DIR, "sdkconfig.%s" % env.subst("$PIOENV")),
 ))
 
+#
+# generate modified Arduino IDF sdkconfig, applying settings from "custom_sdkconfig"
+#
+if config.has_option("env:"+env["PIOENV"], "custom_component_add"):
+    flag_custom_component_add = True
+if config.has_option("env:"+env["PIOENV"], "custom_component_remove"):
+    flag_custom_component_remove = True
+
+if config.has_option("env:"+env["PIOENV"], "custom_sdkconfig"):
+    flag_custom_sdkonfig = True
+if "espidf.custom_sdkconfig" in board:
+    flag_custom_sdkonfig = True
+
+def HandleArduinoIDFsettings(env):
+    def get_MD5_hash(phrase):
+        import hashlib
+        return hashlib.md5((phrase).encode('utf-8')).hexdigest()[:16]
+
+    def custom_sdkconfig_file(string):
+        if not config.has_option("env:"+env["PIOENV"], "custom_sdkconfig"):
+            return ""
+        sdkconfig_entrys = env.GetProjectOption("custom_sdkconfig").splitlines()
+        for file in sdkconfig_entrys:
+            if "http" in file and "://" in file:
+                response = requests.get(file.split(" ")[0])
+                if response.ok:
+                    target = str(response.content.decode('utf-8'))
+                else:
+                    print("Failed to download:", file)
+                    return ""
+                return target
+            if "file://" in file:
+                file_path = join(PROJECT_DIR,file.lstrip("file://").split(os.path.sep)[-1])
+                if os.path.exists(file_path):
+                    with open(file_path, 'r') as file:
+                        target = file.read()
+                else:
+                    print("File not found:", file_path)
+                    return ""
+                return target
+        return ""
+
+
+    custom_sdk_config_flags = ""
+    board_idf_config_flags = ""
+    sdkconfig_file_flags = ""
+    custom_sdkconfig_file_str = ""
+
+    if config.has_option("env:"+env["PIOENV"], "custom_sdkconfig"):
+        flag_custom_sdkonfig = True
+        custom_sdk_config_flags = (env.GetProjectOption("custom_sdkconfig").rstrip("\n")) + "\n"
+        custom_sdkconfig_file_str = custom_sdkconfig_file(sdkconfig_file_flags)
+
+    if "espidf.custom_sdkconfig" in board:
+        board_idf_config_flags = ('\n'.join([element for element in board.get("espidf.custom_sdkconfig", "")])).rstrip("\n") + "\n"
+        flag_custom_sdkonfig = True
+
+    if flag_custom_sdkonfig == True: # TDOO duplicated
+        print("*** Add \"custom_sdkconfig\" settings to IDF sdkconfig.defaults ***")
+        idf_config_flags = custom_sdk_config_flags
+        if custom_sdkconfig_file_str != "":
+            sdkconfig_file_flags = custom_sdkconfig_file_str + "\n"
+            idf_config_flags = sdkconfig_file_flags + idf_config_flags
+        idf_config_flags = board_idf_config_flags + idf_config_flags
+        if flash_frequency != "80m":
+            idf_config_flags = idf_config_flags + "# CONFIG_ESPTOOLPY_FLASHFREQ_80M is not set\n"
+            esptool_flashfreq_y = "CONFIG_ESPTOOLPY_FLASHFREQ_%s=y\n" % flash_frequency.upper()
+            esptool_flashfreq_M = "CONFIG_ESPTOOLPY_FLASHFREQ=\"%s\"\n" % flash_frequency
+            idf_config_flags = idf_config_flags + esptool_flashfreq_y + esptool_flashfreq_M
+        if flash_mode != "qio":
+            idf_config_flags = idf_config_flags + "# CONFIG_ESPTOOLPY_FLASHMODE_QIO is not set\n"
+        esptool_flashmode = "CONFIG_ESPTOOLPY_FLASHMODE_%s=y\n" % flash_mode.upper()
+        if esptool_flashmode not in idf_config_flags:
+            idf_config_flags = idf_config_flags + esptool_flashmode
+        if mcu in ("esp32") and "CONFIG_FREERTOS_UNICORE=y" in idf_config_flags:
+            idf_config_flags = idf_config_flags + "# CONFIG_SPIRAM is not set\n"
+
+        idf_config_flags = idf_config_flags.splitlines()
+        sdkconfig_src = join(ARDUINO_FRMWRK_LIB_DIR,mcu,"sdkconfig")
+
+        def get_flag(line):
+            if line.startswith("#") and "is not set" in line:
+                return line.split(" ")[1]
+            elif not line.startswith("#") and len(line.split("=")) > 1:
+                return line.split("=")[0]
+            else:
+                return None
+
+        with open(sdkconfig_src) as src:
+            sdkconfig_dst = os.path.join(PROJECT_DIR, "sdkconfig.defaults")
+            dst = open(sdkconfig_dst,"w")
+            dst.write("# TASMOTA__"+ get_MD5_hash(''.join(custom_sdk_config_flags).strip() + mcu) +"\n")
+            while line := src.readline():
+                flag = get_flag(line)
+                if flag is None:
+                    dst.write(line)
+                else:
+                    no_match = True
+                    for item in idf_config_flags:
+                        if flag == get_flag(item.replace("\'", "")):
+                            dst.write(item.replace("\'", "")+"\n")
+                            no_match = False
+                            print("Replace:",line,"with:",item.replace("\'", ""))
+                            idf_config_flags.remove(item)
+                    if no_match:
+                        dst.write(line)
+            for item in idf_config_flags: # are there new flags?
+                print("Add:",item.replace("\'", ""))
+                dst.write(item.replace("\'", "")+"\n")
+            dst.close()
+        return
+    else:
+        return
+
+def HandleCOMPONENTsettings(env):
+    if flag_custom_component_add == True or flag_custom_component_remove == True: # todo remove duplicated
+        import yaml
+        from yaml import SafeLoader
+        print("*** \"custom_component\" is used to select managed idf components ***")
+        if flag_custom_component_remove == True:
+            idf_custom_component_remove = env.GetProjectOption("custom_component_remove").splitlines()
+        else:
+            idf_custom_component_remove = ""
+        if flag_custom_component_add == True:
+            idf_custom_component_add = env.GetProjectOption("custom_component_add").splitlines()
+        else:
+            idf_custom_component_add = ""
+
+        # search "idf_component.yml" file
+        try: # 1.st in Arduino framework
+            idf_component_yml_src = os.path.join(ARDUINO_FRAMEWORK_DIR, "idf_component.yml")
+            shutil.copy(join(ARDUINO_FRAMEWORK_DIR,"idf_component.yml"),join(ARDUINO_FRAMEWORK_DIR,"idf_component.yml.orig"))
+            yml_file_dir = idf_component_yml_src
+        except: # 2.nd Project source
+            try:
+                idf_component_yml_src = os.path.join(PROJECT_SRC_DIR, "idf_component.yml")
+                shutil.copy(join(PROJECT_SRC_DIR,"idf_component.yml"),join(PROJECT_SRC_DIR,"idf_component.yml.orig"))
+                yml_file_dir = idf_component_yml_src
+            except: # no idf_component.yml in Project source -> create
+                idf_component_yml_src = os.path.join(PROJECT_SRC_DIR, "idf_component.yml")
+                yml_file_dir = idf_component_yml_src
+                idf_component_yml_str = """
+                    dependencies:
+                      idf: \">=5.1\"
+                """
+                idf_component_yml = yaml.safe_load(idf_component_yml_str)
+                with open(idf_component_yml_src, 'w',) as f :
+                    yaml.dump(idf_component_yml,f) 
+
+        yaml_file=open(idf_component_yml_src,"r")
+        idf_component=yaml.load(yaml_file, Loader=SafeLoader)
+        idf_component_str=json.dumps(idf_component)      # convert to json string
+        idf_component_json=json.loads(idf_component_str) # convert string to json dict
+
+        if idf_custom_component_remove != "":
+            for entry in idf_custom_component_remove:
+                # checking if the entry exists before removing
+                if entry in idf_component_json["dependencies"]:
+                    print("*** Removing component:",entry)
+                    del idf_component_json["dependencies"][entry]
+
+        if idf_custom_component_add != "":
+            for entry in idf_custom_component_add:
+                if len(str(entry)) > 4: # too short or empty entry
+                    # add new entrys to json
+                    if "@" in entry:
+                        idf_comp_entry = str(entry.split("@")[0]).replace(" ", "")
+                        idf_comp_vers = str(entry.split("@")[1]).replace(" ", "")
+                    else:
+                        idf_comp_entry = str(entry).replace(" ", "")
+                        idf_comp_vers = "*"
+                    if idf_comp_entry not in idf_component_json["dependencies"]:
+                        print("*** Adding component:", idf_comp_entry, idf_comp_vers)
+                        new_entry = {idf_comp_entry: {"version": idf_comp_vers}}
+                        idf_component_json["dependencies"].update(new_entry)
+
+        idf_component_yml_file = open(yml_file_dir,"w")
+        yaml.dump(idf_component_json, idf_component_yml_file)
+        idf_component_yml_file.close()
+        # print("JSON from modified idf_component.yml:")
+        # print(json.dumps(idf_component_json))
+        return
+    return
+
+if flag_custom_component_add == True or flag_custom_component_remove == True:
+    HandleCOMPONENTsettings(env)
+
+if flag_custom_sdkonfig == True and "arduino" in env.subst("$PIOFRAMEWORK"):
+    HandleArduinoIDFsettings(env)
+    LIB_SOURCE = os.path.join(ProjectConfig.get_instance().get("platformio", "platforms_dir"), "espressif32", "builder", "build_lib")
+    if not bool(os.path.exists(os.path.join(PROJECT_DIR, ".dummy"))):
+        shutil.copytree(LIB_SOURCE, os.path.join(PROJECT_DIR, ".dummy"))
+    PROJECT_SRC_DIR = os.path.join(PROJECT_DIR, ".dummy")
+    env.Replace(
+        PROJECT_SRC_DIR=PROJECT_SRC_DIR,
+        BUILD_FLAGS="",
+        BUILD_UNFLAGS="",
+        LINKFLAGS="",
+        PIOFRAMEWORK="arduino",
+        ARDUINO_LIB_COMPILE_FLAG="Build",
+    )
+    env["INTEGRATION_EXTRA_DATA"].update({"arduino_lib_compile_flag": env.subst("$ARDUINO_LIB_COMPILE_FLAG")})
 
 def get_project_lib_includes(env):
     project = ProjectAsLibBuilder(env, "$PROJECT_DIR")
@@ -1060,7 +1330,8 @@ def generate_empty_partition_image(binary_path, image_size):
         ),
     )
 
-    env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", empty_partition)
+    if flag_custom_sdkonfig == False:
+        env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", empty_partition)
 
 
 def get_partition_info(pt_path, pt_offset, pt_params):
@@ -1557,7 +1828,8 @@ app_includes = get_app_includes(elf_config)
 # Compile bootloader
 #
 
-env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", build_bootloader(sdk_config))
+if flag_custom_sdkonfig == False:
+    env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", build_bootloader(sdk_config))
 
 #
 # Target: ESP-IDF menuconfig
@@ -1773,6 +2045,78 @@ if os.path.isdir(ulp_dir) and os.listdir(ulp_dir) and mcu not in ("esp32c2", "es
     env.SConscript("ulp.py", exports="env sdk_config project_config app_includes idf_variant")
 
 #
+# Compile Arduino IDF sources
+#
+
+if "arduino" in env.get("PIOFRAMEWORK") and "espidf" not in env.get("PIOFRAMEWORK"):
+    def idf_lib_copy(source, target, env):
+        env_build = join(env["PROJECT_BUILD_DIR"],env["PIOENV"])
+        sdkconfig_h_path = join(env_build,"config","sdkconfig.h")
+        arduino_libs = ARDUINO_FRMWRK_LIB_DIR
+        lib_src = join(env_build,"esp-idf")
+        lib_dst = join(arduino_libs,mcu,"lib")
+        ld_dst = join(arduino_libs,mcu,"ld")
+        mem_var = join(arduino_libs,mcu,board.get("build.arduino.memory_type", (board.get("build.flash_mode", "dio") + "_qspi")))
+        src = [join(lib_src,x) for x in os.listdir(lib_src)]
+        src = [folder for folder in src if not os.path.isfile(folder)] # folders only
+        for folder in src:
+            files = [join(folder,x) for x in os.listdir(folder)]
+            for file in files:
+                if file.strip().endswith(".a"):
+                    shutil.copyfile(file,join(lib_dst,file.split(os.path.sep)[-1]))
+
+        shutil.move(join(lib_dst,"libspi_flash.a"),join(mem_var,"libspi_flash.a"))
+        shutil.move(join(env_build,"memory.ld"),join(ld_dst,"memory.ld"))
+        if mcu == "esp32s3":
+            shutil.move(join(lib_dst,"libesp_psram.a"),join(mem_var,"libesp_psram.a"))
+            shutil.move(join(lib_dst,"libesp_system.a"),join(mem_var,"libesp_system.a"))
+            shutil.move(join(lib_dst,"libfreertos.a"),join(mem_var,"libfreertos.a"))
+            shutil.move(join(lib_dst,"libbootloader_support.a"),join(mem_var,"libbootloader_support.a"))
+            shutil.move(join(lib_dst,"libesp_hw_support.a"),join(mem_var,"libesp_hw_support.a"))
+
+        shutil.copyfile(sdkconfig_h_path,join(mem_var,"include","sdkconfig.h"))
+        if not bool(os.path.isfile(join(arduino_libs,mcu,"sdkconfig.orig"))):
+            shutil.move(join(arduino_libs,mcu,"sdkconfig"),join(arduino_libs,mcu,"sdkconfig.orig"))
+        shutil.copyfile(join(env.subst("$PROJECT_DIR"),"sdkconfig."+env["PIOENV"]),join(arduino_libs,mcu,"sdkconfig"))
+        shutil.copyfile(join(env.subst("$PROJECT_DIR"),"sdkconfig."+env["PIOENV"]),join(arduino_libs,"sdkconfig"))
+        print("*** Copied compiled %s IDF libraries to Arduino framework ***" % idf_variant)
+
+        pio_exe_path = shutil.which("platformio"+(".exe" if IS_WINDOWS else ""))
+        pio_cmd = env["PIOENV"]
+        env.Execute(
+            env.VerboseAction(
+                (
+                    '"%s" run -e ' % pio_exe_path
+                    + " ".join(['"%s"' % pio_cmd])
+                ),
+                "*** Starting Arduino compile %s with custom libraries ***" % pio_cmd,
+            )
+        )
+        if flag_custom_component_add == True or flag_custom_component_remove == True:
+            try:
+                shutil.copy(join(ARDUINO_FRAMEWORK_DIR,"idf_component.yml.orig"),join(ARDUINO_FRAMEWORK_DIR,"idf_component.yml"))
+                print("*** Original Arduino \"idf_component.yml\" restored ***")
+            except:
+                print("*** Original Arduino \"idf_component.yml\" couldnt be restored ***")
+    env.AddPostAction("checkprogsize", idf_lib_copy)
+
+if "espidf" in env.get("PIOFRAMEWORK") and (flag_custom_component_add == True or flag_custom_component_remove == True):
+    def idf_custom_component(source, target, env):
+        try:
+            shutil.copy(join(ARDUINO_FRAMEWORK_DIR,"idf_component.yml.orig"),join(ARDUINO_FRAMEWORK_DIR,"idf_component.yml"))
+            print("*** Original Arduino \"idf_component.yml\" restored ***")
+        except:
+            try:
+                shutil.copy(join(PROJECT_SRC_DIR,"idf_component.yml.orig"),join(PROJECT_SRC_DIR,"idf_component.yml"))
+                print("*** Original \"idf_component.yml\" restored ***")
+            except: # no "idf_component.yml" in source folder
+                try:
+                    os.remove(join(PROJECT_SRC_DIR,"idf_component.yml"))
+                    print("*** pioarduino generated \"idf_component.yml\" removed ***")
+                except:
+                    print("*** \"idf_component.yml\" couldnt be removed ***")
+    env.AddPostAction("checkprogsize", idf_custom_component)
+#
 # Process OTA partition and image
 #
 
@@ -1785,7 +2129,10 @@ ota_partition_params = get_partition_info(
 if ota_partition_params["size"] and ota_partition_params["offset"]:
     # Generate an empty image if OTA is enabled in partition table
     ota_partition_image = os.path.join("$BUILD_DIR", "ota_data_initial.bin")
-    generate_empty_partition_image(ota_partition_image, ota_partition_params["size"])
+    if "arduino" in env.subst("$PIOFRAMEWORK"):
+        ota_partition_image = os.path.join(ARDUINO_FRAMEWORK_DIR, "tools", "partitions", "boot_app0.bin")
+    else:
+        generate_empty_partition_image(ota_partition_image, ota_partition_params["size"])
 
     env.Append(
         FLASH_EXTRA_IMAGES=[
@@ -1797,18 +2144,61 @@ if ota_partition_params["size"] and ota_partition_params["offset"]:
             )
         ]
     )
+    EXTRA_IMG_DIR = join(env.subst("$PROJECT_DIR"), "variants", "tasmota")
+    env.Append(
+        FLASH_EXTRA_IMAGES=[
+            (offset, join(EXTRA_IMG_DIR, img)) for offset, img in board.get("upload.arduino.flash_extra_images", [])
+        ]
+    )
+
+def _parse_size(value):
+    if isinstance(value, int):
+        return value
+    elif value.isdigit():
+        return int(value)
+    elif value.startswith("0x"):
+        return int(value, 16)
+    elif value[-1].upper() in ("K", "M"):
+        base = 1024 if value[-1].upper() == "K" else 1024 * 1024
+        return int(value[:-1]) * base
+    return value
 
 #
 # Configure application partition offset
 #
 
-env.Replace(
+partitions_csv = env.subst("$PARTITIONS_TABLE_CSV")
-    ESP32_APP_OFFSET=get_app_partition_offset(
+result = []
-        env.subst("$PARTITIONS_TABLE_CSV"), partition_table_offset
+next_offset = 0
-    )
+bound = int(board.get("upload.offset_address", "0x10000"), 16) # default 0x10000
-)
+with open(partitions_csv) as fp:
+    for line in fp.readlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        tokens = [t.strip() for t in line.split(",")]
+        if len(tokens) < 5:
+            continue
+        partition = {
+            "name": tokens[0],
+            "type": tokens[1],
+            "subtype": tokens[2],
+            "offset": tokens[3] or next_offset,
+            "size": tokens[4],
+            "flags": tokens[5] if len(tokens) > 5 else None
+        }
+        result.append(partition)
+        next_offset = _parse_size(partition["offset"])
+        if (partition["subtype"] == "ota_0"):
+            bound = next_offset
+        next_offset = (next_offset + bound - 1) & ~(bound - 1)
 
+env.Replace(ESP32_APP_OFFSET=str(hex(bound)))
+
+#
 # Propagate application offset to debug configurations
+#
+
 env["INTEGRATION_EXTRA_DATA"].update(
     {"application_offset": env.subst("$ESP32_APP_OFFSET")}
 )

builder/main.py

@@ -300,6 +300,7 @@ env.Replace(
     ),
 
     ESP32_APP_OFFSET=env.get("INTEGRATION_EXTRA_DATA").get("application_offset"),
+    ARDUINO_LIB_COMPILE_FLAG="Inactive",
 
     PROGSUFFIX=".elf"
 )

examples/arduino-blink/platformio.ini

@@ -7,25 +7,94 @@
 ; Please visit documentation for the other options and examples
 ; http://docs.platformio.org/page/projectconf.html
 
-[env:esp-wrover-kit]
+[env:esp32solo1]
 platform = espressif32
 framework = arduino
-board = esp-wrover-kit
+board = esp32-solo1
+build_flags = -DLED_BUILTIN=2
+custom_component_remove =
+    espressif/esp_hosted
+    espressif/esp_wifi_remote
+    espressif/esp-dsp
+    espressif/esp32-camera
+    espressif/libsodium
+    espressif/esp-modbus
+    espressif/qrcode
+    espressif/esp_insights
+    espressif/esp_diag_data_store
+    espressif/esp_diagnostics
+    espressif/esp_rainmaker
+    espressif/rmaker_common
+
+[env:esp32-c2-devkitm-1]
+platform = espressif32
+framework = arduino
+board = esp32-c2-devkitm-1
 monitor_speed = 115200
-build_flags =
+custom_sdkconfig        = 'CONFIG_IDF_TARGET="esp32c2"'
-	-DLED_BUILTIN=2
+custom_component_remove = espressif/esp_hosted
- 
+                          espressif/esp_wifi_remote
+                          espressif/esp-dsp
+                          espressif/esp32-camera
+                          espressif/libsodium
+                          espressif/esp-modbus
+                          espressif/qrcode
+                          espressif/esp_insights
+                          espressif/esp_diag_data_store
+                          espressif/esp_diagnostics
+                          espressif/esp_rainmaker
+                          espressif/rmaker_common
+
+[env:esp32s3-qio_opi_per]
+; OPI Performance settings -> Display use
+platform = espressif32
+framework = arduino
+board = esp32s3_120_16_8-qio_opi
+custom_sdkconfig            = CONFIG_SPIRAM_MODE_OCT=y
+                              CONFIG_SPIRAM_SPEED_120M=y
+                              CONFIG_LCD_RGB_ISR_IRAM_SAFE=y
+                              CONFIG_GDMA_CTRL_FUNC_IN_IRAM=y
+                              CONFIG_I2S_ISR_IRAM_SAFE=y
+                              CONFIG_GDMA_ISR_IRAM_SAFE=y
+                              CONFIG_SPIRAM_XIP_FROM_PSRAM=y
+                              CONFIG_SPIRAM_FETCH_INSTRUCTIONS=y
+                              CONFIG_SPIRAM_RODATA=y
+                              CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y
+                              CONFIG_ESP32S3_DATA_CACHE_64KB=y
+                              CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y
+custom_component_remove     = espressif/esp_hosted
+                              espressif/esp_wifi_remote
+                              espressif/qrcode
+                              espressif/esp_insights
+                              espressif/esp_diag_data_store
+                              espressif/esp_diagnostics
+                              espressif/esp_rainmaker
+                              espressif/rmaker_common
+custom_component_add        = lvgl/lvgl @ ^9.2.2
+
 [env:esp32-c6-devkitc-1]
 platform = espressif32
 framework = arduino
 board = esp32-c6-devkitc-1
 monitor_speed = 115200
+custom_component_remove = espressif/esp_hosted
+                          espressif/esp_wifi_remote
+                          espressif/mdns
+                          espressif/esp-dsp
+                          espressif/esp_modem
+                          espressif/esp32-camera
 
 [env:esp32-h2-devkitm-1]
 platform = espressif32
 framework = arduino
 board = esp32-h2-devkitm-1
 monitor_speed = 115200
+custom_component_remove = espressif/esp_hosted
+                          espressif/esp_wifi_remote
+                          espressif/mdns
+                          espressif/esp-dsp
+                          espressif/esp_modem
+                          espressif/esp32-camera
 
 [env:esp32-p4]
 platform = espressif32
@@ -33,3 +102,9 @@ framework = arduino
 board = esp32-p4
 build_flags = -DLED_BUILTIN=2
 monitor_speed = 115200
+custom_component_remove = espressif/esp_hosted
+                          espressif/esp_wifi_remote
+                          espressif/mdns
+                          espressif/esp-dsp
+                          espressif/esp_modem
+                          espressif/esp32-camera

examples/arduino-zigbee-switch/src/Zigbee_On_Off_Switch.ino

@@ -33,10 +33,10 @@
 
 #include "Zigbee.h"
 
+/* Zigbee switch configuration */
 #define SWITCH_ENDPOINT_NUMBER 5
 
-/* Switch configuration */
+#define GPIO_INPUT_IO_TOGGLE_SWITCH BOOT_PIN
-#define GPIO_INPUT_IO_TOGGLE_SWITCH 9
 #define PAIR_SIZE(TYPE_STR_PAIR)    (sizeof(TYPE_STR_PAIR) / sizeof(TYPE_STR_PAIR[0]))
 
 typedef enum {
@@ -95,9 +95,6 @@ static void enableGpioInterrupt(bool enabled) {
 /********************* Arduino functions **************************/
 void setup() {
   Serial.begin(115200);
-  while (!Serial) {
-    delay(10);
-  }
 
   //Optional: set Zigbee device name and model
   zbSwitch.setManufacturerAndModel("Espressif", "ZigbeeSwitch");

examples/tasmota_platformio_override.ini

@@ -0,0 +1,37 @@
+[platformio]
+default_envs            = tasmota32-hybrid
+
+[env:tasmota32_base]
+platform                = file://.
+
+[env:tasmota32-hybrid]
+extends                 = env:tasmota32_base
+board                   = esp32
+build_flags             = ${env:tasmota32_base.build_flags}
+                          -DHTTPCLIENT_NOSECURE
+lib_ignore              = ${env:tasmota32_base.lib_ignore}
+                          Micro-RTSP
+                          epdiy
+custom_sdkconfig        = https://raw.githubusercontent.com/pioarduino/sdkconfig/refs/heads/main/sdkconfig_tasmota_esp32
+                          '# CONFIG_ETH_USE_ESP32_EMAC is not set'
+                          '# CONFIG_ETH_PHY_INTERFACE_RMII is not set'
+                          '# CONFIG_ETH_RMII_CLK_INPUT is not set'
+                          '# CONFIG_ETH_RMII_CLK_IN_GPIO is not set'
+custom_component_remove = espressif/esp_hosted
+                          espressif/esp_wifi_remote
+                          espressif/esp-dsp
+                          espressif/network_provisioning
+                          espressif/esp-zboss-lib
+                          espressif/esp-zigbee-lib
+                          espressif/esp_rainmaker
+                          espressif/rmaker_common
+                          espressif/esp_insights
+                          espressif/esp_diag_data_store
+                          espressif/esp_diagnostics
+                          espressif/cbor
+                          espressif/qrcode
+                          espressif/esp-sr
+                          espressif/libsodium
+                          espressif/esp-modbus
+                          chmorgan/esp-libhelix-mp3
+                          espressif/esp32-camera

platform.json

@@ -41,6 +41,12 @@
       "owner": "espressif",
       "version": ""
     },
+    "framework-arduino-c2-skeleton-lib": {
+      "type": "framework",
+      "optional": true,
+      "owner": "espressif",
+      "version": "https://github.com/pioarduino/platform-espressif32/releases/download/53.03.10-rc2/c2_arduino_compile_skeleton.zip"
+    },
     "framework-espidf": {
       "type": "framework",
       "optional": true,

platform.py

@@ -37,11 +37,12 @@ class Espressif32Platform(PlatformBase):
 
         board_config = self.board_config(variables.get("board"))
         mcu = variables.get("board_build.mcu", board_config.get("build.mcu", "esp32"))
+        board_sdkconfig = variables.get("board_espidf.custom_sdkconfig", board_config.get("espidf.custom_sdkconfig", ""))
+        core_variant_board = ''.join(variables.get("board_build.extra_flags", board_config.get("build.extra_flags", "")))
+        core_variant_board = core_variant_board.replace("-D", " ")
+        core_variant_build = (''.join(variables.get("build_flags", []))).replace("-D", " ")
         frameworks = variables.get("pioframework", [])
 
-        if variables.get("custom_sdkconfig") is not None:
-            frameworks.append("espidf")
-
         if "arduino" in frameworks:
             self.packages["framework-arduinoespressif32"]["optional"] = False
             self.packages["framework-arduinoespressif32-libs"]["optional"] = False
@@ -51,6 +52,12 @@ class Espressif32Platform(PlatformBase):
             dyn_lib_url = packjdata['packages'][0]['tools'][0]['systems'][0]['url']
             self.packages["framework-arduinoespressif32-libs"]["version"] = dyn_lib_url
 
+        if variables.get("custom_sdkconfig") is not None or len(str(board_sdkconfig)) > 3:
+            frameworks.append("espidf")
+            self.packages["framework-espidf"]["optional"] = False
+            if mcu == "esp32c2":
+                self.packages["framework-arduino-c2-skeleton-lib"]["optional"] = False
+
         if "buildfs" in targets:
             filesystem = variables.get("board_build.filesystem", "littlefs")
             if filesystem == "littlefs":
@@ -90,10 +97,10 @@ class Espressif32Platform(PlatformBase):
         if "espidf" in frameworks:
             self.packages["toolchain-esp32ulp"]["optional"] = False
             for p in self.packages:
-                if p in ("tool-cmake", "tool-ninja"):
+                if p in ("tool-scons", "tool-cmake", "tool-ninja"):
                     self.packages[p]["optional"] = False
-#                elif p in ("tool-mconf", "tool-idf") and IS_WINDOWS:
+                # elif p in ("tool-mconf", "tool-idf") and IS_WINDOWS:
-#                    self.packages[p]["optional"] = False
+                    # self.packages[p]["optional"] = False
 
         if mcu in ("esp32", "esp32s2", "esp32s3"):
             self.packages["toolchain-xtensa-esp-elf"]["optional"] = False