diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index e48ec17..fed7865 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -37,11 +37,17 @@ from SCons.Script import ( ) from platformio import fs +from platformio.compat import IS_WINDOWS from platformio.proc import exec_command -from platformio.util import get_systype from platformio.builder.tools.piolib import ProjectAsLibBuilder from platformio.package.version import get_original_version, pepver_to_semver +# Added to avoid conflicts between installed Python packages from +# the IDF virtual environment and PlatformIO Core +# Note: This workaround can be safely deleted when PlatformIO 6.1.7 is released +if os.environ.get("PYTHONPATH"): + del os.environ["PYTHONPATH"] + env = DefaultEnvironment() env.SConscript("_embed_files.py", exports="env") @@ -51,8 +57,7 @@ mcu = board.get("build.mcu", "esp32") idf_variant = mcu.lower() # Required until Arduino switches to v5 -IDF5 = platform.get_package_version( - "framework-espidf").split(".")[1].startswith("5") +IDF5 = platform.get_package_version("framework-espidf").split(".")[1].startswith("5") FRAMEWORK_DIR = platform.get_package_dir("framework-espidf") TOOLCHAIN_DIR = platform.get_package_dir( "toolchain-%s" % ("riscv32-esp" if mcu == "esp32c3" else ("xtensa-%s" % mcu)) @@ -224,7 +229,7 @@ def populate_idf_env_vars(idf_env): os.path.join(TOOLCHAIN_DIR, "bin"), platform.get_package_dir("tool-ninja"), os.path.join(platform.get_package_dir("tool-cmake"), "bin"), - os.path.dirname(env.subst("$PYTHONEXE")), + os.path.dirname(get_python_exe()), ] if mcu != "esp32c3": @@ -232,7 +237,7 @@ def populate_idf_env_vars(idf_env): os.path.join(platform.get_package_dir("toolchain-esp32ulp"), "bin"), ) - if "windows" in get_systype(): + if IS_WINDOWS: additional_packages.append(platform.get_package_dir("tool-mconf")) idf_env["PATH"] = os.pathsep.join(additional_packages + [idf_env["PATH"]]) @@ -559,7 +564,7 @@ def generate_project_ld_script(sdk_config, ignore_targets=None): } cmd = ( - '"$PYTHONEXE" "{script}" --input $SOURCE ' + '"$ESPIDF_PYTHONEXE" "{script}" --input $SOURCE ' '--config "{config}" --fragments {fragments} --output $TARGET ' '--kconfig "{kconfig}" --env-file "{env_file}" ' '--libraries-file "{libraries_list}" ' @@ -763,7 +768,7 @@ def build_bootloader(sdk_config): [ "-DIDF_TARGET=" + idf_variant, "-DPYTHON_DEPS_CHECKED=1", - "-DPYTHON=" + env.subst("$PYTHONEXE"), + "-DPYTHON=" + get_python_exe(), "-DIDF_PATH=" + FRAMEWORK_DIR, "-DSDKCONFIG=" + SDKCONFIG_PATH, "-DLEGACY_INCLUDE_COMMON_HEADERS=", @@ -918,7 +923,7 @@ def generate_empty_partition_image(binary_path, image_size): binary_path, None, env.VerboseAction( - '"$PYTHONEXE" "%s" %s $TARGET' + '"$ESPIDF_PYTHONEXE" "%s" %s $TARGET' % ( os.path.join( FRAMEWORK_DIR, @@ -943,7 +948,7 @@ def get_partition_info(pt_path, pt_offset, pt_params): env.Exit(1) cmd = [ - env.subst("$PYTHONEXE"), + get_python_exe(), os.path.join(FRAMEWORK_DIR, "components", "partition_table", "parttool.py"), "-q", "--partition-table-offset", @@ -1000,7 +1005,7 @@ def generate_mbedtls_bundle(sdk_config): FRAMEWORK_DIR, "components", "mbedtls", "esp_crt_bundle" ) - cmd = [env.subst("$PYTHONEXE"), os.path.join(default_crt_dir, "gen_crt_bundle.py")] + cmd = [get_python_exe(), os.path.join(default_crt_dir, "gen_crt_bundle.py")] crt_args = ["--input"] if sdk_config.get("MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL", False): @@ -1051,12 +1056,12 @@ def generate_mbedtls_bundle(sdk_config): def install_python_deps(): - def _get_installed_pip_packages(): + def _get_installed_pip_packages(python_exe_path): result = {} packages = {} pip_output = subprocess.check_output( [ - env.subst("$PYTHONEXE"), + python_exe_path, "-m", "pip", "list", @@ -1087,7 +1092,8 @@ def install_python_deps(): # Remove specific versions for IDF5 as not required deps = {dep: "" for dep in deps} - installed_packages = _get_installed_pip_packages() + python_exe_path = get_python_exe() + installed_packages = _get_installed_pip_packages(python_exe_path) packages_to_install = [] for package, spec in deps.items(): if package not in installed_packages: @@ -1101,22 +1107,17 @@ def install_python_deps(): env.Execute( env.VerboseAction( ( - '"$PYTHONEXE" -m pip install -U ' - + " ".join( - [ - '"%s%s"' % (p, deps[p]) - for p in packages_to_install - ] - ) + '"%s" -m pip install -U ' % python_exe_path + + " ".join(['"%s%s"' % (p, deps[p]) for p in packages_to_install]) ), "Installing ESP-IDF's Python dependencies", ) ) - if "windows" in get_systype() and "windows-curses" not in installed_packages: + if IS_WINDOWS and "windows-curses" not in installed_packages: env.Execute( env.VerboseAction( - "$PYTHONEXE -m pip install windows-curses", + '"%s" -m pip install windows-curses' % python_exe_path, "Installing windows-curses package", ) ) @@ -1128,15 +1129,59 @@ def install_python_deps(): }: env.Execute( env.VerboseAction( - '$PYTHONEXE -m pip install "file://%s/tools/kconfig_new/esp-windows-curses"' - % FRAMEWORK_DIR, + '"%s" -m pip install "file://%s/tools/kconfig_new/esp-windows-curses"' + % (python_exe_path, FRAMEWORK_DIR), "Installing windows-curses package", ) ) +def get_python_exe(): + def _create_venv(venv_dir): + pip_path = os.path.join( + venv_dir, + "Scripts" if IS_WINDOWS else "bin", + "pip" + (".exe" if IS_WINDOWS else ""), + ) + if not os.path.isfile(pip_path): + # Use the built-in PlatformIO Python to create a standalone IDF virtual env + env.Execute( + env.VerboseAction( + '"$PYTHONEXE" -m venv --clear "%s"' % venv_dir, + "Creating a virtual environment for IDF Python dependencies", + ) + ) + + assert os.path.isfile( + pip_path + ), "Error: Failed to create a proper virtual environment. Missing the pip binary!" + + # The name of the IDF venv contains the IDF version to avoid possible conflicts and + # unnecessary reinstallation of Python dependencies in cases when Arduino + # as an IDF component requires a different version of the IDF package and + # hence a different set of Python deps or their versions + idf_version = get_original_version(platform.get_package_version("framework-espidf")) + venv_dir = os.path.join( + env.subst("$PROJECT_CORE_DIR"), "penv", ".espidf-" + idf_version) + + if not os.path.isdir(venv_dir): + _create_venv(venv_dir) + + python_exe_path = os.path.join( + venv_dir, + "Scripts" if IS_WINDOWS else "bin", + "python" + (".exe" if IS_WINDOWS else ""), + ) + + assert os.path.isfile(python_exe_path), ( + "Error: Missing Python executable file `%s`" % python_exe_path + ) + + return python_exe_path + + # -# ESP-IDF requires Python packages with specific versions +# ESP-IDF requires Python packages with specific versions in a virtual environment # install_python_deps() @@ -1235,7 +1280,7 @@ project_codemodel = get_cmake_code_model( "-DIDF_TARGET=" + idf_variant, "-DPYTHON_DEPS_CHECKED=1", "-DEXTRA_COMPONENT_DIRS:PATH=" + ";".join(extra_components), - "-DPYTHON=" + env.subst("$PYTHONEXE"), + "-DPYTHON=" + get_python_exe(), "-DSDKCONFIG=" + SDKCONFIG_PATH, ] + click.parser.split_arg_string(board.get("build.cmake_extra_args", "")), @@ -1373,7 +1418,7 @@ partition_table = env.Command( os.path.join("$BUILD_DIR", "partitions.bin"), "$PARTITIONS_TABLE_CSV", env.VerboseAction( - '"$PYTHONEXE" "%s" -q --offset "%s" --flash-size "%s" $SOURCE $TARGET' + '"$ESPIDF_PYTHONEXE" "%s" -q --offset "%s" --flash-size "%s" $SOURCE $TARGET' % ( os.path.join( FRAMEWORK_DIR, "components", "partition_table", "gen_esp32part.py" @@ -1396,6 +1441,7 @@ env.MergeFlags(project_flags) env.Prepend( CPPPATH=app_includes["plain_includes"], CPPDEFINES=project_defines, + ESPIDF_PYTHONEXE=get_python_exe(), LINKFLAGS=extra_flags, LIBS=libs, FLASH_EXTRA_IMAGES=[