From f1da5442792903af6bdd6d51b7c98b3aaeeefc6c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Apr 2019 19:56:16 +0300 Subject: [PATCH 01/13] Initial commit of PIO Unified Debugger --- .isort.cfg | 2 +- platformio/__main__.py | 16 +- platformio/commands/debug.py | 42 ---- platformio/commands/debug/__init__.py | 15 ++ platformio/commands/debug/client.py | 271 ++++++++++++++++++++++ platformio/commands/debug/command.py | 135 +++++++++++ platformio/commands/debug/helpers.py | 317 ++++++++++++++++++++++++++ platformio/commands/debug/initcfgs.py | 109 +++++++++ platformio/commands/debug/process.py | 72 ++++++ platformio/commands/debug/server.py | 110 +++++++++ platformio/managers/core.py | 9 + platformio/util.py | 7 +- 12 files changed, 1051 insertions(+), 54 deletions(-) delete mode 100644 platformio/commands/debug.py create mode 100644 platformio/commands/debug/__init__.py create mode 100644 platformio/commands/debug/client.py create mode 100644 platformio/commands/debug/command.py create mode 100644 platformio/commands/debug/helpers.py create mode 100644 platformio/commands/debug/initcfgs.py create mode 100644 platformio/commands/debug/process.py create mode 100644 platformio/commands/debug/server.py diff --git a/.isort.cfg b/.isort.cfg index 9b58f629..1ee0d412 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=79 -known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial +known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial, twisted diff --git a/platformio/__main__.py b/platformio/__main__.py index 61db78de..0252aba6 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -14,7 +14,7 @@ import os import sys -from os.path import join +from os.path import isdir, isfile, join from platform import system from traceback import format_exc @@ -24,15 +24,19 @@ from platformio import __version__, exception, maintenance from platformio.util import get_source_dir -class PlatformioCLI(click.MultiCommand): # pylint: disable=R0904 +class PlatformioCLI(click.MultiCommand): def list_commands(self, ctx): cmds = [] - for filename in os.listdir(join(get_source_dir(), "commands")): - if filename.startswith("__init__"): + commands_dir = join(get_source_dir(), "commands") + for name in os.listdir(commands_dir): + if name.startswith("__init__"): continue - if filename.endswith(".py"): - cmds.append(filename[:-3]) + if (isdir(join(commands_dir, name)) + and isfile(join(commands_dir, name, "command.py"))): + cmds.append(name) + elif name.endswith(".py"): + cmds.append(name[:-3]) cmds.sort() return cmds diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py deleted file mode 100644 index e43aeed1..00000000 --- a/platformio/commands/debug.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -from os import getcwd - -import click - -from platformio.managers.core import pioplus_call - - -@click.command( - "debug", - context_settings=dict(ignore_unknown_options=True), - short_help="PIO Unified Debugger") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option("--environment", "-e", metavar="") -@click.option("--verbose", "-v", is_flag=True) -@click.option("--interface", type=click.Choice(["gdb"])) -@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED) -def cli(*args, **kwargs): # pylint: disable=unused-argument - pioplus_call(sys.argv[1:]) diff --git a/platformio/commands/debug/__init__.py b/platformio/commands/debug/__init__.py new file mode 100644 index 00000000..7fba44c2 --- /dev/null +++ b/platformio/commands/debug/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.debug.command import cli diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py new file mode 100644 index 00000000..a0bcc61b --- /dev/null +++ b/platformio/commands/debug/client.py @@ -0,0 +1,271 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re +import time +from hashlib import sha1 +from os.path import abspath, basename, dirname, isdir, join, splitext +from tempfile import mkdtemp + +from twisted.internet import protocol, reactor, stdio, task + +from platformio import app, exception, util +from platformio.commands.debug import helpers, initcfgs +from platformio.commands.debug.process import BaseProcess +from platformio.commands.debug.server import DebugServer +from platformio.telemetry import MeasurementProtocol + +LOG_FILE = None + + +class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes + + PIO_SRC_NAME = ".pioinit" + INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" + + def __init__(self, project_dir, args, debug_options, env_options): + self.project_dir = project_dir + self.args = list(args) + self.debug_options = debug_options + self.env_options = env_options + + self._debug_server = DebugServer(debug_options, env_options) + self._session_id = None + + if not isdir(util.get_cache_dir()): + os.makedirs(util.get_cache_dir()) + self._gdbsrc_dir = mkdtemp( + dir=util.get_cache_dir(), prefix=".piodebug-") + + self._target_is_run = False + self._last_server_activity = 0 + self._auto_continue_timer = None + + def spawn(self, gdb_path, prog_path): + session_hash = gdb_path + prog_path + self._session_id = sha1( + session_hash if util.PY2 else session_hash.encode()).hexdigest() + self._kill_previous_session() + + patterns = { + "PROJECT_DIR": helpers.escape_path(self.project_dir), + "PROG_PATH": helpers.escape_path(prog_path), + "PROG_DIR": helpers.escape_path(dirname(prog_path)), + "PROG_NAME": basename(splitext(prog_path)[0]), + "DEBUG_PORT": self.debug_options['port'], + "UPLOAD_PROTOCOL": self.debug_options['upload_protocol'], + "INIT_BREAK": self.debug_options['init_break'] or "", + "LOAD_CMD": self.debug_options['load_cmd'] or "", + } + + self._debug_server.spawn(patterns) + + if not patterns['DEBUG_PORT']: + patterns['DEBUG_PORT'] = self._debug_server.get_debug_port() + self.generate_pioinit(self._gdbsrc_dir, patterns) + + # start GDB client + args = [ + "piogdb", + "-q", + "--directory", self._gdbsrc_dir, + "--directory", self.project_dir, + "-l", "10" + ] # yapf: disable + args.extend(self.args) + if not gdb_path: + raise exception.DebugInvalidOptions("GDB client is not configured") + gdb_data_dir = self._get_data_dir(gdb_path) + if gdb_data_dir: + args.extend(["--data-directory", gdb_data_dir]) + args.append(patterns['PROG_PATH']) + + return reactor.spawnProcess( + self, gdb_path, args, path=self.project_dir, env=os.environ) + + @staticmethod + def _get_data_dir(gdb_path): + if "msp430" in gdb_path: + return None + gdb_data_dir = abspath(join(dirname(gdb_path), "..", "share", "gdb")) + return gdb_data_dir if isdir(gdb_data_dir) else None + + def generate_pioinit(self, dst_dir, patterns): + server_exe = (self.debug_options.get("server") or {}).get( + "executable", "").lower() + if "jlink" in server_exe: + cfg = initcfgs.GDB_JLINK_INIT_CONFIG + elif "st-util" in server_exe: + cfg = initcfgs.GDB_STUTIL_INIT_CONFIG + elif "mspdebug" in server_exe: + cfg = initcfgs.MSPDEBUG_INIT_CONFIG + elif self.debug_options['require_debug_port']: + cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG + else: + cfg = initcfgs.GDB_DEFAULT_INIT_CONFIG + commands = cfg.split("\n") + + if self.debug_options['init_cmds']: + commands = self.debug_options['init_cmds'] + commands.extend(self.debug_options['extra_cmds']) + + if not any("define pio_reset_target" in cmd for cmd in commands): + commands = [ + "define pio_reset_target", + " echo Warning! Undefined pio_reset_target command\\n", + " mon reset", + "end" + ] + commands # yapf: disable + if not any("define pio_reset_halt_target" in cmd for cmd in commands): + commands = [ + "define pio_reset_halt_target", + " echo Warning! Undefined pio_reset_halt_target command\\n", + " mon reset halt", + "end" + ] + commands # yapf: disable + if not any("define pio_restart_target" in cmd for cmd in commands): + commands += [ + "define pio_restart_target", + " pio_reset_halt_target", + " $INIT_BREAK", + " %s" % ("continue" if patterns['INIT_BREAK'] else "next"), + "end" + ] # yapf: disable + + banner = [ + "echo PlatformIO Unified Debugger > http://bit.ly/pio-debug\\n", + "echo PlatformIO: Initializing remote target...\\n" + ] + footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] + commands = banner + commands + footer + + with open(join(dst_dir, self.PIO_SRC_NAME), "w") as fp: + fp.write("\n".join(self.apply_patterns(commands, patterns))) + + def connectionMade(self): + self._lock_session(self.transport.pid) + + p = protocol.Protocol() + p.dataReceived = self.onStdInData + stdio.StandardIO(p) + + def onStdInData(self, data): + self._last_server_activity = time.time() + if LOG_FILE: + with open(LOG_FILE, "ab") as fp: + fp.write(data) + + if b"-exec-run" in data: + if self._target_is_run: + token, _ = data.split(b"-", 1) + self.outReceived(token + b"^running\n") + return + data = data.replace(b"-exec-run", b"-exec-continue") + + if b"-exec-continue" in data: + self._target_is_run = True + if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"): + self.transport.write(b"pio_reset_target\n") + self.transport.write(data) + + def processEnded(self, reason): # pylint: disable=unused-argument + self._unlock_session() + if self._gdbsrc_dir and isdir(self._gdbsrc_dir): + util.rmtree_(self._gdbsrc_dir) + if self._debug_server: + self._debug_server.terminate() + + reactor.stop() + + def outReceived(self, data): + self._last_server_activity = time.time() + super(GDBClient, self).outReceived(data) + self._handle_error(data) + # go to init break automatically + if self.INIT_COMPLETED_BANNER.encode() in data: + self._auto_continue_timer = task.LoopingCall( + self._auto_exec_continue) + self._auto_continue_timer.start(0.1) + + def errReceived(self, data): + super(GDBClient, self).errReceived(data) + self._handle_error(data) + + def console_log(self, msg): + if helpers.is_mi_mode(self.args): + self.outReceived(('~"%s\\n"\n' % msg).encode()) + else: + self.outReceived(("%s\n" % msg).encode()) + + def _auto_exec_continue(self): + auto_exec_delay = 0.5 # in seconds + if self._last_server_activity > (time.time() - auto_exec_delay): + return + if self._auto_continue_timer: + self._auto_continue_timer.stop() + self._auto_continue_timer = None + + if not self.debug_options['init_break'] or self._target_is_run: + return + self.console_log( + "PlatformIO: Resume the execution to `debug_init_break = %s`" % + self.debug_options['init_break']) + self.transport.write(b"0-exec-continue\n" if helpers. + is_mi_mode(self.args) else b"continue\n") + self._target_is_run = True + + def _handle_error(self, data): + if (self.PIO_SRC_NAME.encode() not in data + or b"Error in sourced" not in data): + return + configuration = {"debug": self.debug_options, "env": self.env_options} + exd = re.sub(r'\\(?!")', "/", json.dumps(configuration)) + exd = re.sub(r'"(?:[a-z]\:)?((/[^"/]+)+)"', lambda m: '"%s"' % join( + *m.group(1).split("/")[-2:]), exd, re.I | re.M) + mp = MeasurementProtocol() + mp['exd'] = "DebugGDBPioInitError: %s" % exd + mp['exf'] = 1 + mp.send("exception") + self.transport.loseConnection() + + def _kill_previous_session(self): + assert self._session_id + pid = None + with app.ContentCache() as cc: + pid = cc.get(self._session_id) + cc.delete(self._session_id) + if not pid: + return + if "windows" in util.get_systype(): + kill = ["Taskkill", "/PID", pid, "/F"] + else: + kill = ["kill", pid] + try: + util.exec_command(kill) + except: # pylint: disable=bare-except + pass + + def _lock_session(self, pid): + if not self._session_id: + return + with app.ContentCache() as cc: + cc.set(self._session_id, str(pid), "1h") + + def _unlock_session(self): + if not self._session_id: + return + with app.ContentCache() as cc: + cc.delete(self._session_id) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py new file mode 100644 index 00000000..918aeb60 --- /dev/null +++ b/platformio/commands/debug/command.py @@ -0,0 +1,135 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-arguments, too-many-statements +# pylint: disable=too-many-locals, too-many-branches + +import os +from os.path import isfile + +import click + +from platformio import exception, util +from platformio.commands.debug import helpers +from platformio.managers.core import inject_contrib_pysite + + +@click.command( + "debug", + context_settings=dict(ignore_unknown_options=True), + short_help="PIO Unified Debugger") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path( + exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("--environment", "-e", metavar="") +@click.option("--verbose", "-v", is_flag=True) +@click.option("--interface", type=click.Choice(["gdb"])) +@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED) +@click.pass_context +def cli(ctx, project_dir, environment, verbose, interface, __unprocessed): + try: + util.ensure_udev_rules() + except NameError: + pass + except exception.InvalidUdevRules as e: + for line in str(e).split("\n") + [""]: + click.echo( + ('~"%s\\n"' if helpers.is_mi_mode(__unprocessed) else "%s") % + line) + + if not util.is_platformio_project(project_dir) and os.getenv("CWD"): + project_dir = os.getenv("CWD") + + with util.cd(project_dir): + env_name = helpers.check_env_name(project_dir, environment) + env_options = helpers.get_env_options(project_dir, env_name) + if not set(env_options.keys()) >= set(["platform", "board"]): + raise exception.ProjectEnvsNotAvailable() + debug_options = helpers.validate_debug_options(ctx, env_options) + assert debug_options + + if not interface: + return helpers.predebug_project(ctx, project_dir, env_name, False, + verbose) + + configuration = helpers.load_configuration(ctx, project_dir, env_name) + if not configuration: + raise exception.DebugInvalidOptions( + "Could not load debug configuration") + + if "--version" in __unprocessed: + result = util.exec_command([configuration['gdb_path'], "--version"]) + if result['returncode'] == 0: + return click.echo(result['out']) + raise exception.PlatformioException("\n".join( + [result['out'], result['err']])) + + debug_options['load_cmd'] = helpers.configure_esp32_load_cmd( + debug_options, configuration) + + rebuild_prog = False + preload = debug_options['load_cmd'] == "preload" + load_mode = debug_options['load_mode'] + if load_mode == "always": + rebuild_prog = ( + preload + or not helpers.has_debug_symbols(configuration['prog_path'])) + elif load_mode == "modified": + rebuild_prog = ( + helpers.is_prog_obsolete(configuration['prog_path']) + or not helpers.has_debug_symbols(configuration['prog_path'])) + else: + rebuild_prog = not isfile(configuration['prog_path']) + + if preload or (not rebuild_prog and load_mode != "always"): + # don't load firmware through debug server + debug_options['load_cmd'] = None + + if rebuild_prog: + if helpers.is_mi_mode(__unprocessed): + output = helpers.GDBBytesIO() + click.echo('~"Preparing firmware for debugging...\\n"') + with helpers.capture_std_streams(output): + helpers.predebug_project(ctx, project_dir, env_name, preload, + verbose) + output.close() + else: + click.echo("Preparing firmware for debugging...") + helpers.predebug_project(ctx, project_dir, env_name, preload, + verbose) + + # save SHA sum of newly created prog + if load_mode == "modified": + helpers.is_prog_obsolete(configuration['prog_path']) + + if not isfile(configuration['prog_path']): + raise exception.DebugInvalidOptions("Program/firmware is missed") + + # run debugging client + inject_contrib_pysite() + from platformio.commands.debug.client import GDBClient, reactor + + client = GDBClient(project_dir, __unprocessed, debug_options, env_options) + client.spawn(configuration['gdb_path'], configuration['prog_path']) + + reactor.run() + + return True diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py new file mode 100644 index 00000000..42a4ef40 --- /dev/null +++ b/platformio/commands/debug/helpers.py @@ -0,0 +1,317 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import sys +import time +from contextlib import contextmanager +from fnmatch import fnmatch +from hashlib import sha1 +from io import BytesIO +from os.path import isfile + +from platformio import VERSION, exception, util +from platformio.commands.platform import \ + platform_install as cmd_platform_install +from platformio.commands.run import cli as cmd_run +from platformio.managers.platform import PlatformFactory + + +class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods + + STDOUT = sys.stdout + + def write(self, text): + for line in text.strip().split("\n"): + self.STDOUT.write('~"%s\\n"\n' % line) + self.STDOUT.flush() + + +def is_mi_mode(args): + return "--interpreter" in " ".join(args) + + +def escape_path(path): + return path.replace("\\", "/") + + +def check_env_name(project_dir, environment): + config = util.load_project_config(project_dir) + envs = [] + for section in config.sections(): + if section.startswith("env:"): + envs.append(section[4:]) + if not envs: + raise exception.ProjectEnvsNotAvailable() + if not environment and config.has_option("platformio", "env_default"): + environment = config.get("platformio", "env_default").split(", ")[0] + if environment: + if environment in envs: + return environment + raise exception.UnknownEnvNames(environment, envs) + return envs[0] + + +def get_env_options(project_dir, environment): + config = util.load_project_config(project_dir) + options = {} + for k, v in config.items("env:%s" % environment): + options[k] = v + return options + + +def validate_debug_options(cmd_ctx, env_options): + + def _cleanup_cmds(cmds): + if not cmds: + return [] + if not isinstance(cmds, list): + cmds = cmds.split("\n") + return [c.strip() for c in cmds if c.strip()] + + try: + platform = PlatformFactory.newPlatform(env_options['platform']) + except exception.UnknownPlatform: + cmd_ctx.invoke( + cmd_platform_install, + platforms=[env_options['platform']], + skip_default_package=True) + platform = PlatformFactory.newPlatform(env_options['platform']) + + board_config = platform.board_config(env_options['board']) + tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool")) + tool_settings = board_config.get("debug", {}).get("tools", {}).get( + tool_name, {}) + server_options = None + + # specific server per a system + if isinstance(tool_settings.get("server", {}), list): + for item in tool_settings['server'][:]: + tool_settings['server'] = item + if util.get_systype() in item.get("system", []): + break + + # user overwrites debug server + if env_options.get("debug_server"): + server_options = { + "cwd": None, + "executable": None, + "arguments": env_options.get("debug_server") + } + if not isinstance(server_options['arguments'], list): + server_options['arguments'] = server_options['arguments'].split( + "\n") + server_options['arguments'] = [ + arg.strip() for arg in server_options['arguments'] if arg.strip() + ] + server_options['executable'] = server_options['arguments'][0] + server_options['arguments'] = server_options['arguments'][1:] + elif "server" in tool_settings: + server_package = tool_settings['server'].get("package") + server_package_dir = platform.get_package_dir( + server_package) if server_package else None + if server_package and not server_package_dir: + platform.install_packages( + with_packages=[server_package], + skip_default_package=True, + silent=True) + server_package_dir = platform.get_package_dir(server_package) + server_options = dict( + cwd=server_package_dir if server_package else None, + executable=tool_settings['server'].get("executable"), + arguments=[ + a.replace("$PACKAGE_DIR", escape_path(server_package_dir)) + if server_package_dir else a + for a in tool_settings['server'].get("arguments", []) + ]) + + extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds")) + extra_cmds.extend(_cleanup_cmds(tool_settings.get("extra_cmds"))) + result = dict( + tool=tool_name, + upload_protocol=env_options.get( + "upload_protocol", + board_config.get("upload", {}).get("protocol")), + load_cmd=env_options.get("debug_load_cmd", + tool_settings.get("load_cmd", "load")), + load_mode=env_options.get("debug_load_mode", + tool_settings.get("load_mode", "always")), + init_break=env_options.get( + "debug_init_break", tool_settings.get("init_break", + "tbreak main")), + init_cmds=_cleanup_cmds( + env_options.get("debug_init_cmds", + tool_settings.get("init_cmds"))), + extra_cmds=extra_cmds, + require_debug_port=tool_settings.get("require_debug_port", False), + port=reveal_debug_port( + env_options.get("debug_port", tool_settings.get("port")), + tool_name, tool_settings), + server=server_options) + return result + + +def predebug_project(ctx, project_dir, env_name, preload, verbose): + ctx.invoke( + cmd_run, + project_dir=project_dir, + environment=[env_name], + target=["__debug"] + (["upload"] if preload else []), + verbose=verbose) + if preload: + time.sleep(5) + + +@contextmanager +def capture_std_streams(stdout, stderr=None): + _stdout = sys.stdout + _stderr = sys.stderr + sys.stdout = stdout + sys.stderr = stderr or stdout + yield + sys.stdout = _stdout + sys.stderr = _stderr + + +def load_configuration(ctx, project_dir, env_name): + output = BytesIO() + with capture_std_streams(output): + ctx.invoke( + cmd_run, + project_dir=project_dir, + environment=[env_name], + target=["idedata"]) + result = output.getvalue().decode() + output.close() + if '"includes":' not in result: + return None + for line in result.split("\n"): + line = line.strip() + if line.startswith('{"') and "cxx_path" in line: + return json.loads(line[:line.rindex("}") + 1]) + return None + + +def configure_esp32_load_cmd(debug_options, configuration): + ignore_conds = [ + debug_options['load_cmd'] != "load", + "xtensa-esp32" not in configuration.get("cc_path", ""), + not configuration.get("flash_extra_images"), not all([ + isfile(item['path']) + for item in configuration.get("flash_extra_images") + ]) + ] + if any(ignore_conds): + return debug_options['load_cmd'] + + mon_cmds = [ + 'monitor program_esp32 "{{{path}}}" {offset} verify'.format( + path=escape_path(item['path']), offset=item['offset']) + for item in configuration.get("flash_extra_images") + ] + mon_cmds.append('monitor program_esp32 "{%s.bin}" 0x10000 verify' % + escape_path(configuration['prog_path'][:-4])) + return "\n".join(mon_cmds) + + +def has_debug_symbols(prog_path): + if not isfile(prog_path): + return False + matched = { + b".debug_info": False, + b".debug_abbrev": False, + b" -Og": False, + b" -g": False, + b"__PLATFORMIO_DEBUG__": (3, 6) > VERSION[:2] + } + with open(prog_path, "rb") as fp: + last_data = b"" + while True: + data = fp.read(1024) + if not data: + break + for pattern, found in matched.items(): + if found: + continue + if pattern in last_data + data: + matched[pattern] = True + last_data = data + return all(matched.values()) + + +def is_prog_obsolete(prog_path): + prog_hash_path = prog_path + ".sha1" + if not isfile(prog_path): + return True + shasum = sha1() + with open(prog_path, "rb") as fp: + while True: + data = fp.read(1024) + if not data: + break + shasum.update(data) + new_digest = shasum.hexdigest() + old_digest = None + if isfile(prog_hash_path): + with open(prog_hash_path, "r") as fp: + old_digest = fp.read() + if new_digest == old_digest: + return False + with open(prog_hash_path, "w") as fp: + fp.write(new_digest) + return True + + +def reveal_debug_port(env_debug_port, tool_name, tool_settings): + + def _get_pattern(): + if not env_debug_port: + return None + if set(["*", "?", "[", "]"]) & set(env_debug_port): + return env_debug_port + return None + + def _is_match_pattern(port): + pattern = _get_pattern() + if not pattern: + return True + return fnmatch(port, pattern) + + def _look_for_serial_port(hwids): + for item in util.get_serialports(filter_hwid=True): + if not _is_match_pattern(item['port']): + continue + port = item['port'] + if tool_name.startswith("blackmagic"): + if "windows" in util.get_systype() and \ + port.startswith("COM") and len(port) > 4: + port = "\\\\.\\%s" % port + if "GDB" in item['description']: + return port + for hwid in hwids: + hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") + if hwid_str in item['hwid']: + return port + return None + + if env_debug_port and not _get_pattern(): + return env_debug_port + if not tool_settings.get("require_debug_port"): + return None + + debug_port = _look_for_serial_port(tool_settings.get("hwids", [])) + if not debug_port: + raise exception.DebugInvalidOptions( + "Please specify `debug_port` for environment") + return debug_port diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py new file mode 100644 index 00000000..81af4a21 --- /dev/null +++ b/platformio/commands/debug/initcfgs.py @@ -0,0 +1,109 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GDB_DEFAULT_INIT_CONFIG = """ +define pio_reset_halt_target + monitor reset halt +end + +define pio_reset_target + monitor reset +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +pio_reset_halt_target +$LOAD_CMD +monitor init +pio_reset_halt_target +""" + +GDB_STUTIL_INIT_CONFIG = """ +define pio_reset_halt_target + monitor halt + monitor reset +end + +define pio_reset_target + monitor reset +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +pio_reset_halt_target +$LOAD_CMD +pio_reset_halt_target +""" + +GDB_JLINK_INIT_CONFIG = """ +define pio_reset_halt_target + monitor halt + monitor reset +end + +define pio_reset_target + monitor reset +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +pio_reset_halt_target +$LOAD_CMD +pio_reset_halt_target +""" + +GDB_BLACKMAGIC_INIT_CONFIG = """ +define pio_reset_halt_target + set language c + set *0xE000ED0C = 0x05FA0004 + set $busy = (*0xE000ED0C & 0x4) + while ($busy) + set $busy = (*0xE000ED0C & 0x4) + end + set language auto +end + +define pio_reset_target + pio_reset_halt_target +end + +target extended-remote $DEBUG_PORT +monitor swdp_scan +attach 1 +set mem inaccessible-by-default off +$INIT_BREAK +$LOAD_CMD + +set language c +set *0xE000ED0C = 0x05FA0004 +set $busy = (*0xE000ED0C & 0x4) +while ($busy) + set $busy = (*0xE000ED0C & 0x4) +end +set language auto +""" + +MSPDEBUG_INIT_CONFIG = """ +define pio_reset_halt_target +end + +define pio_reset_target +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +monitor erase +$LOAD_CMD +pio_reset_halt_target +""" diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py new file mode 100644 index 00000000..3f1fbd22 --- /dev/null +++ b/platformio/commands/debug/process.py @@ -0,0 +1,72 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click +from twisted.internet import protocol + +from platformio import util +from platformio.commands.debug import helpers + +LOG_FILE = None + + +class BaseProcess(protocol.ProcessProtocol, object): + + STDOUT_CHUNK_SIZE = 2048 + + COMMON_PATTERNS = { + "PLATFORMIO_HOME_DIR": helpers.escape_path(util.get_home_dir()), + "PYTHONEXE": os.getenv("PYTHONEXEPATH", "") + } + + def apply_patterns(self, source, patterns=None): + _patterns = self.COMMON_PATTERNS.copy() + _patterns.update(patterns or {}) + + def _replace(text): + for key, value in _patterns.items(): + pattern = "$%s" % key + text = text.replace(pattern, value or "") + return text + + if isinstance(source, util.string_types): + source = _replace(source) + elif isinstance(source, (list, dict)): + items = enumerate(source) if isinstance(source, + list) else source.items() + for key, value in items: + if isinstance(value, util.string_types): + source[key] = _replace(value) + elif isinstance(value, (list, dict)): + source[key] = self.apply_patterns(value, patterns) + + return source + + def outReceived(self, data): + if LOG_FILE: + with open(LOG_FILE, "ab") as fp: + fp.write(data) + while data: + chunk = data[:self.STDOUT_CHUNK_SIZE] + click.echo(chunk, nl=False) + data = data[self.STDOUT_CHUNK_SIZE:] + + @staticmethod + def errReceived(data): + if LOG_FILE: + with open(LOG_FILE, "ab") as fp: + fp.write(data) + click.echo(data, nl=False, err=True) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py new file mode 100644 index 00000000..977fe812 --- /dev/null +++ b/platformio/commands/debug/server.py @@ -0,0 +1,110 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from os.path import isdir, isfile, join + +from twisted.internet import reactor + +from platformio import exception, util +from platformio.commands.debug import helpers +from platformio.commands.debug.process import BaseProcess + + +class DebugServer(BaseProcess): + + def __init__(self, debug_options, env_options): + self.debug_options = debug_options + self.env_options = env_options + + self._debug_port = None + self._transport = None + + def spawn(self, patterns): # pylint: disable=too-many-branches + systype = util.get_systype() + server = self.debug_options.get("server") + if not server: + return None + server = self.apply_patterns(server, patterns) + server_executable = server['executable'] + if not server_executable: + return None + if server['cwd']: + server_executable = join(server['cwd'], server_executable) + if ("windows" in systype and not server_executable.endswith(".exe") + and isfile(server_executable + ".exe")): + server_executable = server_executable + ".exe" + + if not isfile(server_executable): + server_executable = util.where_is_program(server_executable) + if not isfile(server_executable): + raise exception.DebugInvalidOptions( + "\nCould not launch Debug Server '%s'. Please check that it " + "is installed and is included in a system PATH\n\n" + "See documentation or contact contact@platformio.org:\n" + "http://docs.platformio.org/page/plus/debugging.html\n" % + server_executable) + + self._debug_port = ":3333" + openocd_pipe_allowed = all([ + not self.debug_options['port'], "openocd" in server_executable, + self.env_options['platform'] != "riscv" + ]) + if openocd_pipe_allowed: + args = [] + if server['cwd']: + args.extend(["-s", helpers.escape_path(server['cwd'])]) + args.extend([ + "-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled" + ]) + args.extend(server['arguments']) + str_args = " ".join( + [arg if arg.startswith("-") else '"%s"' % arg for arg in args]) + self._debug_port = '| "%s" %s' % ( + helpers.escape_path(server_executable), str_args) + else: + env = os.environ.copy() + # prepend server "lib" folder to LD path + if ("windows" not in systype and server['cwd'] + and isdir(join(server['cwd'], "lib"))): + ld_key = ("DYLD_LIBRARY_PATH" + if "darwin" in systype else "LD_LIBRARY_PATH") + env[ld_key] = join(server['cwd'], "lib") + if os.environ.get(ld_key): + env[ld_key] = "%s:%s" % (env[ld_key], + os.environ.get(ld_key)) + # prepend BIN to PATH + if server['cwd'] and isdir(join(server['cwd'], "bin")): + env['PATH'] = "%s%s%s" % ( + join(server['cwd'], "bin"), os.pathsep, + os.environ.get("PATH", os.environ.get("Path", ""))) + + self._transport = reactor.spawnProcess( + self, + server_executable, [server_executable] + server['arguments'], + path=server['cwd'], + env=env) + if "mspdebug" in server_executable.lower(): + self._debug_port = ":2000" + elif "jlink" in server_executable.lower(): + self._debug_port = ":2331" + + return self._transport + + def get_debug_port(self): + return self._debug_port + + def terminate(self): + if self._transport: + self._transport.signalProcess("KILL") diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 3fd7f1cd..a6598361 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -115,6 +115,15 @@ def shutdown_piohome_servers(): port += 1 +def inject_contrib_pysite(): + from site import addsitedir + contrib_pysite_dir = get_core_package_dir("contrib-pysite") + if contrib_pysite_dir in sys.path: + return + addsitedir(contrib_pysite_dir) + sys.path.insert(0, contrib_pysite_dir) + + def pioplus_call(args, **kwargs): if "windows" in util.get_systype() and sys.version_info < (2, 7, 6): raise exception.PlatformioException( diff --git a/platformio/util.py b/platformio/util.py index dd280ba2..08592355 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -518,11 +518,8 @@ def get_mdns_services(): try: import zeroconf except ImportError: - from site import addsitedir - from platformio.managers.core import get_core_package_dir - contrib_pysite_dir = get_core_package_dir("contrib-pysite") - addsitedir(contrib_pysite_dir) - sys.path.insert(0, contrib_pysite_dir) + from platformio.managers.core import inject_contrib_pysite + inject_contrib_pysite() import zeroconf class mDNSListener(object): From c7d8b50474e2c6e4847739401fa50fc704bd4d7e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Apr 2019 19:58:34 +0300 Subject: [PATCH 02/13] Use generic GDB_MSPDEBUG_INIT_CONFIG --- platformio/commands/debug/client.py | 2 +- platformio/commands/debug/initcfgs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index a0bcc61b..7e9aebb9 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -111,7 +111,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes elif "st-util" in server_exe: cfg = initcfgs.GDB_STUTIL_INIT_CONFIG elif "mspdebug" in server_exe: - cfg = initcfgs.MSPDEBUG_INIT_CONFIG + cfg = initcfgs.GDB_MSPDEBUG_INIT_CONFIG elif self.debug_options['require_debug_port']: cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG else: diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index 81af4a21..62f36c0d 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -94,7 +94,7 @@ end set language auto """ -MSPDEBUG_INIT_CONFIG = """ +GDB_MSPDEBUG_INIT_CONFIG = """ define pio_reset_halt_target end From 948a977fa543a7f70eebaabaa5f836323f0906cf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Apr 2019 20:33:31 +0300 Subject: [PATCH 03/13] Initial commit of PIO Unit Testing --- platformio/commands/test.py | 67 --------- platformio/commands/test/__init__.py | 15 ++ platformio/commands/test/command.py | 181 +++++++++++++++++++++++ platformio/commands/test/embedded.py | 133 +++++++++++++++++ platformio/commands/test/native.py | 39 +++++ platformio/commands/test/processor.py | 200 ++++++++++++++++++++++++++ platformio/exception.py | 9 ++ 7 files changed, 577 insertions(+), 67 deletions(-) delete mode 100644 platformio/commands/test.py create mode 100644 platformio/commands/test/__init__.py create mode 100644 platformio/commands/test/command.py create mode 100644 platformio/commands/test/embedded.py create mode 100644 platformio/commands/test/native.py create mode 100644 platformio/commands/test/processor.py diff --git a/platformio/commands/test.py b/platformio/commands/test.py deleted file mode 100644 index 4c6414c9..00000000 --- a/platformio/commands/test.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -from os import getcwd - -import click - -from platformio.managers.core import pioplus_call - - -@click.command("test", short_help="Local Unit Testing") -@click.option("--environment", "-e", multiple=True, metavar="") -@click.option( - "--filter", - "-f", - multiple=True, - metavar="", - help="Filter tests by a pattern") -@click.option( - "--ignore", - "-i", - multiple=True, - metavar="", - help="Ignore tests by a pattern") -@click.option("--upload-port") -@click.option("--test-port") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option("--without-building", is_flag=True) -@click.option("--without-uploading", is_flag=True) -@click.option( - "--no-reset", - is_flag=True, - help="Disable software reset via Serial.DTR/RST") -@click.option( - "--monitor-rts", - default=None, - type=click.IntRange(0, 1), - help="Set initial RTS line state for Serial Monitor") -@click.option( - "--monitor-dtr", - default=None, - type=click.IntRange(0, 1), - help="Set initial DTR line state for Serial Monitor") -@click.option("--verbose", "-v", is_flag=True) -def cli(*args, **kwargs): # pylint: disable=unused-argument - pioplus_call(sys.argv[1:]) diff --git a/platformio/commands/test/__init__.py b/platformio/commands/test/__init__.py new file mode 100644 index 00000000..6d4c3e2b --- /dev/null +++ b/platformio/commands/test/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.test.command import cli diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py new file mode 100644 index 00000000..49da7bba --- /dev/null +++ b/platformio/commands/test/command.py @@ -0,0 +1,181 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from fnmatch import fnmatch +from os import getcwd, listdir +from os.path import isdir, join +from time import time + +import click + +from platformio import exception, util +from platformio.commands.run import check_project_envs, print_header +from platformio.commands.test.embedded import EmbeddedTestProcessor +from platformio.commands.test.native import NativeTestProcessor + + +@click.command("test", short_help="Unit Testing") +@click.option("--environment", "-e", multiple=True, metavar="") +@click.option( + "--filter", + "-f", + multiple=True, + metavar="", + help="Filter tests by a pattern") +@click.option( + "--ignore", + "-i", + multiple=True, + metavar="", + help="Ignore tests by a pattern") +@click.option("--upload-port") +@click.option("--test-port") +@click.option( + "-d", + "--project-dir", + default=getcwd, + type=click.Path( + exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("--without-building", is_flag=True) +@click.option("--without-uploading", is_flag=True) +@click.option("--without-testing", is_flag=True) +@click.option("--no-reset", is_flag=True) +@click.option( + "--monitor-rts", + default=None, + type=click.IntRange(0, 1), + help="Set initial RTS line state for Serial Monitor") +@click.option( + "--monitor-dtr", + default=None, + type=click.IntRange(0, 1), + help="Set initial DTR line state for Serial Monitor") +@click.option("--verbose", "-v", is_flag=True) +@click.pass_context +def cli( # pylint: disable=redefined-builtin + ctx, environment, ignore, filter, upload_port, test_port, project_dir, + without_building, without_uploading, without_testing, no_reset, + monitor_rts, monitor_dtr, verbose): + with util.cd(project_dir): + test_dir = util.get_projecttest_dir() + if not isdir(test_dir): + raise exception.TestDirNotExists(test_dir) + test_names = get_test_names(test_dir) + projectconf = util.load_project_config() + env_default = None + if projectconf.has_option("platformio", "env_default"): + env_default = util.parse_conf_multi_values( + projectconf.get("platformio", "env_default")) + assert check_project_envs(projectconf, environment or env_default) + + click.echo("Verbose mode can be enabled via `-v, --verbose` option") + click.echo("Collected %d items" % len(test_names)) + + start_time = time() + results = [] + for testname in test_names: + for section in projectconf.sections(): + if not section.startswith("env:"): + continue + + # filter and ignore patterns + patterns = dict(filter=list(filter), ignore=list(ignore)) + for key in patterns: + if projectconf.has_option(section, "test_%s" % key): + patterns[key].extend([ + p.strip() + for p in projectconf.get(section, "test_%s" % + key).split(", ") + if p.strip() + ]) + + envname = section[4:] + skip_conditions = [ + environment and envname not in environment, + not environment and env_default + and envname not in env_default, + testname != "*" and patterns['filter'] and + not any([fnmatch(testname, p) + for p in patterns['filter']]), + testname != "*" + and any([fnmatch(testname, p) + for p in patterns['ignore']]), + ] + if any(skip_conditions): + results.append((None, testname, envname)) + continue + + cls = (NativeTestProcessor if projectconf.get( + section, "platform") == "native" else + EmbeddedTestProcessor) + tp = cls( + ctx, testname, envname, + dict( + project_config=projectconf, + project_dir=project_dir, + upload_port=upload_port, + test_port=test_port, + without_building=without_building, + without_uploading=without_uploading, + without_testing=without_testing, + no_reset=no_reset, + monitor_rts=monitor_rts, + monitor_dtr=monitor_dtr, + verbose=verbose)) + results.append((tp.process(), testname, envname)) + + if without_testing: + return + + click.echo() + print_header("[%s]" % click.style("TEST SUMMARY")) + + passed = True + for result in results: + status, testname, envname = result + status_str = click.style("PASSED", fg="green") + if status is False: + passed = False + status_str = click.style("FAILED", fg="red") + elif status is None: + status_str = click.style("IGNORED", fg="yellow") + + click.echo( + "test/%s/env:%s\t[%s]" % (click.style(testname, fg="yellow"), + click.style(envname, fg="cyan"), + status_str), + err=status is False) + + print_header( + "[%s] Took %.2f seconds" % ( + (click.style("PASSED", fg="green", bold=True) if passed else + click.style("FAILED", fg="red", bold=True)), time() - start_time), + is_error=not passed) + + if not passed: + raise exception.ReturnErrorCode(1) + + +def get_test_names(test_dir): + names = [] + for item in sorted(listdir(test_dir)): + if isdir(join(test_dir, item)): + names.append(item) + if not names: + names = ["*"] + return names diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py new file mode 100644 index 00000000..39e8555d --- /dev/null +++ b/platformio/commands/test/embedded.py @@ -0,0 +1,133 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import sleep + +import click +import serial + +from platformio import exception, util +from platformio.commands.test.processor import TestProcessorBase +from platformio.managers.platform import PlatformFactory + + +class EmbeddedTestProcessor(TestProcessorBase): + + SERIAL_TIMEOUT = 600 + + def process(self): + if not self.options['without_building']: + self.print_progress("Building... (1/3)") + target = ["__test"] + if self.options['without_uploading']: + target.append("checkprogsize") + self.build_or_upload(target) + + if not self.options['without_uploading']: + self.print_progress("Uploading... (2/3)") + target = ["upload"] + if self.options['without_building']: + target.append("nobuild") + else: + target.append("__test") + self.build_or_upload(target) + + if self.options['without_testing']: + return None + + self.print_progress("Testing... (3/3)") + return self.run() + + def run(self): + click.echo("If you don't see any output for the first 10 secs, " + "please reset board (press reset button)") + click.echo() + + try: + ser = serial.Serial( + baudrate=self.get_baudrate(), timeout=self.SERIAL_TIMEOUT) + ser.port = self.get_test_port() + ser.rts = self.options['monitor_rts'] + ser.dtr = self.options['monitor_dtr'] + ser.open() + except serial.SerialException as e: + click.secho(str(e), fg="red", err=True) + return False + + if not self.options['no_reset']: + ser.flushInput() + ser.setDTR(False) + ser.setRTS(False) + sleep(0.1) + ser.setDTR(True) + ser.setRTS(True) + sleep(0.1) + + while True: + line = ser.readline().strip() + + # fix non-ascii output from device + for i, c in enumerate(line[::-1]): + if not isinstance(c, int): + c = ord(c) + if c > 127: + line = line[-i:] + break + + if not line: + continue + if isinstance(line, bytes): + line = line.decode("utf8") + self.on_run_out(line) + if all([l in line for l in ("Tests", "Failures", "Ignored")]): + break + ser.close() + return not self._run_failed + + def get_test_port(self): + # if test port is specified manually or in config + if self.options.get("test_port"): + return self.options.get("test_port") + if self.env_options.get("test_port"): + return self.env_options.get("test_port") + + assert set(["platform", "board"]) & set(self.env_options.keys()) + p = PlatformFactory.newPlatform(self.env_options['platform']) + board_hwids = p.board_config(self.env_options['board']).get( + "build.hwids", []) + port = None + elapsed = 0 + while elapsed < 5 and not port: + for item in util.get_serialports(): + port = item['port'] + for hwid in board_hwids: + hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") + if hwid_str in item['hwid']: + return port + + # check if port is already configured + try: + serial.Serial(port, timeout=self.SERIAL_TIMEOUT).close() + except serial.SerialException: + port = None + + if not port: + sleep(0.25) + elapsed += 0.25 + + if not port: + raise exception.PlatformioException( + "Please specify `test_port` for environment or use " + "global `--test-port` option.") + return port diff --git a/platformio/commands/test/native.py b/platformio/commands/test/native.py new file mode 100644 index 00000000..0945b10f --- /dev/null +++ b/platformio/commands/test/native.py @@ -0,0 +1,39 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os.path import join + +from platformio import util +from platformio.commands.test.processor import TestProcessorBase + + +class NativeTestProcessor(TestProcessorBase): + + def process(self): + if not self.options['without_building']: + self.print_progress("Building... (1/2)") + self.build_or_upload(["__test"]) + if self.options['without_testing']: + return None + self.print_progress("Testing... (2/2)") + return self.run() + + def run(self): + with util.cd(self.options['project_dir']): + build_dir = util.get_projectbuild_dir() + result = util.exec_command([join(build_dir, self.env_name, "program")], + stdout=util.AsyncPipe(self.on_run_out), + stderr=util.AsyncPipe(self.on_run_out)) + assert "returncode" in result + return result['returncode'] == 0 and not self._run_failed diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py new file mode 100644 index 00000000..7c3b84eb --- /dev/null +++ b/platformio/commands/test/processor.py @@ -0,0 +1,200 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atexit +from os import remove +from os.path import isdir, isfile, join +from string import Template + +import click + +from platformio import exception, util +from platformio.commands.run import cli as cmd_run +from platformio.commands.run import print_header + +TRANSPORT_OPTIONS = { + "arduino": { + "include": "#include ", + "object": "", + "putchar": "Serial.write(c)", + "flush": "Serial.flush()", + "begin": "Serial.begin($baudrate)", + "end": "Serial.end()" + }, + "mbed": { + "include": "#include ", + "object": "Serial pc(USBTX, USBRX);", + "putchar": "pc.putc(c)", + "flush": "", + "begin": "pc.baud($baudrate)", + "end": "" + }, + "energia": { + "include": "#include ", + "object": "", + "putchar": "Serial.write(c)", + "flush": "Serial.flush()", + "begin": "Serial.begin($baudrate)", + "end": "Serial.end()" + }, + "espidf": { + "include": "#include ", + "object": "", + "putchar": "putchar(c)", + "flush": "fflush(stdout)", + "begin": "", + "end": "" + }, + "native": { + "include": "#include ", + "object": "", + "putchar": "putchar(c)", + "flush": "fflush(stdout)", + "begin": "", + "end": "" + }, + "custom": { + "include": '#include "unittest_transport.h"', + "object": "", + "putchar": "unittest_uart_putchar(c)", + "flush": "unittest_uart_flush()", + "begin": "unittest_uart_begin()", + "end": "unittest_uart_end()" + } +} + + +class TestProcessorBase(object): + + DEFAULT_BAUDRATE = 115200 + + def __init__(self, cmd_ctx, testname, envname, options): + self.cmd_ctx = cmd_ctx + self.cmd_ctx.meta['piotest_processor'] = True + self.test_name = testname + self.options = options + self.env_name = envname + self.env_options = { + k: v + for k, v in options['project_config'].items("env:" + envname) + } + self._run_failed = False + self._outputcpp_generated = False + + def get_transport(self): + transport = self.env_options.get("framework") + if self.env_options.get("platform") == "native": + transport = "native" + if "test_transport" in self.env_options: + transport = self.env_options['test_transport'] + if transport not in TRANSPORT_OPTIONS: + raise exception.PlatformioException( + "Unknown Unit Test transport `%s`" % transport) + return transport.lower() + + def get_baudrate(self): + return int(self.env_options.get("test_speed", self.DEFAULT_BAUDRATE)) + + def print_progress(self, text, is_error=False): + click.echo() + print_header( + "[test/%s] %s" % (click.style( + self.test_name, fg="yellow", bold=True), text), + is_error=is_error) + + def build_or_upload(self, target): + if not self._outputcpp_generated: + self.generate_outputcpp(util.get_projecttest_dir()) + self._outputcpp_generated = True + + if self.test_name != "*": + self.cmd_ctx.meta['piotest'] = self.test_name + + if not self.options['verbose']: + click.echo("Please wait...") + + return self.cmd_ctx.invoke( + cmd_run, + project_dir=self.options['project_dir'], + upload_port=self.options['upload_port'], + silent=not self.options['verbose'], + environment=[self.env_name], + disable_auto_clean="nobuild" in target, + target=target) + + def process(self): + raise NotImplementedError + + def run(self): + raise NotImplementedError + + def on_run_out(self, line): + if line.endswith(":PASS"): + click.echo( + "%s\t[%s]" % (line[:-5], click.style("PASSED", fg="green"))) + elif ":FAIL" in line: + self._run_failed = True + click.echo("%s\t[%s]" % (line, click.style("FAILED", fg="red"))) + else: + click.echo(line) + + def generate_outputcpp(self, test_dir): + assert isdir(test_dir) + + cpp_tpl = "\n".join([ + "$include", + "#include ", + "", + "$object", + "", + "void output_start(unsigned int baudrate)", + "{", + " $begin;", + "}", + "", + "void output_char(int c)", + "{", + " $putchar;", + "}", + "", + "void output_flush(void)", + "{", + " $flush;", + "}", + "", + "void output_complete(void)", + "{", + " $end;", + "}" + ]) # yapf: disable + + def delete_tmptest_file(file_): + try: + remove(file_) + except: # pylint: disable=bare-except + if isfile(file_): + click.secho( + "Warning: Could not remove temporary file '%s'. " + "Please remove it manually." % file_, + fg="yellow") + + tpl = Template(cpp_tpl).substitute( + TRANSPORT_OPTIONS[self.get_transport()]) + data = Template(tpl).substitute(baudrate=self.get_baudrate()) + + tmp_file = join(test_dir, "output_export.cpp") + with open(tmp_file, "w") as f: + f.write(data) + + atexit.register(delete_tmptest_file, tmp_file) diff --git a/platformio/exception.py b/platformio/exception.py index 7829b648..77837d7a 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -292,3 +292,12 @@ class DebugSupportError(PlatformioException): class DebugInvalidOptions(PlatformioException): pass + + +class TestDirNotExists(PlatformioException): + + MESSAGE = "A test folder '{0}' does not exist.\nPlease create 'test' "\ + "directory in project's root and put a test set.\n"\ + "More details about Unit "\ + "Testing: http://docs.platformio.org/page/plus/"\ + "unit-testing.html" From 3032cade17909bf0b5f08d1c182a2692e66e3416 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Apr 2019 20:46:28 +0300 Subject: [PATCH 04/13] PyLint fixes --- platformio/commands/test/command.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index 49da7bba..a4a3d627 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-arguments, too-many-locals, too-many-branches + from fnmatch import fnmatch from os import getcwd, listdir from os.path import isdir, join From 65354e995d8511caa36b24a8ff337090dbbd5394 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Apr 2019 21:07:28 +0300 Subject: [PATCH 05/13] Initial commit of PIO Home --- .isort.cfg | 2 +- platformio/commands/home/__init__.py | 15 + platformio/commands/home/command.py | 100 +++++++ platformio/commands/home/helpers.py | 69 +++++ platformio/commands/home/rpc/__init__.py | 13 + .../commands/home/rpc/handlers/__init__.py | 13 + platformio/commands/home/rpc/handlers/app.py | 83 +++++ platformio/commands/home/rpc/handlers/ide.py | 42 +++ platformio/commands/home/rpc/handlers/misc.py | 194 ++++++++++++ platformio/commands/home/rpc/handlers/os.py | 153 ++++++++++ .../commands/home/rpc/handlers/piocore.py | 83 +++++ .../commands/home/rpc/handlers/project.py | 283 ++++++++++++++++++ platformio/commands/home/rpc/server.py | 70 +++++ platformio/commands/{home.py => home/web.py} | 29 +- 14 files changed, 1133 insertions(+), 16 deletions(-) create mode 100644 platformio/commands/home/__init__.py create mode 100644 platformio/commands/home/command.py create mode 100644 platformio/commands/home/helpers.py create mode 100644 platformio/commands/home/rpc/__init__.py create mode 100644 platformio/commands/home/rpc/handlers/__init__.py create mode 100644 platformio/commands/home/rpc/handlers/app.py create mode 100644 platformio/commands/home/rpc/handlers/ide.py create mode 100644 platformio/commands/home/rpc/handlers/misc.py create mode 100644 platformio/commands/home/rpc/handlers/os.py create mode 100644 platformio/commands/home/rpc/handlers/piocore.py create mode 100644 platformio/commands/home/rpc/handlers/project.py create mode 100644 platformio/commands/home/rpc/server.py rename platformio/commands/{home.py => home/web.py} (54%) diff --git a/.isort.cfg b/.isort.cfg index 1ee0d412..5f6d6207 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=79 -known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial, twisted +known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,bs4,jsonrpc diff --git a/platformio/commands/home/__init__.py b/platformio/commands/home/__init__.py new file mode 100644 index 00000000..a889291e --- /dev/null +++ b/platformio/commands/home/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.home.command import cli diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py new file mode 100644 index 00000000..0685670a --- /dev/null +++ b/platformio/commands/home/command.py @@ -0,0 +1,100 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import socket +from os.path import isdir + +import click + +from platformio import exception +from platformio.managers.core import (get_core_package_dir, + inject_contrib_pysite) + + +@click.command("home", short_help="PIO Home") +@click.option("--port", type=int, default=8008, help="HTTP port, default=8008") +@click.option( + "--host", + default="127.0.0.1", + help="HTTP host, default=127.0.0.1. " + "You can open PIO Home for inbound connections with --host=0.0.0.0") +@click.option("--no-open", is_flag=True) # pylint: disable=too-many-locals +def cli(port, host, no_open): + # import contrib modules + inject_contrib_pysite() + from autobahn.twisted.resource import WebSocketResource + from twisted.internet import reactor + from twisted.web import server + from platformio.commands.home.rpc.handlers.app import AppRPC + from platformio.commands.home.rpc.handlers.ide import IDERPC + from platformio.commands.home.rpc.handlers.misc import MiscRPC + from platformio.commands.home.rpc.handlers.os import OSRPC + from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC + from platformio.commands.home.rpc.handlers.project import ProjectRPC + from platformio.commands.home.rpc.server import JSONRPCServerFactory + from platformio.commands.home.web import WebRoot + + factory = JSONRPCServerFactory() + factory.addHandler(AppRPC(), namespace="app") + factory.addHandler(IDERPC(), namespace="ide") + factory.addHandler(MiscRPC(), namespace="misc") + factory.addHandler(OSRPC(), namespace="os") + factory.addHandler(PIOCoreRPC(), namespace="core") + factory.addHandler(ProjectRPC(), namespace="project") + + contrib_dir = get_core_package_dir("contrib-piohome") + if not isdir(contrib_dir): + raise exception.PlatformioException("Invalid path to PIO Home Contrib") + root = WebRoot(contrib_dir) + root.putChild(b"wsrpc", WebSocketResource(factory)) + site = server.Site(root) + + # hook for `platformio-node-helpers` + if host == "__do_not_start__": + return + + # if already started + already_started = False + socket.setdefaulttimeout(1) + try: + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) + already_started = True + except: # pylint: disable=bare-except + pass + + home_url = "http://%s:%d" % (host, port) + if not no_open: + if already_started: + click.launch(home_url) + else: + reactor.callLater(1, lambda: click.launch(home_url)) + + click.echo("\n".join([ + "", + " ___I_", + " /\\-_--\\ PlatformIO Home", + "/ \\_-__\\", + "|[]| [] | %s" % home_url, + "|__|____|______________%s" % ("_" * len(host)), + ])) + click.echo("") + click.echo("Open PIO Home in your browser by this URL => %s" % home_url) + + if already_started: + return + + click.echo("PIO Home has been started. Press Ctrl+C to shutdown.") + + reactor.listenTCP(port, site, interface=host) + reactor.run() diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py new file mode 100644 index 00000000..abac9416 --- /dev/null +++ b/platformio/commands/home/helpers.py @@ -0,0 +1,69 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=keyword-arg-before-vararg, arguments-differ + +import os +import socket + +import requests +from twisted.internet import threads +from twisted.internet.defer import ensureDeferred + +from platformio import util + + +class AsyncSession(requests.Session): + + def __init__(self, n=None, *args, **kwargs): + if n: + from twisted.internet import reactor + pool = reactor.getThreadPool() + pool.adjustPoolsize(0, n) + + super(AsyncSession, self).__init__(*args, **kwargs) + + def request(self, *args, **kwargs): + func = super(AsyncSession, self).request + return threads.deferToThread(func, *args, **kwargs) + + def wrap(self, *args, **kwargs): # pylint: disable=no-self-use + return ensureDeferred(*args, **kwargs) + + +@util.memoized(expire=5000) +def requests_session(): + return AsyncSession(n=5) + + +@util.memoized() +def get_core_fullpath(): + return util.where_is_program( + "platformio" + (".exe" if "windows" in util.get_systype() else "")) + + +@util.memoized(expire=10000) +def is_twitter_blocked(): + ip = "104.244.42.1" + timeout = 2 + try: + if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): + requests.get( + "http://%s" % ip, allow_redirects=False, timeout=timeout) + else: + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80)) + return False + except: # pylint: disable=bare-except + pass + return True diff --git a/platformio/commands/home/rpc/__init__.py b/platformio/commands/home/rpc/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/home/rpc/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/home/rpc/handlers/__init__.py b/platformio/commands/home/rpc/handlers/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py new file mode 100644 index 00000000..ce95a218 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/app.py @@ -0,0 +1,83 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import json +from os.path import expanduser, isfile, join + +from platformio import __version__, app, exception, util + + +class AppRPC(object): + + APPSTATE_PATH = join(util.get_home_dir(), "homestate.json") + + @staticmethod + def load_state(): + state = None + try: + if isfile(AppRPC.APPSTATE_PATH): + state = util.load_json(AppRPC.APPSTATE_PATH) + except exception.PlatformioException: + pass + if not isinstance(state, dict): + state = {} + storage = state.get("storage", {}) + + # base data + caller_id = app.get_session_var("caller_id") + storage['cid'] = app.get_cid() + storage['coreVersion'] = __version__ + storage['coreSystype'] = util.get_systype() + storage['coreCaller'] = (str(caller_id).lower() if caller_id else None) + storage['coreSettings'] = { + name: { + "description": data['description'], + "default_value": data['value'], + "value": app.get_setting(name) + } + for name, data in app.DEFAULT_SETTINGS.items() + } + + # encode to UTF-8 + for key in storage['coreSettings']: + if not key.endswith("dir"): + continue + storage['coreSettings'][key][ + 'default_value'] = util.path_to_unicode( + storage['coreSettings'][key]['default_value']) + storage['coreSettings'][key]['value'] = util.path_to_unicode( + storage['coreSettings'][key]['value']) + storage['homeDir'] = util.path_to_unicode(expanduser("~")) + storage['projectsDir'] = storage['coreSettings']['projects_dir'][ + 'value'] + + # skip non-existing recent projects + storage['recentProjects'] = [ + p for p in storage.get("recentProjects", []) + if util.is_platformio_project(p) + ] + + state['storage'] = storage + return state + + @staticmethod + def get_state(): + return AppRPC.load_state() + + def save_state(self, state): + with open(self.APPSTATE_PATH, "w") as fp: + json.dump(state, fp) + return True diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py new file mode 100644 index 00000000..d5c2bf74 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +from jsonrpc.exceptions import JSONRPCDispatchException +from twisted.internet import defer + + +class IDERPC(object): + + def __init__(self): + self._queue = [] + + def send_command(self, command, params): + if not self._queue: + raise JSONRPCDispatchException( + code=4005, message="PIO Home IDE agent is not started") + while self._queue: + self._queue.pop().callback({ + "id": time.time(), + "method": command, + "params": params + }) + + def listen_commands(self): + self._queue.append(defer.Deferred()) + return self._queue[-1] + + def open_project(self, project_dir): + return self.send_command("open_project", project_dir) diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py new file mode 100644 index 00000000..d7deff19 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -0,0 +1,194 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re +import time + +from bs4 import BeautifulSoup +from twisted.internet import defer, reactor + +from platformio import app +from platformio.commands.home import helpers +from platformio.commands.home.rpc.handlers.os import OSRPC + + +class MiscRPC(object): + + def load_latest_tweets(self, username): + cache_key = "piohome_latest_tweets_%s" % username + cache_valid = "7d" + with app.ContentCache() as cc: + cache_data = cc.get(cache_key) + if cache_data: + cache_data = json.loads(cache_data) + # automatically update cache in background every 12 hours + if cache_data['time'] < (time.time() - (3600 * 12)): + reactor.callLater(5, self._preload_latest_tweets, username, + cache_key, cache_valid) + return cache_data['result'] + + result = self._preload_latest_tweets(username, cache_key, cache_valid) + return result + + @defer.inlineCallbacks + def _preload_latest_tweets(self, username, cache_key, cache_valid): + result = yield self._fetch_tweets(username) + with app.ContentCache() as cc: + cc.set(cache_key, + json.dumps({ + "time": int(time.time()), + "result": result + }), cache_valid) + defer.returnValue(result) + + @defer.inlineCallbacks + def _fetch_tweets(self, username): + api_url = ("https://twitter.com/i/profiles/show/%s/timeline/tweets?" + "include_available_features=1&include_entities=1&" + "include_new_items_bar=true") % username + if helpers.is_twitter_blocked(): + api_url = self._get_proxed_uri(api_url) + html_or_json = yield OSRPC.fetch_content( + api_url, headers=self._get_twitter_headers(username)) + # issue with PIO Core < 3.5.3 and ContentCache + if not isinstance(html_or_json, dict): + html_or_json = json.loads(html_or_json) + assert "items_html" in html_or_json + soup = BeautifulSoup(html_or_json['items_html'], "html.parser") + tweet_nodes = soup.find_all( + "div", attrs={ + "class": "tweet", + "data-tweet-id": True + }) + result = yield defer.DeferredList( + [self._parse_tweet_node(node, username) for node in tweet_nodes], + consumeErrors=True) + defer.returnValue([r[1] for r in result if r[0]]) + + @defer.inlineCallbacks + def _parse_tweet_node(self, tweet, username): + # remove non-visible items + for node in tweet.find_all(class_=["invisible", "u-hidden"]): + node.decompose() + twitter_url = "https://twitter.com" + time_node = tweet.find("span", attrs={"data-time": True}) + text_node = tweet.find(class_="tweet-text") + quote_text_node = tweet.find(class_="QuoteTweet-text") + if quote_text_node and not text_node.get_text().strip(): + text_node = quote_text_node + photos = [ + node.get("data-image-url") for node in (tweet.find_all(class_=[ + "AdaptiveMedia-photoContainer", "QuoteMedia-photoContainer" + ]) or []) + ] + urls = [ + node.get("data-expanded-url") + for node in (quote_text_node or text_node).find_all( + class_="twitter-timeline-link", + attrs={"data-expanded-url": True}) + ] + + # fetch data from iframe card + if (not photos or not urls) and tweet.get("data-card2-type"): + iframe_node = tweet.find( + "div", attrs={"data-full-card-iframe-url": True}) + if iframe_node: + iframe_card = yield self._fetch_iframe_card( + twitter_url + iframe_node.get("data-full-card-iframe-url"), + username) + if not photos and iframe_card['photo']: + photos.append(iframe_card['photo']) + if not urls and iframe_card['url']: + urls.append(iframe_card['url']) + if iframe_card['text_node']: + text_node = iframe_card['text_node'] + + if not photos: + photos.append(tweet.find("img", class_="avatar").get("src")) + + def _fetch_text(text_node): + text = text_node.decode_contents(formatter="html").strip() + text = re.sub(r'href="/', 'href="%s/' % twitter_url, text) + if "

" not in text and "", text) + return text + + defer.returnValue({ + "tweetId": + tweet.get("data-tweet-id"), + "tweetUrl": + twitter_url + tweet.get("data-permalink-path"), + "author": + tweet.get("data-name"), + "time": + int(time_node.get("data-time")), + "timeFormatted": + time_node.string, + "text": + _fetch_text(text_node), + "entries": { + "urls": + urls, + "photos": [ + self._get_proxed_uri(uri) + if helpers.is_twitter_blocked() else uri for uri in photos + ] + }, + "isPinned": + "user-pinned" in tweet.get("class") + }) + + @defer.inlineCallbacks + def _fetch_iframe_card(self, url, username): + if helpers.is_twitter_blocked(): + url = self._get_proxed_uri(url) + html = yield OSRPC.fetch_content( + url, headers=self._get_twitter_headers(username), cache_valid="7d") + soup = BeautifulSoup(html, "html.parser") + photo_node = soup.find("img", attrs={"data-src": True}) + url_node = soup.find("a", class_="TwitterCard-container") + text_node = soup.find("div", class_="SummaryCard-content") + if text_node: + text_node.find( + "span", class_="SummaryCard-destination").decompose() + defer.returnValue({ + "photo": + photo_node.get("data-src") if photo_node else None, + "text_node": + text_node, + "url": + url_node.get("href") if url_node else None + }) + + @staticmethod + def _get_proxed_uri(uri): + index = uri.index("://") + return "https://dl.platformio.org/__prx__/" + uri[index + 3:] + + @staticmethod + def _get_twitter_headers(username): + return { + "Accept": + "application/json, text/javascript, */*; q=0.01", + "Referer": + "https://twitter.com/%s" % username, + "User-Agent": + ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit" + "/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8"), + "X-Twitter-Active-User": + "yes", + "X-Requested-With": + "XMLHttpRequest" + } diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py new file mode 100644 index 00000000..f581c469 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/os.py @@ -0,0 +1,153 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import glob +import os +import shutil +import sys +from functools import cmp_to_key +from os.path import expanduser, isdir, isfile, join + +import click +from twisted.internet import defer + +from platformio import app, util +from platformio.commands.home import helpers + + +class OSRPC(object): + + @staticmethod + @defer.inlineCallbacks + def fetch_content(uri, data=None, headers=None, cache_valid=None): + timeout = 2 + if not headers: + headers = { + "User-Agent": + ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) " + "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 " + "Safari/603.3.8") + } + cache_key = (app.ContentCache.key_from_args(uri, data) + if cache_valid else None) + with app.ContentCache() as cc: + if cache_key: + result = cc.get(cache_key) + if result is not None: + defer.returnValue(result) + + # check internet before and resolve issue with 60 seconds timeout + util.internet_on(raise_exception=True) + + session = helpers.requests_session() + if data: + r = yield session.post( + uri, data=data, headers=headers, timeout=timeout) + else: + r = yield session.get(uri, headers=headers, timeout=timeout) + + r.raise_for_status() + result = r.text + if cache_valid: + with app.ContentCache() as cc: + cc.set(cache_key, result, cache_valid) + defer.returnValue(result) + + def request_content(self, uri, data=None, headers=None, cache_valid=None): + if uri.startswith('http'): + return self.fetch_content(uri, data, headers, cache_valid) + if isfile(uri): + with open(uri) as fp: + return fp.read() + return None + + @staticmethod + def open_url(url): + return click.launch(url) + + @staticmethod + def reveal_file(path): + return click.launch( + path.encode(sys.getfilesystemencoding()) if util.PY2 else path, + locate=True) + + @staticmethod + def is_file(path): + return isfile(path) + + @staticmethod + def is_dir(path): + return isdir(path) + + @staticmethod + def make_dirs(path): + return os.makedirs(path) + + @staticmethod + def rename(src, dst): + return os.rename(src, dst) + + @staticmethod + def copy(src, dst): + return shutil.copytree(src, dst) + + @staticmethod + def glob(pathnames, root=None): + if not isinstance(pathnames, list): + pathnames = [pathnames] + result = set() + for pathname in pathnames: + result |= set( + glob.glob(join(root, pathname) if root else pathname)) + return list(result) + + @staticmethod + def list_dir(path): + + def _cmp(x, y): + if x[1] and not y[1]: + return -1 + if not x[1] and y[1]: + return 1 + if x[0].lower() > y[0].lower(): + return 1 + if x[0].lower() < y[0].lower(): + return -1 + return 0 + + items = [] + if path.startswith("~"): + path = expanduser(path) + if not isdir(path): + return items + for item in os.listdir(path): + try: + item_is_dir = isdir(join(path, item)) + if item_is_dir: + os.listdir(join(path, item)) + items.append((item, item_is_dir)) + except OSError: + pass + return sorted(items, key=cmp_to_key(_cmp)) + + @staticmethod + def get_logical_devices(): + items = [] + for item in util.get_logical_devices(): + if item['name']: + item['name'] = util.path_to_unicode(item['name']) + items.append(item) + return items diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py new file mode 100644 index 00000000..85255a9c --- /dev/null +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -0,0 +1,83 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import json +import os +import re +import sys + +from jsonrpc.exceptions import JSONRPCDispatchException +from twisted.internet.utils import getProcessOutputAndValue + +from platformio import __version__, util +from platformio.commands.home import helpers + + +class PIOCoreRPC(object): + + @staticmethod + def call(args, options=None): + json_output = "--json-output" in args + try: + args = [ + arg.encode(sys.getfilesystemencoding()) if isinstance( + arg, util.string_types) else str(arg) for arg in args + ] + except UnicodeError: + raise JSONRPCDispatchException( + code=4002, message="PIO Core: non-ASCII chars in arguments") + d = getProcessOutputAndValue( + helpers.get_core_fullpath(), + args, + path=(options or {}).get("cwd"), + env={k: v + for k, v in os.environ.items() if "%" not in k}) + d.addCallback(PIOCoreRPC._call_callback, json_output) + d.addErrback(PIOCoreRPC._call_errback) + return d + + @staticmethod + def _call_callback(result, json_output=False): + result = list(result) + assert len(result) == 3 + for i in (0, 1): + result[i] = result[i].decode(sys.getfilesystemencoding()).strip() + out, err, code = result + text = ("%s\n\n%s" % (out, err)).strip() + if code != 0: + raise Exception(text) + + if not json_output: + return text + + try: + return json.loads(out) + except ValueError as e: + if "sh: " in out: + return json.loads( + re.sub(r"^sh: [^\n]+$", "", out, flags=re.M).strip()) + raise e + + @staticmethod + def _call_errback(failure): + raise JSONRPCDispatchException( + code=4003, + message="PIO Core Call Error", + data=failure.getErrorMessage()) + + @staticmethod + def version(): + return __version__ diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py new file mode 100644 index 00000000..02961ed8 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/project.py @@ -0,0 +1,283 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os +import shutil +import sys +import time +from os.path import (basename, expanduser, getmtime, isdir, isfile, join, + realpath, sep) + +from jsonrpc.exceptions import JSONRPCDispatchException + +from platformio import exception, util +from platformio.commands.home.rpc.handlers.app import AppRPC +from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.ide.projectgenerator import ProjectGenerator +from platformio.managers.platform import PlatformManager + +try: + from configparser import Error as ConfigParserError +except ImportError: + from ConfigParser import Error as ConfigParserError + + +class ProjectRPC(object): + + @staticmethod + def _get_projects(project_dirs=None): + + def _get_project_data(project_dir): + data = {"boards": [], "libExtraDirs": []} + config = util.load_project_config(project_dir) + + if config.has_section("platformio") and \ + config.has_option("platformio", "lib_extra_dirs"): + data['libExtraDirs'].extend( + util.parse_conf_multi_values( + config.get("platformio", "lib_extra_dirs"))) + + for section in config.sections(): + if not section.startswith("env:"): + continue + if config.has_option(section, "board"): + data['boards'].append(config.get(section, "board")) + if config.has_option(section, "lib_extra_dirs"): + data['libExtraDirs'].extend( + util.parse_conf_multi_values( + config.get(section, "lib_extra_dirs"))) + + # resolve libExtraDirs paths + with util.cd(project_dir): + data['libExtraDirs'] = [ + expanduser(d) if d.startswith("~") else realpath(d) + for d in data['libExtraDirs'] + ] + + # skip non existing folders + data['libExtraDirs'] = [ + d for d in data['libExtraDirs'] if isdir(d) + ] + + return data + + def _path_to_name(path): + return (sep).join(path.split(sep)[-2:]) + + if not project_dirs: + project_dirs = AppRPC.load_state()['storage']['recentProjects'] + + result = [] + pm = PlatformManager() + for project_dir in project_dirs: + data = {} + boards = [] + try: + data = _get_project_data(project_dir) + except exception.NotPlatformIOProject: + continue + except ConfigParserError: + pass + + for board_id in data.get("boards", []): + name = board_id + try: + name = pm.board_config(board_id)['name'] + except (exception.UnknownBoard, exception.UnknownPlatform): + pass + boards.append({"id": board_id, "name": name}) + + result.append({ + "path": + project_dir, + "name": + _path_to_name(project_dir), + "modified": + int(getmtime(project_dir)), + "boards": + boards, + "extraLibStorages": [{ + "name": _path_to_name(d), + "path": d + } for d in data.get("libExtraDirs", [])] + }) + return result + + def get_projects(self, project_dirs=None): + return self._get_projects(project_dirs) + + def init(self, board, framework, project_dir): + assert project_dir + state = AppRPC.load_state() + if not isdir(project_dir): + os.makedirs(project_dir) + args = ["init", "--project-dir", project_dir, "--board", board] + if framework: + args.extend(["--project-option", "framework = %s" % framework]) + if (state['storage']['coreCaller'] and state['storage']['coreCaller'] + in ProjectGenerator.get_supported_ides()): + args.extend(["--ide", state['storage']['coreCaller']]) + d = PIOCoreRPC.call(args) + d.addCallback(self._generate_project_main, project_dir, framework) + return d + + @staticmethod + def _generate_project_main(_, project_dir, framework): + main_content = None + if framework == "arduino": + main_content = "\n".join([ + "#include ", + "", + "void setup() {", + " // put your setup code here, to run once:", + "}", + "", + "void loop() {", + " // put your main code here, to run repeatedly:", + "}" + "" + ]) # yapf: disable + elif framework == "mbed": + main_content = "\n".join([ + "#include ", + "", + "int main() {", + "", + " // put your setup code here, to run once:", + "", + " while(1) {", + " // put your main code here, to run repeatedly:", + " }", + "}", + "" + ]) # yapf: disable + if not main_content: + return project_dir + with util.cd(project_dir): + src_dir = util.get_projectsrc_dir() + main_path = join(src_dir, "main.cpp") + if isfile(main_path): + return project_dir + if not isdir(src_dir): + os.makedirs(src_dir) + with open(main_path, "w") as f: + f.write(main_content.strip()) + return project_dir + + def import_arduino(self, board, use_arduino_libs, arduino_project_dir): + # don't import PIO Project + if util.is_platformio_project(arduino_project_dir): + return arduino_project_dir + + is_arduino_project = any([ + isfile( + join(arduino_project_dir, + "%s.%s" % (basename(arduino_project_dir), ext))) + for ext in ("ino", "pde") + ]) + if not is_arduino_project: + raise JSONRPCDispatchException( + code=4000, + message="Not an Arduino project: %s" % arduino_project_dir) + + state = AppRPC.load_state() + project_dir = join(state['storage']['projectsDir'].decode("utf-8"), + time.strftime("%y%m%d-%H%M%S-") + board) + if not isdir(project_dir): + os.makedirs(project_dir) + args = ["init", "--project-dir", project_dir, "--board", board] + args.extend(["--project-option", "framework = arduino"]) + if use_arduino_libs: + args.extend([ + "--project-option", + "lib_extra_dirs = ~/Documents/Arduino/libraries" + ]) + if (state['storage']['coreCaller'] and state['storage']['coreCaller'] + in ProjectGenerator.get_supported_ides()): + args.extend(["--ide", state['storage']['coreCaller']]) + d = PIOCoreRPC.call(args) + d.addCallback(self._finalize_arduino_import, project_dir, + arduino_project_dir) + return d + + @staticmethod + def _finalize_arduino_import(_, project_dir, arduino_project_dir): + with util.cd(project_dir): + src_dir = util.get_projectsrc_dir() + if isdir(src_dir): + util.rmtree_(src_dir) + shutil.copytree( + arduino_project_dir.encode(sys.getfilesystemencoding()), + src_dir) + return project_dir + + @staticmethod + def get_project_examples(): + result = [] + for manifest in PlatformManager().get_installed(): + examples_dir = join(manifest['__pkg_dir'], "examples") + if not isdir(examples_dir): + continue + items = [] + for project_dir, _, __ in os.walk(examples_dir): + project_description = None + try: + config = util.load_project_config(project_dir) + if config.has_section("platformio") and \ + config.has_option("platformio", "description"): + project_description = config.get( + "platformio", "description") + except (exception.NotPlatformIOProject, + exception.InvalidProjectConf): + continue + + path_tokens = project_dir.split(sep) + items.append({ + "name": + "/".join(path_tokens[path_tokens.index("examples") + 1:]), + "path": + project_dir, + "description": + project_description + }) + result.append({ + "platform": { + "title": manifest['title'], + "version": manifest['version'] + }, + "items": sorted(items, key=lambda item: item['name']) + }) + return sorted(result, key=lambda data: data['platform']['title']) + + @staticmethod + def import_pio(project_dir): + if not project_dir or not util.is_platformio_project(project_dir): + raise JSONRPCDispatchException( + code=4001, + message="Not an PlatformIO project: %s" % project_dir) + new_project_dir = join( + AppRPC.load_state()['storage']['projectsDir'].decode("utf-8"), + time.strftime("%y%m%d-%H%M%S-") + basename(project_dir)) + shutil.copytree(project_dir, new_project_dir) + + state = AppRPC.load_state() + args = ["init", "--project-dir", new_project_dir] + if (state['storage']['coreCaller'] and state['storage']['coreCaller'] + in ProjectGenerator.get_supported_ides()): + args.extend(["--ide", state['storage']['coreCaller']]) + d = PIOCoreRPC.call(args) + d.addCallback(lambda _: new_project_dir) + return d diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py new file mode 100644 index 00000000..cd4f05b9 --- /dev/null +++ b/platformio/commands/home/rpc/server.py @@ -0,0 +1,70 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import jsonrpc +from autobahn.twisted.websocket import (WebSocketServerFactory, + WebSocketServerProtocol) +from jsonrpc.exceptions import JSONRPCDispatchException +from twisted.internet import defer + + +class JSONRPCServerProtocol(WebSocketServerProtocol): + + def onMessage(self, payload, isBinary): # pylint: disable=unused-argument + # print("> %s" % payload) + response = jsonrpc.JSONRPCResponseManager.handle( + payload, self.factory.dispatcher).data + # if error + if "result" not in response: + self.sendJSONResponse(response) + return None + + d = defer.maybeDeferred(lambda: response['result']) + d.addCallback(self._callback, response) + d.addErrback(self._errback, response) + + return None + + def _callback(self, result, response): + response['result'] = result + self.sendJSONResponse(response) + + def _errback(self, failure, response): + if isinstance(failure.value, JSONRPCDispatchException): + e = failure.value + else: + e = JSONRPCDispatchException( + code=4999, message=failure.getErrorMessage()) + del response["result"] + response['error'] = e.error._data # pylint: disable=protected-access + print(response['error']) + self.sendJSONResponse(response) + + def sendJSONResponse(self, response): + # print("< %s" % response) + self.sendMessage(json.dumps(response).encode("utf8")) + + +class JSONRPCServerFactory(WebSocketServerFactory): + + protocol = JSONRPCServerProtocol + + def __init__(self): + super(JSONRPCServerFactory, self).__init__() + self.dispatcher = jsonrpc.Dispatcher() + + def addHandler(self, handler, namespace): + self.dispatcher.build_method_map(handler, prefix="%s." % namespace) diff --git a/platformio/commands/home.py b/platformio/commands/home/web.py similarity index 54% rename from platformio/commands/home.py rename to platformio/commands/home/web.py index cd6b86f6..313ce084 100644 --- a/platformio/commands/home.py +++ b/platformio/commands/home/web.py @@ -12,20 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -import click - -from platformio.managers.core import pioplus_call +from twisted.internet import reactor +from twisted.web import static -@click.command("home", short_help="PIO Home") -@click.option("--port", type=int, default=8008, help="HTTP port, default=8008") -@click.option( - "--host", - default="127.0.0.1", - help="HTTP host, default=127.0.0.1. " - "You can open PIO Home for inbound connections with --host=0.0.0.0") -@click.option("--no-open", is_flag=True) -def cli(*args, **kwargs): # pylint: disable=unused-argument - pioplus_call(sys.argv[1:]) +class WebRoot(static.File): + + def render_GET(self, request): + if request.args.get("__shutdown__", False): + reactor.stop() + return "Server has been stopped" + + request.setHeader("cache-control", + "no-cache, no-store, must-revalidate") + request.setHeader("pragma", "no-cache") + request.setHeader("expires", "0") + return static.File.render_GET(self, request) From 131144ec347e051d6998c258fb631392855d6f8a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 13:00:53 +0300 Subject: [PATCH 06/13] Resolve PyLint "import-error" --- platformio/commands/debug/client.py | 5 ++++- platformio/commands/debug/process.py | 2 +- platformio/commands/debug/server.py | 2 +- platformio/commands/home/command.py | 2 ++ platformio/commands/home/helpers.py | 8 ++++---- platformio/commands/home/rpc/handlers/ide.py | 6 +++--- platformio/commands/home/rpc/handlers/misc.py | 4 ++-- platformio/commands/home/rpc/handlers/os.py | 2 +- platformio/commands/home/rpc/handlers/piocore.py | 10 +++++----- platformio/commands/home/rpc/handlers/project.py | 6 +++--- platformio/commands/home/rpc/server.py | 2 ++ platformio/commands/home/web.py | 4 ++-- 12 files changed, 30 insertions(+), 23 deletions(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 7e9aebb9..ac97243f 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -20,7 +20,10 @@ from hashlib import sha1 from os.path import abspath, basename, dirname, isdir, join, splitext from tempfile import mkdtemp -from twisted.internet import protocol, reactor, stdio, task +from twisted.internet import protocol # pylint: disable=import-error +from twisted.internet import reactor # pylint: disable=import-error +from twisted.internet import stdio # pylint: disable=import-error +from twisted.internet import task # pylint: disable=import-error from platformio import app, exception, util from platformio.commands.debug import helpers, initcfgs diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py index 3f1fbd22..0079e18c 100644 --- a/platformio/commands/debug/process.py +++ b/platformio/commands/debug/process.py @@ -15,7 +15,7 @@ import os import click -from twisted.internet import protocol +from twisted.internet import protocol # pylint: disable=import-error from platformio import util from platformio.commands.debug import helpers diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index 977fe812..ca5c2a2e 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -15,7 +15,7 @@ import os from os.path import isdir, isfile, join -from twisted.internet import reactor +from twisted.internet import reactor # pylint: disable=import-error from platformio import exception, util from platformio.commands.debug import helpers diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 0685670a..917187fb 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -33,9 +33,11 @@ from platformio.managers.core import (get_core_package_dir, def cli(port, host, no_open): # import contrib modules inject_contrib_pysite() + # pylint: disable=import-error from autobahn.twisted.resource import WebSocketResource from twisted.internet import reactor from twisted.web import server + # pylint: enable=import-error from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.ide import IDERPC from platformio.commands.home.rpc.handlers.misc import MiscRPC diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index abac9416..20b9503c 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -18,8 +18,9 @@ import os import socket import requests -from twisted.internet import threads -from twisted.internet.defer import ensureDeferred +from twisted.internet import defer # pylint: disable=import-error +from twisted.internet import reactor # pylint: disable=import-error +from twisted.internet import threads # pylint: disable=import-error from platformio import util @@ -28,7 +29,6 @@ class AsyncSession(requests.Session): def __init__(self, n=None, *args, **kwargs): if n: - from twisted.internet import reactor pool = reactor.getThreadPool() pool.adjustPoolsize(0, n) @@ -39,7 +39,7 @@ class AsyncSession(requests.Session): return threads.deferToThread(func, *args, **kwargs) def wrap(self, *args, **kwargs): # pylint: disable=no-self-use - return ensureDeferred(*args, **kwargs) + return defer.ensureDeferred(*args, **kwargs) @util.memoized(expire=5000) diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index d5c2bf74..5f8f31a4 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -14,8 +14,8 @@ import time -from jsonrpc.exceptions import JSONRPCDispatchException -from twisted.internet import defer +import jsonrpc # pylint: disable=import-error +from twisted.internet import defer # pylint: disable=import-error class IDERPC(object): @@ -25,7 +25,7 @@ class IDERPC(object): def send_command(self, command, params): if not self._queue: - raise JSONRPCDispatchException( + raise jsonrpc.exceptions.JSONRPCDispatchException( code=4005, message="PIO Home IDE agent is not started") while self._queue: self._queue.pop().callback({ diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index d7deff19..d3b53383 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -16,8 +16,8 @@ import json import re import time -from bs4 import BeautifulSoup -from twisted.internet import defer, reactor +from bs4 import BeautifulSoup # pylint: disable=import-error +from twisted.internet import defer, reactor # pylint: disable=import-error from platformio import app from platformio.commands.home import helpers diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index f581c469..780dbec8 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -22,7 +22,7 @@ from functools import cmp_to_key from os.path import expanduser, isdir, isfile, join import click -from twisted.internet import defer +from twisted.internet import defer # pylint: disable=import-error from platformio import app, util from platformio.commands.home import helpers diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 85255a9c..80a642cb 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -19,8 +19,8 @@ import os import re import sys -from jsonrpc.exceptions import JSONRPCDispatchException -from twisted.internet.utils import getProcessOutputAndValue +import jsonrpc # pylint: disable=import-error +from twisted.internet import utils # pylint: disable=import-error from platformio import __version__, util from platformio.commands.home import helpers @@ -37,9 +37,9 @@ class PIOCoreRPC(object): arg, util.string_types) else str(arg) for arg in args ] except UnicodeError: - raise JSONRPCDispatchException( + raise jsonrpc.exceptions.JSONRPCDispatchException( code=4002, message="PIO Core: non-ASCII chars in arguments") - d = getProcessOutputAndValue( + d = utils.getProcessOutputAndValue( helpers.get_core_fullpath(), args, path=(options or {}).get("cwd"), @@ -73,7 +73,7 @@ class PIOCoreRPC(object): @staticmethod def _call_errback(failure): - raise JSONRPCDispatchException( + raise jsonrpc.exceptions.JSONRPCDispatchException( code=4003, message="PIO Core Call Error", data=failure.getErrorMessage()) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 02961ed8..2a4c4032 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -21,7 +21,7 @@ import time from os.path import (basename, expanduser, getmtime, isdir, isfile, join, realpath, sep) -from jsonrpc.exceptions import JSONRPCDispatchException +import jsonrpc # pylint: disable=import-error from platformio import exception, util from platformio.commands.home.rpc.handlers.app import AppRPC @@ -189,7 +189,7 @@ class ProjectRPC(object): for ext in ("ino", "pde") ]) if not is_arduino_project: - raise JSONRPCDispatchException( + raise jsonrpc.exceptions.JSONRPCDispatchException( code=4000, message="Not an Arduino project: %s" % arduino_project_dir) @@ -265,7 +265,7 @@ class ProjectRPC(object): @staticmethod def import_pio(project_dir): if not project_dir or not util.is_platformio_project(project_dir): - raise JSONRPCDispatchException( + raise jsonrpc.exceptions.JSONRPCDispatchException( code=4001, message="Not an PlatformIO project: %s" % project_dir) new_project_dir = join( diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index cd4f05b9..bf339994 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=import-error + import json import jsonrpc diff --git a/platformio/commands/home/web.py b/platformio/commands/home/web.py index 313ce084..df48b1fa 100644 --- a/platformio/commands/home/web.py +++ b/platformio/commands/home/web.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import reactor -from twisted.web import static +from twisted.internet import reactor # pylint: disable=import-error +from twisted.web import static # pylint: disable=import-error class WebRoot(static.File): From 76818448e29c029f4524dd90bdc2d844b794326b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 13:01:52 +0300 Subject: [PATCH 07/13] Ensure PIO Home mimetypes are known --- platformio/commands/home/command.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 917187fb..d1b7d607 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import mimetypes import socket from os.path import isdir @@ -58,6 +59,12 @@ def cli(port, host, no_open): contrib_dir = get_core_package_dir("contrib-piohome") if not isdir(contrib_dir): raise exception.PlatformioException("Invalid path to PIO Home Contrib") + + # Ensure PIO Home mimetypes are known + mimetypes.add_type("text/html", ".html") + mimetypes.add_type("text/css", ".css") + mimetypes.add_type("application/javascript", ".js") + root = WebRoot(contrib_dir) root.putChild(b"wsrpc", WebSocketResource(factory)) site = server.Site(root) From d9b0364aa8e8e5c1d20c1f396a8b30ec1b3a3439 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 13:12:41 +0300 Subject: [PATCH 08/13] Allow overriding a default project "platformio.ini" configuration file --- platformio/commands/debug/command.py | 18 +++++++++++++++--- platformio/commands/debug/helpers.py | 8 ++++---- platformio/commands/test/command.py | 15 ++++++++++++--- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 918aeb60..c604d8b9 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -39,12 +39,22 @@ from platformio.managers.core import inject_contrib_pysite dir_okay=True, writable=True, resolve_path=True)) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) @click.option("--environment", "-e", metavar="") @click.option("--verbose", "-v", is_flag=True) @click.option("--interface", type=click.Choice(["gdb"])) @click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED) @click.pass_context -def cli(ctx, project_dir, environment, verbose, interface, __unprocessed): +def cli(ctx, project_dir, project_conf, environment, verbose, interface, + __unprocessed): try: util.ensure_udev_rules() except NameError: @@ -59,8 +69,10 @@ def cli(ctx, project_dir, environment, verbose, interface, __unprocessed): project_dir = os.getenv("CWD") with util.cd(project_dir): - env_name = helpers.check_env_name(project_dir, environment) - env_options = helpers.get_env_options(project_dir, env_name) + env_name = helpers.check_env_name(project_conf or project_dir, + environment) + env_options = helpers.get_env_options(project_conf or project_dir, + env_name) if not set(env_options.keys()) >= set(["platform", "board"]): raise exception.ProjectEnvsNotAvailable() debug_options = helpers.validate_debug_options(ctx, env_options) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 42a4ef40..757d3ad9 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -46,8 +46,8 @@ def escape_path(path): return path.replace("\\", "/") -def check_env_name(project_dir, environment): - config = util.load_project_config(project_dir) +def check_env_name(project_conf, environment): + config = util.load_project_config(project_conf) envs = [] for section in config.sections(): if section.startswith("env:"): @@ -63,8 +63,8 @@ def check_env_name(project_dir, environment): return envs[0] -def get_env_options(project_dir, environment): - config = util.load_project_config(project_dir) +def get_env_options(project_conf, environment): + config = util.load_project_config(project_conf) options = {} for k, v in config.items("env:%s" % environment): options[k] = v diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index a4a3d627..47f8571f 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -53,6 +53,15 @@ from platformio.commands.test.native import NativeTestProcessor dir_okay=True, writable=True, resolve_path=True)) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @click.option("--without-testing", is_flag=True) @@ -71,14 +80,14 @@ from platformio.commands.test.native import NativeTestProcessor @click.pass_context def cli( # pylint: disable=redefined-builtin ctx, environment, ignore, filter, upload_port, test_port, project_dir, - without_building, without_uploading, without_testing, no_reset, - monitor_rts, monitor_dtr, verbose): + project_conf, without_building, without_uploading, without_testing, + no_reset, monitor_rts, monitor_dtr, verbose): with util.cd(project_dir): test_dir = util.get_projecttest_dir() if not isdir(test_dir): raise exception.TestDirNotExists(test_dir) test_names = get_test_names(test_dir) - projectconf = util.load_project_config() + projectconf = util.load_project_config(project_conf) env_default = None if projectconf.has_option("platformio", "env_default"): env_default = util.parse_conf_multi_values( From 0b2d7806183c817fbb7854c2245cb7b538fe09b2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 15:45:52 +0300 Subject: [PATCH 09/13] Switch to the new ProjectConfig API --- platformio/commands/debug/command.py | 13 +++--- platformio/commands/debug/helpers.py | 26 ++---------- .../commands/home/rpc/handlers/project.py | 19 ++++----- platformio/commands/test/command.py | 42 ++++++++----------- platformio/commands/test/processor.py | 6 +-- 5 files changed, 38 insertions(+), 68 deletions(-) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index c604d8b9..7e1bf62e 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -16,13 +16,14 @@ # pylint: disable=too-many-locals, too-many-branches import os -from os.path import isfile +from os.path import isfile, join import click from platformio import exception, util from platformio.commands.debug import helpers from platformio.managers.core import inject_contrib_pysite +from platformio.project.config import ProjectConfig @click.command( @@ -69,10 +70,12 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, project_dir = os.getenv("CWD") with util.cd(project_dir): - env_name = helpers.check_env_name(project_conf or project_dir, - environment) - env_options = helpers.get_env_options(project_conf or project_dir, - env_name) + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini")) + config.validate(envs=[environment] if environment else None) + + env_name = environment or helpers.get_default_debug_env(config) + env_options = config.items(env=env_name, as_dict=True) if not set(env_options.keys()) >= set(["platform", "board"]): raise exception.ProjectEnvsNotAvailable() debug_options = helpers.validate_debug_options(ctx, env_options) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 757d3ad9..139411ea 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -46,29 +46,9 @@ def escape_path(path): return path.replace("\\", "/") -def check_env_name(project_conf, environment): - config = util.load_project_config(project_conf) - envs = [] - for section in config.sections(): - if section.startswith("env:"): - envs.append(section[4:]) - if not envs: - raise exception.ProjectEnvsNotAvailable() - if not environment and config.has_option("platformio", "env_default"): - environment = config.get("platformio", "env_default").split(", ")[0] - if environment: - if environment in envs: - return environment - raise exception.UnknownEnvNames(environment, envs) - return envs[0] - - -def get_env_options(project_conf, environment): - config = util.load_project_config(project_conf) - options = {} - for k, v in config.items("env:%s" % environment): - options[k] = v - return options +def get_default_debug_env(config): + default_envs = config.default_envs() + return default_envs[0] if default_envs else config.envs()[0] def validate_debug_options(cmd_ctx, env_options): diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 2a4c4032..c3f48fcc 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -28,11 +28,7 @@ from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager - -try: - from configparser import Error as ConfigParserError -except ImportError: - from ConfigParser import Error as ConfigParserError +from platformio.project.config import ProjectConfig class ProjectRPC(object): @@ -42,7 +38,8 @@ class ProjectRPC(object): def _get_project_data(project_dir): data = {"boards": [], "libExtraDirs": []} - config = util.load_project_config(project_dir) + config = ProjectConfig(join(project_dir, "platformio.ini")) + config.validate(validate_options=False) if config.has_section("platformio") and \ config.has_option("platformio", "lib_extra_dirs"): @@ -87,10 +84,8 @@ class ProjectRPC(object): boards = [] try: data = _get_project_data(project_dir) - except exception.NotPlatformIOProject: + except exception.PlatformIOProjectException: continue - except ConfigParserError: - pass for board_id in data.get("boards", []): name = board_id @@ -235,13 +230,13 @@ class ProjectRPC(object): for project_dir, _, __ in os.walk(examples_dir): project_description = None try: - config = util.load_project_config(project_dir) + config = ProjectConfig(join(project_dir, "platformio.ini")) + config.validate(validate_options=False) if config.has_section("platformio") and \ config.has_option("platformio", "description"): project_description = config.get( "platformio", "description") - except (exception.NotPlatformIOProject, - exception.InvalidProjectConf): + except exception.PlatformIOProjectException: continue path_tokens = project_dir.split(sep) diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index 47f8571f..2fc2eca1 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -22,9 +22,10 @@ from time import time import click from platformio import exception, util -from platformio.commands.run import check_project_envs, print_header +from platformio.commands.run import print_header from platformio.commands.test.embedded import EmbeddedTestProcessor from platformio.commands.test.native import NativeTestProcessor +from platformio.project.config import ProjectConfig @click.command("test", short_help="Unit Testing") @@ -87,39 +88,32 @@ def cli( # pylint: disable=redefined-builtin if not isdir(test_dir): raise exception.TestDirNotExists(test_dir) test_names = get_test_names(test_dir) - projectconf = util.load_project_config(project_conf) - env_default = None - if projectconf.has_option("platformio", "env_default"): - env_default = util.parse_conf_multi_values( - projectconf.get("platformio", "env_default")) - assert check_project_envs(projectconf, environment or env_default) + + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini")) + config.validate(envs=environment) click.echo("Verbose mode can be enabled via `-v, --verbose` option") click.echo("Collected %d items" % len(test_names)) - start_time = time() results = [] + start_time = time() + default_envs = config.default_envs() for testname in test_names: - for section in projectconf.sections(): - if not section.startswith("env:"): - continue + for envname in config.envs(): + section = "env:%s" % envname # filter and ignore patterns patterns = dict(filter=list(filter), ignore=list(ignore)) for key in patterns: - if projectconf.has_option(section, "test_%s" % key): - patterns[key].extend([ - p.strip() - for p in projectconf.get(section, "test_%s" % - key).split(", ") - if p.strip() - ]) + if config.has_option(section, "test_%s" % key): + patterns[key].extend( + config.getlist(section, "test_%s" % key)) - envname = section[4:] skip_conditions = [ environment and envname not in environment, - not environment and env_default - and envname not in env_default, + not environment and default_envs + and envname not in default_envs, testname != "*" and patterns['filter'] and not any([fnmatch(testname, p) for p in patterns['filter']]), @@ -131,13 +125,13 @@ def cli( # pylint: disable=redefined-builtin results.append((None, testname, envname)) continue - cls = (NativeTestProcessor if projectconf.get( - section, "platform") == "native" else + cls = (NativeTestProcessor + if config.get(section, "platform") == "native" else EmbeddedTestProcessor) tp = cls( ctx, testname, envname, dict( - project_config=projectconf, + project_config=config, project_dir=project_dir, upload_port=upload_port, test_port=test_port, diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 7c3b84eb..4a4bc9dd 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -85,10 +85,8 @@ class TestProcessorBase(object): self.test_name = testname self.options = options self.env_name = envname - self.env_options = { - k: v - for k, v in options['project_config'].items("env:" + envname) - } + self.env_options = options['project_config'].items( + env=envname, as_dict=True) self._run_failed = False self._outputcpp_generated = False From 446176bf5e57b53678b8eb271c150c7b76679d4a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 15:47:17 +0300 Subject: [PATCH 10/13] Allow to skip ProjectConfig option validation with new "validate_options" argument --- platformio/project/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index d2e77bb7..d4f82c1b 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -259,7 +259,7 @@ class ProjectConfig(object): return [] return self.getlist("platformio", "env_default") - def validate(self, envs=None): + def validate(self, envs=None, validate_options=True): if not isfile(self.path): raise exception.NotPlatformIOProject(self.path) # check envs @@ -267,11 +267,11 @@ class ProjectConfig(object): if not known: raise exception.ProjectEnvsNotAvailable() - unknown = set((envs or []) + self.default_envs()) - known + unknown = set(list(envs or []) + self.default_envs()) - known if unknown: raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known)) - return self.validate_options() + return self.validate_options() if validate_options else True def validate_options(self): warnings = set() From 59848c3115c76a513dc8f779b4c5e143e7abe8cb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 15:49:01 +0300 Subject: [PATCH 11/13] Check unknown build environment passed by user --- platformio/commands/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index fe497f3b..a7327d34 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -80,7 +80,7 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, config = ProjectConfig.get_instance( project_conf or join(project_dir, "platformio.ini")) - config.validate() + config.validate(environment) results = [] start_time = time() From d2449762c23a01f03ea094221033066f5d218f16 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 17:26:10 +0300 Subject: [PATCH 12/13] Move Python or Platform dependent code to "compat" module --- platformio/__main__.py | 4 +-- platformio/app.py | 6 ++-- platformio/builder/tools/piolib.py | 7 ++--- platformio/builder/tools/pioplatform.py | 3 +- platformio/builder/tools/pioupload.py | 4 +-- platformio/builder/tools/piowinhooks.py | 6 ++-- platformio/builder/tools/platformio.py | 3 +- platformio/commands/upgrade.py | 3 +- platformio/compat.py | 42 +++++++++++++++++++++++++ platformio/downloader.py | 9 +++--- platformio/ide/projectgenerator.py | 6 ++-- platformio/managers/core.py | 7 +++-- platformio/managers/lib.py | 5 +-- platformio/managers/package.py | 10 +++--- platformio/managers/platform.py | 3 +- platformio/project/helpers.py | 6 ++-- platformio/util.py | 41 +++++++++--------------- 17 files changed, 98 insertions(+), 67 deletions(-) create mode 100644 platformio/compat.py diff --git a/platformio/__main__.py b/platformio/__main__.py index 9138fe3c..0dc0cca1 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -14,13 +14,13 @@ import os import sys -from platform import system from traceback import format_exc import click from platformio import __version__, exception, maintenance from platformio.commands import PlatformioCLI +from platformio.compat import CYGWIN @click.command( @@ -45,7 +45,7 @@ def process_result(ctx, result, force, caller): # pylint: disable=W0613 def configure(): - if "cygwin" in system().lower(): + if CYGWIN: raise exception.CygwinEnvDetected() # https://urllib3.readthedocs.org diff --git a/platformio/app.py b/platformio/app.py index 247fb65c..c8a071fc 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -25,6 +25,7 @@ from time import time import requests from platformio import exception, lockfile, util +from platformio.compat import PY2, WINDOWS def projects_dir_validate(projects_dir): @@ -173,7 +174,7 @@ class ContentCache(object): if not arg: continue arg = str(arg) - h.update(arg if util.PY2 else arg.encode()) + h.update(arg if PY2 else arg.encode()) return h.hexdigest() def get(self, key): @@ -361,7 +362,6 @@ def get_cid(): uid = uuid.getnode() cid = uuid.UUID(bytes=hashlib.md5(str(uid).encode()).digest()) cid = str(cid) - if ("windows" in util.get_systype() - or os.getuid() > 0): # yapf: disable pylint: disable=no-member + if WINDOWS or os.getuid() > 0: # yapf: disable pylint: disable=no-member set_state_item("cid", cid) return cid diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index e1a0b50b..270f3f21 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -33,6 +33,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, util from platformio.builder.tools import platformio as piotool +from platformio.compat import PY2, WINDOWS from platformio.managers.lib import LibraryManager from platformio.managers.package import PackageManager @@ -91,8 +92,6 @@ class LibBuilderFactory(object): class LibBuilderBase(object): - IS_WINDOWS = "windows" in util.get_systype() - LDF_MODES = ["off", "chain", "deep", "chain+", "deep+"] LDF_MODE_DEFAULT = "chain" @@ -135,7 +134,7 @@ class LibBuilderBase(object): def __contains__(self, path): p1 = self.path p2 = path - if self.IS_WINDOWS: + if WINDOWS: p1 = p1.lower() p2 = p2.lower() return commonprefix((p1 + sep, p2)) == p1 + sep @@ -191,7 +190,7 @@ class LibBuilderBase(object): @property def build_dir(self): lib_hash = hashlib.sha1( - self.path if util.PY2 else self.path.encode()).hexdigest()[:3] + self.path if PY2 else self.path.encode()).hexdigest()[:3] return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path)) @property diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 6427b33c..cc61a709 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -21,6 +21,7 @@ from os.path import isdir, isfile, join from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import exception, util +from platformio.compat import WINDOWS from platformio.managers.platform import PlatformFactory # pylint: disable=too-many-branches, too-many-locals @@ -79,7 +80,7 @@ def LoadPioPlatform(env, variables): env.PrependENVPath( "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir) - if ("windows" not in systype and isdir(join(pkg_dir, "lib")) + if (not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain"): env.PrependENVPath( "DYLD_LIBRARY_PATH" diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 0d5a55ea..a7b2685d 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -26,6 +26,7 @@ from SCons.Script import ARGUMENTS # pylint: disable=import-error from serial import Serial, SerialException from platformio import exception, util +from platformio.compat import WINDOWS # pylint: disable=unused-argument @@ -134,8 +135,7 @@ def AutodetectUploadPort(*args, **kwargs): continue port = item['port'] if upload_protocol.startswith("blackmagic"): - if "windows" in util.get_systype() and \ - port.startswith("COM") and len(port) > 4: + if WINDOWS and port.startswith("COM") and len(port) > 4: port = "\\\\.\\%s" % port if "GDB" in item['description']: return port diff --git a/platformio/builder/tools/piowinhooks.py b/platformio/builder/tools/piowinhooks.py index c893ffd5..1105091f 100644 --- a/platformio/builder/tools/piowinhooks.py +++ b/platformio/builder/tools/piowinhooks.py @@ -18,7 +18,7 @@ from hashlib import md5 from os import makedirs from os.path import isdir, isfile, join -from platformio import util +from platformio.compat import PY2, WINDOWS # Windows CLI has limit with command length to 8192 # Leave 2000 chars for flags and other options @@ -63,7 +63,7 @@ def _file_long_data(env, data): makedirs(build_dir) tmp_file = join( build_dir, - "longcmd-%s" % md5(data if util.PY2 else data.encode()).hexdigest()) + "longcmd-%s" % md5(data if PY2 else data.encode()).hexdigest()) if isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: @@ -76,7 +76,7 @@ def exists(_): def generate(env): - if "windows" not in util.get_systype(): + if not WINDOWS: return None env.Replace(_long_sources_hook=long_sources_hook) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 41f32700..a8272b83 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -27,7 +27,8 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Export # pylint: disable=import-error from SCons.Script import SConscript # pylint: disable=import-error -from platformio.util import glob_escape, pioversion_to_intstr, string_types +from platformio.compat import string_types +from platformio.util import glob_escape, pioversion_to_intstr SRC_HEADER_EXT = ["h", "hpp"] SRC_C_EXT = ["c", "cc", "cpp"] diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index c47ff8dc..6acb83ae 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -20,6 +20,7 @@ import click import requests from platformio import VERSION, __version__, exception, util +from platformio.compat import WINDOWS from platformio.managers.core import shutdown_piohome_servers @@ -68,7 +69,7 @@ def cli(dev): raise exception.UpgradeError("\n".join([str(cmd), str(e)])) permission_errors = ("permission denied", "not permitted") if (any(m in r['err'].lower() for m in permission_errors) - and "windows" not in util.get_systype()): + and not WINDOWS): click.secho( """ ----------------- diff --git a/platformio/compat.py b/platformio/compat.py new file mode 100644 index 00000000..cd50d905 --- /dev/null +++ b/platformio/compat.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +PY2 = sys.version_info[0] == 2 +CYGWIN = sys.platform.startswith('cygwin') +WINDOWS = sys.platform.startswith('win') + + +def get_filesystem_encoding(): + return sys.getfilesystemencoding() or sys.getdefaultencoding() + + +if PY2: + # pylint: disable=undefined-variable + string_types = (str, unicode) + + def is_bytes(x): + return isinstance(x, (buffer, bytearray)) + + def path_to_unicode(path): + return path.decode(get_filesystem_encoding()).encode("utf-8") +else: + string_types = (str, ) + + def is_bytes(x): + return isinstance(x, (bytes, memoryview, bytearray)) + + def path_to_unicode(path): + return path diff --git a/platformio/downloader.py b/platformio/downloader.py index 0b0ef5df..f86b723d 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -15,13 +15,14 @@ from email.utils import parsedate_tz from math import ceil from os.path import getsize, join -from sys import getfilesystemencoding, version_info +from sys import version_info from time import mktime import click import requests from platformio import util +from platformio.compat import PY2, get_filesystem_encoding from platformio.exception import (FDSHASumMismatch, FDSizeMismatch, FDUnrecognizedStatusCode) @@ -45,15 +46,15 @@ class FileDownloader(object): if disposition and "filename=" in disposition: self._fname = disposition[disposition.index("filename=") + 9:].replace('"', "").replace("'", "") - if util.PY2: + if PY2: self._fname = self._fname.encode("utf8") else: self._fname = [p for p in url.split("/") if p][-1] self._destination = self._fname if dest_dir: - if util.PY2: - dest_dir = dest_dir.decode(getfilesystemencoding()) + if PY2: + dest_dir = dest_dir.decode(get_filesystem_encoding()) self.set_destination(join(dest_dir, self._fname)) def set_destination(self, destination): diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 9cbf3028..1216aaee 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -23,6 +23,7 @@ from click.testing import CliRunner from platformio import exception, util from platformio.commands.run import cli as cmd_run +from platformio.compat import PY2, WINDOWS from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_projectlib_dir, get_projectlibdeps_dir, get_projectsrc_dir) @@ -120,7 +121,7 @@ class ProjectGenerator(object): contents = self._render_tpl(tpl_path) self._merge_contents( join(dst_dir, file_name), - contents.encode("utf8") if util.PY2 else contents) + contents.encode("utf8") if PY2 else contents) def _render_tpl(self, tpl_path): return bottle.template( @@ -155,5 +156,4 @@ class ProjectGenerator(object): @staticmethod def _fix_os_path(path): - return (re.sub(r"[\\]+", '\\' * 4, path) - if "windows" in util.get_systype() else path) + return (re.sub(r"[\\]+", '\\' * 4, path) if WINDOWS else path) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 1e469a03..04369712 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,6 +21,7 @@ from time import sleep import requests from platformio import __version__, exception, util +from platformio.compat import PY2, WINDOWS from platformio.managers.package import PackageManager CORE_PACKAGES = { @@ -29,7 +30,7 @@ CORE_PACKAGES = { "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.2.0", "tool-unity": "~1.20403.0", - "tool-scons": "~2.20501.7" if util.PY2 else "~3.30005.0" + "tool-scons": "~2.20501.7" if PY2 else "~3.30005.0" } PIOPLUS_AUTO_UPDATES_MAX = 100 @@ -99,7 +100,7 @@ def update_core_packages(only_check=False, silent=False): if not silent or pm.outdated(pkg_dir, requirements): if name == "tool-pioplus" and not only_check: shutdown_piohome_servers() - if "windows" in util.get_systype(): + if WINDOWS: sleep(1) pm.update(name, requirements, only_check=only_check) return True @@ -126,7 +127,7 @@ def inject_contrib_pysite(): def pioplus_call(args, **kwargs): - if "windows" in util.get_systype() and sys.version_info < (2, 7, 6): + if WINDOWS and sys.version_info < (2, 7, 6): raise exception.PlatformioException( "PlatformIO Core Plus v%s does not run under Python version %s.\n" "Minimum supported version is 2.7.6, please upgrade Python.\n" diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 4cbb34a7..b478f49c 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -23,6 +23,7 @@ from os.path import isdir, join import click from platformio import app, commands, exception, util +from platformio.compat import string_types from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager @@ -120,7 +121,7 @@ class LibraryManager(BasePkgManager): # convert listed items via comma to array for key in ("keywords", "frameworks", "platforms"): if key not in manifest or \ - not isinstance(manifest[key], util.string_types): + not isinstance(manifest[key], string_types): continue manifest[key] = [ i.strip() for i in manifest[key].split(",") if i.strip() @@ -147,7 +148,7 @@ class LibraryManager(BasePkgManager): continue if item[k] == "*": del item[k] - elif isinstance(item[k], util.string_types): + elif isinstance(item[k], string_types): item[k] = [ i.strip() for i in item[k].split(",") if i.strip() ] diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 516a951c..4d637716 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -588,9 +588,8 @@ class PkgInstallerMixin(object): if "__src_url" in cur_manifest: target_dirname = "%s@src-%s" % ( pkg_dirname, - hashlib.md5(cur_manifest['__src_url'] if util. - PY2 else cur_manifest['__src_url'].encode( - )).hexdigest()) + hashlib.md5( + cur_manifest['__src_url'].encode()).hexdigest()) shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: @@ -599,9 +598,8 @@ class PkgInstallerMixin(object): if "__src_url" in tmp_manifest: target_dirname = "%s@src-%s" % ( pkg_dirname, - hashlib.md5(tmp_manifest['__src_url'] if util. - PY2 else tmp_manifest['__src_url'].encode( - )).hexdigest()) + hashlib.md5( + tmp_manifest['__src_url'].encode()).hexdigest()) pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index bfcda3f2..8c0b7de9 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -23,6 +23,7 @@ import click import semantic_version from platformio import __version__, app, exception, util +from platformio.compat import PY2 from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.project.helpers import get_projectboards_dir @@ -389,7 +390,7 @@ class PlatformRunMixin(object): # encode and append variables for key, value in variables.items(): - if util.PY2: + if PY2: cmd.append("%s=%s" % (key.upper(), base64.b64encode(value))) else: cmd.append( diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index d7c19595..66a9c39b 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -13,16 +13,14 @@ # limitations under the License. import os -import sys from hashlib import sha1 from os import walk from os.path import abspath, dirname, expanduser, isdir, isfile, join from platformio import __version__ +from platformio.compat import PY2, WINDOWS from platformio.project.config import ProjectConfig -PY2 = sys.version_info[0] == 2 - def get_project_dir(): return os.getcwd() @@ -132,7 +130,7 @@ def calculate_project_hash(): if path.endswith(check_suffixes): chunks.append(path) chunks_to_str = ",".join(sorted(chunks)) - if sys.platform == "win32": + if WINDOWS: # Fix issue with useless project rebuilding for case insensitive FS. # A case of disk drive can differ... chunks_to_str = chunks_to_str.lower() diff --git a/platformio/util.py b/platformio/util.py index bf8292d7..56db49e5 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -32,19 +32,15 @@ import click import requests from platformio import __apiurl__, __version__, exception +from platformio.compat import path_to_unicode # pylint: disable=unused-import +from platformio.compat import PY2, WINDOWS, string_types from platformio.project.config import ProjectConfig from platformio.project.helpers import ( # pylint: disable=unused-import get_project_dir, get_project_optional_dir, get_projectboards_dir, get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir, get_projectsrc_dir, get_projecttest_dir, is_platformio_project) -# FIXME: check platformio.project.helpers imports - -PY2 = sys.version_info[0] == 2 -if PY2: - string_types = basestring # pylint: disable=undefined-variable -else: - string_types = str +# FIXME: remove import of path_to_unicode class AsyncPipe(Thread): @@ -148,12 +144,6 @@ def singleton(cls): return get_instance -def path_to_unicode(path): - if not PY2: - return path - return path.decode(sys.getfilesystemencoding()).encode("utf-8") - - def load_json(file_path): try: with open(file_path, "r") as f: @@ -180,7 +170,7 @@ def get_home_dir(): home_dir = get_project_optional_dir("home_dir", join(expanduser("~"), ".platformio")) win_home_dir = None - if "windows" in get_systype(): + if WINDOWS: win_home_dir = splitdrive(home_dir)[0] + "\\.platformio" if isdir(win_home_dir): home_dir = win_home_dir @@ -281,7 +271,7 @@ def copy_pythonpath_to_osenv(): _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep) for p in os.sys.path: conditions = [p not in _PYTHONPATH] - if "windows" not in get_systype(): + if not WINDOWS: conditions.append( isdir(join(p, "click")) or isdir(join(p, "platformio"))) if all(conditions): @@ -299,14 +289,12 @@ def get_serial_ports(filter_hwid=False): for p, d, h in comports(): if not p: continue - if "windows" in get_systype(): - if PY2: - try: - d = unicode( # pylint: disable=undefined-variable - d, - errors="ignore") - except TypeError: - pass + if WINDOWS and PY2: + try: + # pylint: disable=undefined-variable + d = unicode(d, errors="ignore") + except TypeError: + pass if not filter_hwid or "VID:PID" in h: result.append({"port": p, "description": d, "hwid": h}) @@ -326,7 +314,7 @@ get_serialports = get_serial_ports def get_logical_devices(): items = [] - if "windows" in get_systype(): + if WINDOWS: try: result = exec_command( ["wmic", "logicaldisk", "get", "name,VolumeName"]).get( @@ -583,9 +571,8 @@ def where_is_program(program, envpath=None): # try OS's built-in commands try: - result = exec_command( - ["where" if "windows" in get_systype() else "which", program], - env=env) + result = exec_command(["where" if WINDOWS else "which", program], + env=env) if result['returncode'] == 0 and isfile(result['out'].strip()): return result['out'].strip() except OSError: From abfee8308e85567467b5546eff9bb0400ddbd3c2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 17:47:02 +0300 Subject: [PATCH 13/13] Switch Python or Platform dependent code to "compat" module --- platformio/commands/debug/client.py | 3 ++- platformio/commands/debug/process.py | 5 +++-- platformio/commands/home/rpc/handlers/app.py | 10 +++++----- platformio/commands/home/rpc/handlers/os.py | 6 +++--- platformio/commands/home/rpc/handlers/piocore.py | 10 +++++----- platformio/commands/home/rpc/handlers/project.py | 5 ++--- 6 files changed, 20 insertions(+), 19 deletions(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index ac97243f..25a57db1 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -29,6 +29,7 @@ from platformio import app, exception, util from platformio.commands.debug import helpers, initcfgs from platformio.commands.debug.process import BaseProcess from platformio.commands.debug.server import DebugServer +from platformio.compat import PY2 from platformio.telemetry import MeasurementProtocol LOG_FILE = None @@ -60,7 +61,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes def spawn(self, gdb_path, prog_path): session_hash = gdb_path + prog_path self._session_id = sha1( - session_hash if util.PY2 else session_hash.encode()).hexdigest() + session_hash if PY2 else session_hash.encode()).hexdigest() self._kill_previous_session() patterns = { diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py index 0079e18c..0d70abe1 100644 --- a/platformio/commands/debug/process.py +++ b/platformio/commands/debug/process.py @@ -19,6 +19,7 @@ from twisted.internet import protocol # pylint: disable=import-error from platformio import util from platformio.commands.debug import helpers +from platformio.compat import string_types LOG_FILE = None @@ -42,13 +43,13 @@ class BaseProcess(protocol.ProcessProtocol, object): text = text.replace(pattern, value or "") return text - if isinstance(source, util.string_types): + if isinstance(source, string_types): source = _replace(source) elif isinstance(source, (list, dict)): items = enumerate(source) if isinstance(source, list) else source.items() for key, value in items: - if isinstance(value, util.string_types): + if isinstance(value, string_types): source[key] = _replace(value) elif isinstance(value, (list, dict)): source[key] = self.apply_patterns(value, patterns) diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index ce95a218..8241b03a 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -18,6 +18,7 @@ import json from os.path import expanduser, isfile, join from platformio import __version__, app, exception, util +from platformio.compat import path_to_unicode class AppRPC(object): @@ -55,12 +56,11 @@ class AppRPC(object): for key in storage['coreSettings']: if not key.endswith("dir"): continue - storage['coreSettings'][key][ - 'default_value'] = util.path_to_unicode( - storage['coreSettings'][key]['default_value']) - storage['coreSettings'][key]['value'] = util.path_to_unicode( + storage['coreSettings'][key]['default_value'] = path_to_unicode( + storage['coreSettings'][key]['default_value']) + storage['coreSettings'][key]['value'] = path_to_unicode( storage['coreSettings'][key]['value']) - storage['homeDir'] = util.path_to_unicode(expanduser("~")) + storage['homeDir'] = path_to_unicode(expanduser("~")) storage['projectsDir'] = storage['coreSettings']['projects_dir'][ 'value'] diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 780dbec8..8b263632 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -17,7 +17,6 @@ from __future__ import absolute_import import glob import os import shutil -import sys from functools import cmp_to_key from os.path import expanduser, isdir, isfile, join @@ -26,6 +25,7 @@ from twisted.internet import defer # pylint: disable=import-error from platformio import app, util from platformio.commands.home import helpers +from platformio.compat import PY2, get_filesystem_encoding, path_to_unicode class OSRPC(object): @@ -81,7 +81,7 @@ class OSRPC(object): @staticmethod def reveal_file(path): return click.launch( - path.encode(sys.getfilesystemencoding()) if util.PY2 else path, + path.encode(get_filesystem_encoding()) if PY2 else path, locate=True) @staticmethod @@ -148,6 +148,6 @@ class OSRPC(object): items = [] for item in util.get_logical_devices(): if item['name']: - item['name'] = util.path_to_unicode(item['name']) + item['name'] = path_to_unicode(item['name']) items.append(item) return items diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 80a642cb..b651b498 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -17,13 +17,13 @@ from __future__ import absolute_import import json import os import re -import sys import jsonrpc # pylint: disable=import-error from twisted.internet import utils # pylint: disable=import-error -from platformio import __version__, util +from platformio import __version__ from platformio.commands.home import helpers +from platformio.compat import get_filesystem_encoding, string_types class PIOCoreRPC(object): @@ -33,8 +33,8 @@ class PIOCoreRPC(object): json_output = "--json-output" in args try: args = [ - arg.encode(sys.getfilesystemencoding()) if isinstance( - arg, util.string_types) else str(arg) for arg in args + arg.encode(get_filesystem_encoding()) if isinstance( + arg, string_types) else str(arg) for arg in args ] except UnicodeError: raise jsonrpc.exceptions.JSONRPCDispatchException( @@ -54,7 +54,7 @@ class PIOCoreRPC(object): result = list(result) assert len(result) == 3 for i in (0, 1): - result[i] = result[i].decode(sys.getfilesystemencoding()).strip() + result[i] = result[i].decode(get_filesystem_encoding()).strip() out, err, code = result text = ("%s\n\n%s" % (out, err)).strip() if code != 0: diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index c3f48fcc..ae2eaa07 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -16,7 +16,6 @@ from __future__ import absolute_import import os import shutil -import sys import time from os.path import (basename, expanduser, getmtime, isdir, isfile, join, realpath, sep) @@ -26,6 +25,7 @@ import jsonrpc # pylint: disable=import-error from platformio import exception, util from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.compat import get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig @@ -215,8 +215,7 @@ class ProjectRPC(object): if isdir(src_dir): util.rmtree_(src_dir) shutil.copytree( - arduino_project_dir.encode(sys.getfilesystemencoding()), - src_dir) + arduino_project_dir.encode(get_filesystem_encoding()), src_dir) return project_dir @staticmethod