Refactor Unified Debugger to native Python Asynchronous I/O stack // Resolve #3793 , Resolve #3595

This commit is contained in:
Ivan Kravets
2021-03-17 17:42:11 +02:00
parent edf724d20d
commit b54a8b40a4
11 changed files with 306 additions and 252 deletions
+13
View File
@@ -0,0 +1,13 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+189
View File
@@ -0,0 +1,189 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import signal
import subprocess
import sys
import time
from platformio import fs
from platformio.compat import (
create_task,
get_locale_encoding,
get_running_loop,
string_types,
)
from platformio.proc import get_pythonexe_path
from platformio.project.helpers import get_project_core_dir
class DebugSubprocessProtocol(asyncio.SubprocessProtocol):
def __init__(self, factory):
self.factory = factory
self._is_exited = False
def connection_made(self, transport):
self.factory.connection_made(transport)
def pipe_data_received(self, fd, data):
pipe_to_cb = [
self.factory.stdin_data_received,
self.factory.stdout_data_received,
self.factory.stderr_data_received,
]
pipe_to_cb[fd](data)
def connection_lost(self, exc):
self.process_exited()
def process_exited(self):
if self._is_exited:
return
self.factory.process_exited()
self._is_exited = True
class DebugBaseProcess:
STDOUT_CHUNK_SIZE = 2048
LOG_FILE = None
COMMON_PATTERNS = {
"PLATFORMIO_HOME_DIR": get_project_core_dir(),
"PLATFORMIO_CORE_DIR": get_project_core_dir(),
"PYTHONEXE": get_pythonexe_path(),
}
def __init__(self):
self.transport = None
self._is_running = False
self._last_activity = 0
self._exit_future = None
self._stdin_read_task = None
self._std_encoding = get_locale_encoding()
async def spawn(self, *args, **kwargs):
wait_until_exit = False
if "wait_until_exit" in kwargs:
wait_until_exit = kwargs["wait_until_exit"]
del kwargs["wait_until_exit"]
for pipe in ("stdin", "stdout", "stderr"):
if pipe not in kwargs:
kwargs[pipe] = subprocess.PIPE
loop = get_running_loop()
await loop.subprocess_exec(
lambda: DebugSubprocessProtocol(self), *args, **kwargs
)
if wait_until_exit:
self._exit_future = loop.create_future()
await self._exit_future
def is_running(self):
return self._is_running
def connection_made(self, transport):
self._is_running = True
self.transport = transport
def connect_stdin_pipe(self):
self._stdin_read_task = create_task(self._read_stdin_pipe())
async def _read_stdin_pipe(self):
loop = get_running_loop()
try:
loop.add_reader(
sys.stdin.fileno(),
lambda: self.stdin_data_received(sys.stdin.buffer.readline()),
)
except NotImplementedError:
while True:
self.stdin_data_received(
await loop.run_in_executor(None, sys.stdin.buffer.readline)
)
def stdin_data_received(self, data):
self._last_activity = time.time()
if self.LOG_FILE:
with open(self.LOG_FILE, "ab") as fp:
fp.write(data)
def stdout_data_received(self, data):
self._last_activity = time.time()
if self.LOG_FILE:
with open(self.LOG_FILE, "ab") as fp:
fp.write(data)
while data:
chunk = data[: self.STDOUT_CHUNK_SIZE]
print(chunk.decode(self._std_encoding, "replace"), end="", flush=True)
data = data[self.STDOUT_CHUNK_SIZE :]
def stderr_data_received(self, data):
self._last_activity = time.time()
if self.LOG_FILE:
with open(self.LOG_FILE, "ab") as fp:
fp.write(data)
print(
data.decode(self._std_encoding, "replace"),
end="",
file=sys.stderr,
flush=True,
)
def process_exited(self):
self._is_running = False
self._last_activity = time.time()
# Allow terminating via SIGINT/CTRL+C
signal.signal(signal.SIGINT, signal.default_int_handler)
if self._stdin_read_task:
self._stdin_read_task.cancel()
self._stdin_read_task = None
if self._exit_future:
self._exit_future.set_result(True)
self._exit_future = None
def apply_patterns(self, source, patterns=None):
_patterns = self.COMMON_PATTERNS.copy()
_patterns.update(patterns or {})
for key, value in _patterns.items():
if key.endswith(("_DIR", "_PATH")):
_patterns[key] = fs.to_unix_path(value)
def _replace(text):
for key, value in _patterns.items():
pattern = "$%s" % key
text = text.replace(pattern, value or "")
return text
if isinstance(source, string_types):
source = _replace(source)
elif isinstance(source, (list, dict)):
items = enumerate(source) if isinstance(source, list) else source.items()
for key, value in items:
if isinstance(value, string_types):
source[key] = _replace(value)
elif isinstance(value, (list, dict)):
source[key] = self.apply_patterns(value, patterns)
return source
def terminate(self):
if not self.is_running() or not self.transport:
return
try:
self.transport.kill()
self.transport.close()
except: # pylint: disable=bare-except
pass
+270
View File
@@ -0,0 +1,270 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
import os
import re
import signal
import tempfile
import time
from platformio import fs, proc, telemetry, util
from platformio.cache import ContentCache
from platformio.compat import get_running_loop, hashlib_encode_data, is_bytes
from platformio.debug import helpers
from platformio.debug.exception import DebugInvalidOptionsError
from platformio.debug.initcfgs import get_gdb_init_config
from platformio.debug.process.base import DebugBaseProcess
from platformio.debug.process.server import DebugServerProcess
from platformio.project.helpers import get_project_cache_dir
class DebugClientProcess(
DebugBaseProcess
): # pylint: disable=too-many-instance-attributes
PIO_SRC_NAME = ".pioinit"
INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed"
def __init__(self, project_dir, args, debug_options, env_options):
super(DebugClientProcess, self).__init__()
self.project_dir = project_dir
self.args = list(args)
self.debug_options = debug_options
self.env_options = env_options
self._server_process = DebugServerProcess(debug_options, env_options)
self._session_id = None
if not os.path.isdir(get_project_cache_dir()):
os.makedirs(get_project_cache_dir())
self._gdbsrc_dir = tempfile.mkdtemp(
dir=get_project_cache_dir(), prefix=".piodebug-"
)
self._target_is_running = False
self._errors_buffer = b""
async def run(self, gdb_path, prog_path):
session_hash = gdb_path + prog_path
self._session_id = hashlib.sha1(hashlib_encode_data(session_hash)).hexdigest()
self._kill_previous_session()
patterns = {
"PROJECT_DIR": self.project_dir,
"PROG_PATH": prog_path,
"PROG_DIR": os.path.dirname(prog_path),
"PROG_NAME": os.path.basename(os.path.splitext(prog_path)[0]),
"DEBUG_PORT": self.debug_options["port"],
"UPLOAD_PROTOCOL": self.debug_options["upload_protocol"],
"INIT_BREAK": self.debug_options["init_break"] or "",
"LOAD_CMDS": "\n".join(self.debug_options["load_cmds"] or []),
}
await self._server_process.run(patterns)
if not patterns["DEBUG_PORT"]:
patterns["DEBUG_PORT"] = self._server_process.get_debug_port()
self.generate_pioinit(self._gdbsrc_dir, patterns)
# start GDB client
args = [
gdb_path,
"-q",
"--directory",
self._gdbsrc_dir,
"--directory",
self.project_dir,
"-l",
"10",
]
args.extend(self.args)
if not gdb_path:
raise DebugInvalidOptionsError("GDB client is not configured")
gdb_data_dir = self._get_data_dir(gdb_path)
if gdb_data_dir:
args.extend(["--data-directory", gdb_data_dir])
args.append(patterns["PROG_PATH"])
await self.spawn(*args, cwd=self.project_dir, wait_until_exit=True)
@staticmethod
def _get_data_dir(gdb_path):
if "msp430" in gdb_path:
return None
gdb_data_dir = os.path.realpath(
os.path.join(os.path.dirname(gdb_path), "..", "share", "gdb")
)
return gdb_data_dir if os.path.isdir(gdb_data_dir) else None
def generate_pioinit(self, dst_dir, patterns):
# default GDB init commands depending on debug tool
commands = get_gdb_init_config(self.debug_options).split("\n")
if self.debug_options["init_cmds"]:
commands = self.debug_options["init_cmds"]
commands.extend(self.debug_options["extra_cmds"])
if not any("define pio_reset_run_target" in cmd for cmd in commands):
commands = [
"define pio_reset_run_target",
" echo Warning! Undefined pio_reset_run_target command\\n",
" monitor reset",
"end",
] + commands
if not any("define pio_reset_halt_target" in cmd for cmd in commands):
commands = [
"define pio_reset_halt_target",
" echo Warning! Undefined pio_reset_halt_target command\\n",
" monitor reset halt",
"end",
] + commands
if not any("define pio_restart_target" in cmd for cmd in commands):
commands += [
"define pio_restart_target",
" pio_reset_halt_target",
" $INIT_BREAK",
" %s" % ("continue" if patterns["INIT_BREAK"] else "next"),
"end",
]
banner = [
"echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n",
"echo PlatformIO: debug_tool = %s\\n" % self.debug_options["tool"],
"echo PlatformIO: Initializing remote target...\\n",
]
footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER]
commands = banner + commands + footer
with open(os.path.join(dst_dir, self.PIO_SRC_NAME), "w") as fp:
fp.write("\n".join(self.apply_patterns(commands, patterns)))
def connection_made(self, transport):
super(DebugClientProcess, self).connection_made(transport)
self._lock_session(transport.get_pid())
# Disable SIGINT and allow GDB's Ctrl+C interrupt
signal.signal(signal.SIGINT, lambda *args, **kwargs: None)
self.connect_stdin_pipe()
def stdin_data_received(self, data):
super(DebugClientProcess, self).stdin_data_received(data)
if b"-exec-run" in data:
if self._target_is_running:
token, _ = data.split(b"-", 1)
self.stdout_data_received(token + b"^running\n")
return
data = data.replace(b"-exec-run", b"-exec-continue")
if b"-exec-continue" in data:
self._target_is_running = True
if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"):
# Allow terminating via SIGINT/CTRL+C
signal.signal(signal.SIGINT, signal.default_int_handler)
self.transport.get_pipe_transport(0).write(b"pio_reset_run_target\n")
self.transport.get_pipe_transport(0).write(data)
def stdout_data_received(self, data):
super(DebugClientProcess, self).stdout_data_received(data)
self._handle_error(data)
# go to init break automatically
if self.INIT_COMPLETED_BANNER.encode() in data:
telemetry.send_event(
"Debug", "Started", telemetry.dump_run_environment(self.env_options)
)
self._auto_exec_continue()
def console_log(self, msg):
if helpers.is_gdbmi_mode():
msg = helpers.escape_gdbmi_stream("~", msg)
self.stdout_data_received(msg if is_bytes(msg) else msg.encode())
def _auto_exec_continue(self):
auto_exec_delay = 0.5 # in seconds
if self._last_activity > (time.time() - auto_exec_delay):
get_running_loop().call_later(0.1, self._auto_exec_continue)
return
if not self.debug_options["init_break"] or self._target_is_running:
return
self.console_log(
"PlatformIO: Resume the execution to `debug_init_break = %s`\n"
% self.debug_options["init_break"]
)
self.console_log(
"PlatformIO: More configuration options -> http://bit.ly/pio-debug\n"
)
self.transport.get_pipe_transport(0).write(
b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n"
)
self._target_is_running = True
def stderr_data_received(self, data):
super(DebugClientProcess, self).stderr_data_received(data)
self._handle_error(data)
def _handle_error(self, data):
self._errors_buffer = (self._errors_buffer + data)[-8192:] # keep last 8 KBytes
if not (
self.PIO_SRC_NAME.encode() in self._errors_buffer
and b"Error in sourced" in self._errors_buffer
):
return
last_erros = self._errors_buffer.decode()
last_erros = " ".join(reversed(last_erros.split("\n")))
last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M)
err = "%s -> %s" % (
telemetry.dump_run_environment(self.env_options),
last_erros,
)
telemetry.send_exception("DebugInitError: %s" % err)
self.transport.close()
def process_exited(self):
self._unlock_session()
if self._gdbsrc_dir and os.path.isdir(self._gdbsrc_dir):
fs.rmtree(self._gdbsrc_dir)
if self._server_process:
self._server_process.terminate()
super(DebugClientProcess, self).process_exited()
def _kill_previous_session(self):
assert self._session_id
pid = None
with ContentCache() as cc:
pid = cc.get(self._session_id)
cc.delete(self._session_id)
if not pid:
return
if "windows" in util.get_systype():
kill = ["Taskkill", "/PID", pid, "/F"]
else:
kill = ["kill", pid]
try:
proc.exec_command(kill)
except: # pylint: disable=bare-except
pass
def _lock_session(self, pid):
if not self._session_id:
return
with ContentCache() as cc:
cc.set(self._session_id, str(pid), "1h")
def _unlock_session(self):
if not self._session_id:
return
with ContentCache() as cc:
cc.delete(self._session_id)
+148
View File
@@ -0,0 +1,148 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import os
import time
from platformio import fs, util
from platformio.debug.exception import DebugInvalidOptionsError
from platformio.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode
from platformio.debug.process.base import DebugBaseProcess
from platformio.proc import where_is_program
class DebugServerProcess(DebugBaseProcess):
def __init__(self, debug_options, env_options):
super(DebugServerProcess, self).__init__()
self.debug_options = debug_options
self.env_options = env_options
self._debug_port = ":3333"
self._ready = False
async def run(self, patterns): # pylint: disable=too-many-branches
systype = util.get_systype()
server = self.debug_options.get("server")
if not server:
return None
server = self.apply_patterns(server, patterns)
server_executable = server["executable"]
if not server_executable:
return None
if server["cwd"]:
server_executable = os.path.join(server["cwd"], server_executable)
if (
"windows" in systype
and not server_executable.endswith(".exe")
and os.path.isfile(server_executable + ".exe")
):
server_executable = server_executable + ".exe"
if not os.path.isfile(server_executable):
server_executable = where_is_program(server_executable)
if not os.path.isfile(server_executable):
raise DebugInvalidOptionsError(
"\nCould not launch Debug Server '%s'. Please check that it "
"is installed and is included in a system PATH\n\n"
"See documentation or contact contact@platformio.org:\n"
"https://docs.platformio.org/page/plus/debugging.html\n"
% server_executable
)
openocd_pipe_allowed = all(
[not self.debug_options["port"], "openocd" in server_executable]
)
openocd_pipe_allowed = False
if openocd_pipe_allowed:
args = []
if server["cwd"]:
args.extend(["-s", server["cwd"]])
args.extend(
["-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled"]
)
args.extend(server["arguments"])
str_args = " ".join(
[arg if arg.startswith("-") else '"%s"' % arg for arg in args]
)
self._debug_port = '| "%s" %s' % (server_executable, str_args)
self._debug_port = fs.to_unix_path(self._debug_port)
return self._debug_port
env = os.environ.copy()
# prepend server "lib" folder to LD path
if (
"windows" not in systype
and server["cwd"]
and os.path.isdir(os.path.join(server["cwd"], "lib"))
):
ld_key = "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH"
env[ld_key] = os.path.join(server["cwd"], "lib")
if os.environ.get(ld_key):
env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key))
# prepend BIN to PATH
if server["cwd"] and os.path.isdir(os.path.join(server["cwd"], "bin")):
env["PATH"] = "%s%s%s" % (
os.path.join(server["cwd"], "bin"),
os.pathsep,
os.environ.get("PATH", os.environ.get("Path", "")),
)
await self.spawn(
*([server_executable] + server["arguments"]), cwd=server["cwd"], env=env
)
if "mspdebug" in server_executable.lower():
self._debug_port = ":2000"
elif "jlink" in server_executable.lower():
self._debug_port = ":2331"
elif "qemu" in server_executable.lower():
self._debug_port = ":1234"
await self._wait_until_ready()
return self._debug_port
async def _wait_until_ready(self):
ready_pattern = self.debug_options.get("server", {}).get("ready_pattern")
timeout = 60 if ready_pattern else 10
elapsed = 0
delay = 0.5
auto_ready_delay = 0.5
while not self._ready and self.is_running() and elapsed < timeout:
await asyncio.sleep(delay)
if not ready_pattern:
self._ready = self._last_activity < (time.time() - auto_ready_delay)
elapsed += delay
def _check_ready_by_pattern(self, data):
if self._ready:
return self._ready
ready_pattern = self.debug_options.get("server", {}).get("ready_pattern")
if ready_pattern:
self._ready = ready_pattern.encode() in data
return self._ready
def get_debug_port(self):
return self._debug_port
def stdout_data_received(self, data):
super(DebugServerProcess, self).stdout_data_received(
escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data
)
self._check_ready_by_pattern(data)
def stderr_data_received(self, data):
super(DebugServerProcess, self).stderr_data_received(data)
self._check_ready_by_pattern(data)