pioduino/esp-idf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: format python files with isort and double-quote-string-fixer

Browse Source
This commit is contained in:
Fu Hanxi
2021-01-26 10:49:01 +08:00
parent dc8402ea61
commit 0146f258d7
276 changed files with 8241 additions and 8162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/ci/python_packages/ttfw_idf/CIScanTests.py
+7 -7
View File
@@ -7,9 +7,9 @@ from collections import defaultdict
from copy import deepcopy
from find_apps import find_apps
from find_build_apps import BUILD_SYSTEMS, BUILD_SYSTEM_CMAKE
from find_build_apps import BUILD_SYSTEM_CMAKE, BUILD_SYSTEMS
from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
from ttfw_idf.IDFAssignTest import ExampleAssignTest, TestAppsAssignTest
from idf_py_actions.constants import SUPPORTED_TARGETS, PREVIEW_TARGETS
TEST_LABELS = {
'example_test': 'BOT_LABEL_EXAMPLE_TEST',
@@ -73,15 +73,15 @@ def main():
default=BUILD_SYSTEM_CMAKE)
parser.add_argument('-c', '--ci-config-file',
required=True,
help="gitlab ci config target-test file")
help='gitlab ci config target-test file')
parser.add_argument('-o', '--output-path',
required=True,
help="output path of the scan result")
parser.add_argument("--exclude", nargs="*",
help='output path of the scan result')
parser.add_argument('--exclude', nargs='*',
help='Ignore specified directory. Can be used multiple times.')
parser.add_argument('--preserve', action="store_true",
parser.add_argument('--preserve', action='store_true',
help='add this flag to preserve artifacts for all apps')
parser.add_argument('--build-all', action="store_true",
parser.add_argument('--build-all', action='store_true',
help='add this flag to build all apps')
args = parser.parse_args()
tools/ci/python_packages/ttfw_idf/DebugUtils.py
+2 -2
View File
@@ -13,12 +13,12 @@
# limitations under the License.
from __future__ import unicode_literals
from io import open
import logging
from io import open
import pexpect
import pygdbmi.gdbcontroller
from tiny_test_fw import Utility
try:
tools/ci/python_packages/ttfw_idf/IDFApp.py
+63 -62
View File
@@ -22,7 +22,8 @@ import sys
from abc import abstractmethod
from tiny_test_fw import App
from .IDFAssignTest import ExampleGroup, TestAppsGroup, UnitTestGroup, IDFCaseGroup, ComponentUTGroup
from .IDFAssignTest import ComponentUTGroup, ExampleGroup, IDFCaseGroup, TestAppsGroup, UnitTestGroup
try:
import gitlab_api
@@ -36,8 +37,8 @@ def parse_encrypted_flag(args, offs, binary):
# If the current entry is a partition, we have to check whether it is
# the one we are looking for or not
try:
if (entry["offset"], entry["file"]) == (offs, binary):
return entry["encrypted"] == "true"
if (entry['offset'], entry['file']) == (offs, binary):
return entry['encrypted'] == 'true'
except (TypeError, KeyError):
# TypeError occurs if the entry is a list, which is possible in JSON
# data structure.
@@ -58,12 +59,12 @@ def parse_flash_settings(path, default_encryption=False):
# The following list only contains the files that need encryption
encrypt_files = []
if file_name == "flasher_args.json":
if file_name == 'flasher_args.json':
# CMake version using build metadata file
with open(path, "r") as f:
with open(path, 'r') as f:
args = json.load(f)
for (offs, binary) in args["flash_files"].items():
for (offs, binary) in args['flash_files'].items():
if offs:
flash_files.append((offs, binary))
encrypted = parse_encrypted_flag(args, offs, binary)
@@ -73,15 +74,15 @@ def parse_flash_settings(path, default_encryption=False):
if (encrypted is None and default_encryption) or encrypted:
encrypt_files.append((offs, binary))
flash_settings = args["flash_settings"]
app_name = os.path.splitext(args["app"]["file"])[0]
flash_settings = args['flash_settings']
app_name = os.path.splitext(args['app']['file'])[0]
else:
# GNU Make version uses download.config arguments file
with open(path, "r") as f:
args = f.readlines()[-1].split(" ")
with open(path, 'r') as f:
args = f.readlines()[-1].split(' ')
flash_settings = {}
for idx in range(0, len(args), 2): # process arguments in pairs
if args[idx].startswith("--"):
if args[idx].startswith('--'):
# strip the -- from the command line argument
flash_settings[args[idx][2:]] = args[idx + 1]
else:
@@ -92,7 +93,7 @@ def parse_flash_settings(path, default_encryption=False):
encrypt_files = flash_files
# we can only guess app name in download.config.
for p in flash_files:
if not os.path.dirname(p[1]) and "partition" not in p[1]:
if not os.path.dirname(p[1]) and 'partition' not in p[1]:
# app bin usually in the same dir with download.config and it's not partition table
app_name = os.path.splitext(p[1])[0]
break
@@ -107,9 +108,9 @@ class Artifacts(object):
# at least one of app_path or config_name is not None. otherwise we can't match artifact
assert app_path or config_name
assert os.path.exists(artifact_index_file)
self.gitlab_inst = gitlab_api.Gitlab(os.getenv("CI_PROJECT_ID"))
self.gitlab_inst = gitlab_api.Gitlab(os.getenv('CI_PROJECT_ID'))
self.dest_root_path = dest_root_path
with open(artifact_index_file, "r") as f:
with open(artifact_index_file, 'r') as f:
artifact_index = json.load(f)
self.artifact_info = self._find_artifact(artifact_index, app_path, config_name, target)
@@ -120,11 +121,11 @@ class Artifacts(object):
if app_path:
# We use endswith here to avoid issue like:
# examples_protocols_mqtt_ws but return a examples_protocols_mqtt_wss failure
match_result = artifact_info["app_dir"].endswith(app_path)
match_result = artifact_info['app_dir'].endswith(app_path)
if config_name:
match_result = match_result and config_name == artifact_info["config"]
match_result = match_result and config_name == artifact_info['config']
if target:
match_result = match_result and target == artifact_info["target"]
match_result = match_result and target == artifact_info['target']
if match_result:
ret = artifact_info
break
@@ -134,15 +135,15 @@ class Artifacts(object):
def _get_app_base_path(self):
if self.artifact_info:
return os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"])
return os.path.join(self.artifact_info['work_dir'], self.artifact_info['build_dir'])
else:
return None
def _get_flash_arg_file(self, base_path, job_id):
if self.artifact_info["build_system"] == "cmake":
flash_arg_file = os.path.join(base_path, "flasher_args.json")
if self.artifact_info['build_system'] == 'cmake':
flash_arg_file = os.path.join(base_path, 'flasher_args.json')
else:
flash_arg_file = os.path.join(base_path, "download.config")
flash_arg_file = os.path.join(base_path, 'download.config')
self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path)
return flash_arg_file
@@ -152,19 +153,19 @@ class Artifacts(object):
# files also appear in the first list
flash_files, _, _, app_name = parse_flash_settings(os.path.join(self.dest_root_path, flash_arg_file))
artifact_files = [os.path.join(base_path, p[1]) for p in flash_files]
artifact_files.append(os.path.join(base_path, app_name + ".elf"))
artifact_files.append(os.path.join(base_path, app_name + '.elf'))
self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
def _download_sdkconfig_file(self, base_path, job_id):
self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")],
self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), 'sdkconfig')],
self.dest_root_path)
def download_artifacts(self):
if not self.artifact_info:
return None
base_path = self._get_app_base_path()
job_id = self.artifact_info["ci_job_id"]
job_id = self.artifact_info['ci_job_id']
# 1. download flash args file
flash_arg_file = self._get_flash_arg_file(base_path, job_id)
@@ -177,15 +178,15 @@ class Artifacts(object):
def download_artifact_files(self, file_names):
if self.artifact_info:
base_path = os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"])
job_id = self.artifact_info["ci_job_id"]
base_path = os.path.join(self.artifact_info['work_dir'], self.artifact_info['build_dir'])
job_id = self.artifact_info['ci_job_id']
# download all binary files
artifact_files = [os.path.join(base_path, fn) for fn in file_names]
self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
# download sdkconfig file
self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")],
self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), 'sdkconfig')],
self.dest_root_path)
else:
base_path = None
@@ -197,13 +198,13 @@ class UnitTestArtifacts(Artifacts):
def _get_app_base_path(self):
if self.artifact_info:
output_dir = self.BUILDS_DIR_RE.sub('output/', self.artifact_info["build_dir"])
return os.path.join(self.artifact_info["app_dir"], output_dir)
output_dir = self.BUILDS_DIR_RE.sub('output/', self.artifact_info['build_dir'])
return os.path.join(self.artifact_info['app_dir'], output_dir)
else:
return None
def _download_sdkconfig_file(self, base_path, job_id):
self.gitlab_inst.download_artifact(job_id, [os.path.join(base_path, "sdkconfig")], self.dest_root_path)
self.gitlab_inst.download_artifact(job_id, [os.path.join(base_path, 'sdkconfig')], self.dest_root_path)
class IDFApp(App.BaseApp):
@@ -212,8 +213,8 @@ class IDFApp(App.BaseApp):
idf applications should inherent from this class and overwrite method get_binary_path.
"""
IDF_DOWNLOAD_CONFIG_FILE = "download.config"
IDF_FLASH_ARGS_FILE = "flasher_args.json"
IDF_DOWNLOAD_CONFIG_FILE = 'download.config'
IDF_FLASH_ARGS_FILE = 'flasher_args.json'
def __init__(self, app_path, config_name=None, target=None, case_group=IDFCaseGroup, artifact_cls=Artifacts):
super(IDFApp, self).__init__(app_path)
@@ -229,11 +230,11 @@ class IDFApp(App.BaseApp):
assert os.path.exists(self.binary_path)
if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path):
if self.IDF_FLASH_ARGS_FILE not in os.listdir(self.binary_path):
msg = ("Neither {} nor {} exists. "
msg = ('Neither {} nor {} exists. '
"Try to run 'make print_flash_cmd | tail -n 1 > {}/{}' "
"or 'idf.py build' "
"for resolving the issue."
"").format(self.IDF_DOWNLOAD_CONFIG_FILE, self.IDF_FLASH_ARGS_FILE,
'for resolving the issue.'
'').format(self.IDF_DOWNLOAD_CONFIG_FILE, self.IDF_FLASH_ARGS_FILE,
self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE)
raise AssertionError(msg)
@@ -252,7 +253,7 @@ class IDFApp(App.BaseApp):
@classmethod
def get_sdk_path(cls): # type: () -> str
idf_path = os.getenv("IDF_PATH")
idf_path = os.getenv('IDF_PATH')
assert idf_path
assert os.path.exists(idf_path)
return idf_path
@@ -263,7 +264,7 @@ class IDFApp(App.BaseApp):
Note: could be overwritten by a derived class to provide other locations or order
"""
return [os.path.join(self.binary_path, "sdkconfig"), os.path.join(self.binary_path, "..", "sdkconfig")]
return [os.path.join(self.binary_path, 'sdkconfig'), os.path.join(self.binary_path, '..', 'sdkconfig')]
def get_sdkconfig(self):
"""
@@ -306,13 +307,13 @@ class IDFApp(App.BaseApp):
if path:
return os.path.join(self.idf_path, path)
else:
raise OSError("Failed to get binary for {}".format(self))
raise OSError('Failed to get binary for {}'.format(self))
def _get_elf_file_path(self):
ret = ""
ret = ''
file_names = os.listdir(self.binary_path)
for fn in file_names:
if os.path.splitext(fn)[1] == ".elf":
if os.path.splitext(fn)[1] == '.elf':
ret = os.path.join(self.binary_path, fn)
return ret
@@ -343,14 +344,14 @@ class IDFApp(App.BaseApp):
# a default encrpytion flag: the macro
# CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT
sdkconfig_dict = self.get_sdkconfig()
default_encryption = "CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT" in sdkconfig_dict
default_encryption = 'CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT' in sdkconfig_dict
flash_files, encrypt_files, flash_settings, _ = parse_flash_settings(path, default_encryption)
# Flash setting "encrypt" only and only if all the files to flash
# must be encrypted. Else, this parameter should be False.
# All files must be encrypted is both file lists are the same
flash_settings["encrypt"] = sorted(flash_files) == sorted(encrypt_files)
flash_settings['encrypt'] = sorted(flash_files) == sorted(encrypt_files)
return self._int_offs_abs_paths(flash_files), self._int_offs_abs_paths(encrypt_files), flash_settings
@@ -363,9 +364,9 @@ class IDFApp(App.BaseApp):
(Called from constructor)
"""
partition_tool = os.path.join(self.idf_path,
"components",
"partition_table",
"gen_esp32part.py")
'components',
'partition_table',
'gen_esp32part.py')
assert os.path.exists(partition_tool)
errors = []
@@ -393,18 +394,18 @@ class IDFApp(App.BaseApp):
p,
os.linesep,
msg) for p, msg in errors])
raise ValueError("No partition table found for IDF binary path: {}{}{}".format(self.binary_path,
raise ValueError('No partition table found for IDF binary path: {}{}{}'.format(self.binary_path,
os.linesep,
traceback_msg))
partition_table = dict()
for line in raw_data.splitlines():
if line[0] != "#":
if line[0] != '#':
try:
_name, _type, _subtype, _offset, _size, _flags = line.split(",")
if _size[-1] == "K":
_name, _type, _subtype, _offset, _size, _flags = line.split(',')
if _size[-1] == 'K':
_size = int(_size[:-1]) * 1024
elif _size[-1] == "M":
elif _size[-1] == 'M':
_size = int(_size[:-1]) * 1024 * 1024
else:
_size = int(_size)
@@ -412,11 +413,11 @@ class IDFApp(App.BaseApp):
except ValueError:
continue
partition_table[_name] = {
"type": _type,
"subtype": _subtype,
"offset": _offset,
"size": _size,
"flags": _flags
'type': _type,
'subtype': _subtype,
'offset': _offset,
'size': _size,
'flags': _flags
}
return partition_table
@@ -444,11 +445,11 @@ class Example(IDFApp):
"""
overrides the parent method to provide exact path of sdkconfig for example tests
"""
return [os.path.join(self.binary_path, "..", "sdkconfig")]
return [os.path.join(self.binary_path, '..', 'sdkconfig')]
def _try_get_binary_from_local_fs(self):
# build folder of example path
path = os.path.join(self.idf_path, self.app_path, "build")
path = os.path.join(self.idf_path, self.app_path, 'build')
if os.path.exists(path):
return path
@@ -456,11 +457,11 @@ class Example(IDFApp):
# Path format: $IDF_PATH/build_examples/app_path_with_underscores/config/target
# (see tools/ci/build_examples.sh)
# For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
app_path_underscored = self.app_path.replace(os.path.sep, "_")
app_path_underscored = self.app_path.replace(os.path.sep, '_')
example_path = os.path.join(self.idf_path, self.case_group.LOCAL_BUILD_DIR)
for dirpath in os.listdir(example_path):
if os.path.basename(dirpath) == app_path_underscored:
path = os.path.join(example_path, dirpath, self.config_name, self.target, "build")
path = os.path.join(example_path, dirpath, self.config_name, self.target, 'build')
if os.path.exists(path):
return path
else:
@@ -476,18 +477,18 @@ class UT(IDFApp):
super(UT, self).__init__(app_path, config_name, target, case_group, artifacts_cls)
def _try_get_binary_from_local_fs(self):
path = os.path.join(self.idf_path, self.app_path, "build")
path = os.path.join(self.idf_path, self.app_path, 'build')
if os.path.exists(path):
return path
# first try to get from build folder of unit-test-app
path = os.path.join(self.idf_path, "tools", "unit-test-app", "build")
path = os.path.join(self.idf_path, 'tools', 'unit-test-app', 'build')
if os.path.exists(path):
# found, use bin in build path
return path
# ``build_unit_test.sh`` will copy binary to output folder
path = os.path.join(self.idf_path, "tools", "unit-test-app", "output", self.target, self.config_name)
path = os.path.join(self.idf_path, 'tools', 'unit-test-app', 'output', self.target, self.config_name)
if os.path.exists(path):
return path
tools/ci/python_packages/ttfw_idf/IDFAssignTest.py
+1 -1
View File
@@ -18,7 +18,7 @@ import gitlab_api
from tiny_test_fw.Utility import CIAssignTest
try:
from idf_py_actions.constants import SUPPORTED_TARGETS, PREVIEW_TARGETS
from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
except ImportError:
SUPPORTED_TARGETS = []
PREVIEW_TARGETS = []
tools/ci/python_packages/ttfw_idf/IDFDUT.py
+83 -84
View File
@@ -13,14 +13,15 @@
# limitations under the License.
""" DUT for IDF applications """
import functools
import os
import os.path
import sys
import re
import functools
import tempfile
import subprocess
import sys
import tempfile
import time
import pexpect
# python2 and python3 queue package name is different
@@ -29,18 +30,16 @@ try:
except ImportError:
import queue as _queue
from serial.tools import list_ports
from tiny_test_fw import DUT, Utility
try:
import esptool
except ImportError: # cheat and use IDF's copy of esptool if available
idf_path = os.getenv("IDF_PATH")
idf_path = os.getenv('IDF_PATH')
if not idf_path or not os.path.exists(idf_path):
raise
sys.path.insert(0, os.path.join(idf_path, "components", "esptool_py", "esptool"))
sys.path.insert(0, os.path.join(idf_path, 'components', 'esptool_py', 'esptool'))
import esptool
@@ -54,14 +53,14 @@ class IDFDUTException(RuntimeError):
class IDFRecvThread(DUT.RecvThread):
PERFORMANCE_PATTERN = re.compile(r"\[Performance]\[(\w+)]: ([^\r\n]+)\r?\n")
PERFORMANCE_PATTERN = re.compile(r'\[Performance]\[(\w+)]: ([^\r\n]+)\r?\n')
EXCEPTION_PATTERNS = [
re.compile(r"(Guru Meditation Error: Core\s+\d panic'ed \([\w].*?\))"),
re.compile(r"(abort\(\) was called at PC 0x[a-fA-F\d]{8} on core \d)"),
re.compile(r"(rst 0x\d+ \(TG\dWDT_SYS_RESET|TGWDT_CPU_RESET\))")
re.compile(r'(abort\(\) was called at PC 0x[a-fA-F\d]{8} on core \d)'),
re.compile(r'(rst 0x\d+ \(TG\dWDT_SYS_RESET|TGWDT_CPU_RESET\))')
]
BACKTRACE_PATTERN = re.compile(r"Backtrace:((\s(0x[0-9a-f]{8}):0x[0-9a-f]{8})+)")
BACKTRACE_ADDRESS_PATTERN = re.compile(r"(0x[0-9a-f]{8}):0x[0-9a-f]{8}")
BACKTRACE_PATTERN = re.compile(r'Backtrace:((\s(0x[0-9a-f]{8}):0x[0-9a-f]{8})+)')
BACKTRACE_ADDRESS_PATTERN = re.compile(r'(0x[0-9a-f]{8}):0x[0-9a-f]{8}')
def __init__(self, read, dut):
super(IDFRecvThread, self).__init__(read, dut)
@@ -71,8 +70,8 @@ class IDFRecvThread(DUT.RecvThread):
def collect_performance(self, comp_data):
matches = self.PERFORMANCE_PATTERN.findall(comp_data)
for match in matches:
Utility.console_log("[Performance][{}]: {}".format(match[0], match[1]),
color="orange")
Utility.console_log('[Performance][{}]: {}'.format(match[0], match[1]),
color='orange')
self.performance_items.put((match[0], match[1]))
def detect_exception(self, comp_data):
@@ -83,7 +82,7 @@ class IDFRecvThread(DUT.RecvThread):
if match:
start = match.end()
self.exceptions.put(match.group(0))
Utility.console_log("[Exception]: {}".format(match.group(0)), color="red")
Utility.console_log('[Exception]: {}'.format(match.group(0)), color='red')
else:
break
@@ -93,18 +92,18 @@ class IDFRecvThread(DUT.RecvThread):
match = self.BACKTRACE_PATTERN.search(comp_data, pos=start)
if match:
start = match.end()
Utility.console_log("[Backtrace]:{}".format(match.group(1)), color="red")
Utility.console_log('[Backtrace]:{}'.format(match.group(1)), color='red')
# translate backtrace
addresses = self.BACKTRACE_ADDRESS_PATTERN.findall(match.group(1))
translated_backtrace = ""
translated_backtrace = ''
for addr in addresses:
ret = self.dut.lookup_pc_address(addr)
if ret:
translated_backtrace += ret + "\n"
translated_backtrace += ret + '\n'
if translated_backtrace:
Utility.console_log("Translated backtrace\n:" + translated_backtrace, color="yellow")
Utility.console_log('Translated backtrace\n:' + translated_backtrace, color='yellow')
else:
Utility.console_log("Failed to translate backtrace", color="yellow")
Utility.console_log('Failed to translate backtrace', color='yellow')
else:
break
@@ -149,7 +148,7 @@ class IDFDUT(DUT.SerialDUT):
# /dev/ttyAMA0 port is listed in Raspberry Pi
# /dev/tty.Bluetooth-Incoming-Port port is listed in Mac
INVALID_PORT_PATTERN = re.compile(r"AMA|Bluetooth")
INVALID_PORT_PATTERN = re.compile(r'AMA|Bluetooth')
# if need to erase NVS partition in start app
ERASE_NVS = True
RECV_THREAD_CLS = IDFRecvThread
@@ -163,7 +162,7 @@ class IDFDUT(DUT.SerialDUT):
@classmethod
def _get_rom(cls):
raise NotImplementedError("This is an abstraction class, method not defined.")
raise NotImplementedError('This is an abstraction class, method not defined.')
@classmethod
def get_mac(cls, app, port):
@@ -200,7 +199,7 @@ class IDFDUT(DUT.SerialDUT):
# Otherwise overwrite it in ESP8266DUT
inst = esptool.ESPLoader.detect_chip(port)
if expected_rom_class and type(inst) != expected_rom_class:
raise RuntimeError("Target not expected")
raise RuntimeError('Target not expected')
return inst.read_mac() is not None, get_target_by_rom_class(type(inst))
except(esptool.FatalError, RuntimeError):
return False, None
@@ -227,7 +226,7 @@ class IDFDUT(DUT.SerialDUT):
# and encrypt_files contains the ones to flash encrypted.
flash_files = self.app.flash_files
encrypt_files = self.app.encrypt_files
encrypt = self.app.flash_settings.get("encrypt", False)
encrypt = self.app.flash_settings.get('encrypt', False)
if encrypt:
flash_files = encrypt_files
encrypt_files = []
@@ -236,12 +235,12 @@ class IDFDUT(DUT.SerialDUT):
for entry in flash_files
if entry not in encrypt_files]
flash_files = [(offs, open(path, "rb")) for (offs, path) in flash_files]
encrypt_files = [(offs, open(path, "rb")) for (offs, path) in encrypt_files]
flash_files = [(offs, open(path, 'rb')) for (offs, path) in flash_files]
encrypt_files = [(offs, open(path, 'rb')) for (offs, path) in encrypt_files]
if erase_nvs:
address = self.app.partition_table["nvs"]["offset"]
size = self.app.partition_table["nvs"]["size"]
address = self.app.partition_table['nvs']['offset']
size = self.app.partition_table['nvs']['size']
nvs_file = tempfile.TemporaryFile()
nvs_file.write(b'\xff' * size)
nvs_file.seek(0)
@@ -252,7 +251,7 @@ class IDFDUT(DUT.SerialDUT):
# Get the CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT macro
# value. If it is set to True, then NVS is always encrypted.
sdkconfig_dict = self.app.get_sdkconfig()
macro_encryption = "CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT" in sdkconfig_dict
macro_encryption = 'CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT' in sdkconfig_dict
# If the macro is not enabled (plain text flash) or all files
# must be encrypted, add NVS to flash_files.
if not macro_encryption or encrypt:
@@ -270,9 +269,9 @@ class IDFDUT(DUT.SerialDUT):
# write_flash expects the parameter encrypt_files to be None and not
# an empty list, so perform the check here
flash_args = FlashArgs({
'flash_size': self.app.flash_settings["flash_size"],
'flash_mode': self.app.flash_settings["flash_mode"],
'flash_freq': self.app.flash_settings["flash_freq"],
'flash_size': self.app.flash_settings['flash_size'],
'flash_mode': self.app.flash_settings['flash_mode'],
'flash_freq': self.app.flash_settings['flash_freq'],
'addr_filename': flash_files,
'encrypt_files': encrypt_files or None,
'no_stub': False,
@@ -328,9 +327,9 @@ class IDFDUT(DUT.SerialDUT):
"""
raise NotImplementedError() # TODO: implement this
# address = self.app.partition_table[partition]["offset"]
size = self.app.partition_table[partition]["size"]
size = self.app.partition_table[partition]['size']
# TODO can use esp.erase_region() instead of this, I think
with open(".erase_partition.tmp", "wb") as f:
with open('.erase_partition.tmp', 'wb') as f:
f.write(chr(0xFF) * size)
@_uses_esptool
@@ -356,18 +355,18 @@ class IDFDUT(DUT.SerialDUT):
"""
if os.path.isabs(output_file) is False:
output_file = os.path.relpath(output_file, self.app.get_log_folder())
if "partition" in kwargs:
partition = self.app.partition_table[kwargs["partition"]]
_address = partition["offset"]
_size = partition["size"]
elif "address" in kwargs and "size" in kwargs:
_address = kwargs["address"]
_size = kwargs["size"]
if 'partition' in kwargs:
partition = self.app.partition_table[kwargs['partition']]
_address = partition['offset']
_size = partition['size']
elif 'address' in kwargs and 'size' in kwargs:
_address = kwargs['address']
_size = kwargs['size']
else:
raise IDFToolError("You must specify 'partition' or ('address' and 'size') to dump flash")
content = esp.read_flash(_address, _size)
with open(output_file, "wb") as f:
with open(output_file, 'wb') as f:
f.write(content)
@classmethod
@@ -405,9 +404,9 @@ class IDFDUT(DUT.SerialDUT):
return ports
def lookup_pc_address(self, pc_addr):
cmd = ["%saddr2line" % self.TOOLCHAIN_PREFIX,
"-pfiaC", "-e", self.app.elf_file, pc_addr]
ret = ""
cmd = ['%saddr2line' % self.TOOLCHAIN_PREFIX,
'-pfiaC', '-e', self.app.elf_file, pc_addr]
ret = ''
try:
translation = subprocess.check_output(cmd)
ret = translation.decode()
@@ -439,7 +438,7 @@ class IDFDUT(DUT.SerialDUT):
def stop_receive(self):
if self.receive_thread:
for name in ["performance_items", "exceptions"]:
for name in ['performance_items', 'exceptions']:
source_queue = getattr(self.receive_thread, name)
dest_queue = getattr(self, name)
self._queue_copy(source_queue, dest_queue)
@@ -447,7 +446,7 @@ class IDFDUT(DUT.SerialDUT):
def get_exceptions(self):
""" Get exceptions detected by DUT receive thread. """
return self._get_from_queue("exceptions")
return self._get_from_queue('exceptions')
def get_performance_items(self):
"""
@@ -456,18 +455,18 @@ class IDFDUT(DUT.SerialDUT):
:return: a list of performance items.
"""
return self._get_from_queue("performance_items")
return self._get_from_queue('performance_items')
def close(self):
super(IDFDUT, self).close()
if not self.allow_dut_exception and self.get_exceptions():
Utility.console_log("DUT exception detected on {}".format(self), color="red")
Utility.console_log('DUT exception detected on {}'.format(self), color='red')
raise IDFDUTException()
class ESP32DUT(IDFDUT):
TARGET = "esp32"
TOOLCHAIN_PREFIX = "xtensa-esp32-elf-"
TARGET = 'esp32'
TOOLCHAIN_PREFIX = 'xtensa-esp32-elf-'
@classmethod
def _get_rom(cls):
@@ -478,8 +477,8 @@ class ESP32DUT(IDFDUT):
class ESP32S2DUT(IDFDUT):
TARGET = "esp32s2"
TOOLCHAIN_PREFIX = "xtensa-esp32s2-elf-"
TARGET = 'esp32s2'
TOOLCHAIN_PREFIX = 'xtensa-esp32s2-elf-'
@classmethod
def _get_rom(cls):
@@ -490,8 +489,8 @@ class ESP32S2DUT(IDFDUT):
class ESP32C3DUT(IDFDUT):
TARGET = "esp32c3"
TOOLCHAIN_PREFIX = "riscv32-esp-elf-"
TARGET = 'esp32c3'
TOOLCHAIN_PREFIX = 'riscv32-esp-elf-'
@classmethod
def _get_rom(cls):
@@ -502,8 +501,8 @@ class ESP32C3DUT(IDFDUT):
class ESP8266DUT(IDFDUT):
TARGET = "esp8266"
TOOLCHAIN_PREFIX = "xtensa-lx106-elf-"
TARGET = 'esp8266'
TOOLCHAIN_PREFIX = 'xtensa-lx106-elf-'
@classmethod
def _get_rom(cls):
@@ -528,34 +527,34 @@ class IDFQEMUDUT(IDFDUT):
QEMU_SERIAL_PORT = 3334
def __init__(self, name, port, log_file, app, allow_dut_exception=False, **kwargs):
self.flash_image = tempfile.NamedTemporaryFile('rb+', suffix=".bin", prefix="qemu_flash_img")
self.flash_image = tempfile.NamedTemporaryFile('rb+', suffix='.bin', prefix='qemu_flash_img')
self.app = app
self.flash_size = 4 * 1024 * 1024
self._write_flash_img()
args = [
"qemu-system-xtensa",
"-nographic",
"-machine", self.TARGET,
"-drive", "file={},if=mtd,format=raw".format(self.flash_image.name),
"-nic", "user,model=open_eth",
"-serial", "tcp::{},server,nowait".format(self.QEMU_SERIAL_PORT),
"-S",
"-global driver=timer.esp32.timg,property=wdt_disable,value=true"]
'qemu-system-xtensa',
'-nographic',
'-machine', self.TARGET,
'-drive', 'file={},if=mtd,format=raw'.format(self.flash_image.name),
'-nic', 'user,model=open_eth',
'-serial', 'tcp::{},server,nowait'.format(self.QEMU_SERIAL_PORT),
'-S',
'-global driver=timer.esp32.timg,property=wdt_disable,value=true']
# TODO(IDF-1242): generate a temporary efuse binary, pass it to QEMU
if "QEMU_BIOS_PATH" in os.environ:
args += ["-L", os.environ["QEMU_BIOS_PATH"]]
if 'QEMU_BIOS_PATH' in os.environ:
args += ['-L', os.environ['QEMU_BIOS_PATH']]
self.qemu = pexpect.spawn(" ".join(args), timeout=self.DEFAULT_EXPECT_TIMEOUT)
self.qemu.expect_exact(b"(qemu)")
self.qemu = pexpect.spawn(' '.join(args), timeout=self.DEFAULT_EXPECT_TIMEOUT)
self.qemu.expect_exact(b'(qemu)')
super(IDFQEMUDUT, self).__init__(name, port, log_file, app, allow_dut_exception=allow_dut_exception, **kwargs)
def _write_flash_img(self):
self.flash_image.seek(0)
self.flash_image.write(b'\x00' * self.flash_size)
for offs, path in self.app.flash_files:
with open(path, "rb") as flash_file:
with open(path, 'rb') as flash_file:
contents = flash_file.read()
self.flash_image.seek(offs)
self.flash_image.write(contents)
@@ -568,7 +567,7 @@ class IDFQEMUDUT(IDFDUT):
@classmethod
def get_mac(cls, app, port):
# TODO(IDF-1242): get this from QEMU/efuse binary
return "11:22:33:44:55:66"
return '11:22:33:44:55:66'
@classmethod
def confirm_dut(cls, port, **kwargs):
@@ -577,30 +576,30 @@ class IDFQEMUDUT(IDFDUT):
def start_app(self, erase_nvs=ERASE_NVS):
# TODO: implement erase_nvs
# since the flash image is generated every time in the constructor, maybe this isn't needed...
self.qemu.sendline(b"cont\n")
self.qemu.expect_exact(b"(qemu)")
self.qemu.sendline(b'cont\n')
self.qemu.expect_exact(b'(qemu)')
def reset(self):
self.qemu.sendline(b"system_reset\n")
self.qemu.expect_exact(b"(qemu)")
self.qemu.sendline(b'system_reset\n')
self.qemu.expect_exact(b'(qemu)')
def erase_partition(self, partition):
raise NotImplementedError("method erase_partition not implemented")
raise NotImplementedError('method erase_partition not implemented')
def erase_flash(self):
raise NotImplementedError("method erase_flash not implemented")
raise NotImplementedError('method erase_flash not implemented')
def dump_flash(self, output_file, **kwargs):
raise NotImplementedError("method dump_flash not implemented")
raise NotImplementedError('method dump_flash not implemented')
@classmethod
def list_available_ports(cls):
return ["socket://localhost:{}".format(cls.QEMU_SERIAL_PORT)]
return ['socket://localhost:{}'.format(cls.QEMU_SERIAL_PORT)]
def close(self):
super(IDFQEMUDUT, self).close()
self.qemu.sendline(b"q\n")
self.qemu.expect_exact(b"(qemu)")
self.qemu.sendline(b'q\n')
self.qemu.expect_exact(b'(qemu)')
for _ in range(self.DEFAULT_EXPECT_TIMEOUT):
if not self.qemu.isalive():
break
@@ -610,5 +609,5 @@ class IDFQEMUDUT(IDFDUT):
class ESP32QEMUDUT(IDFQEMUDUT):
TARGET = "esp32"
TOOLCHAIN_PREFIX = "xtensa-esp32-elf-"
TARGET = 'esp32'
TOOLCHAIN_PREFIX = 'xtensa-esp32-elf-'
tools/ci/python_packages/ttfw_idf/__init__.py
+19 -19
View File
@@ -19,12 +19,12 @@ import re
from copy import deepcopy
import junit_xml
from tiny_test_fw import TinyFW, Utility
from .DebugUtils import OCDBackend, GDBBackend, CustomProcess # noqa: export DebugUtils for users
from .IDFApp import IDFApp, Example, LoadableElfTestApp, UT, TestApp, ComponentUTApp # noqa: export all Apps for users
from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP32C3DUT, ESP8266DUT, ESP32QEMUDUT # noqa: export DUTs for users
from .unity_test_parser import TestResults, TestFormat
from .DebugUtils import CustomProcess, GDBBackend, OCDBackend # noqa: export DebugUtils for users
from .IDFApp import UT, ComponentUTApp, Example, IDFApp, LoadableElfTestApp, TestApp # noqa: export all Apps for users
from .IDFDUT import ESP32C3DUT, ESP32DUT, ESP32QEMUDUT, ESP32S2DUT, ESP8266DUT, IDFDUT # noqa: export DUTs for users
from .unity_test_parser import TestFormat, TestResults
# pass TARGET_DUT_CLS_DICT to Env.py to avoid circular dependency issue.
TARGET_DUT_CLS_DICT = {
@@ -35,7 +35,7 @@ TARGET_DUT_CLS_DICT = {
def format_case_id(target, case_name):
return "{}.{}".format(target, case_name)
return '{}.{}'.format(target, case_name)
try:
@@ -128,13 +128,13 @@ def test_func_generator(func, app, target, ci_target, module, execution_time, le
dut_dict=dut_classes, **kwargs
)
test_func = original_method(func)
test_func.case_info["ID"] = format_case_id(target, test_func.case_info["name"])
test_func.case_info['ID'] = format_case_id(target, test_func.case_info['name'])
return test_func
@ci_target_check
def idf_example_test(app=Example, target="ESP32", ci_target=None, module="examples", execution_time=1,
level="example", erase_nvs=True, config_name=None, **kwargs):
def idf_example_test(app=Example, target='ESP32', ci_target=None, module='examples', execution_time=1,
level='example', erase_nvs=True, config_name=None, **kwargs):
"""
decorator for testing idf examples (with default values for some keyword args).
@@ -155,8 +155,8 @@ def idf_example_test(app=Example, target="ESP32", ci_target=None, module="exampl
@ci_target_check
def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", execution_time=1,
level="unit", erase_nvs=True, **kwargs):
def idf_unit_test(app=UT, target='ESP32', ci_target=None, module='unit-test', execution_time=1,
level='unit', erase_nvs=True, **kwargs):
"""
decorator for testing idf unit tests (with default values for some keyword args).
@@ -176,8 +176,8 @@ def idf_unit_test(app=UT, target="ESP32", ci_target=None, module="unit-test", ex
@ci_target_check
def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc", execution_time=1,
level="integration", erase_nvs=True, config_name=None, **kwargs):
def idf_custom_test(app=TestApp, target='ESP32', ci_target=None, module='misc', execution_time=1,
level='integration', erase_nvs=True, config_name=None, **kwargs):
"""
decorator for idf custom tests (with default values for some keyword args).
@@ -198,8 +198,8 @@ def idf_custom_test(app=TestApp, target="ESP32", ci_target=None, module="misc",
@ci_target_check
def idf_component_unit_test(app=ComponentUTApp, target="ESP32", ci_target=None, module="misc", execution_time=1,
level="integration", erase_nvs=True, config_name=None, **kwargs):
def idf_component_unit_test(app=ComponentUTApp, target='ESP32', ci_target=None, module='misc', execution_time=1,
level='integration', erase_nvs=True, config_name=None, **kwargs):
"""
decorator for idf custom tests (with default values for some keyword args).
@@ -253,11 +253,11 @@ def log_performance(item, value):
:param item: performance item name
:param value: performance value
"""
performance_msg = "[Performance][{}]: {}".format(item, value)
Utility.console_log(performance_msg, "orange")
performance_msg = '[Performance][{}]: {}'.format(item, value)
Utility.console_log(performance_msg, 'orange')
# update to junit test report
current_junit_case = TinyFW.JunitReport.get_current_test_case()
current_junit_case.stdout += performance_msg + "\r\n"
current_junit_case.stdout += performance_msg + '\r\n'
def check_performance(item, value, target):
@@ -300,7 +300,7 @@ def check_performance(item, value, target):
# if no exception was thrown then the performance is met and no need to continue
break
else:
raise AssertionError("Failed to get performance standard for {}".format(item))
raise AssertionError('Failed to get performance standard for {}'.format(item))
MINIMUM_FREE_HEAP_SIZE_RE = re.compile(r'Minimum free heap size: (\d+) bytes')
tools/ci/python_packages/ttfw_idf/unity_test_parser.py
+49 -49
View File
@@ -13,13 +13,13 @@ import re
import junit_xml
_NORMAL_TEST_REGEX = re.compile(r"(?P<file>.+):(?P<line>\d+):(?P<test_name>[^\s:]+):(?P<result>PASS|FAIL|IGNORE)(?:: (?P<message>.+))?")
_UNITY_FIXTURE_VERBOSE_PREFIX_REGEX = re.compile(r"(?P<prefix>TEST\((?P<test_group>[^\s,]+), (?P<test_name>[^\s\)]+)\))(?P<remainder>.+)?$")
_UNITY_FIXTURE_REMAINDER_REGEX = re.compile(r"^(?P<file>.+):(?P<line>\d+)::(?P<result>PASS|FAIL|IGNORE)(?:: (?P<message>.+))?")
_NORMAL_TEST_REGEX = re.compile(r'(?P<file>.+):(?P<line>\d+):(?P<test_name>[^\s:]+):(?P<result>PASS|FAIL|IGNORE)(?:: (?P<message>.+))?')
_UNITY_FIXTURE_VERBOSE_PREFIX_REGEX = re.compile(r'(?P<prefix>TEST\((?P<test_group>[^\s,]+), (?P<test_name>[^\s\)]+)\))(?P<remainder>.+)?$')
_UNITY_FIXTURE_REMAINDER_REGEX = re.compile(r'^(?P<file>.+):(?P<line>\d+)::(?P<result>PASS|FAIL|IGNORE)(?:: (?P<message>.+))?')
_TEST_SUMMARY_BLOCK_REGEX = re.compile(
r"^(?P<num_tests>\d+) Tests (?P<num_failures>\d+) Failures (?P<num_ignored>\d+) Ignored\s*\r?\n(?P<overall_result>OK|FAIL)(?:ED)?", re.MULTILINE
r'^(?P<num_tests>\d+) Tests (?P<num_failures>\d+) Failures (?P<num_ignored>\d+) Ignored\s*\r?\n(?P<overall_result>OK|FAIL)(?:ED)?', re.MULTILINE
)
_TEST_RESULT_ENUM = ["PASS", "FAIL", "IGNORE"]
_TEST_RESULT_ENUM = ['PASS', 'FAIL', 'IGNORE']
class TestFormat(enum.Enum):
@@ -63,14 +63,14 @@ class TestResult:
self,
test_name,
result,
group="default",
file="",
group='default',
file='',
line=0,
message="",
full_line="",
message='',
full_line='',
):
if result not in _TEST_RESULT_ENUM:
raise ValueError("result must be one of {}.".format(_TEST_RESULT_ENUM))
raise ValueError('result must be one of {}.'.format(_TEST_RESULT_ENUM))
self._test_name = test_name
self._result = result
@@ -78,11 +78,11 @@ class TestResult:
self._message = message
self._full_line = full_line
if result != "PASS":
if result != 'PASS':
self._file = file
self._line = line
else:
self._file = ""
self._file = ''
self._line = 0
def file(self):
@@ -150,7 +150,7 @@ class TestResults:
self._parse_unity_fixture_verbose(test_output)
else:
raise ValueError(
"test_format must be one of UNITY_BASIC or UNITY_FIXTURE_VERBOSE."
'test_format must be one of UNITY_BASIC or UNITY_FIXTURE_VERBOSE.'
)
def num_tests(self):
@@ -185,7 +185,7 @@ class TestResults:
return self._tests
def to_junit(
self, suite_name="all_tests",
self, suite_name='all_tests',
):
"""
Convert the tests to JUnit XML.
@@ -207,7 +207,7 @@ class TestResults:
test_case_list = []
for test in self._tests:
if test.result() == "PASS":
if test.result() == 'PASS':
test_case_list.append(
junit_xml.TestCase(name=test.name(), classname=test.group())
)
@@ -218,11 +218,11 @@ class TestResults:
file=test.file(),
line=test.line(),
)
if test.result() == "FAIL":
if test.result() == 'FAIL':
junit_tc.add_failure_info(
message=test.message(), output=test.full_line()
)
elif test.result() == "IGNORE":
elif test.result() == 'IGNORE':
junit_tc.add_skipped_info(
message=test.message(), output=test.full_line()
)
@@ -245,17 +245,17 @@ class TestResults:
"""
match = _TEST_SUMMARY_BLOCK_REGEX.search(unity_output)
if not match:
raise ValueError("A Unity test summary block was not found.")
raise ValueError('A Unity test summary block was not found.')
try:
stats = TestStats()
stats.total = int(match.group("num_tests"))
stats.failed = int(match.group("num_failures"))
stats.ignored = int(match.group("num_ignored"))
stats.total = int(match.group('num_tests'))
stats.failed = int(match.group('num_failures'))
stats.ignored = int(match.group('num_ignored'))
stats.passed = stats.total - stats.failed - stats.ignored
return stats
except ValueError:
raise ValueError("The Unity test summary block was not valid.")
raise ValueError('The Unity test summary block was not valid.')
def _parse_unity_basic(self, unity_output):
"""
@@ -268,13 +268,13 @@ class TestResults:
for test in _NORMAL_TEST_REGEX.finditer(unity_output):
try:
new_test = TestResult(
test.group("test_name"),
test.group("result"),
file=test.group("file"),
line=int(test.group("line")),
message=test.group("message")
if test.group("message") is not None
else "",
test.group('test_name'),
test.group('result'),
file=test.group('file'),
line=int(test.group('line')),
message=test.group('message')
if test.group('message') is not None
else '',
full_line=test.group(0),
)
except ValueError:
@@ -283,10 +283,10 @@ class TestResults:
self._add_new_test(new_test, found_test_stats)
if len(self._tests) == 0:
raise ValueError("No tests were found.")
raise ValueError('No tests were found.')
if found_test_stats != self._test_stats:
raise ValueError("Test output does not match summary block.")
raise ValueError('Test output does not match summary block.')
def _parse_unity_fixture_verbose(self, unity_output):
"""
@@ -309,7 +309,7 @@ class TestResults:
if prefix_match:
# Handle the remaining portion of a test case line after the unity_fixture
# prefix.
remainder = prefix_match.group("remainder")
remainder = prefix_match.group('remainder')
if remainder:
self._parse_unity_fixture_remainder(
prefix_match, remainder, found_test_stats
@@ -324,10 +324,10 @@ class TestResults:
pass
if len(self._tests) == 0:
raise ValueError("No tests were found.")
raise ValueError('No tests were found.')
if found_test_stats != self._test_stats:
raise ValueError("Test output does not match summary block.")
raise ValueError('Test output does not match summary block.')
def _parse_unity_fixture_remainder(self, prefix_match, remainder, test_stats):
"""
@@ -337,26 +337,26 @@ class TestResults:
"""
new_test = None
if remainder == " PASS":
if remainder == ' PASS':
new_test = TestResult(
prefix_match.group("test_name"),
"PASS",
group=prefix_match.group("test_group"),
prefix_match.group('test_name'),
'PASS',
group=prefix_match.group('test_group'),
full_line=prefix_match.group(0),
)
else:
remainder_match = _UNITY_FIXTURE_REMAINDER_REGEX.match(remainder)
if remainder_match:
new_test = TestResult(
prefix_match.group("test_name"),
remainder_match.group("result"),
group=prefix_match.group("test_group"),
file=remainder_match.group("file"),
line=int(remainder_match.group("line")),
message=remainder_match.group("message")
if remainder_match.group("message") is not None
else "",
full_line=prefix_match.group("prefix") + remainder_match.group(0),
prefix_match.group('test_name'),
remainder_match.group('result'),
group=prefix_match.group('test_group'),
file=remainder_match.group('file'),
line=int(remainder_match.group('line')),
message=remainder_match.group('message')
if remainder_match.group('message') is not None
else '',
full_line=prefix_match.group('prefix') + remainder_match.group(0),
)
if new_test is not None:
@@ -365,9 +365,9 @@ class TestResults:
def _add_new_test(self, new_test, test_stats):
"""Add a new test and increment the proper members of test_stats."""
test_stats.total += 1
if new_test.result() == "PASS":
if new_test.result() == 'PASS':
test_stats.passed += 1
elif new_test.result() == "FAIL":
elif new_test.result() == 'FAIL':
test_stats.failed += 1
else:
test_stats.ignored += 1