ci: remove ttfw related files
This commit is contained in:
Fu Hanxi
@@ -1,87 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
class for handling Test Apps. Currently it provides the following features:
|
||||
|
||||
1. get SDK path
|
||||
2. get SDK tools
|
||||
3. parse application info from its path. for example:
|
||||
* provide download info
|
||||
* provide partition table info
|
||||
|
||||
Test Apps should inherent from BaseApp class and overwrite the methods.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
# timestamp used for calculate log folder name
|
||||
LOG_FOLDER_TIMESTAMP = time.time()
|
||||
|
||||
|
||||
class BaseApp(object):
|
||||
"""
|
||||
Base Class for App.
|
||||
Defines the mandatory methods that App need to implement.
|
||||
Also implements some common methods.
|
||||
|
||||
:param app_path: the path for app.
|
||||
:param config_name: app configuration to be tested
|
||||
:param target: build target
|
||||
"""
|
||||
|
||||
def __init__(self, app_path, config_name=None, target=None):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def get_sdk_path(cls):
|
||||
"""
|
||||
get sdk path.
|
||||
|
||||
subclass must overwrite this method.
|
||||
|
||||
:return: abs sdk path
|
||||
"""
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def get_tools(cls):
|
||||
"""
|
||||
get SDK related tools for applications
|
||||
|
||||
subclass must overwrite this method.
|
||||
|
||||
:return: tuple, abs path of each tool
|
||||
"""
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def get_log_folder(cls, test_suite_name):
|
||||
"""
|
||||
By default log folder is ``${SDK_PATH}/TEST_LOGS/${test_suite_name}_${timestamp}``.
|
||||
|
||||
The log folder name is consist once start running, ensure all logs of will be put into the same folder.
|
||||
|
||||
:param test_suite_name: the test suite name, by default it's the base file name for main module
|
||||
:return: the log folder path
|
||||
"""
|
||||
if not test_suite_name:
|
||||
test_suite_name = os.path.splitext(os.path.basename(sys.modules['__main__'].__file__))[0]
|
||||
sdk_path = cls.get_sdk_path()
|
||||
log_folder = os.path.join(sdk_path, 'TEST_LOGS',
|
||||
test_suite_name +
|
||||
time.strftime('_%m%d_%H_%M_%S', time.localtime(LOG_FOLDER_TIMESTAMP)))
|
||||
if not os.path.exists(log_folder):
|
||||
os.makedirs(log_folder)
|
||||
return log_folder
|
||||
|
||||
def process_app_info(self):
|
||||
"""
|
||||
parse built app info for DUTTool
|
||||
|
||||
subclass must overwrite this method.
|
||||
|
||||
:return: required info for specific DUTTool
|
||||
"""
|
||||
pass
|
||||
@@ -1,790 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
DUT provides 3 major groups of features:
|
||||
|
||||
* DUT port feature, provide basic open/close/read/write features
|
||||
* DUT tools, provide extra methods to control the device, like download and start app
|
||||
* DUT expect method, provide features for users to check DUT outputs
|
||||
|
||||
The current design of DUT have 3 classes for one DUT: BaseDUT, DUTPort, DUTTool.
|
||||
|
||||
* BaseDUT class:
|
||||
* defines methods DUT port and DUT tool need to overwrite
|
||||
* provide the expect methods and some other methods based on DUTPort
|
||||
* DUTPort class:
|
||||
* inherent from BaseDUT class
|
||||
* implements the port features by overwriting port methods defined in BaseDUT
|
||||
* DUTTool class:
|
||||
* inherent from one of the DUTPort class
|
||||
* implements the tools features by overwriting tool methods defined in BaseDUT
|
||||
* could add some new methods provided by the tool
|
||||
|
||||
This module implements the BaseDUT class and one of the port class SerialDUT.
|
||||
User should implement their DUTTool classes.
|
||||
If they using different port then need to implement their DUTPort class as well.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import copy
|
||||
import functools
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
# python2 and python3 queue package name is different
|
||||
try:
|
||||
import Queue as _queue
|
||||
except ImportError:
|
||||
import queue as _queue # type: ignore
|
||||
|
||||
try:
|
||||
from typing import Callable, List
|
||||
except ImportError:
|
||||
# Only used for type annotations
|
||||
pass
|
||||
|
||||
import serial
|
||||
from serial.tools import list_ports
|
||||
|
||||
from . import Utility
|
||||
|
||||
|
||||
class ExpectTimeout(ValueError):
|
||||
""" timeout for expect method """
|
||||
pass
|
||||
|
||||
|
||||
class UnsupportedExpectItem(ValueError):
|
||||
""" expect item not supported by the expect method """
|
||||
pass
|
||||
|
||||
|
||||
def _expect_lock(func):
|
||||
@functools.wraps(func)
|
||||
def handler(self, *args, **kwargs):
|
||||
with self.expect_lock:
|
||||
ret = func(self, *args, **kwargs)
|
||||
return ret
|
||||
return handler
|
||||
|
||||
|
||||
def _decode_data(data):
|
||||
""" for python3, if the data is bytes, then decode it to string """
|
||||
if isinstance(data, bytes):
|
||||
# convert bytes to string. This is a bit of a hack, we know that we want to log this
|
||||
# later so encode to the stdout encoding with backslash escapes for anything non-encodable
|
||||
try:
|
||||
return data.decode(sys.stdout.encoding, 'backslashreplace')
|
||||
except UnicodeDecodeError: # Python <3.5 doesn't support backslashreplace
|
||||
return data.decode(sys.stdout.encoding, 'replace')
|
||||
return data
|
||||
|
||||
|
||||
def _pattern_to_string(pattern):
|
||||
try:
|
||||
ret = 'RegEx: ' + pattern.pattern
|
||||
except AttributeError:
|
||||
ret = pattern
|
||||
return ret
|
||||
|
||||
|
||||
class _DataCache(_queue.Queue):
|
||||
"""
|
||||
Data cache based on Queue. Allow users to process data cache based on bytes instead of Queue."
|
||||
"""
|
||||
|
||||
def __init__(self, maxsize=0):
|
||||
_queue.Queue.__init__(self, maxsize=maxsize)
|
||||
self.data_cache = str()
|
||||
|
||||
def _move_from_queue_to_cache(self):
|
||||
"""
|
||||
move all of the available data in the queue to cache
|
||||
|
||||
:return: True if moved any item from queue to data cache, else False
|
||||
"""
|
||||
ret = False
|
||||
while True:
|
||||
try:
|
||||
self.data_cache += _decode_data(self.get(0))
|
||||
ret = True
|
||||
except _queue.Empty:
|
||||
break
|
||||
return ret
|
||||
|
||||
def get_data(self, timeout=0.0):
|
||||
"""
|
||||
get a copy of data from cache.
|
||||
|
||||
:param timeout: timeout for waiting new queue item
|
||||
:return: copy of data cache
|
||||
"""
|
||||
# make sure timeout is non-negative
|
||||
if timeout < 0:
|
||||
timeout = 0
|
||||
|
||||
ret = self._move_from_queue_to_cache()
|
||||
|
||||
if not ret:
|
||||
# we only wait for new data if we can't provide a new data_cache
|
||||
try:
|
||||
data = self.get(timeout=timeout)
|
||||
self.data_cache += _decode_data(data)
|
||||
except _queue.Empty:
|
||||
# don't do anything when on update for cache
|
||||
pass
|
||||
return copy.deepcopy(self.data_cache)
|
||||
|
||||
def flush(self, index=0xFFFFFFFF):
|
||||
"""
|
||||
flush data from cache.
|
||||
|
||||
:param index: if < 0 then don't do flush, otherwise flush data before index
|
||||
:return: None
|
||||
"""
|
||||
# first add data in queue to cache
|
||||
self.get_data()
|
||||
|
||||
if index > 0:
|
||||
self.data_cache = self.data_cache[index:]
|
||||
|
||||
|
||||
class _LogThread(threading.Thread, _queue.Queue):
|
||||
"""
|
||||
We found some SD card on Raspberry Pi could have very bad performance.
|
||||
It could take seconds to save small amount of data.
|
||||
If the DUT receives data and save it as log, then it stops receiving data until log is saved.
|
||||
This could lead to expect timeout.
|
||||
As an workaround to this issue, ``BaseDUT`` class will create a thread to save logs.
|
||||
Then data will be passed to ``expect`` as soon as received.
|
||||
"""
|
||||
def __init__(self):
|
||||
threading.Thread.__init__(self, name='LogThread')
|
||||
_queue.Queue.__init__(self, maxsize=0)
|
||||
self.setDaemon(True)
|
||||
self.flush_lock = threading.Lock()
|
||||
|
||||
def save_log(self, filename, data):
|
||||
"""
|
||||
:param filename: log file name
|
||||
:param data: log data. Must be ``bytes``.
|
||||
"""
|
||||
self.put({'filename': filename, 'data': data})
|
||||
|
||||
def flush_data(self):
|
||||
with self.flush_lock:
|
||||
data_cache = dict()
|
||||
while True:
|
||||
# move all data from queue to data cache
|
||||
try:
|
||||
log = self.get_nowait()
|
||||
try:
|
||||
data_cache[log['filename']] += log['data']
|
||||
except KeyError:
|
||||
data_cache[log['filename']] = log['data']
|
||||
except _queue.Empty:
|
||||
break
|
||||
# flush data
|
||||
for filename in data_cache:
|
||||
with open(filename, 'ab+') as f:
|
||||
f.write(data_cache[filename])
|
||||
|
||||
def run(self):
|
||||
while True:
|
||||
time.sleep(1)
|
||||
self.flush_data()
|
||||
|
||||
|
||||
class RecvThread(threading.Thread):
|
||||
|
||||
CHECK_FUNCTIONS = [] # type: List[Callable]
|
||||
""" DUT subclass can define a few check functions to process received data. """
|
||||
|
||||
def __init__(self, read, dut):
|
||||
super(RecvThread, self).__init__()
|
||||
self.exit_event = threading.Event()
|
||||
self.setDaemon(True)
|
||||
self.read = read
|
||||
self.dut = dut
|
||||
self.data_cache = dut.data_cache
|
||||
self.recorded_data = dut.recorded_data
|
||||
self.record_data_lock = dut.record_data_lock
|
||||
self._line_cache = str()
|
||||
|
||||
def _line_completion(self, data):
|
||||
"""
|
||||
Usually check functions requires to check for one complete line.
|
||||
This method will do line completion for the first line, and strip incomplete last line.
|
||||
"""
|
||||
ret = self._line_cache
|
||||
decoded_data = _decode_data(data)
|
||||
|
||||
# cache incomplete line to later process
|
||||
lines = decoded_data.splitlines(True)
|
||||
last_line = lines[-1]
|
||||
|
||||
if last_line[-1] != '\n':
|
||||
if len(lines) == 1:
|
||||
# only one line and the line is not finished, then append this to cache
|
||||
self._line_cache += lines[-1]
|
||||
ret = str()
|
||||
else:
|
||||
# more than one line and not finished, replace line cache
|
||||
self._line_cache = lines[-1]
|
||||
ret += ''.join(lines[:-1])
|
||||
else:
|
||||
# line finishes, flush cache
|
||||
self._line_cache = str()
|
||||
ret += decoded_data
|
||||
return ret
|
||||
|
||||
def run(self):
|
||||
while not self.exit_event.isSet():
|
||||
raw_data = self.read(1000)
|
||||
if raw_data:
|
||||
# we need to do line completion before call check functions
|
||||
# need to call check functions first
|
||||
# otherwise check functions could be called after cases finished
|
||||
comp_data = self._line_completion(raw_data)
|
||||
for check_function in self.CHECK_FUNCTIONS:
|
||||
check_function(self, comp_data)
|
||||
|
||||
with self.record_data_lock:
|
||||
self.data_cache.put(raw_data)
|
||||
for capture_id in self.recorded_data:
|
||||
self.recorded_data[capture_id].put(raw_data)
|
||||
|
||||
def exit(self):
|
||||
self.exit_event.set()
|
||||
self.join()
|
||||
|
||||
|
||||
class BaseDUT(object):
|
||||
"""
|
||||
:param name: application defined name for port
|
||||
:param port: comport name, used to create DUT port
|
||||
:param log_file: log file name
|
||||
:param app: test app instance
|
||||
:param kwargs: extra args for DUT to create ports
|
||||
"""
|
||||
|
||||
DEFAULT_EXPECT_TIMEOUT = 10
|
||||
MAX_EXPECT_FAILURES_TO_SAVED = 10
|
||||
RECV_THREAD_CLS = RecvThread
|
||||
TARGET = None
|
||||
""" DUT subclass can specify RECV_THREAD_CLS to do add some extra stuff when receive data.
|
||||
For example, DUT can implement exception detect & analysis logic in receive thread subclass. """
|
||||
LOG_THREAD = _LogThread()
|
||||
LOG_THREAD.start()
|
||||
|
||||
def __init__(self, name, port, log_file, app, **kwargs):
|
||||
|
||||
self.expect_lock = threading.Lock()
|
||||
self.name = name
|
||||
self.port = port
|
||||
self.log_file = log_file
|
||||
self.app = app
|
||||
self.data_cache = _DataCache()
|
||||
# the main process of recorded data are done in receive thread
|
||||
# but receive thread could be closed in DUT lifetime (tool methods)
|
||||
# so we keep it in BaseDUT, as their life cycle are same
|
||||
self.recorded_data = dict()
|
||||
self.record_data_lock = threading.RLock()
|
||||
self.receive_thread = None
|
||||
self.expect_failures = []
|
||||
self._port_open()
|
||||
self.start_receive()
|
||||
|
||||
def __str__(self):
|
||||
return 'DUT({}: {})'.format(self.name, str(self.port))
|
||||
|
||||
def _save_expect_failure(self, pattern, data, start_time):
|
||||
"""
|
||||
Save expect failure. If the test fails, then it will print the expect failures.
|
||||
In some cases, user will handle expect exceptions.
|
||||
The expect failures could be false alarm, and test case might generate a lot of such failures.
|
||||
Therefore, we don't print the failure immediately and limit the max size of failure list.
|
||||
"""
|
||||
self.expect_failures.insert(0, {'pattern': pattern, 'data': data,
|
||||
'start': start_time, 'end': time.time()})
|
||||
self.expect_failures = self.expect_failures[:self.MAX_EXPECT_FAILURES_TO_SAVED]
|
||||
|
||||
def _save_dut_log(self, data):
|
||||
"""
|
||||
Save DUT log into file using another thread.
|
||||
This is a workaround for some devices takes long time for file system operations.
|
||||
|
||||
See descriptions in ``_LogThread`` for details.
|
||||
"""
|
||||
self.LOG_THREAD.save_log(self.log_file, data)
|
||||
|
||||
# define for methods need to be overwritten by Port
|
||||
@classmethod
|
||||
def list_available_ports(cls):
|
||||
"""
|
||||
list all available ports.
|
||||
|
||||
subclass (port) must overwrite this method.
|
||||
|
||||
:return: list of available comports
|
||||
"""
|
||||
pass
|
||||
|
||||
def _port_open(self):
|
||||
"""
|
||||
open the port.
|
||||
|
||||
subclass (port) must overwrite this method.
|
||||
|
||||
:return: None
|
||||
"""
|
||||
pass
|
||||
|
||||
def _port_read(self, size=1):
|
||||
"""
|
||||
read form port. This method should not blocking for long time, otherwise receive thread can not exit.
|
||||
|
||||
subclass (port) must overwrite this method.
|
||||
|
||||
:param size: max size to read.
|
||||
:return: read data.
|
||||
"""
|
||||
pass
|
||||
|
||||
def _port_write(self, data):
|
||||
"""
|
||||
write to port.
|
||||
|
||||
subclass (port) must overwrite this method.
|
||||
|
||||
:param data: data to write
|
||||
:return: None
|
||||
"""
|
||||
pass
|
||||
|
||||
def _port_close(self):
|
||||
"""
|
||||
close port.
|
||||
|
||||
subclass (port) must overwrite this method.
|
||||
|
||||
:return: None
|
||||
"""
|
||||
pass
|
||||
|
||||
# methods that need to be overwritten by Tool
|
||||
@classmethod
|
||||
def confirm_dut(cls, port, **kwargs):
|
||||
"""
|
||||
confirm if it's a DUT, usually used by auto detecting DUT in by Env config.
|
||||
|
||||
subclass (tool) must overwrite this method.
|
||||
|
||||
:param port: comport
|
||||
:return: tuple of result (bool), and target (str)
|
||||
"""
|
||||
pass
|
||||
|
||||
def start_app(self):
|
||||
"""
|
||||
usually after we got DUT, we need to do some extra works to let App start.
|
||||
For example, we need to reset->download->reset to let IDF application start on DUT.
|
||||
|
||||
subclass (tool) must overwrite this method.
|
||||
|
||||
:return: None
|
||||
"""
|
||||
pass
|
||||
|
||||
# methods that features raw port methods
|
||||
def start_receive(self):
|
||||
"""
|
||||
Start thread to receive data.
|
||||
|
||||
:return: None
|
||||
"""
|
||||
self.receive_thread = self.RECV_THREAD_CLS(self._port_read, self)
|
||||
self.receive_thread.start()
|
||||
|
||||
def stop_receive(self):
|
||||
"""
|
||||
stop the receiving thread for the port
|
||||
:return: None
|
||||
"""
|
||||
if self.receive_thread:
|
||||
self.receive_thread.exit()
|
||||
self.LOG_THREAD.flush_data()
|
||||
self.receive_thread = None
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
permanently close the port
|
||||
"""
|
||||
self.stop_receive()
|
||||
self._port_close()
|
||||
|
||||
@staticmethod
|
||||
def u_to_bytearray(data):
|
||||
"""
|
||||
if data is not bytearray then it tries to convert it
|
||||
|
||||
:param data: data which needs to be checked and maybe transformed
|
||||
"""
|
||||
if isinstance(data, type(u'')):
|
||||
try:
|
||||
data = data.encode('utf-8')
|
||||
except UnicodeEncodeError as e:
|
||||
print(u'Cannot encode {} of type {}'.format(data, type(data)))
|
||||
raise e
|
||||
return data
|
||||
|
||||
def write(self, data, eol='\r\n', flush=True):
|
||||
"""
|
||||
:param data: data
|
||||
:param eol: end of line pattern.
|
||||
:param flush: if need to flush received data cache before write data.
|
||||
usually we need to flush data before write,
|
||||
make sure processing outputs generated by wrote.
|
||||
:return: None
|
||||
"""
|
||||
# do flush before write
|
||||
if flush:
|
||||
self.data_cache.flush()
|
||||
# do write if cache
|
||||
if data is not None:
|
||||
self._port_write(self.u_to_bytearray(data) + self.u_to_bytearray(eol) if eol else self.u_to_bytearray(data))
|
||||
|
||||
@_expect_lock
|
||||
def read(self, size=0xFFFFFFFF):
|
||||
"""
|
||||
read(size=0xFFFFFFFF)
|
||||
read raw data. NOT suggested to use this method.
|
||||
Only use it if expect method doesn't meet your requirement.
|
||||
|
||||
:param size: read size. default read all data
|
||||
:return: read data
|
||||
"""
|
||||
data = self.data_cache.get_data(0)[:size]
|
||||
self.data_cache.flush(size)
|
||||
return data
|
||||
|
||||
def start_capture_raw_data(self, capture_id='default'):
|
||||
"""
|
||||
Sometime application want to get DUT raw data and use ``expect`` method at the same time.
|
||||
Capture methods provides a way to get raw data without affecting ``expect`` or ``read`` method.
|
||||
|
||||
If you call ``start_capture_raw_data`` with same capture id again, it will restart capture on this ID.
|
||||
|
||||
:param capture_id: ID of capture. You can use different IDs to do different captures at the same time.
|
||||
"""
|
||||
with self.record_data_lock:
|
||||
try:
|
||||
# if start capture on existed ID, we do flush data and restart capture
|
||||
self.recorded_data[capture_id].flush()
|
||||
except KeyError:
|
||||
# otherwise, create new data cache
|
||||
self.recorded_data[capture_id] = _DataCache()
|
||||
|
||||
def stop_capture_raw_data(self, capture_id='default'):
|
||||
"""
|
||||
Stop capture and get raw data.
|
||||
This method should be used after ``start_capture_raw_data`` on the same capture ID.
|
||||
|
||||
:param capture_id: ID of capture.
|
||||
:return: captured raw data between start capture and stop capture.
|
||||
"""
|
||||
with self.record_data_lock:
|
||||
try:
|
||||
ret = self.recorded_data[capture_id].get_data()
|
||||
self.recorded_data.pop(capture_id)
|
||||
except KeyError as e:
|
||||
e.message = 'capture_id does not exist. ' \
|
||||
'You should call start_capture_raw_data with same ID ' \
|
||||
'before calling stop_capture_raw_data'
|
||||
raise e
|
||||
return ret
|
||||
|
||||
# expect related methods
|
||||
|
||||
@staticmethod
|
||||
def _expect_str(data, pattern):
|
||||
"""
|
||||
protected method. check if string is matched in data cache.
|
||||
|
||||
:param data: data to process
|
||||
:param pattern: string
|
||||
:return: pattern if match succeed otherwise None
|
||||
"""
|
||||
index = data.find(pattern)
|
||||
if index != -1:
|
||||
ret = pattern
|
||||
index += len(pattern)
|
||||
else:
|
||||
ret = None
|
||||
return ret, index
|
||||
|
||||
@staticmethod
|
||||
def _expect_re(data, pattern):
|
||||
"""
|
||||
protected method. check if re pattern is matched in data cache
|
||||
|
||||
:param data: data to process
|
||||
:param pattern: compiled RegEx pattern
|
||||
:return: match groups if match succeed otherwise None
|
||||
"""
|
||||
ret = None
|
||||
if isinstance(pattern.pattern, bytes):
|
||||
pattern = re.compile(_decode_data(pattern.pattern))
|
||||
match = pattern.search(data)
|
||||
if match:
|
||||
ret = tuple(x for x in match.groups())
|
||||
index = match.end()
|
||||
else:
|
||||
index = -1
|
||||
return ret, index
|
||||
|
||||
EXPECT_METHOD = [
|
||||
[type(re.compile('')), '_expect_re'],
|
||||
[type(b''), '_expect_str'], # Python 2 & 3 hook to work without 'from builtins import str' from future
|
||||
[type(u''), '_expect_str'],
|
||||
]
|
||||
|
||||
def _get_expect_method(self, pattern):
|
||||
"""
|
||||
protected method. get expect method according to pattern type.
|
||||
|
||||
:param pattern: expect pattern, string or compiled RegEx
|
||||
:return: ``_expect_str`` or ``_expect_re``
|
||||
"""
|
||||
for expect_method in self.EXPECT_METHOD:
|
||||
if isinstance(pattern, expect_method[0]):
|
||||
method = expect_method[1]
|
||||
break
|
||||
else:
|
||||
raise UnsupportedExpectItem()
|
||||
return self.__getattribute__(method)
|
||||
|
||||
@_expect_lock
|
||||
def expect(self, pattern, timeout=DEFAULT_EXPECT_TIMEOUT, full_stdout=False):
|
||||
"""
|
||||
expect(pattern, timeout=DEFAULT_EXPECT_TIMEOUT)
|
||||
expect received data on DUT match the pattern. will raise exception when expect timeout.
|
||||
|
||||
:raise ExpectTimeout: failed to find the pattern before timeout
|
||||
:raise UnsupportedExpectItem: pattern is not string or compiled RegEx
|
||||
|
||||
:param pattern: string or compiled RegEx(string pattern)
|
||||
:param timeout: timeout for expect
|
||||
:param full_stdout: return full stdout until meet expect string/pattern or just matched string
|
||||
:return: string if pattern is string; matched groups if pattern is RegEx
|
||||
"""
|
||||
method = self._get_expect_method(pattern)
|
||||
stdout = ''
|
||||
|
||||
# non-blocking get data for first time
|
||||
data = self.data_cache.get_data(0)
|
||||
start_time = time.time()
|
||||
while True:
|
||||
ret, index = method(data, pattern)
|
||||
if ret is not None:
|
||||
stdout = data[:index]
|
||||
self.data_cache.flush(index)
|
||||
break
|
||||
time_remaining = start_time + timeout - time.time()
|
||||
if time_remaining < 0:
|
||||
break
|
||||
# wait for new data from cache
|
||||
data = self.data_cache.get_data(time_remaining)
|
||||
|
||||
if ret is None:
|
||||
pattern = _pattern_to_string(pattern)
|
||||
self._save_expect_failure(pattern, data, start_time)
|
||||
raise ExpectTimeout(self.name + ': ' + pattern)
|
||||
return stdout if full_stdout else ret
|
||||
|
||||
def _expect_multi(self, expect_all, expect_item_list, timeout):
|
||||
"""
|
||||
protected method. internal logical for expect multi.
|
||||
|
||||
:param expect_all: True or False, expect all items in the list or any in the list
|
||||
:param expect_item_list: expect item list
|
||||
:param timeout: timeout
|
||||
:return: None
|
||||
"""
|
||||
def process_expected_item(item_raw):
|
||||
# convert item raw data to standard dict
|
||||
item = {
|
||||
'pattern': item_raw[0] if isinstance(item_raw, tuple) else item_raw,
|
||||
'method': self._get_expect_method(item_raw[0] if isinstance(item_raw, tuple)
|
||||
else item_raw),
|
||||
'callback': item_raw[1] if isinstance(item_raw, tuple) else None,
|
||||
'index': -1,
|
||||
'ret': None,
|
||||
}
|
||||
return item
|
||||
|
||||
expect_items = [process_expected_item(x) for x in expect_item_list]
|
||||
|
||||
# non-blocking get data for first time
|
||||
data = self.data_cache.get_data(0)
|
||||
|
||||
start_time = time.time()
|
||||
matched_expect_items = list()
|
||||
while True:
|
||||
for expect_item in expect_items:
|
||||
if expect_item not in matched_expect_items:
|
||||
# exclude those already matched
|
||||
expect_item['ret'], expect_item['index'] = \
|
||||
expect_item['method'](data, expect_item['pattern'])
|
||||
if expect_item['ret'] is not None:
|
||||
# match succeed for one item
|
||||
matched_expect_items.append(expect_item)
|
||||
|
||||
# if expect all, then all items need to be matched,
|
||||
# else only one item need to matched
|
||||
if expect_all:
|
||||
match_succeed = len(matched_expect_items) == len(expect_items)
|
||||
else:
|
||||
match_succeed = True if matched_expect_items else False
|
||||
|
||||
time_remaining = start_time + timeout - time.time()
|
||||
if time_remaining < 0 or match_succeed:
|
||||
break
|
||||
else:
|
||||
data = self.data_cache.get_data(time_remaining)
|
||||
|
||||
if match_succeed:
|
||||
# sort matched items according to order of appearance in the input data,
|
||||
# so that the callbacks are invoked in correct order
|
||||
matched_expect_items = sorted(matched_expect_items, key=lambda it: it['index'])
|
||||
# invoke callbacks and flush matched data cache
|
||||
slice_index = -1
|
||||
for expect_item in matched_expect_items:
|
||||
# trigger callback
|
||||
if expect_item['callback']:
|
||||
expect_item['callback'](expect_item['ret'])
|
||||
slice_index = max(slice_index, expect_item['index'])
|
||||
# flush already matched data
|
||||
self.data_cache.flush(slice_index)
|
||||
else:
|
||||
pattern = str([_pattern_to_string(x['pattern']) for x in expect_items])
|
||||
self._save_expect_failure(pattern, data, start_time)
|
||||
raise ExpectTimeout(self.name + ': ' + pattern)
|
||||
|
||||
@_expect_lock
|
||||
def expect_any(self, *expect_items, **timeout):
|
||||
"""
|
||||
expect_any(*expect_items, timeout=DEFAULT_TIMEOUT)
|
||||
expect any of the patterns.
|
||||
will call callback (if provided) if pattern match succeed and then return.
|
||||
will pass match result to the callback.
|
||||
|
||||
:raise ExpectTimeout: failed to match any one of the expect items before timeout
|
||||
:raise UnsupportedExpectItem: pattern in expect_item is not string or compiled RegEx
|
||||
|
||||
:arg expect_items: one or more expect items.
|
||||
string, compiled RegEx pattern or (string or RegEx(string pattern), callback)
|
||||
:keyword timeout: timeout for expect
|
||||
:return: None
|
||||
"""
|
||||
# to be compatible with python2
|
||||
# in python3 we can write f(self, *expect_items, timeout=DEFAULT_TIMEOUT)
|
||||
if 'timeout' not in timeout:
|
||||
timeout['timeout'] = self.DEFAULT_EXPECT_TIMEOUT
|
||||
return self._expect_multi(False, expect_items, **timeout)
|
||||
|
||||
@_expect_lock
|
||||
def expect_all(self, *expect_items, **timeout):
|
||||
"""
|
||||
expect_all(*expect_items, timeout=DEFAULT_TIMEOUT)
|
||||
expect all of the patterns.
|
||||
will call callback (if provided) if all pattern match succeed and then return.
|
||||
will pass match result to the callback.
|
||||
|
||||
:raise ExpectTimeout: failed to match all of the expect items before timeout
|
||||
:raise UnsupportedExpectItem: pattern in expect_item is not string or compiled RegEx
|
||||
|
||||
:arg expect_items: one or more expect items.
|
||||
string, compiled RegEx pattern or (string or RegEx(string pattern), callback)
|
||||
:keyword timeout: timeout for expect
|
||||
:return: None
|
||||
"""
|
||||
# to be compatible with python2
|
||||
# in python3 we can write f(self, *expect_items, timeout=DEFAULT_TIMEOUT)
|
||||
if 'timeout' not in timeout:
|
||||
timeout['timeout'] = self.DEFAULT_EXPECT_TIMEOUT
|
||||
return self._expect_multi(True, expect_items, **timeout)
|
||||
|
||||
@staticmethod
|
||||
def _format_ts(ts):
|
||||
return '{}:{}'.format(time.strftime('%m-%d %H:%M:%S', time.localtime(ts)), str(ts % 1)[2:5])
|
||||
|
||||
def print_debug_info(self):
|
||||
"""
|
||||
Print debug info of current DUT. Currently we will print debug info for expect failures.
|
||||
"""
|
||||
Utility.console_log('DUT debug info for DUT: {}:'.format(self.name), color='orange')
|
||||
|
||||
for failure in self.expect_failures:
|
||||
Utility.console_log(u'\t[pattern]: {}\r\n\t[data]: {}\r\n\t[time]: {} - {}\r\n'
|
||||
.format(failure['pattern'], failure['data'],
|
||||
self._format_ts(failure['start']), self._format_ts(failure['end'])),
|
||||
color='orange')
|
||||
|
||||
|
||||
class SerialDUT(BaseDUT):
|
||||
""" serial with logging received data feature """
|
||||
|
||||
DEFAULT_UART_CONFIG = {
|
||||
'baudrate': 115200,
|
||||
'bytesize': serial.EIGHTBITS,
|
||||
'parity': serial.PARITY_NONE,
|
||||
'stopbits': serial.STOPBITS_ONE,
|
||||
'timeout': 0.05,
|
||||
'xonxoff': False,
|
||||
'rtscts': False,
|
||||
}
|
||||
|
||||
def __init__(self, name, port, log_file, app, **kwargs):
|
||||
self.port_inst = None
|
||||
self.serial_configs = self.DEFAULT_UART_CONFIG.copy()
|
||||
for uart_config_name in self.serial_configs.keys():
|
||||
if uart_config_name in kwargs:
|
||||
self.serial_configs[uart_config_name] = kwargs[uart_config_name]
|
||||
super(SerialDUT, self).__init__(name, port, log_file, app, **kwargs)
|
||||
|
||||
def _format_data(self, data):
|
||||
"""
|
||||
format data for logging. do decode and add timestamp.
|
||||
|
||||
:param data: raw data from read
|
||||
:return: formatted data (str)
|
||||
"""
|
||||
timestamp = '[{}]'.format(self._format_ts(time.time()))
|
||||
formatted_data = timestamp.encode() + b'\r\n' + data + b'\r\n'
|
||||
return formatted_data
|
||||
|
||||
def _port_open(self):
|
||||
self.port_inst = serial.serial_for_url(self.port, **self.serial_configs)
|
||||
|
||||
def _port_close(self):
|
||||
self.port_inst.close()
|
||||
|
||||
def _port_read(self, size=1):
|
||||
data = self.port_inst.read(size)
|
||||
if data:
|
||||
self._save_dut_log(self._format_data(data))
|
||||
return data
|
||||
|
||||
def _port_write(self, data):
|
||||
if isinstance(data, str):
|
||||
data = data.encode()
|
||||
self.port_inst.write(data)
|
||||
|
||||
@classmethod
|
||||
def list_available_ports(cls):
|
||||
return [x.device for x in list_ports.comports()]
|
||||
@@ -1,193 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
""" Test Env, manages DUT, App and EnvConfig, interface for test cases to access these components """
|
||||
import functools
|
||||
import os
|
||||
import threading
|
||||
import traceback
|
||||
|
||||
import netifaces
|
||||
|
||||
from . import EnvConfig
|
||||
|
||||
|
||||
def _synced(func):
|
||||
@functools.wraps(func)
|
||||
def decorator(self, *args, **kwargs):
|
||||
with self.lock:
|
||||
ret = func(self, *args, **kwargs)
|
||||
return ret
|
||||
|
||||
decorator.__doc__ = func.__doc__
|
||||
return decorator
|
||||
|
||||
|
||||
class Env(object):
|
||||
"""
|
||||
test env, manages DUTs and env configs.
|
||||
|
||||
:keyword app: class for default application
|
||||
:keyword dut: class for default DUT
|
||||
:keyword env_tag: test env tag, used to select configs from env config file
|
||||
:keyword env_config_file: test env config file path
|
||||
:keyword test_name: test suite name, used when generate log folder name
|
||||
"""
|
||||
CURRENT_LOG_FOLDER = ''
|
||||
|
||||
def __init__(self,
|
||||
app=None,
|
||||
dut=None,
|
||||
env_tag=None,
|
||||
env_config_file=None,
|
||||
test_suite_name=None,
|
||||
**kwargs):
|
||||
self.app_cls = app
|
||||
self.default_dut_cls = dut
|
||||
self.config = EnvConfig.Config(env_config_file, env_tag)
|
||||
self.log_path = self.app_cls.get_log_folder(test_suite_name)
|
||||
if not os.path.exists(self.log_path):
|
||||
os.makedirs(self.log_path)
|
||||
|
||||
Env.CURRENT_LOG_FOLDER = self.log_path
|
||||
|
||||
self.allocated_duts = dict()
|
||||
self.lock = threading.RLock()
|
||||
|
||||
@_synced
|
||||
def get_dut(self, dut_name, app_path, dut_class=None, app_class=None, app_config_name=None, **dut_init_args):
|
||||
"""
|
||||
get_dut(dut_name, app_path, dut_class=None, app_class=None)
|
||||
|
||||
:param dut_name: user defined name for DUT
|
||||
:param app_path: application path, app instance will use this path to process application info
|
||||
:param dut_class: dut class, if not specified will use default dut class of env
|
||||
:param app_class: app class, if not specified will use default app of env
|
||||
:param app_config_name: app build config
|
||||
:keyword dut_init_args: extra kwargs used when creating DUT instance
|
||||
:return: dut instance
|
||||
"""
|
||||
if dut_name in self.allocated_duts:
|
||||
dut = self.allocated_duts[dut_name]['dut']
|
||||
else:
|
||||
if dut_class is None:
|
||||
dut_class = self.default_dut_cls
|
||||
if app_class is None:
|
||||
app_class = self.app_cls
|
||||
|
||||
app_target = dut_class.TARGET
|
||||
detected_target = None
|
||||
|
||||
try:
|
||||
port = self.config.get_variable(dut_name)
|
||||
if not app_target:
|
||||
result, detected_target = dut_class.confirm_dut(port)
|
||||
except ValueError:
|
||||
# try to auto detect ports
|
||||
allocated_ports = [self.allocated_duts[x]['port'] for x in self.allocated_duts]
|
||||
available_ports = dut_class.list_available_ports()
|
||||
for port in available_ports:
|
||||
if port not in allocated_ports:
|
||||
result, detected_target = dut_class.confirm_dut(port)
|
||||
if result:
|
||||
break
|
||||
else:
|
||||
port = None
|
||||
|
||||
if not app_target:
|
||||
app_target = detected_target
|
||||
if not app_target:
|
||||
raise ValueError("DUT class doesn't specify the target, and autodetection failed")
|
||||
app_inst = app_class(app_path, app_config_name, app_target)
|
||||
|
||||
if port:
|
||||
try:
|
||||
dut_config = self.get_variable(dut_name + '_port_config')
|
||||
except ValueError:
|
||||
dut_config = dict()
|
||||
dut_config.update(dut_init_args)
|
||||
dut = dut_class(dut_name, port,
|
||||
os.path.join(self.log_path, dut_name + '.txt'),
|
||||
app_inst,
|
||||
**dut_config)
|
||||
self.allocated_duts[dut_name] = {'port': port, 'dut': dut}
|
||||
else:
|
||||
raise ValueError('Failed to get DUT')
|
||||
return dut
|
||||
|
||||
@_synced
|
||||
def close_dut(self, dut_name):
|
||||
"""
|
||||
close_dut(dut_name)
|
||||
close one DUT by name if DUT name is valid (the name used by ``get_dut``). otherwise will do nothing.
|
||||
|
||||
:param dut_name: user defined name for DUT
|
||||
:return: None
|
||||
"""
|
||||
try:
|
||||
dut = self.allocated_duts.pop(dut_name)['dut']
|
||||
dut.close()
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
@_synced
|
||||
def get_variable(self, variable_name):
|
||||
"""
|
||||
get_variable(variable_name)
|
||||
get variable from config file. If failed then try to auto-detected it.
|
||||
|
||||
:param variable_name: name of the variable
|
||||
:return: value of variable if successfully found. otherwise None.
|
||||
"""
|
||||
return self.config.get_variable(variable_name)
|
||||
|
||||
PROTO_MAP = {
|
||||
'ipv4': netifaces.AF_INET,
|
||||
'ipv6': netifaces.AF_INET6,
|
||||
'mac': netifaces.AF_LINK,
|
||||
}
|
||||
|
||||
@_synced
|
||||
def get_pc_nic_info(self, nic_name='pc_nic', proto='ipv4'):
|
||||
"""
|
||||
get_pc_nic_info(nic_name="pc_nic")
|
||||
try to get info of a specified NIC and protocol.
|
||||
|
||||
:param nic_name: pc nic name. allows passing variable name, nic name value.
|
||||
:param proto: "ipv4", "ipv6" or "mac"
|
||||
:return: a dict of nic info if successfully found. otherwise None.
|
||||
nic info keys could be different for different protocols.
|
||||
key "addr" is available for both mac, ipv4 and ipv6 pic info.
|
||||
"""
|
||||
interfaces = netifaces.interfaces()
|
||||
if nic_name in interfaces:
|
||||
# the name is in the interface list, we regard it as NIC name
|
||||
if_addr = netifaces.ifaddresses(nic_name)
|
||||
else:
|
||||
# it's not in interface name list, we assume it's variable name
|
||||
_nic_name = self.get_variable(nic_name)
|
||||
if_addr = netifaces.ifaddresses(_nic_name)
|
||||
|
||||
return if_addr[self.PROTO_MAP[proto]][0]
|
||||
|
||||
@_synced
|
||||
def close(self, dut_debug=False):
|
||||
"""
|
||||
close()
|
||||
close all DUTs of the Env.
|
||||
|
||||
:param dut_debug: if dut_debug is True, then print all dut expect failures before close it
|
||||
:return: exceptions during close DUT
|
||||
"""
|
||||
dut_close_errors = []
|
||||
for dut_name in self.allocated_duts:
|
||||
dut = self.allocated_duts[dut_name]['dut']
|
||||
try:
|
||||
if dut_debug:
|
||||
dut.print_debug_info()
|
||||
dut.close()
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
dut_close_errors.append(e)
|
||||
self.allocated_duts = dict()
|
||||
return dut_close_errors
|
||||
@@ -1,63 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
The test env could change when we running test from different computers.
|
||||
Test env config provide ``get_variable`` method to allow user get test environment related variables.
|
||||
It will first try to get variable from config file.
|
||||
If failed, then it will try to auto detect (Not supported yet).
|
||||
|
||||
Config file format is yaml. it's a set of key-value pair. The following is an example of config file::
|
||||
|
||||
Example_WIFI:
|
||||
ap_ssid: "myssid"
|
||||
ap_password: "mypassword"
|
||||
Example_ShieldBox:
|
||||
attenuator_port: "/dev/ttyUSB2"
|
||||
ap_ssid: "myssid"
|
||||
ap_password: "mypassword"
|
||||
|
||||
It will first define the env tag for each environment, then add its key-value pairs.
|
||||
This will prevent test cases from getting configs from other env when there're configs for multiple env in one file.
|
||||
"""
|
||||
import yaml
|
||||
from yaml import Loader as Loader
|
||||
|
||||
|
||||
class Config(object):
|
||||
""" Test Env Config """
|
||||
|
||||
def __init__(self, config_file, env_tag):
|
||||
self.configs = self.load_config_file(config_file, env_tag)
|
||||
|
||||
@staticmethod
|
||||
def load_config_file(config_file, env_name):
|
||||
"""
|
||||
load configs from config file.
|
||||
|
||||
:param config_file: config file path
|
||||
:param env_name: env tag name
|
||||
:return: configs for the test env
|
||||
"""
|
||||
try:
|
||||
with open(config_file) as f:
|
||||
configs = yaml.load(f, Loader=Loader)[env_name]
|
||||
except (OSError, TypeError, IOError, KeyError):
|
||||
configs = dict()
|
||||
return configs
|
||||
|
||||
def get_variable(self, variable_name):
|
||||
"""
|
||||
first try to get from config file. if not found, try to auto detect the variable.
|
||||
|
||||
:param variable_name: name of variable
|
||||
:return: value or None
|
||||
"""
|
||||
try:
|
||||
value = self.configs[variable_name]
|
||||
except KeyError:
|
||||
# TODO: to support auto get variable here
|
||||
value = None
|
||||
if value is None:
|
||||
raise ValueError('Failed to get variable')
|
||||
return value
|
||||
@@ -1,6 +0,0 @@
|
||||
.external_ap: &external_ap
|
||||
ap_ssid: "myssid"
|
||||
ap_password: "mypassword"
|
||||
|
||||
Examples_WIFI:
|
||||
<<: external_ap
|
||||
@@ -1,231 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
""" Interface for test cases. """
|
||||
import functools
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
import junit_xml
|
||||
|
||||
from . import DUT, App, Env, Utility
|
||||
from .Utility import format_case_id
|
||||
|
||||
|
||||
class TestCaseFailed(AssertionError):
|
||||
def __init__(self, *cases):
|
||||
"""
|
||||
Raise this exception if one or more test cases fail in a 'normal' way (ie the test runs but fails, no unexpected exceptions)
|
||||
|
||||
This will avoid dumping the Python stack trace, because the assumption is the junit error info and full job log already has
|
||||
enough information for a developer to debug.
|
||||
|
||||
'cases' argument is the names of one or more test cases
|
||||
"""
|
||||
message = 'Test case{} failed: {}'.format('s' if len(cases) > 1 else '', ', '.join(str(c) for c in cases))
|
||||
super(TestCaseFailed, self).__init__(self, message)
|
||||
|
||||
|
||||
class DefaultEnvConfig(object):
|
||||
"""
|
||||
default test configs. There're 3 places to set configs, priority is (high -> low):
|
||||
|
||||
1. overwrite set by caller of test method
|
||||
2. values set by test_method decorator
|
||||
3. default env config get from this class
|
||||
"""
|
||||
DEFAULT_CONFIG = {
|
||||
'app': App.BaseApp,
|
||||
'dut': DUT.BaseDUT,
|
||||
'env_tag': 'default',
|
||||
'env_config_file': None,
|
||||
'test_suite_name': None,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def set_default_config(cls, **kwargs):
|
||||
"""
|
||||
:param kwargs: configs need to be updated
|
||||
:return: None
|
||||
"""
|
||||
cls.DEFAULT_CONFIG.update(kwargs)
|
||||
|
||||
@classmethod
|
||||
def get_default_config(cls):
|
||||
"""
|
||||
:return: current default config
|
||||
"""
|
||||
return cls.DEFAULT_CONFIG.copy()
|
||||
|
||||
|
||||
set_default_config = DefaultEnvConfig.set_default_config
|
||||
get_default_config = DefaultEnvConfig.get_default_config
|
||||
|
||||
|
||||
MANDATORY_INFO = {
|
||||
'execution_time': 1,
|
||||
'env_tag': 'default',
|
||||
'category': 'function',
|
||||
'ignore': False,
|
||||
}
|
||||
|
||||
|
||||
class JunitReport(object):
|
||||
# wrapper for junit test report
|
||||
# TODO: JunitReport methods are not thread safe (although not likely to be used this way).
|
||||
|
||||
JUNIT_FILE_NAME = 'XUNIT_RESULT.xml'
|
||||
JUNIT_DEFAULT_TEST_SUITE = 'test-suite'
|
||||
JUNIT_TEST_SUITE = junit_xml.TestSuite(JUNIT_DEFAULT_TEST_SUITE,
|
||||
hostname=socket.gethostname(),
|
||||
timestamp=datetime.utcnow().isoformat())
|
||||
JUNIT_CURRENT_TEST_CASE = None
|
||||
_TEST_CASE_CREATED_TS = 0
|
||||
|
||||
@classmethod
|
||||
def output_report(cls, junit_file_path):
|
||||
""" Output current test result to file. """
|
||||
with open(os.path.join(junit_file_path, cls.JUNIT_FILE_NAME), 'w') as f:
|
||||
junit_xml.to_xml_report_file(f, [cls.JUNIT_TEST_SUITE], prettyprint=False)
|
||||
|
||||
@classmethod
|
||||
def get_current_test_case(cls):
|
||||
"""
|
||||
By default, the test framework will handle junit test report automatically.
|
||||
While some test case might want to update some info to test report.
|
||||
They can use this method to get current test case created by test framework.
|
||||
|
||||
:return: current junit test case instance created by ``JunitTestReport.create_test_case``
|
||||
"""
|
||||
return cls.JUNIT_CURRENT_TEST_CASE
|
||||
|
||||
@classmethod
|
||||
def test_case_finish(cls, test_case):
|
||||
"""
|
||||
Append the test case to test suite so it can be output to file.
|
||||
Execution time will be automatically updated (compared to ``create_test_case``).
|
||||
"""
|
||||
test_case.elapsed_sec = time.time() - cls._TEST_CASE_CREATED_TS
|
||||
cls.JUNIT_TEST_SUITE.test_cases.append(test_case)
|
||||
|
||||
@classmethod
|
||||
def create_test_case(cls, name):
|
||||
"""
|
||||
Extend ``junit_xml.TestCase`` with:
|
||||
|
||||
1. save create test case so it can be get by ``get_current_test_case``
|
||||
2. log create timestamp, so ``elapsed_sec`` can be auto updated in ``test_case_finish``.
|
||||
|
||||
:param name: test case name
|
||||
:return: instance of ``junit_xml.TestCase``
|
||||
"""
|
||||
# set stdout to empty string, so we can always append string to stdout.
|
||||
# It won't affect output logic. If stdout is empty, it won't be put to report.
|
||||
test_case = junit_xml.TestCase(name, stdout='')
|
||||
cls.JUNIT_CURRENT_TEST_CASE = test_case
|
||||
cls._TEST_CASE_CREATED_TS = time.time()
|
||||
return test_case
|
||||
|
||||
@classmethod
|
||||
def update_performance(cls, performance_items):
|
||||
"""
|
||||
Update performance results to ``stdout`` of current test case.
|
||||
|
||||
:param performance_items: a list of performance items. each performance item is a key-value pair.
|
||||
"""
|
||||
assert cls.JUNIT_CURRENT_TEST_CASE
|
||||
|
||||
for item in performance_items:
|
||||
cls.JUNIT_CURRENT_TEST_CASE.stdout += '[Performance][{}]: {}\n'.format(item[0], item[1])
|
||||
|
||||
|
||||
def test_method(**kwargs):
|
||||
"""
|
||||
decorator for test case function.
|
||||
The following keyword arguments are pre-defined.
|
||||
Any other keyword arguments will be regarded as filter for the test case,
|
||||
able to access them by ``case_info`` attribute of test method.
|
||||
|
||||
:keyword app: class for test app. see :doc:`App <App>` for details
|
||||
:keyword dut: class for current dut. see :doc:`DUT <DUT>` for details
|
||||
:keyword env_tag: name for test environment, used to select configs from config file
|
||||
:keyword env_config_file: test env config file. usually will not set this keyword when define case
|
||||
:keyword test_suite_name: test suite name, used for generating log folder name and adding xunit format test result.
|
||||
usually will not set this keyword when define case
|
||||
:keyword junit_report_by_case: By default the test fw will handle junit report generation.
|
||||
In some cases, one test function might test many test cases.
|
||||
If this flag is set, test case can update junit report by its own.
|
||||
"""
|
||||
def test(test_func):
|
||||
|
||||
case_info = MANDATORY_INFO.copy()
|
||||
case_info['name'] = case_info['ID'] = test_func.__name__
|
||||
case_info['junit_report_by_case'] = False
|
||||
case_info.update(kwargs)
|
||||
|
||||
@functools.wraps(test_func)
|
||||
def handle_test(extra_data=None, **overwrite):
|
||||
"""
|
||||
create env, run test and record test results
|
||||
|
||||
:param extra_data: extra data that runner or main passed to test case
|
||||
:param overwrite: args that runner or main want to overwrite
|
||||
:return: None
|
||||
"""
|
||||
# create env instance
|
||||
env_config = DefaultEnvConfig.get_default_config()
|
||||
for key in kwargs:
|
||||
if key in env_config:
|
||||
env_config[key] = kwargs[key]
|
||||
|
||||
env_config.update(overwrite)
|
||||
env_inst = Env.Env(**env_config)
|
||||
|
||||
# prepare for xunit test results
|
||||
junit_file_path = env_inst.app_cls.get_log_folder(env_config['test_suite_name'])
|
||||
junit_test_case = JunitReport.create_test_case(format_case_id(case_info['ID'],
|
||||
target=env_inst.default_dut_cls.TARGET))
|
||||
result = False
|
||||
unexpected_error = False
|
||||
try:
|
||||
Utility.console_log('starting running test: ' + test_func.__name__, color='green')
|
||||
# execute test function
|
||||
test_func(env_inst, extra_data)
|
||||
# if finish without exception, test result is True
|
||||
result = True
|
||||
except TestCaseFailed as e:
|
||||
junit_test_case.add_failure_info(str(e))
|
||||
except Exception as e:
|
||||
Utility.handle_unexpected_exception(junit_test_case, e)
|
||||
unexpected_error = True
|
||||
finally:
|
||||
# do close all DUTs, if result is False then print DUT debug info
|
||||
close_errors = env_inst.close(dut_debug=(not result))
|
||||
# We have a hook in DUT close, allow DUT to raise error to fail test case.
|
||||
# For example, we don't allow DUT exception (reset) during test execution.
|
||||
# We don't want to implement in exception detection in test function logic,
|
||||
# as we need to add it to every test case.
|
||||
# We can implement it in DUT receive thread,
|
||||
# and raise exception in DUT close to fail test case if reset detected.
|
||||
if close_errors:
|
||||
for error in close_errors:
|
||||
junit_test_case.add_failure_info('env close error: {}'.format(error))
|
||||
result = False
|
||||
if not case_info['junit_report_by_case'] or unexpected_error:
|
||||
JunitReport.test_case_finish(junit_test_case)
|
||||
|
||||
# end case and output result
|
||||
JunitReport.output_report(junit_file_path)
|
||||
|
||||
if result:
|
||||
Utility.console_log('Test Succeed: ' + test_func.__name__, color='green')
|
||||
else:
|
||||
Utility.console_log(('Test Fail: ' + test_func.__name__), color='red')
|
||||
return result
|
||||
|
||||
handle_test.case_info = case_info
|
||||
handle_test.test_method = True
|
||||
return handle_test
|
||||
return test
|
||||
@@ -1,332 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
Common logic to assign test cases to CI jobs.
|
||||
|
||||
Some background knowledge about Gitlab CI and use flow in esp-idf:
|
||||
|
||||
* Gitlab CI jobs are static in ``.gitlab-ci.yml``. We can't dynamically create test jobs
|
||||
* For test job running on DUT, we use ``tags`` to select runners with different test environment
|
||||
* We have ``assign_test`` stage, will collect cases, and then assign them to correct test jobs
|
||||
* ``assign_test`` will fail if failed to assign any cases
|
||||
* with ``assign_test``, we can:
|
||||
* dynamically filter test case we want to test
|
||||
* alert user if they forget to add CI jobs and guide how to add test jobs
|
||||
* the last step of ``assign_test`` is to output config files, then test jobs will run these cases
|
||||
|
||||
The Basic logic to assign test cases is as follow:
|
||||
|
||||
1. do search all the cases
|
||||
2. do filter case (if filter is specified by @bot)
|
||||
3. put cases to different groups according to rule of ``Group``
|
||||
* try to put them in existed groups
|
||||
* if failed then create a new group and add this case
|
||||
4. parse and filter the test jobs from CI config file
|
||||
5. try to assign all groups to jobs according to tags
|
||||
6. output config files for jobs
|
||||
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
import yaml
|
||||
|
||||
try:
|
||||
from yaml import CLoader
|
||||
has_cloader = True
|
||||
except ImportError:
|
||||
has_cloader = False
|
||||
|
||||
from yaml import Loader
|
||||
|
||||
from . import CaseConfig, GitlabCIJob, SearchCases, console_log
|
||||
|
||||
|
||||
class Group(object):
|
||||
MAX_EXECUTION_TIME = 30
|
||||
MAX_CASE = 15
|
||||
SORT_KEYS = ['env_tag']
|
||||
# Matching CI job rules could be different from the way we want to group test cases.
|
||||
# For example, when assign unit test cases, different test cases need to use different test functions.
|
||||
# We need to put them into different groups.
|
||||
# But these groups can be assigned to jobs with same tags, as they use the same test environment.
|
||||
CI_JOB_MATCH_KEYS = SORT_KEYS
|
||||
|
||||
def __init__(self, case):
|
||||
self.execution_time = 0
|
||||
self.case_list = [case]
|
||||
self.filters = dict(zip(self.SORT_KEYS, [self._get_case_attr(case, x) for x in self.SORT_KEYS]))
|
||||
# we use ci_job_match_keys to match CI job tags. It's a set of required tags.
|
||||
self.ci_job_match_keys = self._get_match_keys(case)
|
||||
|
||||
@staticmethod
|
||||
def _get_case_attr(case, attr):
|
||||
# we might use different type for case (dict or test_func)
|
||||
# this method will do get attribute form cases
|
||||
return case.case_info[attr]
|
||||
|
||||
def _get_match_keys(self, case):
|
||||
keys = []
|
||||
for attr in self.CI_JOB_MATCH_KEYS:
|
||||
val = self._get_case_attr(case, attr)
|
||||
if isinstance(val, list):
|
||||
keys.extend(val)
|
||||
else:
|
||||
keys.append(val)
|
||||
return set(keys)
|
||||
|
||||
def accept_new_case(self):
|
||||
"""
|
||||
check if allowed to add any case to this group
|
||||
|
||||
:return: True or False
|
||||
"""
|
||||
max_time = (sum([self._get_case_attr(x, 'execution_time') for x in self.case_list])
|
||||
< self.MAX_EXECUTION_TIME)
|
||||
max_case = (len(self.case_list) < self.MAX_CASE)
|
||||
return max_time and max_case
|
||||
|
||||
def add_case(self, case):
|
||||
"""
|
||||
add case to current group
|
||||
|
||||
:param case: test case
|
||||
:return: True if add succeed, else False
|
||||
"""
|
||||
added = False
|
||||
if self.accept_new_case():
|
||||
for key in self.filters:
|
||||
if self._get_case_attr(case, key) != self.filters[key]:
|
||||
break
|
||||
else:
|
||||
self.case_list.append(case)
|
||||
added = True
|
||||
return added
|
||||
|
||||
def add_extra_case(self, case):
|
||||
"""
|
||||
By default (``add_case`` method), cases will only be added when have equal values of all filters with group.
|
||||
But in some cases, we also want to add cases which are not best fit.
|
||||
For example, one group has can run cases require (A, B). It can also accept cases require (A, ) and (B, ).
|
||||
When assign failed by best fit, we will use this method to try if we can assign all failed cases.
|
||||
|
||||
If subclass want to retry, they need to overwrite this method.
|
||||
Logic can be applied to handle such scenario could be different for different cases.
|
||||
|
||||
:return: True if accepted else False
|
||||
"""
|
||||
pass
|
||||
|
||||
def output(self):
|
||||
"""
|
||||
output data for job configs
|
||||
|
||||
:return: {"Filter": case filter, "CaseConfig": list of case configs for cases in this group}
|
||||
"""
|
||||
output_data = {
|
||||
'Filter': self.filters,
|
||||
'CaseConfig': [{'name': self._get_case_attr(x, 'name')} for x in self.case_list],
|
||||
}
|
||||
return output_data
|
||||
|
||||
|
||||
class AssignTest(object):
|
||||
"""
|
||||
Auto assign tests to CI jobs.
|
||||
|
||||
:param test_case_paths: path of test case file(s)
|
||||
:param ci_config_file: path of ``.gitlab-ci.yml``
|
||||
"""
|
||||
# subclass need to rewrite CI test job pattern, to filter all test jobs
|
||||
CI_TEST_JOB_PATTERN = re.compile(r'^test_.+')
|
||||
# by default we only run function in CI, as other tests could take long time
|
||||
DEFAULT_FILTER = {
|
||||
'category': 'function',
|
||||
'ignore': False,
|
||||
'supported_in_ci': True,
|
||||
}
|
||||
|
||||
def __init__(self, test_case_paths, ci_config_file, case_group=Group):
|
||||
self.test_case_paths = test_case_paths
|
||||
self.test_case_file_pattern = None
|
||||
self.test_cases = []
|
||||
self.jobs = self._parse_gitlab_ci_config(ci_config_file)
|
||||
self.case_group = case_group
|
||||
|
||||
@staticmethod
|
||||
def _handle_parallel_attribute(job_name, job):
|
||||
jobs_out = []
|
||||
try:
|
||||
for i in range(job['parallel']):
|
||||
jobs_out.append(GitlabCIJob.Job(job, job_name + '_{}'.format(i + 1)))
|
||||
except KeyError:
|
||||
# Gitlab don't allow to set parallel to 1.
|
||||
# to make test job name same ($CI_JOB_NAME_$CI_NODE_INDEX),
|
||||
# we append "_" to jobs don't have parallel attribute
|
||||
jobs_out.append(GitlabCIJob.Job(job, job_name + '_'))
|
||||
return jobs_out
|
||||
|
||||
def _parse_gitlab_ci_config(self, ci_config_file):
|
||||
|
||||
with open(ci_config_file, 'r') as f:
|
||||
ci_config = yaml.load(f, Loader=CLoader if has_cloader else Loader)
|
||||
|
||||
job_list = list()
|
||||
for job_name in ci_config:
|
||||
if 'pytest' in job_name:
|
||||
continue
|
||||
if self.CI_TEST_JOB_PATTERN.search(job_name) is not None:
|
||||
job_list.extend(self._handle_parallel_attribute(job_name, ci_config[job_name]))
|
||||
|
||||
job_list.sort(key=lambda x: x['name'])
|
||||
return job_list
|
||||
|
||||
def search_cases(self, case_filter=None):
|
||||
"""
|
||||
:param case_filter: filter for test cases. the filter to use is default filter updated with case_filter param.
|
||||
:return: filtered test case list
|
||||
"""
|
||||
_case_filter = self.DEFAULT_FILTER.copy()
|
||||
if case_filter:
|
||||
_case_filter.update(case_filter)
|
||||
test_methods = SearchCases.Search.search_test_cases(self.test_case_paths, self.test_case_file_pattern)
|
||||
return CaseConfig.filter_test_cases(test_methods, _case_filter)
|
||||
|
||||
def _group_cases(self):
|
||||
"""
|
||||
separate all cases into groups according group rules. each group will be executed by one CI job.
|
||||
|
||||
:return: test case groups.
|
||||
"""
|
||||
groups = []
|
||||
for case in self.test_cases:
|
||||
for group in groups:
|
||||
# add to current group
|
||||
if group.add_case(case):
|
||||
break
|
||||
else:
|
||||
# create new group
|
||||
groups.append(self.case_group(case))
|
||||
return groups
|
||||
|
||||
def _assign_failed_cases(self, assigned_groups, failed_groups):
|
||||
""" try to assign failed cases to already assigned test groups """
|
||||
still_failed_groups = []
|
||||
failed_cases = []
|
||||
for group in failed_groups:
|
||||
failed_cases.extend(group.case_list)
|
||||
for case in failed_cases:
|
||||
# first try to assign to already assigned groups
|
||||
for group in assigned_groups:
|
||||
if group.add_extra_case(case):
|
||||
break
|
||||
else:
|
||||
# if failed, group the failed cases
|
||||
for group in still_failed_groups:
|
||||
if group.add_case(case):
|
||||
break
|
||||
else:
|
||||
still_failed_groups.append(self.case_group(case))
|
||||
return still_failed_groups
|
||||
|
||||
@staticmethod
|
||||
def _apply_bot_filter():
|
||||
"""
|
||||
we support customize CI test with bot.
|
||||
here we process from and return the filter which ``_search_cases`` accepts.
|
||||
|
||||
:return: filter for search test cases
|
||||
"""
|
||||
res = dict()
|
||||
for bot_filter in [os.getenv('BOT_CASE_FILTER'), os.getenv('BOT_TARGET_FILTER')]:
|
||||
if bot_filter:
|
||||
res.update(json.loads(bot_filter))
|
||||
return res
|
||||
|
||||
def _apply_bot_test_count(self):
|
||||
"""
|
||||
Bot could also pass test count.
|
||||
If filtered cases need to be tested for several times, then we do duplicate them here.
|
||||
"""
|
||||
test_count = os.getenv('BOT_TEST_COUNT')
|
||||
if test_count:
|
||||
test_count = int(test_count)
|
||||
self.test_cases *= test_count
|
||||
|
||||
@staticmethod
|
||||
def _count_groups_by_keys(test_groups):
|
||||
"""
|
||||
Count the number of test groups by job match keys.
|
||||
It's an important information to update CI config file.
|
||||
"""
|
||||
group_count = dict()
|
||||
for group in test_groups:
|
||||
key = ','.join(group.ci_job_match_keys)
|
||||
try:
|
||||
group_count[key] += 1
|
||||
except KeyError:
|
||||
group_count[key] = 1
|
||||
return group_count
|
||||
|
||||
def assign_cases(self):
|
||||
"""
|
||||
separate test cases to groups and assign test cases to CI jobs.
|
||||
|
||||
:raise AssertError: if failed to assign any case to CI job.
|
||||
:return: None
|
||||
"""
|
||||
failed_to_assign = []
|
||||
assigned_groups = []
|
||||
case_filter = self._apply_bot_filter()
|
||||
self.test_cases = self.search_cases(case_filter)
|
||||
self._apply_bot_test_count()
|
||||
test_groups = self._group_cases()
|
||||
|
||||
for group in test_groups:
|
||||
for job in self.jobs:
|
||||
if job.match_group(group):
|
||||
job.assign_group(group)
|
||||
assigned_groups.append(group)
|
||||
break
|
||||
else:
|
||||
failed_to_assign.append(group)
|
||||
|
||||
if failed_to_assign:
|
||||
failed_to_assign = self._assign_failed_cases(assigned_groups, failed_to_assign)
|
||||
|
||||
# print debug info
|
||||
# total requirement of current pipeline
|
||||
required_group_count = self._count_groups_by_keys(test_groups)
|
||||
console_log('Required job count by tags:')
|
||||
for tags in required_group_count:
|
||||
console_log('\t{}: {}'.format(tags, required_group_count[tags]))
|
||||
|
||||
# number of unused jobs
|
||||
not_used_jobs = [job for job in self.jobs if 'case group' not in job]
|
||||
if not_used_jobs:
|
||||
console_log('{} jobs not used. Please check if you define too much jobs'.format(len(not_used_jobs)), 'O')
|
||||
for job in not_used_jobs:
|
||||
console_log('\t{}'.format(job['name']), 'O')
|
||||
|
||||
# failures
|
||||
if failed_to_assign:
|
||||
console_log('Too many test cases vs jobs to run. '
|
||||
'Please increase parallel count in .gitlab/ci/target-test.yml '
|
||||
'for jobs with specific tags:', 'R')
|
||||
failed_group_count = self._count_groups_by_keys(failed_to_assign)
|
||||
for tags in failed_group_count:
|
||||
console_log('\t{}: {}'.format(tags, failed_group_count[tags]), 'R')
|
||||
raise RuntimeError('Failed to assign test case to CI jobs')
|
||||
|
||||
def output_configs(self, output_path):
|
||||
"""
|
||||
:param output_path: path to output config files for each CI job
|
||||
:return: None
|
||||
"""
|
||||
if not os.path.exists(output_path):
|
||||
os.makedirs(output_path)
|
||||
for job in self.jobs:
|
||||
job.output_config(output_path)
|
||||
@@ -1,246 +0,0 @@
|
||||
# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Processing case config files.
|
||||
This is mainly designed for CI, we need to auto create and assign test jobs.
|
||||
|
||||
Template Config File::
|
||||
|
||||
TestConfig:
|
||||
app:
|
||||
package: ttfw_idf
|
||||
class: Example
|
||||
dut:
|
||||
path:
|
||||
class:
|
||||
config_file: /somewhere/config_file_for_runner
|
||||
test_name: CI_test_job_1
|
||||
|
||||
Filter:
|
||||
chip: ESP32
|
||||
env_tag: default
|
||||
|
||||
CaseConfig:
|
||||
- name: test_examples_protocol_https_request
|
||||
# optional
|
||||
extra_data: some extra data passed to case with kwarg extra_data
|
||||
overwrite: # overwrite test configs
|
||||
app:
|
||||
package: ttfw_idf
|
||||
class: Example
|
||||
- name: xxx
|
||||
"""
|
||||
import importlib
|
||||
|
||||
import yaml
|
||||
|
||||
try:
|
||||
from yaml import CLoader as Loader
|
||||
except ImportError:
|
||||
from yaml import Loader as Loader
|
||||
|
||||
from . import TestCase
|
||||
|
||||
|
||||
def _convert_to_lower_case_bytes(item):
|
||||
"""
|
||||
bot filter is always lower case string.
|
||||
this function will convert to all string to lower case.
|
||||
Note: Unicode strings are converted to bytes.
|
||||
"""
|
||||
if isinstance(item, (tuple, list)):
|
||||
output = [_convert_to_lower_case_bytes(v) for v in item]
|
||||
elif isinstance(item, type(b'')):
|
||||
output = item.lower()
|
||||
elif isinstance(item, type(u'')):
|
||||
output = item.encode().lower()
|
||||
else:
|
||||
output = item
|
||||
return output
|
||||
|
||||
|
||||
def _filter_one_case(test_method, case_filter):
|
||||
""" Apply filter for one case (the filter logic is the same as described in ``filter_test_cases``) """
|
||||
filter_result = True
|
||||
# filter keys are lower case. Do map lower case keys with original keys.
|
||||
key_mapping = {x.lower(): x for x in test_method.case_info.keys()}
|
||||
|
||||
for orig_key in case_filter:
|
||||
key = key_mapping[orig_key]
|
||||
if key in test_method.case_info:
|
||||
# the filter key is both in case and filter
|
||||
# we need to check if they match
|
||||
filter_item = _convert_to_lower_case_bytes(case_filter[orig_key])
|
||||
accepted_item = _convert_to_lower_case_bytes(test_method.case_info[key])
|
||||
|
||||
if isinstance(filter_item, (tuple, list)) \
|
||||
and isinstance(accepted_item, (tuple, list)):
|
||||
# both list/tuple, check if they have common item
|
||||
filter_result = True if set(filter_item) & set(accepted_item) else False
|
||||
elif isinstance(filter_item, (tuple, list)):
|
||||
# filter item list/tuple, check if case accepted value in filter item list/tuple
|
||||
filter_result = True if accepted_item in filter_item else False
|
||||
elif isinstance(accepted_item, (tuple, list)):
|
||||
# accepted item list/tuple, check if case filter value is in accept item list/tuple
|
||||
filter_result = True if filter_item in accepted_item else False
|
||||
else:
|
||||
if type(filter_item) != type(accepted_item):
|
||||
# This will catch silent ignores of test cases when Unicode and bytes are compared
|
||||
raise AssertionError(filter_item, '!=', accepted_item)
|
||||
# both string/int, just do string compare
|
||||
filter_result = (filter_item == accepted_item)
|
||||
else:
|
||||
# key in filter only, which means the case supports all values for this filter key, match succeed
|
||||
pass
|
||||
if not filter_result:
|
||||
# match failed
|
||||
break
|
||||
return filter_result
|
||||
|
||||
|
||||
def filter_test_cases(test_methods, case_filter):
|
||||
"""
|
||||
filter test case. filter logic:
|
||||
|
||||
1. if filter key both in case attribute and filter:
|
||||
* if both value is string/int, then directly compare
|
||||
* if one is list/tuple, the other one is string/int, then check if string/int is in list/tuple
|
||||
* if both are list/tuple, then check if they have common item
|
||||
2. if only case attribute or filter have the key, filter succeed
|
||||
3. will do case insensitive compare for string
|
||||
|
||||
for example, the following are match succeed scenarios
|
||||
(the rule is symmetric, result is same if exchange values for user filter and case attribute):
|
||||
|
||||
* user case filter is ``chip: ["esp32", "esp32c"]``, case doesn't have ``chip`` attribute
|
||||
* user case filter is ``chip: ["esp32", "esp32c"]``, case attribute is ``chip: "esp32"``
|
||||
* user case filter is ``chip: "esp32"``, case attribute is ``chip: "esp32"``
|
||||
|
||||
:param test_methods: a list of test methods functions
|
||||
:param case_filter: case filter
|
||||
:return: filtered test methods
|
||||
"""
|
||||
filtered_test_methods = []
|
||||
for test_method in test_methods:
|
||||
if _filter_one_case(test_method, case_filter):
|
||||
filtered_test_methods.append(test_method)
|
||||
return filtered_test_methods
|
||||
|
||||
|
||||
class Parser(object):
|
||||
DEFAULT_CONFIG = {
|
||||
'TestConfig': dict(),
|
||||
'Filter': dict(),
|
||||
'CaseConfig': [{'extra_data': None}],
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def parse_config_file(cls, config_file):
|
||||
"""
|
||||
parse from config file and then update to default config.
|
||||
|
||||
:param config_file: config file path
|
||||
:return: configs
|
||||
"""
|
||||
configs = cls.DEFAULT_CONFIG.copy()
|
||||
if config_file:
|
||||
with open(config_file, 'r') as f:
|
||||
configs.update(yaml.load(f, Loader=Loader))
|
||||
return configs
|
||||
|
||||
@classmethod
|
||||
def handle_overwrite_args(cls, overwrite):
|
||||
"""
|
||||
handle overwrite configs. import module from path and then get the required class.
|
||||
|
||||
:param overwrite: overwrite args
|
||||
:return: dict of (original key: class)
|
||||
"""
|
||||
output = dict()
|
||||
for key in overwrite:
|
||||
module = importlib.import_module(overwrite[key]['package'])
|
||||
output[key] = module.__getattribute__(overwrite[key]['class'])
|
||||
return output
|
||||
|
||||
@classmethod
|
||||
def apply_config(cls, test_methods, config_file):
|
||||
"""
|
||||
apply config for test methods
|
||||
|
||||
:param test_methods: a list of test methods functions
|
||||
:param config_file: case filter file
|
||||
:return: filtered cases
|
||||
"""
|
||||
configs = cls.parse_config_file(config_file)
|
||||
test_case_list = []
|
||||
for _config in configs['CaseConfig']:
|
||||
_filter = configs['Filter'].copy()
|
||||
_overwrite = cls.handle_overwrite_args(_config.pop('overwrite', dict()))
|
||||
_extra_data = _config.pop('extra_data', None)
|
||||
_filter.update(_config)
|
||||
|
||||
# Try get target from yml
|
||||
try:
|
||||
_target = _filter['target']
|
||||
except KeyError:
|
||||
_target = None
|
||||
else:
|
||||
_overwrite.update({'target': _target})
|
||||
|
||||
for test_method in test_methods:
|
||||
if _filter_one_case(test_method, _filter):
|
||||
try:
|
||||
dut_dict = test_method.case_info['dut_dict']
|
||||
except (AttributeError, KeyError):
|
||||
dut_dict = None
|
||||
|
||||
if dut_dict and _target:
|
||||
dut = test_method.case_info.get('dut')
|
||||
if _target.upper() in dut_dict:
|
||||
if dut and dut in dut_dict.values(): # don't overwrite special cases
|
||||
_overwrite.update({'dut': dut_dict[_target.upper()]})
|
||||
else:
|
||||
raise ValueError('target {} is not in the specified dut_dict'.format(_target))
|
||||
test_case_list.append(TestCase.TestCase(test_method, _extra_data, **_overwrite))
|
||||
return test_case_list
|
||||
|
||||
|
||||
class Generator(object):
|
||||
""" Case config file generator """
|
||||
|
||||
def __init__(self):
|
||||
self.default_config = {
|
||||
'TestConfig': dict(),
|
||||
'Filter': dict(),
|
||||
}
|
||||
|
||||
def set_default_configs(self, test_config, case_filter):
|
||||
"""
|
||||
:param test_config: "TestConfig" value
|
||||
:param case_filter: "Filter" value
|
||||
:return: None
|
||||
"""
|
||||
self.default_config = {'TestConfig': test_config, 'Filter': case_filter}
|
||||
|
||||
def generate_config(self, case_configs, output_file):
|
||||
"""
|
||||
:param case_configs: "CaseConfig" value
|
||||
:param output_file: output file path
|
||||
:return: None
|
||||
"""
|
||||
config = self.default_config.copy()
|
||||
config.update({'CaseConfig': case_configs})
|
||||
with open(output_file, 'w') as f:
|
||||
yaml.dump(config, f)
|
||||
@@ -1,65 +0,0 @@
|
||||
# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
class Job(dict):
|
||||
"""
|
||||
Gitlab CI job
|
||||
|
||||
:param job: job data loaded from .gitlab-ci.yml
|
||||
:param job_name: job name
|
||||
"""
|
||||
def __init__(self, job, job_name):
|
||||
super(Job, self).__init__(job)
|
||||
self['name'] = job_name
|
||||
self.tags = set(self['tags'])
|
||||
|
||||
def match_group(self, group):
|
||||
"""
|
||||
Match group by tags of job.
|
||||
All filters values of group should be included in tags.
|
||||
|
||||
:param group: case group to match
|
||||
:return: True or False
|
||||
"""
|
||||
match_result = False
|
||||
if 'case group' not in self and group.ci_job_match_keys == self.tags:
|
||||
# group not assigned and all tags match
|
||||
match_result = True
|
||||
return match_result
|
||||
|
||||
def assign_group(self, group):
|
||||
"""
|
||||
assign a case group to a test job.
|
||||
|
||||
:param group: the case group to assign
|
||||
"""
|
||||
self['case group'] = group
|
||||
|
||||
def output_config(self, file_path):
|
||||
"""
|
||||
output test config to the given path.
|
||||
file name will be job_name.yml
|
||||
|
||||
:param file_path: output file path
|
||||
:return: None
|
||||
"""
|
||||
file_name = os.path.join(file_path, self['name'] + '.yml')
|
||||
if 'case group' in self:
|
||||
with open(file_name, 'w') as f:
|
||||
yaml.safe_dump(self['case group'].output(), f, encoding='utf-8', default_flow_style=False)
|
||||
@@ -1,144 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
""" search test cases from a given file or path """
|
||||
import copy
|
||||
import fnmatch
|
||||
import os
|
||||
import types
|
||||
from typing import List
|
||||
|
||||
from . import load_source
|
||||
|
||||
|
||||
class Search:
|
||||
"""
|
||||
This class is used as a class singleton. all the member functions are `classmethod`
|
||||
"""
|
||||
TEST_CASE_FILE_PATTERN = '*_test.py'
|
||||
SUPPORT_REPLICATE_CASES_KEY = ['target']
|
||||
|
||||
# this attribute would be modified while running
|
||||
missing_import_warnings: List[str] = []
|
||||
|
||||
@classmethod
|
||||
def _search_cases_from_file(cls, file_name):
|
||||
""" get test cases from test case .py file """
|
||||
|
||||
print('Try to get cases from: ' + file_name)
|
||||
test_functions = []
|
||||
try:
|
||||
# search case no need to run the functions
|
||||
# mock missing modules would help us get the test case function objects
|
||||
mod = load_source(file_name, mock_missing=True)
|
||||
for func in [mod.__getattribute__(x) for x in dir(mod)
|
||||
if isinstance(mod.__getattribute__(x), types.FunctionType)]:
|
||||
try:
|
||||
# test method decorator will add test_method attribute to test function
|
||||
if func.test_method:
|
||||
test_functions.append(func)
|
||||
except AttributeError:
|
||||
continue
|
||||
except ImportError as e:
|
||||
warning_str = 'ImportError: \r\n\tFile:' + file_name + '\r\n\tError:' + str(e)
|
||||
cls.missing_import_warnings.append(warning_str)
|
||||
|
||||
test_functions_out = []
|
||||
for case in test_functions:
|
||||
test_functions_out += cls.replicate_case(case)
|
||||
|
||||
for i, test_function in enumerate(test_functions_out):
|
||||
print('\t{}. {} <{}>'.format(i + 1, test_function.case_info['name'], test_function.case_info['target']))
|
||||
test_function.case_info['app_dir'] = os.path.dirname(file_name)
|
||||
test_function.case_info['script_path'] = file_name
|
||||
return test_functions_out
|
||||
|
||||
@classmethod
|
||||
def _search_test_case_files(cls, test_case, file_pattern):
|
||||
""" search all test case files recursively of a path """
|
||||
|
||||
if not os.path.exists(test_case):
|
||||
raise OSError(f'test case path "{test_case}" not exist')
|
||||
if os.path.isdir(test_case):
|
||||
test_case_files = []
|
||||
for root, _, file_names in os.walk(test_case):
|
||||
for filename in fnmatch.filter(file_names, file_pattern):
|
||||
test_case_files.append(os.path.join(root, filename))
|
||||
else:
|
||||
test_case_files = [test_case]
|
||||
return test_case_files
|
||||
|
||||
@classmethod
|
||||
def replicate_case(cls, case):
|
||||
"""
|
||||
Replicate cases according to its filter values.
|
||||
If one case has specified filter chip=(ESP32, ESP32C),
|
||||
it will create 2 cases, one for ESP32 and on for ESP32C.
|
||||
Once the cases are replicated, it's easy to filter those we want to execute.
|
||||
|
||||
:param case: the original case
|
||||
:return: a list of replicated cases
|
||||
"""
|
||||
replicate_config = []
|
||||
for key in case.case_info:
|
||||
if key == 'ci_target': # ci_target is used to filter target, should not be duplicated.
|
||||
continue
|
||||
if isinstance(case.case_info[key], (list, tuple)):
|
||||
replicate_config.append(key)
|
||||
|
||||
def _replicate_for_key(cases, replicate_key, replicate_list):
|
||||
def deepcopy_func(f, name=None):
|
||||
fn = types.FunctionType(f.__code__, f.__globals__, name if name else f.__name__,
|
||||
f.__defaults__, f.__closure__)
|
||||
fn.__dict__.update(copy.deepcopy(f.__dict__))
|
||||
return fn
|
||||
|
||||
case_out = []
|
||||
for inner_case in cases:
|
||||
for value in replicate_list:
|
||||
new_case = deepcopy_func(inner_case)
|
||||
new_case.case_info[replicate_key] = value
|
||||
case_out.append(new_case)
|
||||
return case_out
|
||||
|
||||
replicated_cases = [case]
|
||||
while replicate_config:
|
||||
if not replicate_config:
|
||||
break
|
||||
key = replicate_config.pop()
|
||||
if key in cls.SUPPORT_REPLICATE_CASES_KEY:
|
||||
replicated_cases = _replicate_for_key(replicated_cases, key, case.case_info[key])
|
||||
|
||||
# mark the cases with targets not in ci_target
|
||||
for case in replicated_cases:
|
||||
ci_target = case.case_info['ci_target']
|
||||
if not ci_target or case.case_info['target'] in ci_target:
|
||||
case.case_info['supported_in_ci'] = True
|
||||
else:
|
||||
case.case_info['supported_in_ci'] = False
|
||||
|
||||
return replicated_cases
|
||||
|
||||
@classmethod
|
||||
def search_test_cases(cls, test_case_paths, test_case_file_pattern=None):
|
||||
"""
|
||||
search all test cases from a folder or file, and then do case replicate.
|
||||
|
||||
:param test_case_paths: test case file(s) paths
|
||||
:param test_case_file_pattern: unix filename pattern
|
||||
:return: a list of replicated test methods
|
||||
"""
|
||||
if not isinstance(test_case_paths, list):
|
||||
test_case_paths = [test_case_paths]
|
||||
test_case_files = []
|
||||
for path in test_case_paths:
|
||||
test_case_files.extend(
|
||||
cls._search_test_case_files(path, test_case_file_pattern or cls.TEST_CASE_FILE_PATTERN))
|
||||
test_cases = []
|
||||
for test_case_file in test_case_files:
|
||||
test_cases += cls._search_cases_from_file(test_case_file)
|
||||
|
||||
if cls.missing_import_warnings:
|
||||
raise ImportError('\n\n'.join(cls.missing_import_warnings))
|
||||
|
||||
return test_cases
|
||||
@@ -1,58 +0,0 @@
|
||||
# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import yaml
|
||||
|
||||
try:
|
||||
from yaml import CLoader as Loader
|
||||
except ImportError:
|
||||
from yaml import Loader as Loader
|
||||
|
||||
|
||||
class TestCase(object):
|
||||
"""
|
||||
Test Case Object, mainly used with runner.
|
||||
runner can parse all test cases from a given path, set data and config for test case in prepare stage.
|
||||
TestCase instance will record these data, provide run method to let runner execute test case.
|
||||
|
||||
:param test_method: test function
|
||||
:param extra_data: data passed to test function
|
||||
:param overwrite_args: kwargs that overwrite original test case configs
|
||||
"""
|
||||
DEFAULT_CASE_DOC = dict()
|
||||
|
||||
def __init__(self, test_method, extra_data, **overwrite_args):
|
||||
self.test_method = test_method
|
||||
self.extra_data = extra_data
|
||||
self.overwrite_args = overwrite_args
|
||||
|
||||
def run(self):
|
||||
""" execute the test case """
|
||||
return self.test_method(self.extra_data, **self.overwrite_args)
|
||||
|
||||
def document(self):
|
||||
"""
|
||||
generate test case document.
|
||||
parse the case doc with yaml parser and update to original case attributes.
|
||||
|
||||
:return: case document, dict of case attributes and values
|
||||
"""
|
||||
doc_string = self.test_method.__doc__
|
||||
try:
|
||||
doc = yaml.load(doc_string, Loader=Loader)
|
||||
except (AttributeError, OSError, UnicodeDecodeError):
|
||||
doc = self.DEFAULT_CASE_DOC
|
||||
doc.update(self.test_method.env_args)
|
||||
doc.update(self.test_method.accepted_filter)
|
||||
return doc
|
||||
@@ -1,134 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import logging
|
||||
import os.path
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from .. import Env
|
||||
|
||||
_COLOR_CODES = {
|
||||
'white': u'\033[0m',
|
||||
'red': u'\033[31m',
|
||||
'green': u'\033[32m',
|
||||
'orange': u'\033[33m',
|
||||
'blue': u'\033[34m',
|
||||
'purple': u'\033[35m',
|
||||
'W': u'\033[0m',
|
||||
'R': u'\033[31m',
|
||||
'G': u'\033[32m',
|
||||
'O': u'\033[33m',
|
||||
'B': u'\033[34m',
|
||||
'P': u'\033[35m'
|
||||
}
|
||||
|
||||
|
||||
def _get_log_file_name():
|
||||
if Env.Env.CURRENT_LOG_FOLDER:
|
||||
file_name = os.path.join(Env.Env.CURRENT_LOG_FOLDER, 'console.txt')
|
||||
else:
|
||||
raise OSError('env log folder does not exist, will not save to log file')
|
||||
return file_name
|
||||
|
||||
|
||||
def format_timestamp():
|
||||
ts = time.time()
|
||||
return '{}:{}'.format(time.strftime('%m-%d %H:%M:%S', time.localtime(ts)), str(ts % 1)[2:5])
|
||||
|
||||
|
||||
def console_log(data, color='white', end='\n'):
|
||||
"""
|
||||
log data to console.
|
||||
(if not flush console log, Gitlab-CI won't update logs during job execution)
|
||||
|
||||
:param data: data content
|
||||
:param color: color
|
||||
"""
|
||||
if color not in _COLOR_CODES:
|
||||
color = 'white'
|
||||
color_codes = _COLOR_CODES[color]
|
||||
if isinstance(data, type(b'')):
|
||||
data = data.decode('utf-8', 'replace')
|
||||
print(color_codes + data, end=end)
|
||||
if color not in ['white', 'W']:
|
||||
# reset color to white for later logs
|
||||
print(_COLOR_CODES['white'] + u'\r')
|
||||
sys.stdout.flush()
|
||||
log_data = '[{}] '.format(format_timestamp()) + data
|
||||
try:
|
||||
log_file = _get_log_file_name()
|
||||
with open(log_file, 'a+') as f:
|
||||
f.write(log_data + end)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
__LOADED_MODULES = dict() # type: ignore
|
||||
# we should only load one module once.
|
||||
# if we load one module twice,
|
||||
# python will regard the same object loaded in the first time and second time as different objects.
|
||||
# it will lead to strange errors like `isinstance(object, type_of_this_object)` return False
|
||||
|
||||
|
||||
def load_source(path, mock_missing=False):
|
||||
"""
|
||||
Dynamic loading python file. Note that this function SHOULD NOT be used to replace ``import``.
|
||||
It should only be used when the package path is only available in runtime.
|
||||
|
||||
:param path: The path of python file
|
||||
:param mock_missing: If True, will mock the module if the module is not found.
|
||||
:return: Loaded object
|
||||
"""
|
||||
path = os.path.realpath(path)
|
||||
# load name need to be unique, otherwise it will update the already loaded module
|
||||
load_name = str(len(__LOADED_MODULES))
|
||||
try:
|
||||
return __LOADED_MODULES[path]
|
||||
except KeyError:
|
||||
folder = os.path.dirname(path)
|
||||
sys.path.append(folder)
|
||||
from importlib.machinery import SourceFileLoader
|
||||
try:
|
||||
ret = SourceFileLoader(load_name, path).load_module()
|
||||
except ModuleNotFoundError as e:
|
||||
if not mock_missing:
|
||||
raise
|
||||
|
||||
# mock the missing third-party libs. Don't use it when real-testing
|
||||
while True:
|
||||
sys.modules[e.name] = MagicMock()
|
||||
logging.warning('Mocking python module %s', e.name)
|
||||
try:
|
||||
ret = SourceFileLoader(load_name, path).load_module()
|
||||
break
|
||||
except ModuleNotFoundError as f:
|
||||
e = f # another round
|
||||
except SyntaxError:
|
||||
# well, let's ignore it... non of our business
|
||||
return None
|
||||
finally:
|
||||
sys.path.remove(folder)
|
||||
__LOADED_MODULES[path] = ret
|
||||
return ret
|
||||
|
||||
|
||||
def handle_unexpected_exception(junit_test_case, exception):
|
||||
"""
|
||||
Helper to log & add junit result details for an unexpected exception encountered
|
||||
when running a test case.
|
||||
|
||||
Should always be called from inside an except: block
|
||||
"""
|
||||
traceback.print_exc()
|
||||
# AssertionError caused by an 'assert' statement has an empty string as its 'str' form
|
||||
e_str = str(exception) if str(exception) else repr(exception)
|
||||
junit_test_case.add_failure_info('Unexpected exception: {}\n{}'.format(e_str, traceback.format_exc()))
|
||||
|
||||
|
||||
def format_case_id(case_name, target='esp32', config='default'):
|
||||
return '{}.{}.{}'.format(target, config, case_name)
|
||||
@@ -1,124 +0,0 @@
|
||||
# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Command line interface to run test cases from a given path.
|
||||
|
||||
* search and run test cases of a given path
|
||||
* config file which support to filter test cases and passing data to test case
|
||||
|
||||
Use ``python Runner.py test_case_path -c config_file -e env_config_file`` to run test cases.
|
||||
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
from fnmatch import fnmatch
|
||||
|
||||
from tiny_test_fw.TinyFW import JunitReport, set_default_config
|
||||
from tiny_test_fw.Utility import CaseConfig, SearchCases, console_log
|
||||
|
||||
|
||||
class Runner(threading.Thread):
|
||||
"""
|
||||
:param test_case_paths: test case file or folder
|
||||
:param case_config: case config file, allow to filter test cases and pass data to test case
|
||||
:param env_config_file: env config file
|
||||
"""
|
||||
|
||||
def __init__(self, test_case_paths, case_config, env_config_file=None, known_failure_cases_file=None):
|
||||
super(Runner, self).__init__()
|
||||
self.setDaemon(True)
|
||||
if case_config:
|
||||
test_suite_name = os.path.splitext(os.path.basename(case_config))[0]
|
||||
else:
|
||||
test_suite_name = 'TestRunner'
|
||||
set_default_config(env_config_file=env_config_file, test_suite_name=test_suite_name)
|
||||
test_methods = SearchCases.Search.search_test_cases(test_case_paths)
|
||||
self.test_cases = CaseConfig.Parser.apply_config(test_methods, case_config)
|
||||
self.known_failure_cases = self._get_config_cases(known_failure_cases_file)
|
||||
|
||||
@staticmethod
|
||||
def _get_config_cases(config_file):
|
||||
res = set()
|
||||
if not config_file or not os.path.isfile(config_file):
|
||||
return res
|
||||
|
||||
for line in open(config_file).readlines():
|
||||
if not line:
|
||||
continue
|
||||
if not line.strip():
|
||||
continue
|
||||
without_comments = line.split('#')[0].strip()
|
||||
if without_comments:
|
||||
res.add(without_comments)
|
||||
return res
|
||||
|
||||
def run(self):
|
||||
for case in self.test_cases:
|
||||
case.run()
|
||||
|
||||
@staticmethod
|
||||
def is_known_issue(tc_name, known_cases):
|
||||
for case in known_cases:
|
||||
if tc_name == case:
|
||||
return True
|
||||
if fnmatch(tc_name, case):
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_test_result(self):
|
||||
_res = True
|
||||
console_log('Test Results:')
|
||||
for tc in JunitReport.JUNIT_TEST_SUITE.test_cases:
|
||||
if tc.failures:
|
||||
if self.is_known_issue(tc.name, self.known_failure_cases):
|
||||
console_log(' Known Failure: ' + tc.name, color='orange')
|
||||
else:
|
||||
console_log(' Test Fail: ' + tc.name, color='red')
|
||||
_res = False
|
||||
else:
|
||||
console_log(' Test Succeed: ' + tc.name, color='green')
|
||||
|
||||
return _res
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('test_cases', nargs='+',
|
||||
help='test case folders or files')
|
||||
parser.add_argument('--case_config', '-c', default=None,
|
||||
help='case filter/config file')
|
||||
parser.add_argument('--env_config_file', '-e', default=None,
|
||||
help='test env config file')
|
||||
parser.add_argument('--known_failure_cases_file', default=None,
|
||||
help='known failure cases file')
|
||||
args = parser.parse_args()
|
||||
|
||||
test_cases = [os.path.join(os.getenv('IDF_PATH'), path)
|
||||
if not os.path.isabs(path) else path for path in args.test_cases]
|
||||
runner = Runner(test_cases, args.case_config, args.env_config_file, args.known_failure_cases_file)
|
||||
runner.start()
|
||||
|
||||
while True:
|
||||
try:
|
||||
runner.join(1)
|
||||
if not runner.is_alive():
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
print('exit by Ctrl-C')
|
||||
break
|
||||
if not runner.get_test_result():
|
||||
sys.exit(1)
|
||||
@@ -1,43 +0,0 @@
|
||||
# Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http:#www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
""" example of writing test with TinyTestFW """
|
||||
import re
|
||||
|
||||
import ttfw_idf
|
||||
from tiny_test_fw import TinyFW
|
||||
|
||||
|
||||
@ttfw_idf.idf_example_test(env_tag='Example_WIFI')
|
||||
def test_examples_protocol_https_request(env, extra_data):
|
||||
"""
|
||||
steps: |
|
||||
1. join AP
|
||||
2. connect to www.howsmyssl.com:443
|
||||
3. send http request
|
||||
"""
|
||||
dut1 = env.get_dut('https_request', 'examples/protocols/https_request', dut_class=ttfw_idf.ESP32DUT)
|
||||
dut1.start_app()
|
||||
dut1.expect(re.compile(r'Connecting to www.howsmyssl.com:443'), timeout=30)
|
||||
dut1.expect('Performing the SSL/TLS handshake')
|
||||
dut1.expect('Certificate verified.', timeout=15)
|
||||
dut1.expect_all(re.compile(r'Cipher suite is TLS-ECDHE-RSA-WITH-AES-128-GCM-SHA256'),
|
||||
'Reading HTTP response',
|
||||
timeout=20)
|
||||
dut1.expect(re.compile(r'Completed (\d) requests'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
TinyFW.set_default_config(env_config_file='EnvConfigTemplate.yml', dut=ttfw_idf.IDFDUT)
|
||||
test_examples_protocol_https_request()
|
||||
@@ -1,26 +0,0 @@
|
||||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line.
|
||||
SPHINXOPTS =
|
||||
SPHINXAPI = sphinx-apidoc
|
||||
SPHINXAPISRC = ..
|
||||
SPHINXBUILD = python -msphinx
|
||||
SPHINXPROJ = TinyTestFW
|
||||
SOURCEDIR = .
|
||||
BUILDDIR = _build
|
||||
|
||||
# define the files to be excluded here
|
||||
EXCLUEDLIST = "$(SPHINXAPISRC)/example.py"
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile
|
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile
|
||||
@$(SPHINXAPI) -o $(SOURCEDIR) $(SPHINXAPISRC) $(EXCLUEDLIST)
|
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
@@ -1,158 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# TinyTestFW documentation build configuration file, created by
|
||||
# sphinx-quickstart on Thu Sep 21 20:19:12 2017.
|
||||
#
|
||||
# This file is execfile()d with the current directory set to its
|
||||
# containing dir.
|
||||
#
|
||||
# Note that not all possible configuration values are present in this
|
||||
# autogenerated file.
|
||||
#
|
||||
# All configuration values have a default; values that are commented out
|
||||
# serve to show the default.
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
#
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.abspath('..'))
|
||||
|
||||
# import sphinx_rtd_theme
|
||||
|
||||
|
||||
# -- General configuration ------------------------------------------------
|
||||
|
||||
# If your documentation needs a minimal Sphinx version, state it here.
|
||||
#
|
||||
# needs_sphinx = '1.0'
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
extensions = ['sphinx.ext.autodoc',
|
||||
'sphinx.ext.viewcode',
|
||||
'plantweb.directive']
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
# The suffix(es) of source filenames.
|
||||
# You can specify multiple suffix as a list of string:
|
||||
#
|
||||
# source_suffix = ['.rst', '.md']
|
||||
source_suffix = '.rst'
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = 'index'
|
||||
|
||||
# General information about the project.
|
||||
project = u'TinyTestFW'
|
||||
copyright = u'2017, Espressif'
|
||||
author = u'Espressif'
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
# built documents.
|
||||
#
|
||||
# The short X.Y version.
|
||||
version = u'0.1'
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
release = u'0.1'
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
#
|
||||
# This is also used if you do content translation via gettext catalogs.
|
||||
# Usually you set "language" from the command line for these cases.
|
||||
language = None
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# This patterns also effect to html_static_path and html_extra_path
|
||||
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = 'sphinx'
|
||||
|
||||
# If true, `todo` and `todoList` produce output, else they produce nothing.
|
||||
todo_include_todos = False
|
||||
|
||||
|
||||
# -- Options for HTML output ----------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
#
|
||||
html_theme = 'sphinx_rtd_theme'
|
||||
|
||||
# Theme options are theme-specific and customize the look and feel of a theme
|
||||
# further. For a list of options available for each theme, see the
|
||||
# documentation.
|
||||
#
|
||||
# html_theme_options = {}
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ['_static']
|
||||
|
||||
|
||||
# -- Options for HTMLHelp output ------------------------------------------
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = 'TinyTestFWdoc'
|
||||
|
||||
|
||||
# -- Options for LaTeX output ---------------------------------------------
|
||||
|
||||
latex_elements = {
|
||||
# The paper size ('letterpaper' or 'a4paper').
|
||||
#
|
||||
# 'papersize': 'letterpaper',
|
||||
|
||||
# The font size ('10pt', '11pt' or '12pt').
|
||||
#
|
||||
# 'pointsize': '10pt',
|
||||
|
||||
# Additional stuff for the LaTeX preamble.
|
||||
#
|
||||
# 'preamble': '',
|
||||
|
||||
# Latex figure (float) alignment
|
||||
#
|
||||
# 'figure_align': 'htbp',
|
||||
}
|
||||
|
||||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title,
|
||||
# author, documentclass [howto, manual, or own class]).
|
||||
latex_documents = [
|
||||
(master_doc, 'TinyTestFW.tex', u'TinyTestFW Documentation',
|
||||
u'He Yinling', 'manual'),
|
||||
]
|
||||
|
||||
|
||||
# -- Options for manual page output ---------------------------------------
|
||||
|
||||
# One entry per manual page. List of tuples
|
||||
# (source start file, name, description, authors, manual section).
|
||||
man_pages = [
|
||||
(master_doc, 'tinytestfw', u'TinyTestFW Documentation',
|
||||
[author], 1)
|
||||
]
|
||||
|
||||
|
||||
# -- Options for Texinfo output -------------------------------------------
|
||||
|
||||
# Grouping the document tree into Texinfo files. List of tuples
|
||||
# (source start file, target name, title, author,
|
||||
# dir menu entry, description, category)
|
||||
texinfo_documents = [
|
||||
(master_doc, 'TinyTestFW', u'TinyTestFW Documentation',
|
||||
author, 'TinyTestFW', 'One line description of project.',
|
||||
'Miscellaneous'),
|
||||
]
|
||||
@@ -1,204 +0,0 @@
|
||||
.. TinyTestFW documentation master file, created by
|
||||
sphinx-quickstart on Thu Sep 21 20:19:12 2017.
|
||||
You can adapt this file completely to your liking, but it should at least
|
||||
contain the root `toctree` directive.
|
||||
|
||||
Welcome to TinyTestFW's documentation!
|
||||
======================================
|
||||
|
||||
We have a lot of test which depends on interact with DUT via communication port.
|
||||
Usually we send command to the port and then check response to see if the test succeed.
|
||||
TinyTestFW is designed for such scenarios.
|
||||
It supports ESP-IDF applications and can be adapted to other applications by writing new bundles.
|
||||
|
||||
Example
|
||||
-------
|
||||
|
||||
Let's first check a simple example::
|
||||
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
test_fw_path = os.getenv("TEST_FW_PATH")
|
||||
if test_fw_path:
|
||||
sys.path.insert(0, test_fw_path)
|
||||
|
||||
import TinyFW
|
||||
from IDF import IDFApp, IDFDUT
|
||||
|
||||
|
||||
@TinyFW.test_method(app=IDFApp.Example, dut=IDFDUT.IDFDUT, env_tag="Example_WIFI",
|
||||
chip="ESP32", module="examples", execution_time=1)
|
||||
def test_examples_protocol_https_request(env, extra_data):
|
||||
"""
|
||||
steps: |
|
||||
1. join AP
|
||||
2. connect to www.howsmyssl.com:443
|
||||
3. send http request
|
||||
"""
|
||||
dut1 = env.get_dut("https_request", "examples/protocols/https_request")
|
||||
dut1.start_app()
|
||||
dut1.expect("Connecting to www.howsmyssl.com:443", timeout=30)
|
||||
dut1.expect("Performing the SSL/TLS handshake")
|
||||
dut1.expect("Certificate verified.", timeout=15)
|
||||
dut1.expect_all(re.compile(r"Cipher suite is TLS-ECDHE-RSA-WITH-AES-128-GCM-SHA256"),
|
||||
"Reading HTTP response",
|
||||
timeout=20)
|
||||
dut1.expect(re.compile(r"Completed (\d) requests"))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
TinyFW.set_default_config(env_config_file="EnvConfigTemplate.yml")
|
||||
test_examples_protocol_https_request()
|
||||
|
||||
|
||||
SOP for adding test cases
|
||||
-------------------------
|
||||
|
||||
1. Import test framework:
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
* We assume ``TEST_FW_PATH`` is pre-defined before running the tests
|
||||
* Then we can import python packages and files from ``TEST_FW_PATH``
|
||||
|
||||
2. Define test case:
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
1. Define test case ``test_xxx(env, extra_data)``
|
||||
* env: instance of test env, see :doc:`Test Env <Env>` for details
|
||||
* extra_data: extra data passed from test case caller
|
||||
2. Add decorator for test case
|
||||
* add decorator ``TinyFW.test_method`` to test method
|
||||
* define default case configs and filters in decorator, see :doc:`TinyFW.test_method <TinyFW>`
|
||||
|
||||
3. Execute test cases:
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
* define in ``main`` section and execute from this file
|
||||
1. set preset configs(optional). If the config is not define in case decorator, it will use the preset configs.
|
||||
2. call test case method:
|
||||
* if you don't pass any arguments, it will use default values
|
||||
* you can pass ``extra_data`` to test case by adding ``extra_data=some_data`` as kwarg of test case method.
|
||||
default value for extra_data is None.
|
||||
* you can overwrite test case config by adding them as kwarg of test case method.
|
||||
It will overwrite preset configs and case default configs.
|
||||
|
||||
Examples::
|
||||
|
||||
test_examples_protocol_https_request(extra_data=["data1", "data2"], dut=SomeOtherDUT, env_tag="OtherEnv")
|
||||
|
||||
* or, use ``runner`` to execute. see :doc:`runner <Runner>` for details
|
||||
|
||||
Test FW features
|
||||
----------------
|
||||
|
||||
1. Test Environment:
|
||||
1. DUT: DUT class provides methods to interact with DUT
|
||||
* read/write through port
|
||||
* expect method which supports expect one or multiple string or RegEx
|
||||
* tool methods provided by the tool bundle, like ``start_app``, ``reset``
|
||||
2. App:
|
||||
* provide some specific features to the test application of DUT, for example:
|
||||
* SDK path
|
||||
* SDK tools
|
||||
* application information like partition table, download configs
|
||||
3. Environment Configs:
|
||||
* support get env configs from config file or auto-detect from current PC
|
||||
* provide ``get_variable`` method to get variables
|
||||
2. Allow to customize components (DUT, App) to support different devices
|
||||
3. Integrate to CI:
|
||||
* provide interfaces for Gitlab-CI
|
||||
* provide ``search case`` and ``runner`` interfaces, able to integrate with other CI
|
||||
|
||||
|
||||
Class Diagram
|
||||
=============
|
||||
.. uml::
|
||||
|
||||
class BaseDUT {
|
||||
{field} app
|
||||
{method} expect
|
||||
{method} expect_any
|
||||
{method} expect_all
|
||||
{method} read
|
||||
{method} write
|
||||
{method} start_receive
|
||||
{method} stop_receive
|
||||
{method} close
|
||||
}
|
||||
class SerialDUT {
|
||||
{method} _port_read
|
||||
{method} _port_write
|
||||
{method} _port_open
|
||||
{method} _port_close
|
||||
}
|
||||
class IDFDUT {
|
||||
{method} reset
|
||||
{method} start_app
|
||||
}
|
||||
class BaseApp {
|
||||
{method} get_sdk_path
|
||||
{method} get_log_folder
|
||||
}
|
||||
class IDFApp {
|
||||
{field} flash_files
|
||||
{field} flash_settings
|
||||
{field} partition_table
|
||||
}
|
||||
class Example {
|
||||
{method} get_binary_path
|
||||
}
|
||||
class EnvConfig {
|
||||
{method} get_variable
|
||||
}
|
||||
class Env {
|
||||
{field} config
|
||||
{field} allocated_duts
|
||||
{field} app_cls
|
||||
{method} get_dut
|
||||
{method} close_dut
|
||||
{method} get_variable
|
||||
{method} get_pc_nic_info
|
||||
{method} close
|
||||
}
|
||||
|
||||
SerialDUT --|> BaseDUT
|
||||
IDFDUT --|> SerialDUT
|
||||
IDFApp --|> BaseApp
|
||||
Example --|> IDFApp
|
||||
Env *-- EnvConfig
|
||||
Env *-- BaseDUT
|
||||
Env o-- BaseApp
|
||||
BaseDUT o-- BaseApp
|
||||
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Contents:
|
||||
|
||||
modules
|
||||
|
||||
Dependencies
|
||||
============
|
||||
|
||||
Support for both Python2 and Python3 (tested on python 2.7.13 and 3.6.2).
|
||||
|
||||
The following 3rd party lib is required:
|
||||
|
||||
* pyserial
|
||||
* pyyaml
|
||||
* junit_xml
|
||||
* netifaces
|
||||
* matplotlib (if use Utility.LineChart)
|
||||
|
||||
These libraries can be installed by running ``pip install --user -r requirements.txt`` in tiny-test-fw directory.
|
||||
|
||||
To build document, we need to install ``Sphinx``, ``plantweb`` and ``sphinx-rtd-theme`` (you may replace this with your own theme). ``plantweb`` requires internet access during building document.
|
||||
|
||||
|
||||
Indices and tables
|
||||
==================
|
||||
|
||||
* :ref:`genindex`
|
||||
* :ref:`modindex`
|
||||
* :ref:`search`
|
||||
@@ -1,540 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
""" IDF Test Applications """
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from abc import abstractmethod
|
||||
|
||||
from tiny_test_fw import App
|
||||
|
||||
from .IDFAssignTest import ComponentUTGroup, ExampleGroup, IDFCaseGroup, TestAppsGroup, UnitTestGroup
|
||||
|
||||
try:
|
||||
import gitlab_api
|
||||
except ImportError:
|
||||
gitlab_api = None
|
||||
|
||||
try:
|
||||
from typing import Any, Dict, List, Optional, Tuple, Type # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def parse_encrypted_flag(args, offs, binary): # type: (Dict, str, str) -> Any
|
||||
# Find partition entries (e.g. the entries with an offset and a file)
|
||||
for _, entry in args.items():
|
||||
# If the current entry is a partition, we have to check whether it is
|
||||
# the one we are looking for or not
|
||||
try:
|
||||
if (entry['offset'], entry['file']) == (offs, binary):
|
||||
return entry['encrypted'] == 'true'
|
||||
except (TypeError, KeyError):
|
||||
# TypeError occurs if the entry is a list, which is possible in JSON
|
||||
# data structure.
|
||||
# KeyError occurs if the entry doesn't have "encrypted" field.
|
||||
continue
|
||||
|
||||
# The entry was not found, return None. The caller will have to check
|
||||
# CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT macro
|
||||
return None
|
||||
|
||||
|
||||
def parse_flash_settings(path, default_encryption=False): # type: (str, bool) -> Tuple[List[Tuple[str, str]], List[Tuple[str, str]], Dict, Any]
|
||||
file_name = os.path.basename(path)
|
||||
|
||||
# For compatibility reasons, this list contains all the files to be
|
||||
# flashed
|
||||
flash_files = []
|
||||
# The following list only contains the files that need encryption
|
||||
encrypt_files = []
|
||||
|
||||
if file_name == 'flasher_args.json':
|
||||
# CMake version using build metadata file
|
||||
with open(path, 'r') as f:
|
||||
args = json.load(f)
|
||||
|
||||
for (offs, binary) in args['flash_files'].items():
|
||||
if offs:
|
||||
flash_files.append((offs, binary))
|
||||
encrypted = parse_encrypted_flag(args, offs, binary)
|
||||
|
||||
# default_encryption should be taken into account if and only if
|
||||
# encrypted flag is not provided in the JSON file.
|
||||
if (encrypted is None and default_encryption) or encrypted:
|
||||
encrypt_files.append((offs, binary))
|
||||
|
||||
flash_settings = args['flash_settings']
|
||||
app_name = os.path.splitext(args['app']['file'])[0]
|
||||
else:
|
||||
# GNU Make version uses download.config arguments file
|
||||
with open(path, 'r') as f:
|
||||
args = f.readlines()[-1].split(' ')
|
||||
flash_settings = {}
|
||||
for idx in range(0, len(args), 2): # process arguments in pairs
|
||||
if args[idx].startswith('--'):
|
||||
# strip the -- from the command line argument
|
||||
flash_settings[args[idx][2:]] = args[idx + 1]
|
||||
else:
|
||||
# offs, filename
|
||||
flash_files.append((args[idx], args[idx + 1]))
|
||||
# Parameter default_encryption tells us if the files need encryption
|
||||
if default_encryption:
|
||||
encrypt_files = flash_files
|
||||
# we can only guess app name in download.config.
|
||||
for p in flash_files:
|
||||
if not os.path.dirname(p[1]) and 'partition' not in p[1]:
|
||||
# app bin usually in the same dir with download.config and it's not partition table
|
||||
app_name = os.path.splitext(p[1])[0]
|
||||
break
|
||||
else:
|
||||
app_name = None
|
||||
return flash_files, encrypt_files, flash_settings, app_name
|
||||
|
||||
|
||||
class Artifacts(object):
|
||||
def __init__(self, dest_root_path, artifact_index_file, app_path, config_name, target):
|
||||
# type: (str, str, str, str, str) -> None
|
||||
assert gitlab_api
|
||||
# at least one of app_path or config_name is not None. otherwise we can't match artifact
|
||||
assert app_path or config_name
|
||||
assert os.path.exists(artifact_index_file)
|
||||
self.gitlab_inst = gitlab_api.Gitlab(os.getenv('CI_PROJECT_ID'))
|
||||
self.dest_root_path = dest_root_path
|
||||
with open(artifact_index_file, 'r') as f:
|
||||
artifact_index = json.load(f)
|
||||
self.artifact_info = self._find_artifact(artifact_index, app_path, config_name, target)
|
||||
|
||||
@staticmethod
|
||||
def _find_artifact(artifact_index, app_path, config_name, target): # type: ignore
|
||||
for artifact_info in artifact_index:
|
||||
match_result = True
|
||||
if app_path:
|
||||
# We use endswith here to avoid issue like:
|
||||
# examples_protocols_mqtt_ws but return a examples_protocols_mqtt_wss failure
|
||||
match_result = artifact_info['app_dir'].endswith(app_path)
|
||||
if config_name:
|
||||
match_result = match_result and config_name == artifact_info['config']
|
||||
if target:
|
||||
match_result = match_result and target == artifact_info['target']
|
||||
if match_result:
|
||||
ret = artifact_info
|
||||
break
|
||||
else:
|
||||
ret = None
|
||||
return ret
|
||||
|
||||
def _get_app_base_path(self): # type: () -> Any
|
||||
if self.artifact_info:
|
||||
return os.path.join(self.artifact_info['work_dir'], self.artifact_info['build_dir'])
|
||||
else:
|
||||
return None
|
||||
|
||||
def _get_flash_arg_file(self, base_path, job_id): # type: (str, str) -> str
|
||||
if self.artifact_info['build_system'] == 'cmake':
|
||||
flash_arg_file = os.path.join(base_path, 'flasher_args.json')
|
||||
else:
|
||||
flash_arg_file = os.path.join(base_path, 'download.config')
|
||||
|
||||
self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path)
|
||||
return flash_arg_file
|
||||
|
||||
def _download_binary_files(self, base_path, job_id, flash_arg_file): # type: (str, str, str) -> None
|
||||
# Let's ignore the second value returned (encrypt_files) as these
|
||||
# files also appear in the first list
|
||||
flash_files, _, _, app_name = parse_flash_settings(os.path.join(self.dest_root_path, flash_arg_file))
|
||||
artifact_files = [os.path.join(base_path, p[1]) for p in flash_files]
|
||||
artifact_files.append(os.path.join(base_path, app_name + '.elf'))
|
||||
|
||||
bootloader_path = os.path.join(base_path, 'bootloader', 'bootloader.bin')
|
||||
if bootloader_path not in artifact_files:
|
||||
artifact_files.append(bootloader_path)
|
||||
|
||||
self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
|
||||
|
||||
def _download_sdkconfig_file(self, base_path, job_id): # type: (str, str) -> None
|
||||
self.gitlab_inst.download_artifact(job_id, [os.path.join(base_path, 'sdkconfig')],
|
||||
self.dest_root_path)
|
||||
|
||||
def download_artifacts(self): # type: () -> Any
|
||||
if not self.artifact_info:
|
||||
return None
|
||||
base_path = self._get_app_base_path()
|
||||
job_id = self.artifact_info['ci_job_id']
|
||||
# 1. download flash args file
|
||||
flash_arg_file = self._get_flash_arg_file(base_path, job_id)
|
||||
|
||||
# 2. download all binary files
|
||||
self._download_binary_files(base_path, job_id, flash_arg_file)
|
||||
|
||||
# 3. download sdkconfig file
|
||||
self._download_sdkconfig_file(base_path, job_id)
|
||||
return base_path
|
||||
|
||||
def download_artifact_files(self, file_names): # type: (List[str]) -> Any
|
||||
if self.artifact_info:
|
||||
base_path = os.path.join(self.artifact_info['work_dir'], self.artifact_info['build_dir'])
|
||||
job_id = self.artifact_info['ci_job_id']
|
||||
|
||||
# download all binary files
|
||||
artifact_files = [os.path.join(base_path, fn) for fn in file_names]
|
||||
self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
|
||||
|
||||
# download sdkconfig file
|
||||
self.gitlab_inst.download_artifact(job_id, [os.path.join(base_path, 'sdkconfig')],
|
||||
self.dest_root_path)
|
||||
else:
|
||||
base_path = None
|
||||
return base_path
|
||||
|
||||
|
||||
class UnitTestArtifacts(Artifacts):
|
||||
BUILDS_DIR_RE = re.compile(r'^builds/')
|
||||
|
||||
def _get_app_base_path(self): # type: () -> Any
|
||||
if self.artifact_info:
|
||||
output_dir = self.BUILDS_DIR_RE.sub('output/', self.artifact_info['build_dir'])
|
||||
return os.path.join(self.artifact_info['app_dir'], output_dir)
|
||||
else:
|
||||
return None
|
||||
|
||||
def _download_sdkconfig_file(self, base_path, job_id): # type: (str, str) -> None
|
||||
self.gitlab_inst.download_artifact(job_id, [os.path.join(base_path, 'sdkconfig')], self.dest_root_path)
|
||||
|
||||
|
||||
class IDFApp(App.BaseApp):
|
||||
"""
|
||||
Implements common esp-idf application behavior.
|
||||
idf applications should inherent from this class and overwrite method get_binary_path.
|
||||
"""
|
||||
|
||||
IDF_DOWNLOAD_CONFIG_FILE = 'download.config'
|
||||
IDF_FLASH_ARGS_FILE = 'flasher_args.json'
|
||||
|
||||
def __init__(self, app_path, config_name=None, target=None, case_group=IDFCaseGroup, artifact_cls=Artifacts): # type: ignore
|
||||
super(IDFApp, self).__init__(app_path)
|
||||
self.app_path = app_path # type: (str)
|
||||
self.config_name = config_name # type: (str)
|
||||
self.target = target # type: (str)
|
||||
self.idf_path = self.get_sdk_path() # type: (str)
|
||||
self.case_group = case_group
|
||||
self.artifact_cls = artifact_cls
|
||||
self.binary_path = self.get_binary_path()
|
||||
self.elf_file = self._get_elf_file_path()
|
||||
self._elf_file_sha256 = None # type: (Optional[str])
|
||||
assert os.path.exists(self.binary_path)
|
||||
if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path):
|
||||
if self.IDF_FLASH_ARGS_FILE not in os.listdir(self.binary_path):
|
||||
msg = ('Neither {} nor {} exists. '
|
||||
"Try to run 'make print_flash_cmd | tail -n 1 > {}/{}' "
|
||||
"or 'idf.py build' "
|
||||
'for resolving the issue.'
|
||||
'').format(self.IDF_DOWNLOAD_CONFIG_FILE, self.IDF_FLASH_ARGS_FILE,
|
||||
self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE)
|
||||
raise AssertionError(msg)
|
||||
|
||||
# In order to keep backward compatibility, flash_files is unchanged.
|
||||
# However, we now have a new attribute encrypt_files.
|
||||
self.flash_files, self.encrypt_files, self.flash_settings = self._parse_flash_download_config()
|
||||
self.partition_table = self._parse_partition_table()
|
||||
|
||||
def __str__(self): # type: () -> str
|
||||
parts = ['app<{}>'.format(self.app_path)]
|
||||
if self.config_name:
|
||||
parts.append('config<{}>'.format(self.config_name))
|
||||
if self.target:
|
||||
parts.append('target<{}>'.format(self.target))
|
||||
return ' '.join(parts)
|
||||
|
||||
@classmethod
|
||||
def get_sdk_path(cls): # type: () -> str
|
||||
idf_path = os.getenv('IDF_PATH')
|
||||
assert idf_path
|
||||
assert os.path.exists(idf_path)
|
||||
return idf_path
|
||||
|
||||
def _get_sdkconfig_paths(self): # type: () -> List[str]
|
||||
"""
|
||||
returns list of possible paths where sdkconfig could be found
|
||||
|
||||
Note: could be overwritten by a derived class to provide other locations or order
|
||||
"""
|
||||
return [os.path.join(self.binary_path, 'sdkconfig'), os.path.join(self.binary_path, '..', 'sdkconfig')]
|
||||
|
||||
def get_sdkconfig(self): # type: () -> Dict
|
||||
"""
|
||||
reads sdkconfig and returns a dictionary with all configured variables
|
||||
|
||||
:raise: AssertionError: if sdkconfig file does not exist in defined paths
|
||||
"""
|
||||
d = {}
|
||||
sdkconfig_file = None
|
||||
for i in self._get_sdkconfig_paths():
|
||||
if os.path.exists(i):
|
||||
sdkconfig_file = i
|
||||
break
|
||||
assert sdkconfig_file is not None
|
||||
with open(sdkconfig_file) as f:
|
||||
for line in f:
|
||||
configs = line.split('=')
|
||||
if len(configs) == 2:
|
||||
d[configs[0]] = configs[1].rstrip()
|
||||
return d
|
||||
|
||||
def get_sdkconfig_config_value(self, config_key): # type: (str) -> Any
|
||||
sdkconfig_dict = self.get_sdkconfig()
|
||||
value = None
|
||||
if (config_key in sdkconfig_dict):
|
||||
value = sdkconfig_dict[config_key]
|
||||
return value
|
||||
|
||||
@abstractmethod
|
||||
def _try_get_binary_from_local_fs(self): # type: () -> Optional[str]
|
||||
pass
|
||||
|
||||
def get_binary_path(self): # type: () -> str
|
||||
path = self._try_get_binary_from_local_fs()
|
||||
if path:
|
||||
return path
|
||||
|
||||
artifacts = self.artifact_cls(self.idf_path,
|
||||
self.case_group.get_artifact_index_file(),
|
||||
self.app_path, self.config_name, self.target)
|
||||
if isinstance(self, LoadableElfTestApp):
|
||||
assert self.app_files
|
||||
path = artifacts.download_artifact_files(self.app_files)
|
||||
else:
|
||||
path = artifacts.download_artifacts()
|
||||
|
||||
if path:
|
||||
return os.path.join(self.idf_path, path)
|
||||
else:
|
||||
raise OSError('Failed to get binary for {}'.format(self))
|
||||
|
||||
def _get_elf_file_path(self): # type: () -> str
|
||||
ret = ''
|
||||
file_names = os.listdir(self.binary_path)
|
||||
for fn in file_names:
|
||||
if os.path.splitext(fn)[1] == '.elf':
|
||||
ret = os.path.join(self.binary_path, fn)
|
||||
return ret
|
||||
|
||||
def _int_offs_abs_paths(self, files_list): # type: (List[tuple[str, str]]) -> List[Tuple[int, str]]
|
||||
return [(int(offs, 0),
|
||||
os.path.join(self.binary_path, file_path.strip()))
|
||||
for (offs, file_path) in files_list]
|
||||
|
||||
def _parse_flash_download_config(self): # type: () -> Tuple[List[tuple[int, str]], List[tuple[int, str]], Dict]
|
||||
"""
|
||||
Parse flash download config from build metadata files
|
||||
|
||||
Sets self.flash_files, self.flash_settings
|
||||
|
||||
(Called from constructor)
|
||||
|
||||
Returns (flash_files, encrypt_files, flash_settings)
|
||||
"""
|
||||
|
||||
if self.IDF_FLASH_ARGS_FILE in os.listdir(self.binary_path):
|
||||
# CMake version using build metadata file
|
||||
path = os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE)
|
||||
else:
|
||||
# GNU Make version uses download.config arguments file
|
||||
path = os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE)
|
||||
|
||||
# If the JSON doesn't find the encrypted flag for our files, provide
|
||||
# a default encrpytion flag: the macro
|
||||
# CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT
|
||||
sdkconfig_dict = self.get_sdkconfig()
|
||||
default_encryption = 'CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT' in sdkconfig_dict
|
||||
|
||||
flash_files, encrypt_files, flash_settings, _ = parse_flash_settings(path, default_encryption)
|
||||
|
||||
# Flash setting "encrypt" only and only if all the files to flash
|
||||
# must be encrypted. Else, this parameter should be False.
|
||||
# All files must be encrypted is both file lists are the same
|
||||
flash_settings['encrypt'] = sorted(flash_files) == sorted(encrypt_files)
|
||||
|
||||
return self._int_offs_abs_paths(flash_files), self._int_offs_abs_paths(encrypt_files), flash_settings
|
||||
|
||||
def _parse_partition_table(self): # type: ignore
|
||||
"""
|
||||
Parse partition table contents based on app binaries
|
||||
|
||||
Returns partition_table data
|
||||
|
||||
(Called from constructor)
|
||||
"""
|
||||
partition_tool = os.path.join(self.idf_path,
|
||||
'components',
|
||||
'partition_table',
|
||||
'gen_esp32part.py')
|
||||
assert os.path.exists(partition_tool)
|
||||
|
||||
errors = []
|
||||
# self.flash_files is sorted based on offset in order to have a consistent result with different versions of
|
||||
# Python
|
||||
for (_, path) in sorted(self.flash_files, key=lambda elem: elem[0]):
|
||||
if 'partition' in os.path.split(path)[1]:
|
||||
partition_file = os.path.join(self.binary_path, path)
|
||||
|
||||
process = subprocess.Popen([sys.executable, partition_tool, partition_file],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
(raw_data, raw_error) = process.communicate()
|
||||
if isinstance(raw_error, bytes):
|
||||
raw_error = raw_error.decode()
|
||||
if 'Traceback' in raw_error:
|
||||
# Some exception occurred. It is possible that we've tried the wrong binary file.
|
||||
errors.append((path, raw_error))
|
||||
continue
|
||||
|
||||
if isinstance(raw_data, bytes):
|
||||
raw_data = raw_data.decode()
|
||||
break
|
||||
else:
|
||||
traceback_msg = os.linesep.join(['{} {}:{}{}'.format(partition_tool,
|
||||
p,
|
||||
os.linesep,
|
||||
msg) for p, msg in errors])
|
||||
raise ValueError('No partition table found for IDF binary path: {}{}{}'.format(self.binary_path,
|
||||
os.linesep,
|
||||
traceback_msg))
|
||||
|
||||
partition_table = dict()
|
||||
for line in raw_data.splitlines():
|
||||
if line[0] != '#':
|
||||
try:
|
||||
_name, _type, _subtype, _offset, _size, _flags = line.split(',')
|
||||
if _size[-1] == 'K':
|
||||
_size = int(_size[:-1]) * 1024
|
||||
elif _size[-1] == 'M':
|
||||
_size = int(_size[:-1]) * 1024 * 1024
|
||||
else:
|
||||
_size = int(_size)
|
||||
_offset = int(_offset, 0)
|
||||
except ValueError:
|
||||
continue
|
||||
partition_table[_name] = {
|
||||
'type': _type,
|
||||
'subtype': _subtype,
|
||||
'offset': _offset,
|
||||
'size': _size,
|
||||
'flags': _flags
|
||||
}
|
||||
|
||||
return partition_table
|
||||
|
||||
def get_elf_sha256(self): # type: () -> Optional[str]
|
||||
if self._elf_file_sha256:
|
||||
return self._elf_file_sha256
|
||||
|
||||
sha256 = hashlib.sha256()
|
||||
with open(self.elf_file, 'rb') as f:
|
||||
sha256.update(f.read())
|
||||
self._elf_file_sha256 = sha256.hexdigest()
|
||||
return self._elf_file_sha256
|
||||
|
||||
|
||||
class Example(IDFApp):
|
||||
def __init__(self, app_path, config_name='default', target='esp32', case_group=ExampleGroup, artifacts_cls=Artifacts):
|
||||
# type: (str, str, str, Type[ExampleGroup], Type[Artifacts]) -> None
|
||||
if not config_name:
|
||||
config_name = 'default'
|
||||
if not target:
|
||||
target = 'esp32'
|
||||
super(Example, self).__init__(app_path, config_name, target, case_group, artifacts_cls)
|
||||
|
||||
def _try_get_binary_from_local_fs(self): # type: () -> Optional[str]
|
||||
# build folder of example path
|
||||
path = os.path.join(self.idf_path, self.app_path, 'build')
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
# new style build dir
|
||||
path = os.path.join(self.idf_path, self.app_path, f'build_{self.target}_{self.config_name}')
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
# Search for CI build folders.
|
||||
# Path format: $IDF_PATH/<app_dir>/build_<target>_<config>
|
||||
build_dir = f'build_{self.target}_{self.config_name}'
|
||||
example_path = os.path.join(self.idf_path, self.app_path, build_dir)
|
||||
if os.path.exists(example_path):
|
||||
return path
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class UT(IDFApp):
|
||||
def __init__(self, app_path, config_name='default', target='esp32', case_group=UnitTestGroup, artifacts_cls=UnitTestArtifacts):
|
||||
# type: (str, str, str, Type[UnitTestGroup], Type[UnitTestArtifacts]) -> None
|
||||
if not config_name:
|
||||
config_name = 'default'
|
||||
if not target:
|
||||
target = 'esp32'
|
||||
super(UT, self).__init__(app_path, config_name, target, case_group, artifacts_cls)
|
||||
|
||||
def _try_get_binary_from_local_fs(self): # type: () -> Optional[str]
|
||||
path = os.path.join(self.idf_path, self.app_path, 'build')
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
# first try to get from build folder of unit-test-app
|
||||
path = os.path.join(self.idf_path, 'tools', 'unit-test-app', 'build')
|
||||
if os.path.exists(path):
|
||||
# found, use bin in build path
|
||||
return path
|
||||
|
||||
# ``build_unit_test.sh`` will copy binary to output folder
|
||||
path = os.path.join(self.idf_path, 'tools', 'unit-test-app', 'output', self.target, self.config_name)
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class TestApp(Example):
|
||||
def __init__(self, app_path, config_name='default', target='esp32', case_group=TestAppsGroup, artifacts_cls=Artifacts):
|
||||
# type: (str, str, str, Type[TestAppsGroup], Type[Artifacts]) -> None
|
||||
super(TestApp, self).__init__(app_path, config_name, target, case_group, artifacts_cls)
|
||||
|
||||
|
||||
class ComponentUTApp(TestApp):
|
||||
def __init__(self, app_path, config_name='default', target='esp32', case_group=ComponentUTGroup, artifacts_cls=Artifacts):
|
||||
# type: (str, str, str, Type[ComponentUTGroup], Type[Artifacts]) -> None
|
||||
super(ComponentUTApp, self).__init__(app_path, config_name, target, case_group, artifacts_cls)
|
||||
|
||||
|
||||
class LoadableElfTestApp(TestApp):
|
||||
def __init__(self, app_path, app_files, config_name='default', target='esp32', case_group=TestAppsGroup, artifacts_cls=Artifacts):
|
||||
# type: (str, List[str], str, str, Type[TestAppsGroup], Type[Artifacts]) -> None
|
||||
# add arg `app_files` for loadable elf test_app.
|
||||
# Such examples only build elf files, so it doesn't generate flasher_args.json.
|
||||
# So we can't get app files from config file. Test case should pass it to application.
|
||||
super(IDFApp, self).__init__(app_path)
|
||||
self.app_path = app_path
|
||||
self.app_files = app_files
|
||||
self.config_name = config_name or 'default'
|
||||
self.target = target or 'esp32'
|
||||
self.idf_path = self.get_sdk_path()
|
||||
self.case_group = case_group
|
||||
self.artifact_cls = artifacts_cls
|
||||
self.binary_path = self.get_binary_path()
|
||||
self.elf_file = self._get_elf_file_path()
|
||||
assert os.path.exists(self.binary_path)
|
||||
|
||||
|
||||
class SSC(IDFApp):
|
||||
def get_binary_path(self): # type: () -> str
|
||||
# TODO: to implement SSC get binary path
|
||||
return self.app_path
|
||||
|
||||
|
||||
class AT(IDFApp):
|
||||
def get_binary_path(self): # type: () -> str
|
||||
# TODO: to implement AT get binary path
|
||||
return self.app_path
|
||||
@@ -1,354 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""
|
||||
Command line tool to assign tests to CI test jobs.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from copy import deepcopy
|
||||
|
||||
import yaml
|
||||
|
||||
try:
|
||||
from yaml import CLoader as Loader
|
||||
except ImportError:
|
||||
from yaml import Loader as Loader # type: ignore
|
||||
|
||||
import gitlab_api
|
||||
from tiny_test_fw.Utility import CIAssignTest
|
||||
|
||||
try:
|
||||
from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
|
||||
except ImportError:
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
|
||||
|
||||
from idf_py_actions.constants import PREVIEW_TARGETS, SUPPORTED_TARGETS
|
||||
|
||||
IDF_PATH_FROM_ENV = os.getenv('IDF_PATH', '')
|
||||
|
||||
|
||||
class IDFCaseGroup(CIAssignTest.Group):
|
||||
BUILD_JOB_NAMES = None
|
||||
|
||||
@classmethod
|
||||
def get_artifact_index_file(cls):
|
||||
if IDF_PATH_FROM_ENV:
|
||||
artifact_index_file = os.path.join(IDF_PATH_FROM_ENV, 'artifact_index.json')
|
||||
else:
|
||||
artifact_index_file = 'artifact_index.json'
|
||||
return artifact_index_file
|
||||
|
||||
|
||||
class IDFAssignTest(CIAssignTest.AssignTest):
|
||||
DEFAULT_FILTER = {
|
||||
'category': 'function',
|
||||
'ignore': False,
|
||||
'supported_in_ci': True,
|
||||
'nightly_run': False,
|
||||
}
|
||||
|
||||
def __init__(self, test_case_path, ci_config_file, case_group=IDFCaseGroup):
|
||||
super(IDFAssignTest, self).__init__(test_case_path, ci_config_file, case_group)
|
||||
|
||||
def format_build_log_path(self, parallel_num):
|
||||
return 'list_job_{}.txt'.format(parallel_num)
|
||||
|
||||
def create_artifact_index_file(self, project_id=None, pipeline_id=None):
|
||||
if project_id is None:
|
||||
project_id = os.getenv('CI_PROJECT_ID')
|
||||
if pipeline_id is None:
|
||||
pipeline_id = os.getenv('CI_PIPELINE_ID')
|
||||
gitlab_inst = gitlab_api.Gitlab(project_id)
|
||||
|
||||
artifact_index_list = []
|
||||
for build_job_name in self.case_group.BUILD_JOB_NAMES:
|
||||
job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
|
||||
for job_info in job_info_list:
|
||||
parallel_num = job_info['parallel_num'] or 1 # Could be None if "parallel_num" not defined for the job
|
||||
raw_data = gitlab_inst.download_artifact(job_info['id'],
|
||||
[self.format_build_log_path(parallel_num)])[0]
|
||||
build_info_list = [json.loads(line) for line in raw_data.decode().splitlines()]
|
||||
for build_info in build_info_list:
|
||||
build_info['ci_job_id'] = job_info['id']
|
||||
artifact_index_list.append(build_info)
|
||||
artifact_index_file = self.case_group.get_artifact_index_file()
|
||||
with open(artifact_index_file, 'w') as f:
|
||||
json.dump(artifact_index_list, f)
|
||||
|
||||
def search_cases(self, case_filter=None):
|
||||
_filter = deepcopy(case_filter) if case_filter else {}
|
||||
if 'NIGHTLY_RUN' in os.environ or 'BOT_LABEL_NIGHTLY_RUN' in os.environ:
|
||||
_filter.update({'nightly_run': True})
|
||||
return super().search_cases(_filter)
|
||||
|
||||
|
||||
class ExampleGroup(IDFCaseGroup):
|
||||
SORT_KEYS = CI_JOB_MATCH_KEYS = ['env_tag', 'target']
|
||||
|
||||
EXAMPLE_TARGETS = SUPPORTED_TARGETS + PREVIEW_TARGETS
|
||||
BUILD_JOB_NAMES = ['build_examples_cmake_{}'.format(target) for target in EXAMPLE_TARGETS] # type: ignore
|
||||
|
||||
|
||||
class TestAppsGroup(ExampleGroup):
|
||||
TEST_APP_TARGETS = SUPPORTED_TARGETS + PREVIEW_TARGETS
|
||||
BUILD_JOB_NAMES = ['build_test_apps_{}'.format(target) for target in TEST_APP_TARGETS] # type: ignore
|
||||
|
||||
|
||||
class ComponentUTGroup(TestAppsGroup):
|
||||
UNIT_TEST_TARGETS = SUPPORTED_TARGETS + PREVIEW_TARGETS
|
||||
BUILD_JOB_NAMES = ['build_component_ut_{}'.format(target) for target in UNIT_TEST_TARGETS] # type: ignore
|
||||
|
||||
|
||||
class UnitTestGroup(IDFCaseGroup):
|
||||
SORT_KEYS = ['test environment', 'tags', 'chip_target']
|
||||
CI_JOB_MATCH_KEYS = ['test environment']
|
||||
|
||||
UNIT_TEST_TARGETS = SUPPORTED_TARGETS + PREVIEW_TARGETS
|
||||
BUILD_JOB_NAMES = ['build_esp_idf_tests_cmake_{}'.format(target) for target in UNIT_TEST_TARGETS] # type: ignore
|
||||
|
||||
MAX_CASE = 50
|
||||
ATTR_CONVERT_TABLE = {
|
||||
'execution_time': 'execution time'
|
||||
}
|
||||
DUT_CLS_NAME = {
|
||||
'esp32': 'ESP32DUT',
|
||||
'esp32s2': 'ESP32S2DUT',
|
||||
'esp32s3': 'ESP32S3DUT',
|
||||
'esp32c2': 'ESP32C2DUT',
|
||||
'esp32c3': 'ESP32C3DUT',
|
||||
'esp32c6': 'ESP32C6DUT',
|
||||
'esp32h2': 'ESP32H2DUT',
|
||||
'esp8266': 'ESP8266DUT',
|
||||
}
|
||||
|
||||
def __init__(self, case):
|
||||
super(UnitTestGroup, self).__init__(case)
|
||||
for tag in self._get_case_attr(case, 'tags'):
|
||||
self.ci_job_match_keys.add(tag)
|
||||
|
||||
@staticmethod
|
||||
def _get_case_attr(case, attr):
|
||||
if attr in UnitTestGroup.ATTR_CONVERT_TABLE:
|
||||
attr = UnitTestGroup.ATTR_CONVERT_TABLE[attr]
|
||||
return case[attr]
|
||||
|
||||
def add_extra_case(self, case):
|
||||
""" If current group contains all tags required by case, then add succeed """
|
||||
added = False
|
||||
if self.accept_new_case():
|
||||
for key in self.filters:
|
||||
if self._get_case_attr(case, key) != self.filters[key]:
|
||||
if key == 'tags':
|
||||
if set(self._get_case_attr(case, key)).issubset(set(self.filters[key])):
|
||||
continue
|
||||
break
|
||||
else:
|
||||
self.case_list.append(case)
|
||||
added = True
|
||||
return added
|
||||
|
||||
def _create_extra_data(self, test_cases, test_function):
|
||||
"""
|
||||
For unit test case, we need to copy some attributes of test cases into config file.
|
||||
So unit test function knows how to run the case.
|
||||
"""
|
||||
case_data = []
|
||||
for case in test_cases:
|
||||
one_case_data = {
|
||||
'config': self._get_case_attr(case, 'config'),
|
||||
'name': self._get_case_attr(case, 'summary'),
|
||||
'reset': self._get_case_attr(case, 'reset'),
|
||||
'timeout': self._get_case_attr(case, 'timeout'),
|
||||
}
|
||||
|
||||
if test_function in ['run_multiple_devices_cases', 'run_multiple_stage_cases']:
|
||||
try:
|
||||
one_case_data['child case num'] = self._get_case_attr(case, 'child case num')
|
||||
except KeyError as e:
|
||||
print('multiple devices/stages cases must contains at least two test functions')
|
||||
print('case name: {}'.format(one_case_data['name']))
|
||||
raise e
|
||||
|
||||
case_data.append(one_case_data)
|
||||
return case_data
|
||||
|
||||
def _divide_case_by_test_function(self):
|
||||
"""
|
||||
divide cases of current test group by test function they need to use
|
||||
|
||||
:return: dict of list of cases for each test functions
|
||||
"""
|
||||
case_by_test_function = {
|
||||
'run_multiple_devices_cases': [],
|
||||
'run_multiple_stage_cases': [],
|
||||
'run_unit_test_cases': [],
|
||||
}
|
||||
|
||||
for case in self.case_list:
|
||||
if case['multi_device'] == 'Yes':
|
||||
case_by_test_function['run_multiple_devices_cases'].append(case)
|
||||
elif case['multi_stage'] == 'Yes':
|
||||
case_by_test_function['run_multiple_stage_cases'].append(case)
|
||||
else:
|
||||
case_by_test_function['run_unit_test_cases'].append(case)
|
||||
return case_by_test_function
|
||||
|
||||
def output(self):
|
||||
"""
|
||||
output data for job configs
|
||||
|
||||
:return: {"Filter": case filter, "CaseConfig": list of case configs for cases in this group}
|
||||
"""
|
||||
|
||||
target = self._get_case_attr(self.case_list[0], 'chip_target')
|
||||
if target:
|
||||
overwrite = {
|
||||
'dut': {
|
||||
'package': 'ttfw_idf',
|
||||
'class': self.DUT_CLS_NAME[target],
|
||||
}
|
||||
}
|
||||
else:
|
||||
overwrite = dict()
|
||||
|
||||
case_by_test_function = self._divide_case_by_test_function()
|
||||
|
||||
output_data = {
|
||||
# we don't need filter for test function, as UT uses a few test functions for all cases
|
||||
'CaseConfig': [
|
||||
{
|
||||
'name': test_function,
|
||||
'extra_data': self._create_extra_data(test_cases, test_function),
|
||||
'overwrite': overwrite,
|
||||
} for test_function, test_cases in case_by_test_function.items() if test_cases
|
||||
],
|
||||
}
|
||||
return output_data
|
||||
|
||||
|
||||
class ExampleAssignTest(IDFAssignTest):
|
||||
CI_TEST_JOB_PATTERN = re.compile(r'^example_test_.+')
|
||||
|
||||
def __init__(self, test_case_path, ci_config_file):
|
||||
super(ExampleAssignTest, self).__init__(test_case_path, ci_config_file, case_group=ExampleGroup)
|
||||
|
||||
|
||||
class TestAppsAssignTest(IDFAssignTest):
|
||||
CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+')
|
||||
|
||||
def __init__(self, test_case_path, ci_config_file):
|
||||
super(TestAppsAssignTest, self).__init__(test_case_path, ci_config_file, case_group=TestAppsGroup)
|
||||
|
||||
|
||||
class ComponentUTAssignTest(IDFAssignTest):
|
||||
CI_TEST_JOB_PATTERN = re.compile(r'^component_ut_test_.+')
|
||||
|
||||
def __init__(self, test_case_path, ci_config_file):
|
||||
super(ComponentUTAssignTest, self).__init__(test_case_path, ci_config_file, case_group=ComponentUTGroup)
|
||||
|
||||
|
||||
class UnitTestAssignTest(IDFAssignTest):
|
||||
CI_TEST_JOB_PATTERN = re.compile(r'^UT_.+')
|
||||
|
||||
def __init__(self, test_case_path, ci_config_file):
|
||||
super(UnitTestAssignTest, self).__init__(test_case_path, ci_config_file, case_group=UnitTestGroup)
|
||||
|
||||
def search_cases(self, case_filter=None):
|
||||
"""
|
||||
For unit test case, we don't search for test functions.
|
||||
The unit test cases is stored in a yaml file which is created in job build-idf-test.
|
||||
"""
|
||||
|
||||
def find_by_suffix(suffix, path):
|
||||
res = []
|
||||
for root, _, files in os.walk(path):
|
||||
for file in files:
|
||||
if file.endswith(suffix):
|
||||
res.append(os.path.join(root, file))
|
||||
return res
|
||||
|
||||
def get_test_cases_from_yml(yml_file):
|
||||
try:
|
||||
with open(yml_file) as fr:
|
||||
raw_data = yaml.load(fr, Loader=Loader)
|
||||
test_cases = raw_data['test cases']
|
||||
except (IOError, KeyError):
|
||||
return []
|
||||
else:
|
||||
return test_cases
|
||||
|
||||
test_cases = []
|
||||
for path in self.test_case_paths:
|
||||
if os.path.isdir(path):
|
||||
for yml_file in find_by_suffix('.yml', path):
|
||||
test_cases.extend(get_test_cases_from_yml(yml_file))
|
||||
elif os.path.isfile(path) and path.endswith('.yml'):
|
||||
test_cases.extend(get_test_cases_from_yml(path))
|
||||
else:
|
||||
print('Test case path is invalid. Should only happen when use @bot to skip unit test.')
|
||||
|
||||
# filter keys are lower case. Do map lower case keys with original keys.
|
||||
try:
|
||||
key_mapping = {x.lower(): x for x in test_cases[0].keys()}
|
||||
except IndexError:
|
||||
key_mapping = dict()
|
||||
if case_filter:
|
||||
for key in case_filter:
|
||||
filtered_cases = []
|
||||
for case in test_cases:
|
||||
try:
|
||||
mapped_key = key_mapping[key]
|
||||
# bot converts string to lower case
|
||||
if isinstance(case[mapped_key], str):
|
||||
_value = case[mapped_key].lower()
|
||||
else:
|
||||
_value = case[mapped_key]
|
||||
if _value in case_filter[key]:
|
||||
filtered_cases.append(case)
|
||||
except KeyError:
|
||||
# case don't have this key, regard as filter success
|
||||
filtered_cases.append(case)
|
||||
test_cases = filtered_cases
|
||||
# sort cases with configs and test functions
|
||||
# in later stage cases with similar attributes are more likely to be assigned to the same job
|
||||
# it will reduce the count of flash DUT operations
|
||||
test_cases.sort(key=lambda x: x['config'] + x['multi_stage'] + x['multi_device'])
|
||||
return test_cases
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('case_group', choices=['example_test', 'custom_test', 'unit_test', 'component_ut'])
|
||||
parser.add_argument('test_case_paths', nargs='+', help='test case folder or file')
|
||||
parser.add_argument('-c', '--config', default=os.path.join(IDF_PATH_FROM_ENV, '.gitlab', 'ci', 'target-test.yml'),
|
||||
help='gitlab ci config file')
|
||||
parser.add_argument('-o', '--output', help='output path of config files')
|
||||
parser.add_argument('--pipeline_id', '-p', type=int, default=None, help='pipeline_id')
|
||||
parser.add_argument('--test-case-file-pattern', help='file name pattern used to find Python test case files')
|
||||
args = parser.parse_args()
|
||||
|
||||
SUPPORTED_TARGETS.extend(PREVIEW_TARGETS)
|
||||
|
||||
test_case_paths = [os.path.join(IDF_PATH_FROM_ENV, path) if not os.path.isabs(path) else path for path in
|
||||
args.test_case_paths] # type: ignore
|
||||
args_list = [test_case_paths, args.config]
|
||||
if args.case_group == 'example_test':
|
||||
assigner = ExampleAssignTest(*args_list)
|
||||
elif args.case_group == 'custom_test':
|
||||
assigner = TestAppsAssignTest(*args_list)
|
||||
elif args.case_group == 'unit_test':
|
||||
assigner = UnitTestAssignTest(*args_list)
|
||||
elif args.case_group == 'component_ut':
|
||||
assigner = ComponentUTAssignTest(*args_list)
|
||||
else:
|
||||
raise SystemExit(1) # which is impossible
|
||||
|
||||
if args.test_case_file_pattern:
|
||||
assigner.CI_TEST_JOB_PATTERN = re.compile(r'{}'.format(args.test_case_file_pattern))
|
||||
|
||||
assigner.assign_cases()
|
||||
assigner.output_configs(args.output)
|
||||
assigner.create_artifact_index_file()
|
||||
@@ -1,917 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
""" DUT for IDF applications """
|
||||
import collections
|
||||
import functools
|
||||
import io
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import pexpect
|
||||
import serial
|
||||
|
||||
# python2 and python3 queue package name is different
|
||||
try:
|
||||
import Queue as _queue
|
||||
except ImportError:
|
||||
import queue as _queue # type: ignore
|
||||
|
||||
from serial.tools import list_ports
|
||||
from tiny_test_fw import DUT, Utility
|
||||
|
||||
try:
|
||||
import esptool
|
||||
except ImportError: # cheat and use IDF's copy of esptool if available
|
||||
idf_path = os.getenv('IDF_PATH')
|
||||
if not idf_path or not os.path.exists(idf_path):
|
||||
raise
|
||||
sys.path.insert(0, os.path.join(idf_path, 'components', 'esptool_py', 'esptool'))
|
||||
import esptool
|
||||
|
||||
try:
|
||||
# esptool>=4.0
|
||||
detect_chip = esptool.cmds.detect_chip
|
||||
FatalError = esptool.util.FatalError
|
||||
targets = esptool.targets
|
||||
except (AttributeError, ModuleNotFoundError):
|
||||
# esptool<4.0
|
||||
detect_chip = esptool.ESPLoader.detect_chip
|
||||
FatalError = esptool.FatalError
|
||||
targets = esptool
|
||||
|
||||
import espefuse
|
||||
import espsecure
|
||||
|
||||
|
||||
class IDFToolError(OSError):
|
||||
pass
|
||||
|
||||
|
||||
class IDFDUTException(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class IDFRecvThread(DUT.RecvThread):
|
||||
|
||||
PERFORMANCE_PATTERN = re.compile(r'\[Performance]\[(\w+)]: ([^\r\n]+)\r?\n')
|
||||
EXCEPTION_PATTERNS = [
|
||||
re.compile(r"(Guru Meditation Error: Core\s+\d panic'ed \([\w].*?\))"),
|
||||
re.compile(r'(abort\(\) was called at PC 0x[a-fA-F\d]{8} on core \d)'),
|
||||
re.compile(r'(rst 0x\d+ \(TG\dWDT_SYS_RESET|TGWDT_CPU_RESET\))')
|
||||
]
|
||||
BACKTRACE_PATTERN = re.compile(r'Backtrace:((\s(0x[0-9a-f]{8}):0x[0-9a-f]{8})+)')
|
||||
BACKTRACE_ADDRESS_PATTERN = re.compile(r'(0x[0-9a-f]{8}):0x[0-9a-f]{8}')
|
||||
|
||||
def __init__(self, read, dut):
|
||||
super(IDFRecvThread, self).__init__(read, dut)
|
||||
self.exceptions = _queue.Queue()
|
||||
self.performance_items = _queue.Queue()
|
||||
|
||||
def collect_performance(self, comp_data):
|
||||
matches = self.PERFORMANCE_PATTERN.findall(comp_data)
|
||||
for match in matches:
|
||||
Utility.console_log('[Performance][{}]: {}'.format(match[0], match[1]), color='orange')
|
||||
self.performance_items.put((match[0], match[1]))
|
||||
|
||||
def detect_exception(self, comp_data):
|
||||
for pattern in self.EXCEPTION_PATTERNS:
|
||||
start = 0
|
||||
while True:
|
||||
match = pattern.search(comp_data, pos=start)
|
||||
if match:
|
||||
start = match.end()
|
||||
self.exceptions.put(match.group(0))
|
||||
Utility.console_log('[Exception]: {}'.format(match.group(0)), color='red')
|
||||
else:
|
||||
break
|
||||
|
||||
def detect_backtrace(self, comp_data):
|
||||
start = 0
|
||||
while True:
|
||||
match = self.BACKTRACE_PATTERN.search(comp_data, pos=start)
|
||||
if match:
|
||||
start = match.end()
|
||||
Utility.console_log('[Backtrace]:{}'.format(match.group(1)), color='red')
|
||||
# translate backtrace
|
||||
addresses = self.BACKTRACE_ADDRESS_PATTERN.findall(match.group(1))
|
||||
translated_backtrace = ''
|
||||
for addr in addresses:
|
||||
ret = self.dut.lookup_pc_address(addr)
|
||||
if ret:
|
||||
translated_backtrace += ret + '\n'
|
||||
if translated_backtrace:
|
||||
Utility.console_log('Translated backtrace\n:' + translated_backtrace, color='yellow')
|
||||
else:
|
||||
Utility.console_log('Failed to translate backtrace', color='yellow')
|
||||
else:
|
||||
break
|
||||
|
||||
CHECK_FUNCTIONS = [collect_performance, detect_exception, detect_backtrace]
|
||||
|
||||
|
||||
def _uses_esptool(func):
|
||||
""" Suspend listener thread, connect with esptool,
|
||||
call target function with esptool instance,
|
||||
then resume listening for output
|
||||
"""
|
||||
@functools.wraps(func)
|
||||
def handler(self, *args, **kwargs):
|
||||
self.stop_receive()
|
||||
|
||||
settings = self.port_inst.get_settings()
|
||||
|
||||
try:
|
||||
if not self.rom_inst:
|
||||
if not self.secure_boot_en:
|
||||
self.rom_inst = detect_chip(self.port_inst)
|
||||
else:
|
||||
self.rom_inst = self.get_rom()(self.port_inst)
|
||||
self.rom_inst.connect('hard_reset')
|
||||
|
||||
if (self.secure_boot_en):
|
||||
esp = self.rom_inst
|
||||
esp.flash_spi_attach(0)
|
||||
else:
|
||||
esp = self.rom_inst.run_stub()
|
||||
|
||||
ret = func(self, esp, *args, **kwargs)
|
||||
# do hard reset after use esptool
|
||||
esp.hard_reset()
|
||||
finally:
|
||||
# always need to restore port settings
|
||||
self.port_inst.apply_settings(settings)
|
||||
|
||||
self.start_receive()
|
||||
|
||||
return ret
|
||||
return handler
|
||||
|
||||
|
||||
class IDFDUT(DUT.SerialDUT):
|
||||
""" IDF DUT, extends serial with esptool methods
|
||||
|
||||
(Becomes aware of IDFApp instance which holds app-specific data)
|
||||
"""
|
||||
|
||||
# /dev/ttyAMA0 port is listed in Raspberry Pi
|
||||
# /dev/tty.Bluetooth-Incoming-Port port is listed in Mac
|
||||
INVALID_PORT_PATTERN = re.compile(r'AMA|Bluetooth')
|
||||
# if need to erase NVS partition in start app
|
||||
ERASE_NVS = True
|
||||
RECV_THREAD_CLS = IDFRecvThread
|
||||
|
||||
def __init__(self, name, port, log_file, app, allow_dut_exception=False, **kwargs):
|
||||
super(IDFDUT, self).__init__(name, port, log_file, app, **kwargs)
|
||||
self.allow_dut_exception = allow_dut_exception
|
||||
self.exceptions = _queue.Queue()
|
||||
self.performance_items = _queue.Queue()
|
||||
self.rom_inst = None
|
||||
self.secure_boot_en = self.app.get_sdkconfig_config_value('CONFIG_SECURE_BOOT') and \
|
||||
not self.app.get_sdkconfig_config_value('CONFIG_EFUSE_VIRTUAL')
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
raise NotImplementedError('This is an abstraction class, method not defined.')
|
||||
|
||||
@classmethod
|
||||
def get_mac(cls, app, port):
|
||||
"""
|
||||
get MAC address via esptool
|
||||
|
||||
:param app: application instance (to get tool)
|
||||
:param port: serial port as string
|
||||
:return: MAC address or None
|
||||
"""
|
||||
esp = None
|
||||
try:
|
||||
esp = cls.get_rom()(port)
|
||||
esp.connect()
|
||||
return esp.read_mac()
|
||||
except RuntimeError:
|
||||
return None
|
||||
finally:
|
||||
if esp:
|
||||
# do hard reset after use esptool
|
||||
esp.hard_reset()
|
||||
esp._port.close()
|
||||
|
||||
@classmethod
|
||||
def confirm_dut(cls, port, **kwargs):
|
||||
inst = None
|
||||
try:
|
||||
expected_rom_class = cls.get_rom()
|
||||
except NotImplementedError:
|
||||
expected_rom_class = None
|
||||
|
||||
try:
|
||||
# TODO: check whether 8266 works with this logic
|
||||
# Otherwise overwrite it in ESP8266DUT
|
||||
inst = detect_chip(port)
|
||||
if expected_rom_class and type(inst) != expected_rom_class:
|
||||
raise RuntimeError('Target not expected')
|
||||
return inst.read_mac() is not None, get_target_by_rom_class(type(inst))
|
||||
except (FatalError, RuntimeError):
|
||||
return False, None
|
||||
finally:
|
||||
if inst is not None:
|
||||
inst._port.close()
|
||||
|
||||
def _try_flash(self, erase_nvs):
|
||||
"""
|
||||
Called by start_app()
|
||||
|
||||
:return: None
|
||||
"""
|
||||
flash_files = []
|
||||
encrypt_files = []
|
||||
try:
|
||||
# Open the files here to prevents us from having to seek back to 0
|
||||
# each time. Before opening them, we have to organize the lists the
|
||||
# way esptool.write_flash needs:
|
||||
# If encrypt is provided, flash_files contains all the files to
|
||||
# flash.
|
||||
# Else, flash_files contains the files to be flashed as plain text
|
||||
# and encrypt_files contains the ones to flash encrypted.
|
||||
flash_files = self.app.flash_files
|
||||
encrypt_files = self.app.encrypt_files
|
||||
encrypt = self.app.flash_settings.get('encrypt', False)
|
||||
if encrypt:
|
||||
flash_files = encrypt_files
|
||||
encrypt_files = []
|
||||
else:
|
||||
flash_files = [entry
|
||||
for entry in flash_files
|
||||
if entry not in encrypt_files]
|
||||
|
||||
flash_files = [(offs, open(path, 'rb')) for (offs, path) in flash_files]
|
||||
encrypt_files = [(offs, open(path, 'rb')) for (offs, path) in encrypt_files]
|
||||
|
||||
if erase_nvs:
|
||||
address = self.app.partition_table['nvs']['offset']
|
||||
size = self.app.partition_table['nvs']['size']
|
||||
nvs_file = tempfile.TemporaryFile()
|
||||
nvs_file.write(b'\xff' * size)
|
||||
nvs_file.seek(0)
|
||||
if not isinstance(address, int):
|
||||
address = int(address, 0)
|
||||
# We have to check whether this file needs to be added to
|
||||
# flash_files list or encrypt_files.
|
||||
# Get the CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT macro
|
||||
# value. If it is set to True, then NVS is always encrypted.
|
||||
sdkconfig_dict = self.app.get_sdkconfig()
|
||||
macro_encryption = 'CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT' in sdkconfig_dict
|
||||
# If the macro is not enabled (plain text flash) or all files
|
||||
# must be encrypted, add NVS to flash_files.
|
||||
if not macro_encryption or encrypt:
|
||||
flash_files.append((address, nvs_file))
|
||||
else:
|
||||
encrypt_files.append((address, nvs_file))
|
||||
|
||||
self.write_flash_data(flash_files, encrypt_files, False, encrypt)
|
||||
finally:
|
||||
for (_, f) in flash_files:
|
||||
f.close()
|
||||
for (_, f) in encrypt_files:
|
||||
f.close()
|
||||
|
||||
@_uses_esptool
|
||||
def write_flash_data(self, esp, flash_files=None, encrypt_files=None, ignore_flash_encryption_efuse_setting=True, encrypt=False):
|
||||
"""
|
||||
Try flashing at a particular baud rate.
|
||||
|
||||
Structured this way so @_uses_esptool will reconnect each time
|
||||
:return: None
|
||||
"""
|
||||
last_error = None
|
||||
for baud_rate in [921600, 115200]:
|
||||
try:
|
||||
# fake flasher args object, this is a hack until
|
||||
# esptool Python API is improved
|
||||
class FlashArgs(object):
|
||||
def __init__(self, attributes):
|
||||
for key, value in attributes.items():
|
||||
self.__setattr__(key, value)
|
||||
|
||||
# write_flash expects the parameter encrypt_files to be None and not
|
||||
# an empty list, so perform the check here
|
||||
flash_args = FlashArgs({
|
||||
'flash_size': self.app.flash_settings['flash_size'],
|
||||
'flash_mode': self.app.flash_settings['flash_mode'],
|
||||
'flash_freq': self.app.flash_settings['flash_freq'],
|
||||
'addr_filename': flash_files or None,
|
||||
'encrypt_files': encrypt_files or None,
|
||||
'no_stub': self.secure_boot_en,
|
||||
'compress': not self.secure_boot_en,
|
||||
'verify': False,
|
||||
'encrypt': encrypt,
|
||||
'ignore_flash_encryption_efuse_setting': ignore_flash_encryption_efuse_setting,
|
||||
'erase_all': False,
|
||||
'after': 'no_reset',
|
||||
'force': False,
|
||||
'chip': esp.CHIP_NAME.lower().replace('-', ''),
|
||||
})
|
||||
|
||||
esp.change_baud(baud_rate)
|
||||
esptool.detect_flash_size(esp, flash_args)
|
||||
esptool.write_flash(esp, flash_args)
|
||||
break
|
||||
except RuntimeError as e:
|
||||
last_error = e
|
||||
else:
|
||||
raise last_error
|
||||
|
||||
def image_info(self, path_to_file):
|
||||
"""
|
||||
get hash256 of app
|
||||
|
||||
:param: path: path to file
|
||||
:return: sha256 appended to app
|
||||
"""
|
||||
|
||||
old_stdout = sys.stdout
|
||||
new_stdout = io.StringIO()
|
||||
sys.stdout = new_stdout
|
||||
|
||||
class Args(object):
|
||||
def __init__(self, attributes):
|
||||
for key, value in attributes.items():
|
||||
self.__setattr__(key, value)
|
||||
|
||||
args = Args({
|
||||
'chip': self.TARGET,
|
||||
'filename': path_to_file,
|
||||
})
|
||||
esptool.image_info(args)
|
||||
output = new_stdout.getvalue()
|
||||
sys.stdout = old_stdout
|
||||
return output
|
||||
|
||||
def start_app(self, erase_nvs=ERASE_NVS):
|
||||
"""
|
||||
download and start app.
|
||||
|
||||
:param: erase_nvs: whether erase NVS partition during flash
|
||||
:return: None
|
||||
"""
|
||||
self._try_flash(erase_nvs)
|
||||
|
||||
def start_app_no_enc(self):
|
||||
"""
|
||||
download and start app.
|
||||
|
||||
:param: erase_nvs: whether erase NVS partition during flash
|
||||
:return: None
|
||||
"""
|
||||
flash_files = self.app.flash_files + self.app.encrypt_files
|
||||
self.write_flash(flash_files)
|
||||
|
||||
def write_flash(self, flash_files=None, encrypt_files=None, ignore_flash_encryption_efuse_setting=True, encrypt=False):
|
||||
"""
|
||||
Flash files
|
||||
|
||||
:return: None
|
||||
"""
|
||||
flash_offs_files = []
|
||||
encrypt_offs_files = []
|
||||
try:
|
||||
if flash_files:
|
||||
flash_offs_files = [(offs, open(path, 'rb')) for (offs, path) in flash_files]
|
||||
|
||||
if encrypt_files:
|
||||
encrypt_offs_files = [(offs, open(path, 'rb')) for (offs, path) in encrypt_files]
|
||||
|
||||
self.write_flash_data(flash_offs_files, encrypt_offs_files, ignore_flash_encryption_efuse_setting, encrypt)
|
||||
finally:
|
||||
for (_, f) in flash_offs_files:
|
||||
f.close()
|
||||
for (_, f) in encrypt_offs_files:
|
||||
f.close()
|
||||
|
||||
def bootloader_flash(self):
|
||||
"""
|
||||
download bootloader.
|
||||
|
||||
:return: None
|
||||
"""
|
||||
bootloader_path = os.path.join(self.app.binary_path, 'bootloader', 'bootloader.bin')
|
||||
offs = int(self.app.get_sdkconfig()['CONFIG_BOOTLOADER_OFFSET_IN_FLASH'], 0)
|
||||
flash_files = [(offs, bootloader_path)]
|
||||
self.write_flash(flash_files)
|
||||
|
||||
@_uses_esptool
|
||||
def reset(self, esp):
|
||||
"""
|
||||
hard reset DUT
|
||||
|
||||
:return: None
|
||||
"""
|
||||
# decorator `_use_esptool` will do reset
|
||||
# so we don't need to do anything in this method
|
||||
pass
|
||||
|
||||
@_uses_esptool
|
||||
def erase_partition(self, esp, partition):
|
||||
"""
|
||||
:param partition: partition name to erase
|
||||
:return: None
|
||||
"""
|
||||
address = self.app.partition_table[partition]['offset']
|
||||
size = self.app.partition_table[partition]['size']
|
||||
esp.erase_region(address, size)
|
||||
|
||||
@_uses_esptool
|
||||
def erase_flash(self, esp):
|
||||
"""
|
||||
erase the flash completely
|
||||
|
||||
:return: None
|
||||
"""
|
||||
esp.erase_flash()
|
||||
|
||||
@_uses_esptool
|
||||
def dump_flash(self, esp, output_file, **kwargs):
|
||||
"""
|
||||
dump flash
|
||||
|
||||
:param output_file: output file name, if relative path, will use sdk path as base path.
|
||||
:keyword partition: partition name, dump the partition.
|
||||
``partition`` is preferred than using ``address`` and ``size``.
|
||||
:keyword address: dump from address (need to be used with size)
|
||||
:keyword size: dump size (need to be used with address)
|
||||
:return: None
|
||||
"""
|
||||
if os.path.isabs(output_file) is False:
|
||||
output_file = os.path.relpath(output_file, self.app.get_log_folder())
|
||||
if 'partition' in kwargs:
|
||||
partition = self.app.partition_table[kwargs['partition']]
|
||||
_address = partition['offset']
|
||||
_size = partition['size']
|
||||
elif 'address' in kwargs and 'size' in kwargs:
|
||||
_address = kwargs['address']
|
||||
_size = kwargs['size']
|
||||
else:
|
||||
raise IDFToolError("You must specify 'partition' or ('address' and 'size') to dump flash")
|
||||
|
||||
content = esp.read_flash(_address, _size)
|
||||
with open(output_file, 'wb') as f:
|
||||
f.write(content)
|
||||
|
||||
@staticmethod
|
||||
def _sort_usb_ports(ports):
|
||||
"""
|
||||
Move the usb ports to the very beginning
|
||||
:param ports: list of ports
|
||||
:return: list of ports with usb ports at beginning
|
||||
"""
|
||||
usb_ports = []
|
||||
rest_ports = []
|
||||
for port in ports:
|
||||
if 'usb' in port.lower():
|
||||
usb_ports.append(port)
|
||||
else:
|
||||
rest_ports.append(port)
|
||||
return usb_ports + rest_ports
|
||||
|
||||
@classmethod
|
||||
def list_available_ports(cls):
|
||||
# It will return other kinds of ports as well, such as ttyS* ports.
|
||||
# Give the usb ports higher priority
|
||||
ports = cls._sort_usb_ports([x.device for x in list_ports.comports()])
|
||||
espport = os.getenv('ESPPORT')
|
||||
if not espport:
|
||||
# It's a little hard filter out invalid port with `serial.tools.list_ports.grep()`:
|
||||
# The check condition in `grep` is: `if r.search(port) or r.search(desc) or r.search(hwid)`.
|
||||
# This means we need to make all 3 conditions fail, to filter out the port.
|
||||
# So some part of the filters will not be straight forward to users.
|
||||
# And negative regular expression (`^((?!aa|bb|cc).)*$`) is not easy to understand.
|
||||
# Filter out invalid port by our own will be much simpler.
|
||||
return [x for x in ports if not cls.INVALID_PORT_PATTERN.search(x)]
|
||||
|
||||
# On MacOs with python3.6: type of espport is already utf8
|
||||
if isinstance(espport, type(u'')):
|
||||
port_hint = espport
|
||||
else:
|
||||
port_hint = espport.decode('utf8')
|
||||
|
||||
# If $ESPPORT is a valid port, make it appear first in the list
|
||||
if port_hint in ports:
|
||||
ports.remove(port_hint)
|
||||
return [port_hint] + ports
|
||||
|
||||
# On macOS, user may set ESPPORT to /dev/tty.xxx while
|
||||
# pySerial lists only the corresponding /dev/cu.xxx port
|
||||
if sys.platform == 'darwin' and 'tty.' in port_hint:
|
||||
port_hint = port_hint.replace('tty.', 'cu.')
|
||||
if port_hint in ports:
|
||||
ports.remove(port_hint)
|
||||
return [port_hint] + ports
|
||||
|
||||
return ports
|
||||
|
||||
def lookup_pc_address(self, pc_addr):
|
||||
cmd = ['%saddr2line' % self.TOOLCHAIN_PREFIX,
|
||||
'-pfiaC', '-e', self.app.elf_file, pc_addr]
|
||||
ret = ''
|
||||
try:
|
||||
translation = subprocess.check_output(cmd)
|
||||
ret = translation.decode()
|
||||
except OSError:
|
||||
pass
|
||||
return ret
|
||||
|
||||
@staticmethod
|
||||
def _queue_read_all(source_queue):
|
||||
output = []
|
||||
while True:
|
||||
try:
|
||||
output.append(source_queue.get(timeout=0))
|
||||
except _queue.Empty:
|
||||
break
|
||||
return output
|
||||
|
||||
def _queue_copy(self, source_queue, dest_queue):
|
||||
data = self._queue_read_all(source_queue)
|
||||
for d in data:
|
||||
dest_queue.put(d)
|
||||
|
||||
def _get_from_queue(self, queue_name):
|
||||
self_queue = getattr(self, queue_name)
|
||||
if self.receive_thread:
|
||||
recv_thread_queue = getattr(self.receive_thread, queue_name)
|
||||
self._queue_copy(recv_thread_queue, self_queue)
|
||||
return self._queue_read_all(self_queue)
|
||||
|
||||
def stop_receive(self):
|
||||
if self.receive_thread:
|
||||
for name in ['performance_items', 'exceptions']:
|
||||
source_queue = getattr(self.receive_thread, name)
|
||||
dest_queue = getattr(self, name)
|
||||
self._queue_copy(source_queue, dest_queue)
|
||||
super(IDFDUT, self).stop_receive()
|
||||
|
||||
def get_exceptions(self):
|
||||
""" Get exceptions detected by DUT receive thread. """
|
||||
return self._get_from_queue('exceptions')
|
||||
|
||||
def get_performance_items(self):
|
||||
"""
|
||||
DUT receive thread will automatic collect performance results with pattern ``[Performance][name]: value\n``.
|
||||
This method is used to get all performance results.
|
||||
|
||||
:return: a list of performance items.
|
||||
"""
|
||||
return self._get_from_queue('performance_items')
|
||||
|
||||
def close(self):
|
||||
super(IDFDUT, self).close()
|
||||
if not self.allow_dut_exception and self.get_exceptions():
|
||||
raise IDFDUTException('DUT exception detected on {}'.format(self))
|
||||
|
||||
|
||||
class ESP32DUT(IDFDUT):
|
||||
TARGET = 'esp32'
|
||||
TOOLCHAIN_PREFIX = 'xtensa-esp32-elf-'
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
return targets.ESP32ROM
|
||||
|
||||
|
||||
class ESP32S2DUT(IDFDUT):
|
||||
TARGET = 'esp32s2'
|
||||
TOOLCHAIN_PREFIX = 'xtensa-esp32s2-elf-'
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
return targets.ESP32S2ROM
|
||||
|
||||
|
||||
class ESP32S3DUT(IDFDUT):
|
||||
TARGET = 'esp32s3'
|
||||
TOOLCHAIN_PREFIX = 'xtensa-esp32s3-elf-'
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
return targets.ESP32S3ROM
|
||||
|
||||
def erase_partition(self, esp, partition):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class ESP32C2DUT(IDFDUT):
|
||||
TARGET = 'esp32c2'
|
||||
TOOLCHAIN_PREFIX = 'riscv32-esp-elf-'
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
return targets.ESP32C2ROM
|
||||
|
||||
|
||||
class ESP32C3DUT(IDFDUT):
|
||||
TARGET = 'esp32c3'
|
||||
TOOLCHAIN_PREFIX = 'riscv32-esp-elf-'
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
return targets.ESP32C3ROM
|
||||
|
||||
|
||||
class ESP32C6DUT(IDFDUT):
|
||||
TARGET = 'esp32c6'
|
||||
TOOLCHAIN_PREFIX = 'riscv32-esp-elf-'
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
return targets.ESP32C6ROM
|
||||
|
||||
|
||||
class ESP32H2DUT(IDFDUT):
|
||||
TARGET = 'esp32h2'
|
||||
TOOLCHAIN_PREFIX = 'riscv32-esp-elf-'
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
return targets.ESP32H2ROM
|
||||
|
||||
|
||||
class ESP8266DUT(IDFDUT):
|
||||
TARGET = 'esp8266'
|
||||
TOOLCHAIN_PREFIX = 'xtensa-lx106-elf-'
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
return targets.ESP8266ROM
|
||||
|
||||
|
||||
def get_target_by_rom_class(cls):
|
||||
for c in [ESP32DUT, ESP32S2DUT, ESP32S3DUT, ESP32C2DUT, ESP32C3DUT, ESP32C6DUT, ESP32H2DUT, ESP8266DUT, IDFQEMUDUT]:
|
||||
if c.get_rom() == cls:
|
||||
return c.TARGET
|
||||
return None
|
||||
|
||||
|
||||
class IDFQEMUDUT(IDFDUT):
|
||||
TARGET = None
|
||||
TOOLCHAIN_PREFIX = None
|
||||
ERASE_NVS = True
|
||||
DEFAULT_EXPECT_TIMEOUT = 30 # longer timeout, since app startup takes more time in QEMU (due to slow SHA emulation)
|
||||
QEMU_SERIAL_PORT = 3334
|
||||
|
||||
def __init__(self, name, port, log_file, app, allow_dut_exception=False, **kwargs):
|
||||
self.flash_image = tempfile.NamedTemporaryFile('rb+', suffix='.bin', prefix='qemu_flash_img')
|
||||
self.app = app
|
||||
self.flash_size = 4 * 1024 * 1024
|
||||
self._write_flash_img()
|
||||
|
||||
args = [
|
||||
'qemu-system-xtensa',
|
||||
'-nographic',
|
||||
'-machine', self.TARGET,
|
||||
'-drive', 'file={},if=mtd,format=raw'.format(self.flash_image.name),
|
||||
'-nic', 'user,model=open_eth',
|
||||
'-serial', 'tcp::{},server,nowait'.format(self.QEMU_SERIAL_PORT),
|
||||
'-S',
|
||||
'-global driver=timer.esp32.timg,property=wdt_disable,value=true']
|
||||
# TODO(IDF-1242): generate a temporary efuse binary, pass it to QEMU
|
||||
|
||||
if 'QEMU_BIOS_PATH' in os.environ:
|
||||
args += ['-L', os.environ['QEMU_BIOS_PATH']]
|
||||
|
||||
self.qemu = pexpect.spawn(' '.join(args), timeout=self.DEFAULT_EXPECT_TIMEOUT)
|
||||
self.qemu.expect_exact(b'(qemu)')
|
||||
super(IDFQEMUDUT, self).__init__(name, port, log_file, app, allow_dut_exception=allow_dut_exception, **kwargs)
|
||||
|
||||
def _write_flash_img(self):
|
||||
self.flash_image.seek(0)
|
||||
self.flash_image.write(b'\x00' * self.flash_size)
|
||||
for offs, path in self.app.flash_files:
|
||||
with open(path, 'rb') as flash_file:
|
||||
contents = flash_file.read()
|
||||
self.flash_image.seek(offs)
|
||||
self.flash_image.write(contents)
|
||||
self.flash_image.flush()
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
return targets.ESP32ROM
|
||||
|
||||
@classmethod
|
||||
def get_mac(cls, app, port):
|
||||
# TODO(IDF-1242): get this from QEMU/efuse binary
|
||||
return '11:22:33:44:55:66'
|
||||
|
||||
@classmethod
|
||||
def confirm_dut(cls, port, **kwargs):
|
||||
return True, cls.TARGET
|
||||
|
||||
def start_app(self, erase_nvs=ERASE_NVS):
|
||||
# TODO: implement erase_nvs
|
||||
# since the flash image is generated every time in the constructor, maybe this isn't needed...
|
||||
self.qemu.sendline(b'cont\n')
|
||||
self.qemu.expect_exact(b'(qemu)')
|
||||
|
||||
def reset(self):
|
||||
self.qemu.sendline(b'system_reset\n')
|
||||
self.qemu.expect_exact(b'(qemu)')
|
||||
|
||||
def erase_partition(self, partition):
|
||||
raise NotImplementedError('method erase_partition not implemented')
|
||||
|
||||
def erase_flash(self):
|
||||
raise NotImplementedError('method erase_flash not implemented')
|
||||
|
||||
def dump_flash(self, output_file, **kwargs):
|
||||
raise NotImplementedError('method dump_flash not implemented')
|
||||
|
||||
@classmethod
|
||||
def list_available_ports(cls):
|
||||
return ['socket://localhost:{}'.format(cls.QEMU_SERIAL_PORT)]
|
||||
|
||||
def close(self):
|
||||
super(IDFQEMUDUT, self).close()
|
||||
self.qemu.sendline(b'q\n')
|
||||
self.qemu.expect_exact(b'(qemu)')
|
||||
for _ in range(self.DEFAULT_EXPECT_TIMEOUT):
|
||||
if not self.qemu.isalive():
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
self.qemu.terminate(force=True)
|
||||
|
||||
|
||||
class ESP32QEMUDUT(IDFQEMUDUT):
|
||||
TARGET = 'esp32' # type: ignore
|
||||
TOOLCHAIN_PREFIX = 'xtensa-esp32-elf-' # type: ignore
|
||||
|
||||
|
||||
class IDFFPGADUT(IDFDUT):
|
||||
TARGET = None # type: str
|
||||
TOOLCHAIN_PREFIX = None # type: str
|
||||
ERASE_NVS = True
|
||||
FLASH_ENCRYPT_SCHEME = None # type: str
|
||||
FLASH_ENCRYPT_CNT_KEY = None # type: str
|
||||
FLASH_ENCRYPT_CNT_VAL = 0
|
||||
FLASH_ENCRYPT_PURPOSE = None # type: str
|
||||
SECURE_BOOT_EN_KEY = None # type: str
|
||||
SECURE_BOOT_EN_VAL = 0
|
||||
FLASH_SECTOR_SIZE = 4096
|
||||
|
||||
def __init__(self, name, port, log_file, app, allow_dut_exception=False, efuse_reset_port=None, **kwargs):
|
||||
super(IDFFPGADUT, self).__init__(name, port, log_file, app, allow_dut_exception=allow_dut_exception, **kwargs)
|
||||
self.esp = self.get_rom()(port)
|
||||
self.efuses = None
|
||||
self.efuse_operations = None
|
||||
self.efuse_reset_port = efuse_reset_port
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
raise NotImplementedError('This is an abstraction class, method not defined.')
|
||||
|
||||
def erase_partition(self, esp, partition):
|
||||
raise NotImplementedError()
|
||||
|
||||
def enable_efuses(self):
|
||||
# We use an extra COM port to reset the efuses on FPGA.
|
||||
# Connect DTR pin of the COM port to the efuse reset pin on daughter board
|
||||
# Set EFUSEPORT env variable to the extra COM port
|
||||
if not self.efuse_reset_port:
|
||||
raise RuntimeError('EFUSEPORT not specified')
|
||||
|
||||
# Stop any previous serial port operation
|
||||
self.stop_receive()
|
||||
if self.secure_boot_en:
|
||||
self.esp.connect()
|
||||
self.efuses, self.efuse_operations = espefuse.get_efuses(self.esp, False, False, True)
|
||||
|
||||
def burn_efuse(self, field, val):
|
||||
if not self.efuse_operations:
|
||||
self.enable_efuses()
|
||||
BurnEfuseArgs = collections.namedtuple('burn_efuse_args', ['name_value_pairs'])
|
||||
args = BurnEfuseArgs({field: val})
|
||||
self.efuse_operations.burn_efuse(self.esp, self.efuses, args)
|
||||
|
||||
def burn_efuse_key(self, key, purpose, block):
|
||||
if not self.efuse_operations:
|
||||
self.enable_efuses()
|
||||
BurnKeyArgs = collections.namedtuple('burn_key_args',
|
||||
['keyfile', 'keypurpose', 'block',
|
||||
'force_write_always', 'no_write_protect', 'no_read_protect'])
|
||||
args = BurnKeyArgs([key],
|
||||
[purpose],
|
||||
[block],
|
||||
False, False, False)
|
||||
self.efuse_operations.burn_key(self.esp, self.efuses, args)
|
||||
|
||||
def burn_efuse_key_digest(self, key, purpose, block):
|
||||
if not self.efuse_operations:
|
||||
self.enable_efuses()
|
||||
BurnDigestArgs = collections.namedtuple('burn_key_digest_args',
|
||||
['keyfile', 'keypurpose', 'block',
|
||||
'force_write_always', 'no_write_protect', 'no_read_protect'])
|
||||
args = BurnDigestArgs([open(key, 'rb')],
|
||||
[purpose],
|
||||
[block],
|
||||
False, False, True)
|
||||
self.efuse_operations.burn_key_digest(self.esp, self.efuses, args)
|
||||
|
||||
def reset_efuses(self):
|
||||
if not self.efuse_reset_port:
|
||||
raise RuntimeError('EFUSEPORT not specified')
|
||||
with serial.Serial(self.efuse_reset_port) as efuseport:
|
||||
print('Resetting efuses')
|
||||
efuseport.dtr = 0
|
||||
self.port_inst.setRTS(1)
|
||||
self.port_inst.setRTS(0)
|
||||
time.sleep(1)
|
||||
efuseport.dtr = 1
|
||||
self.efuse_operations = None
|
||||
self.efuses = None
|
||||
|
||||
def sign_data(self, data_file, key_files, version, append_signature=0):
|
||||
SignDataArgs = collections.namedtuple('sign_data_args',
|
||||
['datafile','keyfile','output', 'version', 'append_signatures'])
|
||||
outfile = tempfile.NamedTemporaryFile()
|
||||
args = SignDataArgs(data_file, key_files, outfile.name, str(version), append_signature)
|
||||
espsecure.sign_data(args)
|
||||
outfile.seek(0)
|
||||
return outfile.read()
|
||||
|
||||
|
||||
class ESP32C3FPGADUT(IDFFPGADUT):
|
||||
TARGET = 'esp32c3'
|
||||
TOOLCHAIN_PREFIX = 'riscv32-esp-elf-'
|
||||
FLASH_ENCRYPT_SCHEME = 'AES-XTS'
|
||||
FLASH_ENCRYPT_CNT_KEY = 'SPI_BOOT_CRYPT_CNT'
|
||||
FLASH_ENCRYPT_CNT_VAL = 1
|
||||
FLASH_ENCRYPT_PURPOSE = 'XTS_AES_128_KEY'
|
||||
SECURE_BOOT_EN_KEY = 'SECURE_BOOT_EN'
|
||||
SECURE_BOOT_EN_VAL = 1
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
return targets.ESP32C3ROM
|
||||
|
||||
def erase_partition(self, esp, partition):
|
||||
raise NotImplementedError()
|
||||
|
||||
def flash_encrypt_burn_cnt(self):
|
||||
self.burn_efuse(self.FLASH_ENCRYPT_CNT_KEY, self.FLASH_ENCRYPT_CNT_VAL)
|
||||
|
||||
def flash_encrypt_burn_key(self, key, block=0):
|
||||
self.burn_efuse_key(key, self.FLASH_ENCRYPT_PURPOSE, 'BLOCK_KEY%d' % block)
|
||||
|
||||
def flash_encrypt_get_scheme(self):
|
||||
return self.FLASH_ENCRYPT_SCHEME
|
||||
|
||||
def secure_boot_burn_en_bit(self):
|
||||
self.burn_efuse(self.SECURE_BOOT_EN_KEY, self.SECURE_BOOT_EN_VAL)
|
||||
|
||||
def secure_boot_burn_digest(self, digest, key_index=0, block=0):
|
||||
self.burn_efuse_key_digest(digest, 'SECURE_BOOT_DIGEST%d' % key_index, 'BLOCK_KEY%d' % block)
|
||||
|
||||
@classmethod
|
||||
def confirm_dut(cls, port, **kwargs):
|
||||
return True, cls.TARGET
|
||||
|
||||
|
||||
class ESP32S3FPGADUT(IDFFPGADUT):
|
||||
TARGET = 'esp32s3'
|
||||
TOOLCHAIN_PREFIX = 'xtensa-esp32s3-elf-'
|
||||
FLASH_ENCRYPT_SCHEME = 'AES-XTS'
|
||||
FLASH_ENCRYPT_CNT_KEY = 'SPI_BOOT_CRYPT_CNT'
|
||||
FLASH_ENCRYPT_CNT_VAL = 1
|
||||
FLASH_ENCRYPT_PURPOSE = 'XTS_AES_128_KEY'
|
||||
SECURE_BOOT_EN_KEY = 'SECURE_BOOT_EN'
|
||||
SECURE_BOOT_EN_VAL = 1
|
||||
|
||||
@classmethod
|
||||
def get_rom(cls):
|
||||
return targets.ESP32S3ROM
|
||||
|
||||
def erase_partition(self, esp, partition):
|
||||
raise NotImplementedError()
|
||||
|
||||
def flash_encrypt_burn_cnt(self):
|
||||
self.burn_efuse(self.FLASH_ENCRYPT_CNT_KEY, self.FLASH_ENCRYPT_CNT_VAL)
|
||||
|
||||
def flash_encrypt_burn_key(self, key, block=0):
|
||||
self.burn_efuse_key(key, self.FLASH_ENCRYPT_PURPOSE, 'BLOCK_KEY%d' % block)
|
||||
|
||||
def flash_encrypt_get_scheme(self):
|
||||
return self.FLASH_ENCRYPT_SCHEME
|
||||
|
||||
def secure_boot_burn_en_bit(self):
|
||||
self.burn_efuse(self.SECURE_BOOT_EN_KEY, self.SECURE_BOOT_EN_VAL)
|
||||
|
||||
def secure_boot_burn_digest(self, digest, key_index=0, block=0):
|
||||
self.burn_efuse_key_digest(digest, 'SECURE_BOOT_DIGEST%d' % key_index, 'BLOCK_KEY%d' % block)
|
||||
|
||||
@classmethod
|
||||
def confirm_dut(cls, port, **kwargs):
|
||||
return True, cls.TARGET
|
||||
@@ -1,312 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import functools
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from copy import deepcopy
|
||||
|
||||
import junit_xml
|
||||
from tiny_test_fw import TinyFW, Utility
|
||||
|
||||
from .IDFApp import UT, ComponentUTApp, Example, IDFApp, LoadableElfTestApp, TestApp # noqa: export all Apps for users
|
||||
from .IDFDUT import (ESP32C2DUT, ESP32C3DUT, ESP32C3FPGADUT, ESP32C6DUT, ESP32DUT, # noqa: export DUTs for users
|
||||
ESP32H2DUT, ESP32QEMUDUT, ESP32S2DUT, ESP32S3DUT, ESP32S3FPGADUT, ESP8266DUT, IDFDUT)
|
||||
from .unity_test_parser import TestFormat, TestResults
|
||||
|
||||
# pass TARGET_DUT_CLS_DICT to Env.py to avoid circular dependency issue.
|
||||
TARGET_DUT_CLS_DICT = {
|
||||
'ESP32': ESP32DUT,
|
||||
'ESP32S2': ESP32S2DUT,
|
||||
'ESP32S3': ESP32S3DUT,
|
||||
'ESP32C2': ESP32C2DUT,
|
||||
'ESP32C3': ESP32C3DUT,
|
||||
'ESP32C3FPGA': ESP32C3FPGADUT,
|
||||
'ESP32S3FPGA': ESP32S3FPGADUT,
|
||||
'ESP32C6': ESP32C6DUT,
|
||||
'ESP32H2': ESP32H2DUT,
|
||||
}
|
||||
|
||||
|
||||
try:
|
||||
string_type = basestring # type: ignore
|
||||
except NameError:
|
||||
string_type = str
|
||||
|
||||
|
||||
def upper_list_or_str(text):
|
||||
"""
|
||||
Return the uppercase of list of string or string. Return itself for other
|
||||
data types
|
||||
:param text: list or string, other instance will be returned immediately
|
||||
:return: uppercase of list of string
|
||||
"""
|
||||
if isinstance(text, string_type):
|
||||
return [text.upper()]
|
||||
elif isinstance(text, list):
|
||||
return [item.upper() for item in text]
|
||||
else:
|
||||
return text
|
||||
|
||||
|
||||
def local_test_check(decorator_target):
|
||||
# Try to get the sdkconfig.json to read the IDF_TARGET value.
|
||||
# If not set, will set to ESP32.
|
||||
# For CI jobs, this is a fake procedure, the true target and dut will be
|
||||
# overwritten by the job config YAML file.
|
||||
idf_target = 'ESP32' # default if sdkconfig not found or not readable
|
||||
if os.getenv('CI_JOB_ID'): # Only auto-detect target when running locally
|
||||
return idf_target
|
||||
|
||||
decorator_target = upper_list_or_str(decorator_target)
|
||||
expected_json_path = os.path.join('build', 'config', 'sdkconfig.json')
|
||||
if os.path.exists(expected_json_path):
|
||||
sdkconfig = json.load(open(expected_json_path))
|
||||
try:
|
||||
idf_target = sdkconfig['IDF_TARGET'].upper()
|
||||
except KeyError:
|
||||
logging.debug('IDF_TARGET not in {}. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path)))
|
||||
else:
|
||||
logging.debug('IDF_TARGET: {}'.format(idf_target))
|
||||
else:
|
||||
logging.debug('{} not found. IDF_TARGET set to esp32'.format(os.path.abspath(expected_json_path)))
|
||||
|
||||
if isinstance(decorator_target, list):
|
||||
if idf_target not in decorator_target:
|
||||
fpga_target = ''.join((idf_target, 'FPGA'))
|
||||
if fpga_target not in decorator_target:
|
||||
raise ValueError('IDF_TARGET set to {}, not in decorator target value'.format(idf_target))
|
||||
else:
|
||||
idf_target = fpga_target
|
||||
else:
|
||||
if idf_target != decorator_target:
|
||||
raise ValueError('IDF_TARGET set to {}, not equal to decorator target value'.format(idf_target))
|
||||
return idf_target
|
||||
|
||||
|
||||
def get_dut_class(target, dut_class_dict, erase_nvs=None):
|
||||
if target not in dut_class_dict:
|
||||
raise Exception('target can only be {%s} (case insensitive)' % ', '.join(dut_class_dict.keys()))
|
||||
|
||||
dut = dut_class_dict[target.upper()]
|
||||
if erase_nvs:
|
||||
dut.ERASE_NVS = 'erase_nvs'
|
||||
return dut
|
||||
|
||||
|
||||
def ci_target_check(func):
|
||||
@functools.wraps(func)
|
||||
def wrapper(**kwargs):
|
||||
target = upper_list_or_str(kwargs.get('target', []))
|
||||
ci_target = upper_list_or_str(kwargs.get('ci_target', []))
|
||||
if not set(ci_target).issubset(set(target)):
|
||||
raise ValueError('ci_target must be a subset of target')
|
||||
|
||||
return func(**kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, nightly_run, **kwargs):
|
||||
target = upper_list_or_str(target)
|
||||
test_target = local_test_check(target)
|
||||
if 'additional_duts' in kwargs:
|
||||
dut_classes = deepcopy(TARGET_DUT_CLS_DICT)
|
||||
dut_classes.update(kwargs['additional_duts'])
|
||||
else:
|
||||
dut_classes = TARGET_DUT_CLS_DICT
|
||||
dut = get_dut_class(test_target, dut_classes, erase_nvs)
|
||||
original_method = TinyFW.test_method(
|
||||
app=app, dut=dut, target=target, ci_target=upper_list_or_str(ci_target),
|
||||
module=module, execution_time=execution_time, level=level, erase_nvs=erase_nvs,
|
||||
dut_dict=dut_classes, nightly_run=nightly_run, **kwargs
|
||||
)
|
||||
test_func = original_method(func)
|
||||
return test_func
|
||||
|
||||
|
||||
@ci_target_check
|
||||
def idf_example_test(app=Example, target='ESP32', ci_target=None, module='examples', execution_time=1,
|
||||
level='example', erase_nvs=True, config_name=None, nightly_run=False, **kwargs):
|
||||
"""
|
||||
decorator for testing idf examples (with default values for some keyword args).
|
||||
|
||||
:param app: test application class
|
||||
:param target: target supported, string or list
|
||||
:param ci_target: target auto run in CI, if None than all target will be tested, None, string or list
|
||||
:param module: module, string
|
||||
:param execution_time: execution time in minutes, int
|
||||
:param level: test level, could be used to filter test cases, string
|
||||
:param erase_nvs: if need to erase_nvs in DUT.start_app()
|
||||
:param config_name: if specified, name of the app configuration
|
||||
:param kwargs: other keyword args
|
||||
:return: test method
|
||||
"""
|
||||
def test(func):
|
||||
return test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, nightly_run,
|
||||
**kwargs)
|
||||
return test
|
||||
|
||||
|
||||
@ci_target_check
|
||||
def idf_unit_test(app=UT, target='ESP32', ci_target=None, module='unit-test', execution_time=1,
|
||||
level='unit', erase_nvs=True, nightly_run=False, **kwargs):
|
||||
"""
|
||||
decorator for testing idf unit tests (with default values for some keyword args).
|
||||
|
||||
:param app: test application class
|
||||
:param target: target supported, string or list
|
||||
:param ci_target: target auto run in CI, if None than all target will be tested, None, string or list
|
||||
:param module: module, string
|
||||
:param execution_time: execution time in minutes, int
|
||||
:param level: test level, could be used to filter test cases, string
|
||||
:param erase_nvs: if need to erase_nvs in DUT.start_app()
|
||||
:param kwargs: other keyword args
|
||||
:return: test method
|
||||
"""
|
||||
|
||||
def test(func):
|
||||
return test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, nightly_run,
|
||||
**kwargs)
|
||||
return test
|
||||
|
||||
|
||||
@ci_target_check
|
||||
def idf_custom_test(app=TestApp, target='ESP32', ci_target=None, module='misc', execution_time=1,
|
||||
level='integration', erase_nvs=True, config_name=None, nightly_run=False, **kwargs):
|
||||
"""
|
||||
decorator for idf custom tests (with default values for some keyword args).
|
||||
|
||||
:param app: test application class
|
||||
:param target: target supported, string or list
|
||||
:param ci_target: target auto run in CI, if None than all target will be tested, None, string or list
|
||||
:param module: module, string
|
||||
:param execution_time: execution time in minutes, int
|
||||
:param level: test level, could be used to filter test cases, string
|
||||
:param erase_nvs: if need to erase_nvs in DUT.start_app()
|
||||
:param config_name: if specified, name of the app configuration
|
||||
:param kwargs: other keyword args
|
||||
:return: test method
|
||||
"""
|
||||
|
||||
def test(func):
|
||||
return test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, nightly_run,
|
||||
**kwargs)
|
||||
return test
|
||||
|
||||
|
||||
@ci_target_check
|
||||
def idf_component_unit_test(app=ComponentUTApp, target='ESP32', ci_target=None, module='misc', execution_time=1,
|
||||
level='integration', erase_nvs=True, config_name=None, nightly_run=False, **kwargs):
|
||||
"""
|
||||
decorator for idf custom tests (with default values for some keyword args).
|
||||
|
||||
:param app: test application class
|
||||
:param target: target supported, string or list
|
||||
:param ci_target: target auto run in CI, if None than all target will be tested, None, string or list
|
||||
:param module: module, string
|
||||
:param execution_time: execution time in minutes, int
|
||||
:param level: test level, could be used to filter test cases, string
|
||||
:param erase_nvs: if need to erase_nvs in DUT.start_app()
|
||||
:param config_name: if specified, name of the app configuration
|
||||
:param kwargs: other keyword args
|
||||
:return: test method
|
||||
"""
|
||||
|
||||
def test(func):
|
||||
return test_func_generator(func, app, target, ci_target, module, execution_time, level, erase_nvs, nightly_run,
|
||||
**kwargs)
|
||||
|
||||
return test
|
||||
|
||||
|
||||
class ComponentUTResult:
|
||||
"""
|
||||
Function Class, parse component unit test results
|
||||
"""
|
||||
|
||||
results_list = defaultdict(list) # type: dict[str, list[junit_xml.TestSuite]]
|
||||
|
||||
"""
|
||||
For origin unity test cases with macro "TEST", please set "test_format" to "TestFormat.UNITY_FIXTURE_VERBOSE".
|
||||
For IDF unity test cases with macro "TEST CASE", please set "test_format" to "TestFormat.UNITY_BASIC".
|
||||
"""
|
||||
@staticmethod
|
||||
def parse_result(stdout, test_format=TestFormat.UNITY_FIXTURE_VERBOSE):
|
||||
try:
|
||||
results = TestResults(stdout, test_format)
|
||||
except (ValueError, TypeError) as e:
|
||||
raise ValueError('Error occurs when parsing the component unit test stdout to JUnit report: ' + str(e))
|
||||
|
||||
group_name = results.tests()[0].group()
|
||||
ComponentUTResult.results_list[group_name].append(results.to_junit())
|
||||
with open(os.path.join(os.getenv('LOG_PATH', ''), '{}_XUNIT_RESULT.xml'.format(group_name)), 'w') as fw:
|
||||
junit_xml.to_xml_report_file(fw, ComponentUTResult.results_list[group_name])
|
||||
|
||||
if results.num_failed():
|
||||
# raise exception if any case fails
|
||||
err_msg = 'Failed Cases:\n'
|
||||
for test_case in results.test_iter():
|
||||
if test_case.result() == 'FAIL':
|
||||
err_msg += '\t{}: {}'.format(test_case.name(), test_case.message())
|
||||
raise AssertionError(err_msg)
|
||||
|
||||
|
||||
def log_performance(item, value):
|
||||
"""
|
||||
do print performance with pre-defined format to console
|
||||
|
||||
:param item: performance item name
|
||||
:param value: performance value
|
||||
"""
|
||||
performance_msg = '[Performance][{}]: {}'.format(item, value)
|
||||
Utility.console_log(performance_msg, 'orange')
|
||||
# update to junit test report
|
||||
current_junit_case = TinyFW.JunitReport.get_current_test_case()
|
||||
current_junit_case.stdout += performance_msg + '\r\n'
|
||||
|
||||
|
||||
def check_performance(item, value, target):
|
||||
"""
|
||||
check if idf performance meet pass standard
|
||||
|
||||
:param item: performance item name
|
||||
:param value: performance item value
|
||||
:param target: target chip
|
||||
:raise: AssertionError: if check fails
|
||||
"""
|
||||
|
||||
def _find_perf_item(path):
|
||||
with open(path, 'r') as f:
|
||||
data = f.read()
|
||||
match = re.search(r'#define\s+IDF_PERFORMANCE_(MIN|MAX)_{}\s+([\d.]+)'.format(item.upper()), data)
|
||||
return match.group(1), float(match.group(2))
|
||||
|
||||
def _check_perf(op, standard_value):
|
||||
if op == 'MAX':
|
||||
ret = value <= standard_value
|
||||
else:
|
||||
ret = value >= standard_value
|
||||
if not ret:
|
||||
raise AssertionError("[Performance] {} value is {}, doesn't meet pass standard {}"
|
||||
.format(item, value, standard_value))
|
||||
|
||||
path_prefix = os.path.join(IDFApp.get_sdk_path(), 'components', 'idf_test', 'include')
|
||||
performance_files = (os.path.join(path_prefix, target, 'idf_performance_target.h'),
|
||||
os.path.join(path_prefix, 'idf_performance.h'))
|
||||
|
||||
for performance_file in performance_files:
|
||||
try:
|
||||
op, standard = _find_perf_item(performance_file)
|
||||
except (IOError, AttributeError):
|
||||
# performance file doesn't exist or match is not found in it
|
||||
continue
|
||||
|
||||
_check_perf(op, standard)
|
||||
# if no exception was thrown then the performance is met and no need to continue
|
||||
break
|
||||
else:
|
||||
raise AssertionError('Failed to get performance standard for {}'.format(item))
|
||||
@@ -1,377 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""
|
||||
Modification version of https://github.com/ETCLabs/unity-test-parser/blob/develop/unity_test_parser.py
|
||||
since only python 3.6 or higher version have ``enum.auto()``
|
||||
|
||||
unity_test_parser.py
|
||||
|
||||
Parse the output of the Unity Test Framework for C. Parsed results are held in the TestResults
|
||||
object format, which can then be converted to various XML formats.
|
||||
"""
|
||||
|
||||
import enum
|
||||
import re
|
||||
|
||||
import junit_xml
|
||||
|
||||
_NORMAL_TEST_REGEX = re.compile(r'(?P<file>.+):(?P<line>\d+):(?P<test_name>[^\s:]+):(?P<result>PASS|FAIL|IGNORE)(?:: (?P<message>.+))?')
|
||||
_UNITY_FIXTURE_VERBOSE_PREFIX_REGEX = re.compile(r'(?P<prefix>TEST\((?P<test_group>[^\s,]+), (?P<test_name>[^\s\)]+)\))(?P<remainder>.+)?$')
|
||||
_UNITY_FIXTURE_REMAINDER_REGEX = re.compile(r'^(?P<file>.+):(?P<line>\d+)::(?P<result>PASS|FAIL|IGNORE)(?:: (?P<message>.+))?')
|
||||
_TEST_SUMMARY_BLOCK_REGEX = re.compile(
|
||||
r'^(?P<num_tests>\d+) Tests (?P<num_failures>\d+) Failures (?P<num_ignored>\d+) Ignored\s*\r?\n(?P<overall_result>OK|FAIL)(?:ED)?', re.MULTILINE
|
||||
)
|
||||
_TEST_RESULT_ENUM = ['PASS', 'FAIL', 'IGNORE']
|
||||
|
||||
|
||||
class TestFormat(enum.Enum):
|
||||
"""Represents the flavor of Unity used to produce a given output."""
|
||||
|
||||
UNITY_BASIC = 0
|
||||
# UNITY_FIXTURE = enum.auto()
|
||||
UNITY_FIXTURE_VERBOSE = 1
|
||||
|
||||
|
||||
globals().update(TestFormat.__members__)
|
||||
|
||||
|
||||
class TestStats:
|
||||
"""Statistics about a test collection"""
|
||||
|
||||
def __init__(self):
|
||||
self.total = 0
|
||||
self.passed = 0
|
||||
self.failed = 0
|
||||
self.ignored = 0
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, self.__class__):
|
||||
return (self.total == other.total
|
||||
and self.passed == other.passed
|
||||
and self.failed == other.failed
|
||||
and self.ignored == other.ignored)
|
||||
return False
|
||||
|
||||
|
||||
class TestResult:
|
||||
"""
|
||||
Class representing the result of a single test.
|
||||
|
||||
Contains the test name, its result (either PASS, FAIL or IGNORE), the file and line number if
|
||||
the test result was not PASS, and an optional message.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
test_name,
|
||||
result,
|
||||
group='default',
|
||||
file='',
|
||||
line=0,
|
||||
message='',
|
||||
full_line='',
|
||||
):
|
||||
if result not in _TEST_RESULT_ENUM:
|
||||
raise ValueError('result must be one of {}.'.format(_TEST_RESULT_ENUM))
|
||||
|
||||
self._test_name = test_name
|
||||
self._result = result
|
||||
self._group = group
|
||||
self._message = message
|
||||
self._full_line = full_line
|
||||
|
||||
if result != 'PASS':
|
||||
self._file = file
|
||||
self._line = line
|
||||
else:
|
||||
self._file = ''
|
||||
self._line = 0
|
||||
|
||||
def file(self):
|
||||
"""The file name - returns empty string if the result is PASS."""
|
||||
return self._file
|
||||
|
||||
def line(self):
|
||||
"""The line number - returns 0 if the result is PASS."""
|
||||
return self._line
|
||||
|
||||
def name(self):
|
||||
"""The test name."""
|
||||
return self._test_name
|
||||
|
||||
def result(self):
|
||||
"""The test result, one of PASS, FAIL or IGNORED."""
|
||||
return self._result
|
||||
|
||||
def group(self):
|
||||
"""
|
||||
The test group, if applicable.
|
||||
|
||||
For basic Unity output, this will always be "default".
|
||||
"""
|
||||
return self._group
|
||||
|
||||
def message(self):
|
||||
"""The accompanying message - returns empty string if the result is PASS."""
|
||||
return self._message
|
||||
|
||||
def full_line(self):
|
||||
"""The original, full line of unit test output that this object was created from."""
|
||||
return self._full_line
|
||||
|
||||
|
||||
class TestResults:
|
||||
"""
|
||||
Class representing Unity test results.
|
||||
|
||||
After being initialized with raw test output, it parses the output and represents it as a list
|
||||
of TestResult objects which can be inspected or converted to other types of output, e.g. JUnit
|
||||
XML.
|
||||
"""
|
||||
|
||||
def __init__(self, test_output, test_format=TestFormat.UNITY_BASIC):
|
||||
"""
|
||||
Create a new TestResults object from Unity test output.
|
||||
|
||||
Keyword arguments:
|
||||
test_output -- The full test console output, must contain the overall result and summary
|
||||
block at the bottom.
|
||||
|
||||
Optional arguments:
|
||||
test_format -- TestFormat enum representing the flavor of Unity used to create the output.
|
||||
|
||||
Exceptions:
|
||||
ValueError, if the test output is not formatted properly.
|
||||
"""
|
||||
self._tests = []
|
||||
self._test_stats = self._find_summary_block(test_output)
|
||||
|
||||
if test_format is TestFormat.UNITY_BASIC:
|
||||
self._parse_unity_basic(test_output)
|
||||
elif test_format is TestFormat.UNITY_FIXTURE_VERBOSE:
|
||||
self._parse_unity_fixture_verbose(test_output)
|
||||
else:
|
||||
raise ValueError(
|
||||
'test_format must be one of UNITY_BASIC or UNITY_FIXTURE_VERBOSE.'
|
||||
)
|
||||
|
||||
def num_tests(self):
|
||||
"""The total number of tests parsed."""
|
||||
return self._test_stats.total
|
||||
|
||||
def num_passed(self):
|
||||
"""The number of tests with result PASS."""
|
||||
return self._test_stats.passed
|
||||
|
||||
def num_failed(self):
|
||||
"""The number of tests with result FAIL."""
|
||||
return self._test_stats.failed
|
||||
|
||||
def num_ignored(self):
|
||||
"""The number of tests with result IGNORE."""
|
||||
return self._test_stats.ignored
|
||||
|
||||
def test_iter(self):
|
||||
"""Get an iterator for iterating over individual tests.
|
||||
|
||||
Returns an iterator over TestResult objects.
|
||||
|
||||
Example:
|
||||
for test in unity_results.test_iter():
|
||||
print(test.name())
|
||||
"""
|
||||
return iter(self._tests)
|
||||
|
||||
def tests(self):
|
||||
"""Get a list of all the tests (TestResult objects)."""
|
||||
return self._tests
|
||||
|
||||
def to_junit(
|
||||
self, suite_name='all_tests',
|
||||
):
|
||||
"""
|
||||
Convert the tests to JUnit XML.
|
||||
|
||||
Returns a junit_xml.TestSuite containing all of the test cases. One test suite will be
|
||||
generated with the name given in suite_name. Unity Fixture test groups are mapped to the
|
||||
classname attribute of test cases; for basic Unity output there will be one class named
|
||||
"default".
|
||||
|
||||
Optional arguments:
|
||||
suite_name -- The name to use for the "name" and "package" attributes of the testsuite element.
|
||||
|
||||
Sample output:
|
||||
<testsuite disabled="0" errors="0" failures="1" name="[suite_name]" package="[suite_name]" skipped="0" tests="8" time="0">
|
||||
<testcase classname="test_group_1" name="group_1_test" />
|
||||
<testcase classname="test_group_2" name="group_2_test" />
|
||||
</testsuite>
|
||||
"""
|
||||
test_case_list = []
|
||||
|
||||
for test in self._tests:
|
||||
if test.result() == 'PASS':
|
||||
test_case_list.append(
|
||||
junit_xml.TestCase(name=test.name(), classname=test.group())
|
||||
)
|
||||
else:
|
||||
junit_tc = junit_xml.TestCase(
|
||||
name=test.name(),
|
||||
classname=test.group(),
|
||||
file=test.file(),
|
||||
line=test.line(),
|
||||
)
|
||||
if test.result() == 'FAIL':
|
||||
junit_tc.add_failure_info(
|
||||
message=test.message(), output=test.full_line()
|
||||
)
|
||||
elif test.result() == 'IGNORE':
|
||||
junit_tc.add_skipped_info(
|
||||
message=test.message(), output=test.full_line()
|
||||
)
|
||||
test_case_list.append(junit_tc)
|
||||
|
||||
return junit_xml.TestSuite(
|
||||
name=suite_name, package=suite_name, test_cases=test_case_list
|
||||
)
|
||||
|
||||
def _find_summary_block(self, unity_output):
|
||||
"""
|
||||
Find and parse the test summary block.
|
||||
|
||||
Unity prints a test summary block at the end of a test run of the form:
|
||||
-----------------------
|
||||
X Tests Y Failures Z Ignored
|
||||
[PASS|FAIL]
|
||||
|
||||
Returns the contents of the test summary block as a TestStats object.
|
||||
"""
|
||||
match = _TEST_SUMMARY_BLOCK_REGEX.search(unity_output)
|
||||
if not match:
|
||||
raise ValueError('A Unity test summary block was not found.')
|
||||
|
||||
try:
|
||||
stats = TestStats()
|
||||
stats.total = int(match.group('num_tests'))
|
||||
stats.failed = int(match.group('num_failures'))
|
||||
stats.ignored = int(match.group('num_ignored'))
|
||||
stats.passed = stats.total - stats.failed - stats.ignored
|
||||
return stats
|
||||
except ValueError:
|
||||
raise ValueError('The Unity test summary block was not valid.')
|
||||
|
||||
def _parse_unity_basic(self, unity_output):
|
||||
"""
|
||||
Parse basic unity output.
|
||||
|
||||
This is of the form file:line:test_name:result[:optional_message]
|
||||
"""
|
||||
found_test_stats = TestStats()
|
||||
|
||||
for test in _NORMAL_TEST_REGEX.finditer(unity_output):
|
||||
try:
|
||||
new_test = TestResult(
|
||||
test.group('test_name'),
|
||||
test.group('result'),
|
||||
file=test.group('file'),
|
||||
line=int(test.group('line')),
|
||||
message=test.group('message')
|
||||
if test.group('message') is not None
|
||||
else '',
|
||||
full_line=test.group(0),
|
||||
)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
self._add_new_test(new_test, found_test_stats)
|
||||
|
||||
if len(self._tests) == 0:
|
||||
raise ValueError('No tests were found.')
|
||||
|
||||
if found_test_stats != self._test_stats:
|
||||
raise ValueError('Test output does not match summary block.')
|
||||
|
||||
def _parse_unity_fixture_verbose(self, unity_output):
|
||||
"""
|
||||
Parse the output of the unity_fixture add-in invoked with the -v flag.
|
||||
|
||||
This is a more complex operation than basic unity output, because the output for a single
|
||||
test can span multiple lines. There is a prefix of the form "TEST(test_group, test_name)"
|
||||
that always exists on the first line for a given test. Immediately following that can be a
|
||||
pass or fail message, or some number of diagnostic messages followed by a pass or fail
|
||||
message.
|
||||
"""
|
||||
found_test_stats = TestStats()
|
||||
|
||||
line_iter = iter(unity_output.splitlines())
|
||||
try:
|
||||
line = next(line_iter)
|
||||
while True:
|
||||
prefix_match = _UNITY_FIXTURE_VERBOSE_PREFIX_REGEX.search(line)
|
||||
line = next(line_iter)
|
||||
if prefix_match:
|
||||
# Handle the remaining portion of a test case line after the unity_fixture
|
||||
# prefix.
|
||||
remainder = prefix_match.group('remainder')
|
||||
if remainder:
|
||||
self._parse_unity_fixture_remainder(
|
||||
prefix_match, remainder, found_test_stats
|
||||
)
|
||||
# Handle any subsequent lines with more information on the same test case.
|
||||
while not _UNITY_FIXTURE_VERBOSE_PREFIX_REGEX.search(line):
|
||||
self._parse_unity_fixture_remainder(
|
||||
prefix_match, line, found_test_stats
|
||||
)
|
||||
line = next(line_iter)
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
if len(self._tests) == 0:
|
||||
raise ValueError('No tests were found.')
|
||||
|
||||
if found_test_stats != self._test_stats:
|
||||
raise ValueError('Test output does not match summary block.')
|
||||
|
||||
def _parse_unity_fixture_remainder(self, prefix_match, remainder, test_stats):
|
||||
"""
|
||||
Parse the remainder of a Unity Fixture test case.
|
||||
|
||||
Can be on the same line as the prefix or on subsequent lines.
|
||||
"""
|
||||
new_test = None
|
||||
|
||||
if remainder == ' PASS':
|
||||
new_test = TestResult(
|
||||
prefix_match.group('test_name'),
|
||||
'PASS',
|
||||
group=prefix_match.group('test_group'),
|
||||
full_line=prefix_match.group(0),
|
||||
)
|
||||
else:
|
||||
remainder_match = _UNITY_FIXTURE_REMAINDER_REGEX.match(remainder)
|
||||
if remainder_match:
|
||||
new_test = TestResult(
|
||||
prefix_match.group('test_name'),
|
||||
remainder_match.group('result'),
|
||||
group=prefix_match.group('test_group'),
|
||||
file=remainder_match.group('file'),
|
||||
line=int(remainder_match.group('line')),
|
||||
message=remainder_match.group('message')
|
||||
if remainder_match.group('message') is not None
|
||||
else '',
|
||||
full_line=prefix_match.group('prefix') + remainder_match.group(0),
|
||||
)
|
||||
|
||||
if new_test is not None:
|
||||
self._add_new_test(new_test, test_stats)
|
||||
|
||||
def _add_new_test(self, new_test, test_stats):
|
||||
"""Add a new test and increment the proper members of test_stats."""
|
||||
test_stats.total += 1
|
||||
if new_test.result() == 'PASS':
|
||||
test_stats.passed += 1
|
||||
elif new_test.result() == 'FAIL':
|
||||
test_stats.failed += 1
|
||||
else:
|
||||
test_stats.ignored += 1
|
||||
|
||||
self._tests.append(new_test)
|
||||
Reference in New Issue
Block a user