pioduino/esp-idf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CI: assign unit test cases according to sdkconfig:

Browse Source 
We have built unit-test-app with different configs.
Currently we use the config name as tags to match runners.
It's not easy to add new configs (need to update tags to existed runners).

Now we'll parse required test runner tags from `sdkconfig` file.
For example, if config enables `CONFIG_SPIRAM_SUPPORT`, then it requires `psram` tag.
This will make adding new configs easier.

In this commit we change the one behavior of assign test: match keys of
cases should be exactly the same with job tags. This fixes cases select
jobs include their tags, and jobs requires those tags can't be assigned.
This commit is contained in:
He Yin Ling
2018-05-15 11:51:56 +08:00
committed by bot
parent 358c822d6a
commit 3b24bc42d2
8 changed files with 175 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/unit-test-app/tools/UnitTestParser.py
+44 -25
View File
@@ -20,13 +20,8 @@ TEST_CASE_PATTERN = {
"test environment": "UT_T1_1",
"reset": "",
"expected result": "1. set succeed",
"cmd set": "test_unit_test_case"
}
CONFIG_FILE_PATTERN = {
"Config": {"execute count": 1, "execute order": "in order"},
"DUT": [],
"Filter": [{"Add": {"ID": []}}]
"cmd set": "test_unit_test_case",
"Test App": "UT",
}
@@ -39,11 +34,12 @@ class Parser(object):
# file path (relative to idf path)
TAG_DEF_FILE = os.path.join("tools", "unit-test-app", "tools", "TagDefinition.yml")
MODULE_DEF_FILE = os.path.join("tools", "unit-test-app", "tools", "ModuleDefinition.yml")
CONFIG_DEPENDENCY_FILE = os.path.join("tools", "unit-test-app", "tools", "ConfigDependency.yml")
MODULE_ARTIFACT_FILE = os.path.join("components", "idf_test", "ModuleDefinition.yml")
TEST_CASE_FILE = os.path.join("components", "idf_test", "unit_test", "TestCaseAll.yml")
UT_BIN_FOLDER = os.path.join("tools", "unit-test-app", "builds")
UT_BIN_FOLDER = os.path.join("tools", "unit-test-app", "output")
ELF_FILE = "unit-test-app.elf"
APP_NAME_PREFIX = "UT_"
SDKCONFIG_FILE = "sdkconfig"
def __init__(self, idf_path=os.getenv("IDF_PATH")):
self.test_env_tags = {}
@@ -52,21 +48,24 @@ class Parser(object):
self.idf_path = idf_path
self.tag_def = yaml.load(open(os.path.join(idf_path, self.TAG_DEF_FILE), "r"))
self.module_map = yaml.load(open(os.path.join(idf_path, self.MODULE_DEF_FILE), "r"))
self.config_dependency = yaml.load(open(os.path.join(idf_path, self.CONFIG_DEPENDENCY_FILE), "r"))
# used to check if duplicated test case names
self.test_case_names = set()
self.parsing_errors = []
def parse_test_cases_from_elf(self, elf_file, app_name):
def parse_test_cases_for_one_config(self, config_output_folder, config_name):
"""
parse test cases from elf and save test cases need to be executed to unit test folder
:param elf_file: elf file path
:param app_name: built unit test app name
:param config_output_folder: build folder of this config
:param config_name: built unit test config name
"""
elf_file = os.path.join(config_output_folder, self.ELF_FILE)
subprocess.check_output('xtensa-esp32-elf-objdump -t {} | grep test_desc > case_address.tmp'.format(elf_file),
shell=True)
subprocess.check_output('xtensa-esp32-elf-objdump -s {} > section_table.tmp'.format(elf_file), shell=True)
table = CreateSectionTable.SectionTable("section_table.tmp")
tags = self.parse_tags(os.path.join(config_output_folder, self.SDKCONFIG_FILE))
test_cases = []
with open("case_address.tmp", "r") as f:
for line in f:
@@ -82,17 +81,17 @@ class Parser(object):
name = table.get_string("any", name_addr)
desc = table.get_string("any", desc_addr)
file_name = table.get_string("any", file_name_addr)
tc = self.parse_one_test_case(name, desc, file_name, app_name)
tc = self.parse_one_test_case(name, desc, file_name, config_name, tags)
# check if duplicated case names
# we need to use it to select case,
# if duplicated IDs, Unity could select incorrect case to run
# and we need to check all cases no matter if it's going te be executed by CI
# also add app_name here, we allow same case for different apps
if (tc["summary"] + app_name) in self.test_case_names:
if (tc["summary"] + config_name) in self.test_case_names:
self.parsing_errors.append("duplicated test case ID: " + tc["summary"])
else:
self.test_case_names.add(tc["summary"] + app_name)
self.test_case_names.add(tc["summary"] + config_name)
if tc["CI ready"] == "Yes":
# update test env list and the cases of same env list
@@ -150,13 +149,32 @@ class Parser(object):
pass
return p
def parse_one_test_case(self, name, description, file_name, app_name):
def parse_tags(self, sdkconfig_file):
"""
Some test configs could requires different DUTs.
For example, if CONFIG_SPIRAM_SUPPORT is enabled, we need WROVER-Kit to run test.
This method will get tags for runners according to ConfigDependency.yml(maps tags to sdkconfig).
:param sdkconfig_file: sdkconfig file of the unit test config
:return: required tags for runners
"""
required_tags = []
with open(sdkconfig_file, "r") as f:
configs_raw_data = f.read()
configs = configs_raw_data.splitlines(False)
for tag in self.config_dependency:
if self.config_dependency[tag] in configs:
required_tags.append(tag)
return required_tags
def parse_one_test_case(self, name, description, file_name, config_name, tags):
"""
parse one test case
:param name: test case name (summary)
:param description: test case description (tag string)
:param file_name: the file defines this test case
:param app_name: built unit test app name
:param config_name: built unit test app name
:param tags: tags to select runners
:return: parsed test case
"""
prop = self.parse_case_properities(description)
@@ -179,7 +197,7 @@ class Parser(object):
self.file_name_cache[file_name_hash])
test_case = deepcopy(TEST_CASE_PATTERN)
test_case.update({"Test App": self.APP_NAME_PREFIX + app_name,
test_case.update({"config": config_name,
"module": self.module_map[prop["module"]]['module'],
"CI ready": "No" if prop["ignore"] == "Yes" else "Yes",
"ID": tc_id,
@@ -191,7 +209,8 @@ class Parser(object):
"summary": name,
"multi_device": prop["multi_device"],
"multi_stage": prop["multi_stage"],
"timeout": int(prop["timeout"])})
"timeout": int(prop["timeout"]),
"tags": tags})
return test_case
def dump_test_cases(self, test_cases):
@@ -212,12 +231,12 @@ class Parser(object):
""" parse test cases from multiple built unit test apps """
test_cases = []
test_app_folder = os.path.join(self.idf_path, self.UT_BIN_FOLDER)
test_apps = os.listdir(test_app_folder)
for app in test_apps:
elf_file = os.path.join(test_app_folder, app, self.ELF_FILE)
if os.path.exists(elf_file):
test_cases.extend(self.parse_test_cases_from_elf(elf_file, app))
output_folder = os.path.join(self.idf_path, self.UT_BIN_FOLDER)
test_configs = os.listdir(output_folder)
for config in test_configs:
config_output_folder = os.path.join(output_folder, config)
if os.path.exists(config_output_folder):
test_cases.extend(self.parse_test_cases_for_one_config(config_output_folder, config))
self.dump_test_cases(test_cases)