dfc7860ab1
We now use hwmv2 to list boards instead of relying on twister specific config files. One yaml files (twister.yaml for now) will have all the data needed for all possible targets and variations of a board reusing most of the data where possible and variations can override the top level data. Twister keeps track of 'aliases' of boards and identifies that for example native_sim is the same as native_sim/native, so either names will be possible in both test yaml files or on the command line, however, the reporting will always use the full name, so no there is no confusion about what is being tested/built. Signed-off-by: Anas Nashif <anas.nashif@intel.com>
271 lines
11 KiB
Python
271 lines
11 KiB
Python
# vim: set syntax=python ts=4 :
|
|
#
|
|
# Copyright (c) 2018-2022 Intel Corporation
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
import copy
|
|
import scl
|
|
import warnings
|
|
from typing import Union
|
|
from twisterlib.error import ConfigurationError
|
|
|
|
def extract_fields_from_arg_list(target_fields: set, arg_list: Union[str, list]):
|
|
"""
|
|
Given a list of "FIELD=VALUE" args, extract values of args with a
|
|
given field name and return the remaining args separately.
|
|
"""
|
|
extracted_fields = {f : list() for f in target_fields}
|
|
other_fields = []
|
|
|
|
if isinstance(arg_list, str):
|
|
args = arg_list.strip().split()
|
|
else:
|
|
args = arg_list
|
|
|
|
for field in args:
|
|
try:
|
|
name, val = field.split("=", 1)
|
|
except ValueError:
|
|
# Can't parse this. Just pass it through
|
|
other_fields.append(field)
|
|
continue
|
|
|
|
if name in target_fields:
|
|
extracted_fields[name].append(val.strip('\'"'))
|
|
else:
|
|
# Move to other_fields
|
|
other_fields.append(field)
|
|
|
|
return extracted_fields, other_fields
|
|
|
|
class TwisterConfigParser:
|
|
"""Class to read testsuite yaml files with semantic checking
|
|
"""
|
|
|
|
testsuite_valid_keys = {"tags": {"type": "set", "required": False},
|
|
"type": {"type": "str", "default": "integration"},
|
|
"extra_args": {"type": "list"},
|
|
"extra_configs": {"type": "list"},
|
|
"extra_conf_files": {"type": "list", "default": []},
|
|
"extra_overlay_confs" : {"type": "list", "default": []},
|
|
"extra_dtc_overlay_files": {"type": "list", "default": []},
|
|
"required_snippets": {"type": "list"},
|
|
"build_only": {"type": "bool", "default": False},
|
|
"build_on_all": {"type": "bool", "default": False},
|
|
"skip": {"type": "bool", "default": False},
|
|
"slow": {"type": "bool", "default": False},
|
|
"timeout": {"type": "int", "default": 60},
|
|
"min_ram": {"type": "int", "default": 8},
|
|
"modules": {"type": "list", "default": []},
|
|
"depends_on": {"type": "set"},
|
|
"min_flash": {"type": "int", "default": 32},
|
|
"arch_allow": {"type": "set"},
|
|
"arch_exclude": {"type": "set"},
|
|
"extra_sections": {"type": "list", "default": []},
|
|
"integration_platforms": {"type": "list", "default": []},
|
|
"ignore_faults": {"type": "bool", "default": False },
|
|
"ignore_qemu_crash": {"type": "bool", "default": False },
|
|
"testcases": {"type": "list", "default": []},
|
|
"platform_type": {"type": "list", "default": []},
|
|
"platform_exclude": {"type": "set"},
|
|
"platform_allow": {"type": "set"},
|
|
"platform_key": {"type": "list", "default": []},
|
|
"simulation_exclude": {"type": "list", "default": []},
|
|
"toolchain_exclude": {"type": "set"},
|
|
"toolchain_allow": {"type": "set"},
|
|
"filter": {"type": "str"},
|
|
"levels": {"type": "list", "default": []},
|
|
"harness": {"type": "str", "default": "test"},
|
|
"harness_config": {"type": "map", "default": {}},
|
|
"seed": {"type": "int", "default": 0},
|
|
"sysbuild": {"type": "bool", "default": False}
|
|
}
|
|
|
|
def __init__(self, filename, schema):
|
|
"""Instantiate a new TwisterConfigParser object
|
|
|
|
@param filename Source .yaml file to read
|
|
"""
|
|
self.data = {}
|
|
self.schema = schema
|
|
self.filename = filename
|
|
self.scenarios = {}
|
|
self.common = {}
|
|
|
|
def load(self):
|
|
data = scl.yaml_load_verify(self.filename, self.schema)
|
|
self.data = data
|
|
|
|
if 'tests' in self.data:
|
|
self.scenarios = self.data['tests']
|
|
if 'common' in self.data:
|
|
self.common = self.data['common']
|
|
return data
|
|
|
|
def _cast_value(self, value, typestr):
|
|
if isinstance(value, str):
|
|
v = value.strip()
|
|
if typestr == "str":
|
|
return v
|
|
|
|
elif typestr == "float":
|
|
return float(value)
|
|
|
|
elif typestr == "int":
|
|
return int(value)
|
|
|
|
elif typestr == "bool":
|
|
return value
|
|
|
|
elif typestr.startswith("list"):
|
|
if isinstance(value, list):
|
|
return value
|
|
elif isinstance(value, str):
|
|
vs = v.split()
|
|
|
|
if len(vs) > 1:
|
|
warnings.warn(
|
|
"Space-separated lists are deprecated, use YAML lists instead",
|
|
DeprecationWarning)
|
|
|
|
if len(typestr) > 4 and typestr[4] == ":":
|
|
return [self._cast_value(vsi, typestr[5:]) for vsi in vs]
|
|
else:
|
|
return vs
|
|
else:
|
|
raise ValueError
|
|
|
|
elif typestr.startswith("set"):
|
|
if isinstance(value, list):
|
|
return set(value)
|
|
elif isinstance(value, str):
|
|
vs = v.split()
|
|
|
|
if len(vs) > 1:
|
|
warnings.warn(
|
|
"Space-separated lists are deprecated, use YAML lists instead",
|
|
DeprecationWarning)
|
|
|
|
if len(typestr) > 3 and typestr[3] == ":":
|
|
return {self._cast_value(vsi, typestr[4:]) for vsi in vs}
|
|
else:
|
|
return set(vs)
|
|
else:
|
|
raise ValueError
|
|
|
|
elif typestr.startswith("map"):
|
|
return value
|
|
else:
|
|
raise ConfigurationError(
|
|
self.filename, "unknown type '%s'" % value)
|
|
|
|
def get_scenario(self, name):
|
|
"""Get a dictionary representing the keys/values within a scenario
|
|
|
|
@param name The scenario in the .yaml file to retrieve data from
|
|
@return A dictionary containing the scenario key-value pairs with
|
|
type conversion and default values filled in per valid_keys
|
|
"""
|
|
|
|
# "CONF_FILE", "OVERLAY_CONFIG", and "DTC_OVERLAY_FILE" fields from each
|
|
# of the extra_args lines
|
|
extracted_common = {}
|
|
extracted_testsuite = {}
|
|
|
|
d = {}
|
|
for k, v in self.common.items():
|
|
if k == "extra_args":
|
|
# Pull out these fields and leave the rest
|
|
extracted_common, d[k] = extract_fields_from_arg_list(
|
|
{"CONF_FILE", "OVERLAY_CONFIG", "DTC_OVERLAY_FILE"}, v
|
|
)
|
|
else:
|
|
# Copy common value to avoid mutating it with test specific values below
|
|
d[k] = copy.copy(v)
|
|
|
|
for k, v in self.scenarios[name].items():
|
|
if k == "extra_args":
|
|
# Pull out these fields and leave the rest
|
|
extracted_testsuite, v = extract_fields_from_arg_list(
|
|
{"CONF_FILE", "OVERLAY_CONFIG", "DTC_OVERLAY_FILE"}, v
|
|
)
|
|
if k in d:
|
|
if k == "filter":
|
|
d[k] = "(%s) and (%s)" % (d[k], v)
|
|
elif k not in ("extra_conf_files", "extra_overlay_confs",
|
|
"extra_dtc_overlay_files"):
|
|
if isinstance(d[k], str) and isinstance(v, list):
|
|
d[k] = d[k].split() + v
|
|
elif isinstance(d[k], list) and isinstance(v, str):
|
|
d[k] += v.split()
|
|
elif isinstance(d[k], list) and isinstance(v, list):
|
|
d[k] += v
|
|
elif isinstance(d[k], str) and isinstance(v, str):
|
|
d[k] += " " + v
|
|
else:
|
|
# replace value if not str/list (e.g. integer)
|
|
d[k] = v
|
|
else:
|
|
d[k] = v
|
|
|
|
# Compile conf files in to a single list. The order to apply them is:
|
|
# (1) CONF_FILEs extracted from common['extra_args']
|
|
# (2) common['extra_conf_files']
|
|
# (3) CONF_FILES extracted from scenarios[name]['extra_args']
|
|
# (4) scenarios[name]['extra_conf_files']
|
|
d["extra_conf_files"] = \
|
|
extracted_common.get("CONF_FILE", []) + \
|
|
self.common.get("extra_conf_files", []) + \
|
|
extracted_testsuite.get("CONF_FILE", []) + \
|
|
self.scenarios[name].get("extra_conf_files", [])
|
|
|
|
# Repeat the above for overlay confs and DTC overlay files
|
|
d["extra_overlay_confs"] = \
|
|
extracted_common.get("OVERLAY_CONFIG", []) + \
|
|
self.common.get("extra_overlay_confs", []) + \
|
|
extracted_testsuite.get("OVERLAY_CONFIG", []) + \
|
|
self.scenarios[name].get("extra_overlay_confs", [])
|
|
|
|
d["extra_dtc_overlay_files"] = \
|
|
extracted_common.get("DTC_OVERLAY_FILE", []) + \
|
|
self.common.get("extra_dtc_overlay_files", []) + \
|
|
extracted_testsuite.get("DTC_OVERLAY_FILE", []) + \
|
|
self.scenarios[name].get("extra_dtc_overlay_files", [])
|
|
|
|
if any({len(x) > 0 for x in extracted_common.values()}) or \
|
|
any({len(x) > 0 for x in extracted_testsuite.values()}):
|
|
warnings.warn(
|
|
"Do not specify CONF_FILE, OVERLAY_CONFIG, or DTC_OVERLAY_FILE "
|
|
"in extra_args. This feature is deprecated and will soon "
|
|
"result in an error. Use extra_conf_files, extra_overlay_confs "
|
|
"or extra_dtc_overlay_files YAML fields instead",
|
|
DeprecationWarning
|
|
)
|
|
|
|
for k, kinfo in self.testsuite_valid_keys.items():
|
|
if k not in d:
|
|
if "required" in kinfo:
|
|
required = kinfo["required"]
|
|
else:
|
|
required = False
|
|
|
|
if required:
|
|
raise ConfigurationError(
|
|
self.filename,
|
|
"missing required value for '%s' in test '%s'" %
|
|
(k, name))
|
|
else:
|
|
if "default" in kinfo:
|
|
default = kinfo["default"]
|
|
else:
|
|
default = self._cast_value("", kinfo["type"])
|
|
d[k] = default
|
|
else:
|
|
try:
|
|
d[k] = self._cast_value(d[k], kinfo["type"])
|
|
except ValueError:
|
|
raise ConfigurationError(
|
|
self.filename, "bad %s value '%s' for key '%s' in name '%s'" %
|
|
(kinfo["type"], d[k], k, name))
|
|
|
|
return d
|