11e656bb6a
Added integration_toolchains to allow building/testing with multiple toolchains available in the environment. This changes the output structure and adds another level in the path under twister_out signifying the toolchain used. The toolchain used (variant) is also part of the json output now. Signed-off-by: Anas Nashif <anas.nashif@intel.com>
653 lines
23 KiB
Python
653 lines
23 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright (c) 2020 Intel Corporation
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
# pylint: disable=line-too-long
|
|
"""
|
|
Tests for testinstance class
|
|
"""
|
|
|
|
from contextlib import nullcontext
|
|
import os
|
|
import sys
|
|
import pytest
|
|
import mock
|
|
|
|
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
|
|
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
|
|
|
|
from pylib.twister.twisterlib.platform import Simulator
|
|
from twisterlib.statuses import TwisterStatus
|
|
from twisterlib.testinstance import TestInstance
|
|
from twisterlib.error import BuildError
|
|
from twisterlib.runner import TwisterRunner
|
|
from twisterlib.handlers import QEMUHandler
|
|
from expr_parser import reserved
|
|
|
|
|
|
TESTDATA_PART_1 = [
|
|
(False, False, "console", None, "qemu", False, [], (False, True)),
|
|
(False, False, "console", "native", "qemu", False, [], (False, True)),
|
|
(True, False, "console", "native", "nsim", False, [], (True, False)),
|
|
(True, True, "console", "native", "renode", False, [], (True, False)),
|
|
(False, False, "sensor", "native", "", False, [], (True, False)),
|
|
(False, False, "sensor", None, "", False, [], (True, False)),
|
|
(False, True, "sensor", "native", "", True, [], (True, False)),
|
|
]
|
|
@pytest.mark.parametrize(
|
|
"build_only, slow, harness, platform_type, platform_sim, device_testing,fixture, expected",
|
|
TESTDATA_PART_1
|
|
)
|
|
def test_check_build_or_run(
|
|
class_testplan,
|
|
all_testsuites_dict,
|
|
platforms_list,
|
|
build_only,
|
|
slow,
|
|
harness,
|
|
platform_type,
|
|
platform_sim,
|
|
device_testing,
|
|
fixture,
|
|
expected
|
|
):
|
|
"""" Test to check the conditions for build_only and run scenarios
|
|
Scenario 1: Test when different parameters are passed, build_only and run are set correctly
|
|
Scenario 2: Test if build_only is enabled when the OS is Windows"""
|
|
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
testsuite = class_testplan.testsuites.get('scripts/tests/twister/test_data/testsuites/tests/'
|
|
'test_a/test_a.check_1')
|
|
print(testsuite)
|
|
|
|
class_testplan.platforms = platforms_list
|
|
platform = class_testplan.get_platform("demo_board_2")
|
|
platform.type = platform_type
|
|
platform.simulators = [Simulator({"name": platform_sim})] if platform_sim else []
|
|
testsuite.harness = harness
|
|
testsuite.build_only = build_only
|
|
testsuite.slow = slow
|
|
|
|
testinstance = TestInstance(testsuite, platform, 'zephyr', class_testplan.env.outdir)
|
|
env = mock.Mock(
|
|
options=mock.Mock(
|
|
device_testing=False,
|
|
enable_slow=slow,
|
|
fixtures=fixture,
|
|
filter="",
|
|
sim_name=platform_sim
|
|
)
|
|
)
|
|
run = testinstance.check_runnable(env.options)
|
|
_, r = expected
|
|
assert run == r
|
|
|
|
with mock.patch('os.name', 'nt'):
|
|
# path to QEMU binary is not in QEMU_BIN_PATH environment variable
|
|
run = testinstance.check_runnable(env.options)
|
|
assert not run
|
|
|
|
# mock path to QEMU binary in QEMU_BIN_PATH environment variable
|
|
with mock.patch('os.environ', {'QEMU_BIN_PATH': ''}):
|
|
run = testinstance.check_runnable(env.options)
|
|
_, r = expected
|
|
assert run == r
|
|
|
|
|
|
TESTDATA_PART_2 = [
|
|
(True, True, True, ["demo_board_2/unit_testing"], "native",
|
|
None, '\nCONFIG_COVERAGE=y\nCONFIG_COVERAGE_DUMP=y\nCONFIG_ASAN=y\nCONFIG_UBSAN=y'),
|
|
(True, False, True, ["demo_board_2/unit_testing"], "native",
|
|
None, '\nCONFIG_COVERAGE=y\nCONFIG_COVERAGE_DUMP=y\nCONFIG_ASAN=y'),
|
|
(False, False, True, ["demo_board_2/unit_testing"], 'native',
|
|
None, '\nCONFIG_COVERAGE=y\nCONFIG_COVERAGE_DUMP=y'),
|
|
(True, False, True, ["demo_board_2/unit_testing"], 'mcu',
|
|
None, '\nCONFIG_COVERAGE=y\nCONFIG_COVERAGE_DUMP=y'),
|
|
(False, False, False, ["demo_board_2/unit_testing"], 'native', None, ''),
|
|
(False, False, True, ['demo_board_1'], 'native', None, ''),
|
|
(True, False, False, ["demo_board_2"], 'native', None, '\nCONFIG_ASAN=y'),
|
|
(False, True, False, ["demo_board_2"], 'native', None, '\nCONFIG_UBSAN=y'),
|
|
(False, False, False, ["demo_board_2"], 'native',
|
|
["CONFIG_LOG=y"], 'CONFIG_LOG=y'),
|
|
(False, False, False, ["demo_board_2"], 'native',
|
|
["arch:x86:CONFIG_LOG=y"], 'CONFIG_LOG=y'),
|
|
(False, False, False, ["demo_board_2"], 'native',
|
|
["arch:arm:CONFIG_LOG=y"], ''),
|
|
(False, False, False, ["demo_board_2"], 'native',
|
|
["platform:demo_board_2/unit_testing:CONFIG_LOG=y"], 'CONFIG_LOG=y'),
|
|
(False, False, False, ["demo_board_2"], 'native',
|
|
["platform:demo_board_1:CONFIG_LOG=y"], ''),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'enable_asan, enable_ubsan, enable_coverage, coverage_platform, platform_type,'
|
|
' extra_configs, expected_content',
|
|
TESTDATA_PART_2
|
|
)
|
|
def test_create_overlay(
|
|
class_testplan,
|
|
all_testsuites_dict,
|
|
platforms_list,
|
|
enable_asan,
|
|
enable_ubsan,
|
|
enable_coverage,
|
|
coverage_platform,
|
|
platform_type,
|
|
extra_configs,
|
|
expected_content
|
|
):
|
|
"""Test correct content is written to testcase_extra.conf based on if conditions."""
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
testcase = class_testplan.testsuites.get('scripts/tests/twister/test_data/testsuites/samples/'
|
|
'test_app/sample_test.app')
|
|
|
|
if extra_configs:
|
|
testcase.extra_configs = extra_configs
|
|
|
|
class_testplan.platforms = platforms_list
|
|
platform = class_testplan.get_platform("demo_board_2")
|
|
|
|
testinstance = TestInstance(testcase, platform, 'zephyr', class_testplan.env.outdir)
|
|
platform.type = platform_type
|
|
assert testinstance.create_overlay(platform, enable_asan, enable_ubsan, enable_coverage, coverage_platform) == expected_content
|
|
|
|
def test_calculate_sizes(class_testplan, all_testsuites_dict, platforms_list):
|
|
""" Test Calculate sizes method for zephyr elf"""
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
testcase = class_testplan.testsuites.get('scripts/tests/twister/test_data/testsuites/samples/'
|
|
'test_app/sample_test.app')
|
|
class_testplan.platforms = platforms_list
|
|
platform = class_testplan.get_platform("demo_board_2")
|
|
testinstance = TestInstance(testcase, platform, 'zephyr', class_testplan.env.outdir)
|
|
|
|
with pytest.raises(BuildError):
|
|
assert testinstance.calculate_sizes() == "Missing/multiple output ELF binary"
|
|
|
|
TESTDATA_PART_3 = [
|
|
(
|
|
'CONFIG_ARCH_HAS_THREAD_LOCAL_STORAGE and' \
|
|
' CONFIG_TOOLCHAIN_SUPPORTS_THREAD_LOCAL_STORAGE and' \
|
|
' not (CONFIG_TOOLCHAIN_ARCMWDT_SUPPORTS_THREAD_LOCAL_STORAGE and CONFIG_USERSPACE)',
|
|
['kconfig']
|
|
),
|
|
(
|
|
'(dt_compat_enabled("st,stm32-flash-controller") or' \
|
|
' dt_compat_enabled("st,stm32h7-flash-controller")) and' \
|
|
' dt_label_with_parent_compat_enabled("storage_partition", "fixed-partitions")',
|
|
['dts']
|
|
),
|
|
(
|
|
'((CONFIG_FLASH_HAS_DRIVER_ENABLED and not CONFIG_TRUSTED_EXECUTION_NONSECURE) and' \
|
|
' dt_label_with_parent_compat_enabled("storage_partition", "fixed-partitions")) or' \
|
|
' (CONFIG_FLASH_HAS_DRIVER_ENABLED and CONFIG_TRUSTED_EXECUTION_NONSECURE and' \
|
|
' dt_label_with_parent_compat_enabled("slot1_ns_partition", "fixed-partitions"))',
|
|
['dts', 'kconfig']
|
|
),
|
|
(
|
|
'((CONFIG_CPU_AARCH32_CORTEX_R or CONFIG_CPU_CORTEX_M) and' \
|
|
' CONFIG_CPU_HAS_FPU and TOOLCHAIN_HAS_NEWLIB == 1) or CONFIG_ARCH_POSIX',
|
|
['full']
|
|
)
|
|
]
|
|
|
|
@pytest.mark.parametrize("filter_expr, expected_stages", TESTDATA_PART_3)
|
|
def test_which_filter_stages(filter_expr, expected_stages):
|
|
logic_keys = reserved.keys()
|
|
stages = TwisterRunner.get_cmake_filter_stages(filter_expr, logic_keys)
|
|
assert sorted(stages) == sorted(expected_stages)
|
|
|
|
|
|
@pytest.fixture(name='testinstance')
|
|
def sample_testinstance(all_testsuites_dict, class_testplan, platforms_list, request):
|
|
testsuite_path = 'scripts/tests/twister/test_data/testsuites'
|
|
if request.param['testsuite_kind'] == 'sample':
|
|
testsuite_path += '/samples/test_app/sample_test.app'
|
|
elif request.param['testsuite_kind'] == 'tests':
|
|
testsuite_path += '/tests/test_a/test_a.check_1'
|
|
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
testsuite = class_testplan.testsuites.get(testsuite_path)
|
|
class_testplan.platforms = platforms_list
|
|
platform = class_testplan.get_platform(request.param.get('board_name', 'demo_board_2'))
|
|
|
|
testinstance = TestInstance(testsuite, platform, 'zephyr', class_testplan.env.outdir)
|
|
return testinstance
|
|
|
|
|
|
TESTDATA_1 = [
|
|
(False),
|
|
(True),
|
|
]
|
|
|
|
@pytest.mark.parametrize('detailed_test_id', TESTDATA_1)
|
|
def test_testinstance_init(all_testsuites_dict, class_testplan, platforms_list, detailed_test_id):
|
|
testsuite_path = 'scripts/tests/twister/test_data/testsuites/samples/test_app/sample_test.app'
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
testsuite = class_testplan.testsuites.get(testsuite_path)
|
|
testsuite.detailed_test_id = detailed_test_id
|
|
class_testplan.platforms = platforms_list
|
|
platform = class_testplan.get_platform("demo_board_2/unit_testing")
|
|
|
|
testinstance = TestInstance(testsuite, platform, 'zephyr', class_testplan.env.outdir)
|
|
|
|
if detailed_test_id:
|
|
assert testinstance.build_dir == os.path.join(class_testplan.env.outdir, platform.normalized_name, 'zephyr', testsuite_path)
|
|
else:
|
|
assert testinstance.build_dir == os.path.join(class_testplan.env.outdir, platform.normalized_name, 'zephyr', testsuite.source_dir_rel, testsuite.name)
|
|
|
|
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'sample'}], indirect=True)
|
|
def test_testinstance_record(testinstance):
|
|
testinstance.testcases = [mock.Mock()]
|
|
recording = [ {'field_1': 'recording_1_1', 'field_2': 'recording_1_2'},
|
|
{'field_1': 'recording_2_1', 'field_2': 'recording_2_2'}
|
|
]
|
|
with mock.patch(
|
|
'builtins.open',
|
|
mock.mock_open(read_data='')
|
|
) as mock_file, \
|
|
mock.patch(
|
|
'csv.DictWriter.writerow',
|
|
mock.Mock()
|
|
) as mock_writeheader, \
|
|
mock.patch(
|
|
'csv.DictWriter.writerows',
|
|
mock.Mock()
|
|
) as mock_writerows:
|
|
testinstance.record(recording)
|
|
|
|
print(mock_file.mock_calls)
|
|
|
|
mock_file.assert_called_with(
|
|
os.path.join(testinstance.build_dir, 'recording.csv'),
|
|
'w'
|
|
)
|
|
|
|
mock_writeheader.assert_has_calls([mock.call({ k:k for k in recording[0]})])
|
|
mock_writerows.assert_has_calls([mock.call(recording)])
|
|
|
|
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'sample'}], indirect=True)
|
|
def test_testinstance_add_filter(testinstance):
|
|
reason = 'dummy reason'
|
|
filter_type = 'dummy type'
|
|
|
|
testinstance.add_filter(reason, filter_type)
|
|
|
|
assert {'type': filter_type, 'reason': reason} in testinstance.filters
|
|
assert testinstance.status == TwisterStatus.FILTER
|
|
assert testinstance.reason == reason
|
|
assert testinstance.filter_type == filter_type
|
|
|
|
|
|
def test_testinstance_init_cases(all_testsuites_dict, class_testplan, platforms_list):
|
|
testsuite_path = 'scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1'
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
testsuite = class_testplan.testsuites.get(testsuite_path)
|
|
class_testplan.platforms = platforms_list
|
|
platform = class_testplan.get_platform("demo_board_2")
|
|
|
|
testinstance = TestInstance(testsuite, platform, 'zephyr', class_testplan.env.outdir)
|
|
|
|
testinstance.init_cases()
|
|
|
|
assert all(
|
|
[
|
|
any(
|
|
[
|
|
tcc.name == tc.name and tcc.freeform == tc.freeform \
|
|
for tcc in testinstance.testsuite.testcases
|
|
]
|
|
) for tc in testsuite.testcases
|
|
]
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'sample'}], indirect=True)
|
|
def test_testinstance_get_run_id(testinstance):
|
|
res = testinstance._get_run_id()
|
|
|
|
assert isinstance(res, str)
|
|
|
|
|
|
TESTDATA_2 = [
|
|
('another reason', 'another reason'),
|
|
(None, 'dummy reason'),
|
|
]
|
|
|
|
@pytest.mark.parametrize('reason, expected_reason', TESTDATA_2)
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'tests'}], indirect=True)
|
|
def test_testinstance_add_missing_case_status(testinstance, reason, expected_reason):
|
|
testinstance.reason = 'dummy reason'
|
|
|
|
status = TwisterStatus.PASS
|
|
|
|
assert len(testinstance.testcases) > 1, 'Selected testsuite does not have enough testcases.'
|
|
|
|
testinstance.testcases[0].status = TwisterStatus.STARTED
|
|
testinstance.testcases[-1].status = TwisterStatus.NONE
|
|
|
|
testinstance.add_missing_case_status(status, reason)
|
|
|
|
assert testinstance.testcases[0].status == TwisterStatus.FAIL
|
|
assert testinstance.testcases[-1].status == TwisterStatus.PASS
|
|
assert testinstance.testcases[-1].reason == expected_reason
|
|
|
|
|
|
def test_testinstance_dunders(all_testsuites_dict, class_testplan, platforms_list):
|
|
testsuite_path = 'scripts/tests/twister/test_data/testsuites/samples/test_app/sample_test.app'
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
testsuite = class_testplan.testsuites.get(testsuite_path)
|
|
class_testplan.platforms = platforms_list
|
|
platform = class_testplan.get_platform("demo_board_2")
|
|
|
|
testinstance = TestInstance(testsuite, platform, 'zephyr', class_testplan.env.outdir)
|
|
testinstance_copy = TestInstance(testsuite, platform, 'zephyr', class_testplan.env.outdir)
|
|
|
|
d = testinstance.__getstate__()
|
|
|
|
d['name'] = 'dummy name'
|
|
testinstance_copy.__setstate__(d)
|
|
|
|
d['name'] = 'another name'
|
|
testinstance.__setstate__(d)
|
|
|
|
assert testinstance < testinstance_copy
|
|
|
|
testinstance_copy.__setstate__(d)
|
|
|
|
assert not testinstance < testinstance_copy
|
|
assert not testinstance_copy < testinstance
|
|
|
|
assert testinstance.__repr__() == f'<TestSuite {testsuite_path} on demo_board_2/unit_testing>'
|
|
|
|
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'tests'}], indirect=True)
|
|
def test_testinstance_set_case_status_by_name(testinstance):
|
|
name = 'test_a.check_1.2a'
|
|
status = TwisterStatus.PASS
|
|
reason = 'dummy reason'
|
|
|
|
tc = testinstance.set_case_status_by_name(name, status, reason)
|
|
|
|
assert tc.name == name
|
|
assert tc.status == status
|
|
assert tc.reason == reason
|
|
|
|
tc = testinstance.set_case_status_by_name(name, status, None)
|
|
|
|
assert tc.reason == reason
|
|
|
|
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'tests'}], indirect=True)
|
|
def test_testinstance_add_testcase(testinstance):
|
|
name = 'test_a.check_1.3a'
|
|
freeform = True
|
|
|
|
tc = testinstance.add_testcase(name, freeform)
|
|
|
|
assert tc in testinstance.testcases
|
|
|
|
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'tests'}], indirect=True)
|
|
def test_testinstance_get_case_by_name(testinstance):
|
|
name = 'test_a.check_1.2a'
|
|
|
|
tc = testinstance.get_case_by_name(name)
|
|
|
|
assert tc.name == name
|
|
|
|
name = 'test_a.check_1.3a'
|
|
|
|
tc = testinstance.get_case_by_name(name)
|
|
|
|
assert tc is None
|
|
|
|
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'tests'}], indirect=True)
|
|
def test_testinstance_get_case_or_create(caplog, testinstance):
|
|
name = 'test_a.check_1.2a'
|
|
|
|
tc = testinstance.get_case_or_create(name)
|
|
|
|
assert tc.name == name
|
|
|
|
name = 'test_a.check_1.3a'
|
|
|
|
tc = testinstance.get_case_or_create(name)
|
|
|
|
assert tc.name == name
|
|
assert 'Could not find a matching testcase for test_a.check_1.3a' in caplog.text
|
|
|
|
|
|
TESTDATA_3 = [
|
|
(None, 'nonexistent harness', False),
|
|
('nonexistent fixture', 'console', False),
|
|
(None, 'console', True),
|
|
('dummy fixture', 'console', True),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'fixture, harness, expected_can_run',
|
|
TESTDATA_3,
|
|
ids=['improper harness', 'fixture not in list', 'no fixture specified', 'fixture in list']
|
|
)
|
|
def test_testinstance_testsuite_runnable(
|
|
all_testsuites_dict,
|
|
class_testplan,
|
|
fixture,
|
|
harness,
|
|
expected_can_run
|
|
):
|
|
testsuite_path = 'scripts/tests/twister/test_data/testsuites/samples/test_app/sample_test.app'
|
|
class_testplan.testsuites = all_testsuites_dict
|
|
testsuite = class_testplan.testsuites.get(testsuite_path)
|
|
|
|
testsuite.harness = harness
|
|
testsuite.harness_config['fixture'] = fixture
|
|
|
|
fixtures = ['dummy fixture']
|
|
|
|
can_run = TestInstance.testsuite_runnable(testsuite, fixtures)\
|
|
|
|
assert can_run == expected_can_run
|
|
|
|
|
|
TESTDATA_4 = [
|
|
(True, mock.ANY, mock.ANY, mock.ANY, None, [], False),
|
|
(False, True, mock.ANY, mock.ANY, 'device', [], True),
|
|
(False, False, 'qemu', mock.ANY, 'qemu', ['QEMU_PIPE=1'], True),
|
|
(False, False, 'armfvp', mock.ANY, 'armfvp', [], True),
|
|
(False, False, None, 'unit', 'unit', ['COVERAGE=1'], True),
|
|
(False, False, None, 'dummy type', '', [], False),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'preexisting_handler, device_testing, platform_sim, testsuite_type,' \
|
|
' expected_handler_type, expected_handler_args, expected_handler_ready',
|
|
TESTDATA_4,
|
|
ids=['preexisting handler', 'device testing', 'qemu simulation',
|
|
'non-qemu simulation with exec', 'unit teting', 'no handler']
|
|
)
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'tests'}], indirect=True)
|
|
def test_testinstance_setup_handler(
|
|
testinstance,
|
|
preexisting_handler,
|
|
device_testing,
|
|
platform_sim,
|
|
testsuite_type,
|
|
expected_handler_type,
|
|
expected_handler_args,
|
|
expected_handler_ready
|
|
):
|
|
testinstance.handler = mock.Mock() if preexisting_handler else None
|
|
testinstance.platform.simulators = [Simulator({"name": platform_sim, "exec": 'dummy exec'})] if platform_sim else []
|
|
testinstance.testsuite.type = testsuite_type
|
|
env = mock.Mock(
|
|
options=mock.Mock(
|
|
device_testing=device_testing,
|
|
enable_coverage=True,
|
|
sim_name=platform_sim,
|
|
)
|
|
)
|
|
|
|
with mock.patch.object(QEMUHandler, 'get_fifo', return_value=1), \
|
|
mock.patch('shutil.which', return_value=True):
|
|
testinstance.setup_handler(env)
|
|
|
|
if expected_handler_type:
|
|
assert testinstance.handler.type_str == expected_handler_type
|
|
assert testinstance.handler.ready == expected_handler_ready
|
|
assert all([arg in testinstance.handler.args for arg in expected_handler_args])
|
|
|
|
|
|
TESTDATA_5 = [
|
|
('nt', 'renode', mock.ANY, mock.ANY,
|
|
mock.ANY, mock.ANY, mock.ANY,
|
|
mock.ANY, mock.ANY, mock.ANY, mock.ANY, False),
|
|
('linux', mock.ANY, mock.ANY, mock.ANY,
|
|
True, mock.ANY, mock.ANY,
|
|
mock.ANY, mock.ANY, mock.ANY, mock.ANY, False),
|
|
('linux', mock.ANY, mock.ANY, mock.ANY,
|
|
False, True, mock.ANY,
|
|
False, mock.ANY, mock.ANY, mock.ANY, False),
|
|
('linux', 'qemu', mock.ANY, mock.ANY,
|
|
False, mock.ANY, 'pytest',
|
|
mock.ANY, 'not runnable', mock.ANY, None, True),
|
|
('linux', 'renode', 'renode', True,
|
|
False, mock.ANY, 'console',
|
|
mock.ANY, 'not runnable', [], None, True),
|
|
('linux', 'renode', 'renode', False,
|
|
False, mock.ANY, 'not pytest',
|
|
mock.ANY, 'not runnable', mock.ANY, None, False),
|
|
('linux', 'qemu', mock.ANY, mock.ANY,
|
|
False, mock.ANY, 'console',
|
|
mock.ANY, 'not runnable', ['?'], mock.Mock(duts=[mock.Mock(platform='demo_board_2', fixtures=[])]), True),
|
|
]
|
|
|
|
@pytest.mark.parametrize(
|
|
'os_name, platform_sim, platform_sim_exec, exec_exists,' \
|
|
' testsuite_build_only, testsuite_slow, testsuite_harness,' \
|
|
' enable_slow, filter, fixtures, hardware_map, expected',
|
|
TESTDATA_5,
|
|
ids=['windows', 'build only', 'skip slow', 'pytest harness', 'sim', 'no sim', 'hardware map']
|
|
)
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'tests'}], indirect=True)
|
|
def test_testinstance_check_runnable(
|
|
testinstance,
|
|
os_name,
|
|
platform_sim,
|
|
platform_sim_exec,
|
|
exec_exists,
|
|
testsuite_build_only,
|
|
testsuite_slow,
|
|
testsuite_harness,
|
|
enable_slow,
|
|
filter,
|
|
fixtures,
|
|
hardware_map,
|
|
expected
|
|
):
|
|
testinstance.platform.simulators = [Simulator({"name": platform_sim, "exec": platform_sim_exec})]
|
|
testinstance.testsuite.build_only = testsuite_build_only
|
|
testinstance.testsuite.slow = testsuite_slow
|
|
testinstance.testsuite.harness = testsuite_harness
|
|
|
|
env = mock.Mock(
|
|
options=mock.Mock(
|
|
device_testing=False,
|
|
enable_slow=enable_slow,
|
|
fixtures=fixtures,
|
|
filter=filter,
|
|
sim_name=platform_sim
|
|
)
|
|
)
|
|
with mock.patch('os.name', os_name), \
|
|
mock.patch('shutil.which', return_value=exec_exists):
|
|
res = testinstance.check_runnable(env.options, hardware_map)
|
|
|
|
assert res == expected
|
|
|
|
|
|
TESTDATA_6 = [
|
|
(True, 'build.log'),
|
|
(False, ''),
|
|
]
|
|
|
|
@pytest.mark.parametrize('from_buildlog, expected_buildlog_filepath', TESTDATA_6)
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'tests'}], indirect=True)
|
|
def test_testinstance_calculate_sizes(testinstance, from_buildlog, expected_buildlog_filepath):
|
|
expected_elf_filepath = 'dummy.elf'
|
|
expected_extra_sections = []
|
|
expected_warning = True
|
|
testinstance.get_elf_file = mock.Mock(return_value='dummy.elf')
|
|
testinstance.get_buildlog_file = mock.Mock(return_value='build.log')
|
|
|
|
sc_mock = mock.Mock()
|
|
mock_sc = mock.Mock(return_value=sc_mock)
|
|
|
|
with mock.patch('twisterlib.testinstance.SizeCalculator', mock_sc):
|
|
res = testinstance.calculate_sizes(from_buildlog, expected_warning)
|
|
|
|
assert res == sc_mock
|
|
mock_sc.assert_called_once_with(
|
|
elf_filename=expected_elf_filepath,
|
|
extra_sections=expected_extra_sections,
|
|
buildlog_filepath=expected_buildlog_filepath,
|
|
generate_warning=expected_warning
|
|
)
|
|
|
|
|
|
TESTDATA_7 = [
|
|
(True, None),
|
|
(False, BuildError),
|
|
]
|
|
|
|
@pytest.mark.parametrize('sysbuild, expected_error', TESTDATA_7)
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'tests'}], indirect=True)
|
|
def test_testinstance_get_elf_file(caplog, tmp_path, testinstance, sysbuild, expected_error):
|
|
sysbuild_dir = tmp_path / 'sysbuild'
|
|
sysbuild_dir.mkdir()
|
|
zephyr_dir = sysbuild_dir / 'zephyr'
|
|
zephyr_dir.mkdir()
|
|
sysbuild_elf = zephyr_dir / 'dummy.elf'
|
|
sysbuild_elf.write_bytes(b'0')
|
|
sysbuild_elf2 = zephyr_dir / 'dummy2.elf'
|
|
sysbuild_elf2.write_bytes(b'0')
|
|
|
|
testinstance.sysbuild = sysbuild
|
|
testinstance.domains = mock.Mock(
|
|
get_default_domain=mock.Mock(
|
|
return_value=mock.Mock(
|
|
build_dir=sysbuild_dir
|
|
)
|
|
)
|
|
)
|
|
|
|
with pytest.raises(expected_error) if expected_error else nullcontext():
|
|
testinstance.get_elf_file()
|
|
|
|
if expected_error is None:
|
|
assert 'multiple ELF files detected: ' in caplog.text
|
|
|
|
|
|
TESTDATA_8 = [
|
|
(True, None),
|
|
(False, BuildError),
|
|
]
|
|
|
|
@pytest.mark.parametrize('create_build_log, expected_error', TESTDATA_8)
|
|
@pytest.mark.parametrize('testinstance', [{'testsuite_kind': 'tests'}], indirect=True)
|
|
def test_testinstance_get_buildlog_file(tmp_path, testinstance, create_build_log, expected_error):
|
|
if create_build_log:
|
|
build_dir = tmp_path / 'build'
|
|
build_dir.mkdir()
|
|
build_log = build_dir / 'build.log'
|
|
build_log.write_text('')
|
|
testinstance.build_dir = build_dir
|
|
|
|
with pytest.raises(expected_error) if expected_error else nullcontext():
|
|
res = testinstance.get_buildlog_file()
|
|
|
|
if expected_error is None:
|
|
assert res == str(build_log)
|