<div dir="ltr"><div dir="ltr">On Mon, Nov 14, 2022 at 11:54 AM Juraj Linkeš <juraj.linkes@pantheon.tech> wrote: </div><div class="gmail_quote"><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">This is the base class that all test suites inherit from. The base class implements methods common to all test suites. The derived test suites implement tests and any particular setup needed for the suite or tests. Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech> --- dts/conf.yaml | 4 + dts/framework/config/__init__.py | 33 ++- dts/framework/config/conf_yaml_schema.json | 49 ++++ dts/framework/dts.py | 29 +++ dts/framework/exception.py | 65 ++++++ dts/framework/settings.py | 25 +++ dts/framework/test_case.py | 246 +++++++++++++++++++++ 7 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 dts/framework/test_case.py diff --git a/dts/conf.yaml b/dts/conf.yaml index 976888a88e..0b0f2c59b0 100644 --- a/dts/conf.yaml +++ b/dts/conf.yaml @@ -7,6 +7,10 @@ executions: os: linux cpu: native compiler: gcc + perf: false + func: true + test_suites: + - hello_world system_under_test: "SUT 1" nodes: - name: "SUT 1" diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index 344d697a69..8874b10030 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -11,7 +11,7 @@ import pathlib from dataclasses import dataclass from enum import Enum, auto, unique -from typing import Any, Iterable +from typing import Any, Iterable, TypedDict import warlock # type: ignore import yaml @@ -186,9 +186,34 @@ def from_dict(d: dict) -> "BuildTargetConfiguration": ) +class TestSuiteConfigDict(TypedDict): + suite: str + cases: list[str] + + +@dataclass(slots=True, frozen=True) +class TestSuiteConfig: + test_suite: str + test_cases: list[str] + + @staticmethod + def from_dict( + entry: str | TestSuiteConfigDict, + ) -> "TestSuiteConfig": + if isinstance(entry, str): + return TestSuiteConfig(test_suite=entry, test_cases=[]) + elif isinstance(entry, dict): + return TestSuiteConfig(test_suite=entry["suite"], test_cases=entry["cases"]) + else: + raise TypeError(f"{type(entry)} is not valid for a test suite config.") + + @dataclass(slots=True, frozen=True) class ExecutionConfiguration: build_targets: list[BuildTargetConfiguration] + perf: bool + func: bool + test_suites: list[TestSuiteConfig] system_under_test: NodeConfiguration @staticmethod @@ -196,11 +221,17 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration": build_targets: list[BuildTargetConfiguration] = list( map(BuildTargetConfiguration.from_dict, d["build_targets"]) ) + test_suites: list[TestSuiteConfig] = list( + map(TestSuiteConfig.from_dict, d["test_suites"]) + ) sut_name = d["system_under_test"] assert sut_name in node_map, f"Unknown SUT {sut_name} in execution {d}" return ExecutionConfiguration( build_targets=build_targets, + perf=d["perf"], + func=d["func"], + test_suites=test_suites, system_under_test=node_map[sut_name], ) diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json index c59d3e30e6..e37ced65fe 100644 --- a/dts/framework/config/conf_yaml_schema.json +++ b/dts/framework/config/conf_yaml_schema.json @@ -63,6 +63,31 @@ } }, "additionalProperties": false + }, + "test_suite": { + "type": "string", + "enum": [ + "hello_world" + ] + }, + "test_target": { + "type": "object", + "properties": { + "suite": { + "$ref": "#/definitions/test_suite" + }, + "cases": { + "type": "array", + "items": { + "type": "string" + }, + "minimum": 1 + } + }, + "required": [ + "suite" + ], + "additionalProperties": false } }, "type": "object", @@ -130,6 +155,27 @@ }, "minimum": 1 }, + "perf": { + "type": "boolean", + "description": "Enable performance testing" + }, + "func": { + "type": "boolean", + "description": "Enable functional testing" + }, + "test_suites": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/definitions/test_suite" + }, + { + "$ref": "#/definitions/test_target" + } + ] + } + }, "system_under_test": { "$ref": "#/definitions/node_name" } @@ -137,6 +183,9 @@ "additionalProperties": false, "required": [ "build_targets", + "perf", + "func", + "test_suites", "system_under_test" ] }, diff --git a/dts/framework/dts.py b/dts/framework/dts.py index a7c243a5c3..ba3f4b4168 100644 --- a/dts/framework/dts.py +++ b/dts/framework/dts.py @@ -15,6 +15,7 @@ from .logger import DTSLOG, getLogger from .settings import SETTINGS from .stats_reporter import TestStats +from .test_case import TestCase from .test_result import Result from .utils import check_dts_python_version @@ -129,6 +130,34 @@ def run_suite( Use the given build_target to run the test suite with possibly only a subset of tests. If no subset is specified, run all tests. """ + for test_suite_config in execution.test_suites: + result.test_suite = test_suite_config.test_suite + full_suite_path = f"tests.TestSuite_{test_suite_config.test_suite}" + testcase_classes = TestCase.get_testcases(full_suite_path) + dts_logger.debug( + f"Found testcase classes '{testcase_classes}' in '{full_suite_path}'" + ) + for testcase_class in testcase_classes: + testcase = testcase_class( + sut_node, test_suite_config.test_suite, build_target, execution + ) + + testcase.init_log() + testcase.set_requested_cases(SETTINGS.test_cases) + testcase.set_requested_cases(test_suite_config.test_cases) + + <a href="http://dts_logger.info" rel="noreferrer" target="_blank">dts_logger.info</a>(f"Running test suite '{testcase_class.__name__}'") + try: + testcase.execute_setup_all() + testcase.execute_test_cases() + <a href="http://dts_logger.info" rel="noreferrer" target="_blank">dts_logger.info</a>( + f"Finished running test suite '{testcase_class.__name__}'" + ) + result.copy_suite(testcase.get_result()) + test_stats.save(result) # this was originally after teardown + + finally: </blockquote><div> </div><div>You should probably move the "finished" log message down here, so that it always runs.</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex"> + testcase.execute_tear_downall() def quit_execution(nodes: Iterable[Node], return_code: ReturnCode) -> None: diff --git a/dts/framework/exception.py b/dts/framework/exception.py index 93d99432ae..a35eeff640 100644 --- a/dts/framework/exception.py +++ b/dts/framework/exception.py @@ -29,6 +29,10 @@ class ReturnCode(IntEnum): DPDK_BUILD_ERR = 10 NODE_SETUP_ERR = 20 NODE_CLEANUP_ERR = 21 + SUITE_SETUP_ERR = 30 + SUITE_EXECUTION_ERR = 31 + TESTCASE_VERIFY_ERR = 32 + SUITE_CLEANUP_ERR = 33 class DTSError(Exception): @@ -153,6 +157,67 @@ def __init__(self): ) +class TestSuiteNotFound(DTSError): + """ + Raised when a configured test suite cannot be imported. + """ + + return_code: ClassVar[ReturnCode] = ReturnCode.SUITE_SETUP_ERR + + +class SuiteSetupError(DTSError): + """ + Raised when an error occurs during suite setup. + """ + + return_code: ClassVar[ReturnCode] = ReturnCode.SUITE_SETUP_ERR + + def __init__(self): + super(SuiteSetupError, self).__init__("An error occurred during suite setup.") + + +class SuiteExecutionError(DTSError): + """ + Raised when an error occurs during suite execution. + """ + + return_code: ClassVar[ReturnCode] = ReturnCode.SUITE_EXECUTION_ERR + + def __init__(self): + super(SuiteExecutionError, self).__init__( + "An error occurred during suite execution." + ) + + +class VerifyError(DTSError): + """ + To be used within the test cases to verify if a command output + is as it was expected. + """ + + value: str + return_code: ClassVar[ReturnCode] = ReturnCode.TESTCASE_VERIFY_ERR + + def __init__(self, value: str): + self.value = value + + def __str__(self) -> str: + return repr(self.value) + + +class SuiteCleanupError(DTSError): + """ + Raised when an error occurs during suite cleanup. + """ + + return_code: ClassVar[ReturnCode] = ReturnCode.SUITE_CLEANUP_ERR + + def __init__(self): + super(SuiteCleanupError, self).__init__( + "An error occurred during suite cleanup." + ) + + def convert_exception(exception: type[DTSError]) -> Callable[..., Callable[..., None]]: """ When a non-DTS exception is raised while executing the decorated function, diff --git a/dts/framework/settings.py b/dts/framework/settings.py index e2bf3d2ce4..069f28ce81 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -64,6 +64,8 @@ class _Settings: skip_setup: bool dpdk_ref: Path compile_timeout: float + test_cases: list + re_run: int def _get_parser() -> argparse.ArgumentParser: @@ -138,6 +140,25 @@ def _get_parser() -> argparse.ArgumentParser: help="[DTS_COMPILE_TIMEOUT] The timeout for compiling DPDK.", ) + parser.add_argument( + "--test-cases", + action=_env_arg("DTS_TESTCASES"), + default="", + required=False, + help="[DTS_TESTCASES] Comma-separated list of testcases to execute", + ) + + parser.add_argument( + "--re-run", + "--re_run", + action=_env_arg("DTS_RERUN"), + default=0, + type=int, + required=False, + help="[DTS_RERUN] Re-run tests the specified amount of times if a test failure " + "occurs", + ) + return parser @@ -151,6 +172,10 @@ def _get_settings() -> _Settings: skip_setup=(parsed_args.skip_setup == "Y"), dpdk_ref=parsed_args.dpdk_ref, compile_timeout=parsed_args.compile_timeout, + test_cases=parsed_args.test_cases.split(",") + if parsed_args.test_cases != "" + else [], + re_run=parsed_args.re_run, ) diff --git a/dts/framework/test_case.py b/dts/framework/test_case.py new file mode 100644 index 0000000000..0479f795bb --- /dev/null +++ b/dts/framework/test_case.py @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. + +""" +A base class for creating DTS test cases. +""" + +import importlib +import inspect +import re +import time +import traceback + +from .exception import ( + SSHTimeoutError, + SuiteCleanupError, + SuiteExecutionError, + SuiteSetupError, + TestSuiteNotFound, + VerifyError, + convert_exception, +) +from .logger import getLogger +from .settings import SETTINGS +from .test_result import Result + + +class TestCase(object): + def __init__(self, sut_node, suitename, target, execution): + self.sut_node = sut_node + self.suite_name = suitename + self.target = target + + # local variable + self._requested_tests = [] + self._subtitle = None + + # result object for save suite result + self._suite_result = Result() + self._suite_result.sut = self.sut_node.config.hostname + self._suite_result.target = target + self._suite_result.test_suite = self.suite_name + if self._suite_result is None: + raise ValueError("Result object should not None") + + self._enable_func = execution.func + + # command history + self.setup_history = list() + self.test_history = list() + + def init_log(self): + # get log handler + class_name = self.__class__.__name__ + self.logger = getLogger(class_name) + + def set_up_all(self): + pass + + def set_up(self): + pass + + def tear_down(self): + pass + + def tear_down_all(self): + pass + + def verify(self, passed, description): + if not passed: + raise VerifyError(description) + + def _get_functional_cases(self): + """ + Get all functional test cases. + """ + return self._get_test_cases(r"test_(?!perf_)") + + def _has_it_been_requested(self, test_case, test_name_regex): + """ + Check whether test case has been requested for validation. + """ + name_matches = re.match(test_name_regex, test_case.__name__) + if self._requested_tests: + return name_matches and test_case.__name__ in self._requested_tests + + return name_matches + + def set_requested_cases(self, case_list): + """ + Pass down input cases list for check + """ + self._requested_tests += case_list + + def _get_test_cases(self, test_name_regex): + """ + Return case list which name matched regex. + """ + self.logger.debug(f"Searching for testcases in {self.__class__}") + for test_case_name in dir(self): + test_case = getattr(self, test_case_name) + if callable(test_case) and self._has_it_been_requested( + test_case, test_name_regex + ): + yield test_case + + @convert_exception(SuiteSetupError) + def execute_setup_all(self): + """ + Execute suite setup_all function before cases. + """ + try: + self.set_up_all() + return True + except Exception as v: + self.logger.error("set_up_all failed:\n" + traceback.format_exc()) + # record all cases blocked + if self._enable_func: + for case_obj in self._get_functional_cases(): + self._suite_result.test_case = case_obj.__name__ + self._suite_result.test_case_blocked( + "set_up_all failed: {}".format(str(v)) + ) + return False + + def _execute_test_case(self, case_obj): + """ + Execute specified test case in specified suite. If any exception occurred in + validation process, save the result and tear down this case. + """ + case_name = case_obj.__name__ + self._suite_result.test_case = case_obj.__name__ + + case_result = True + try: + <a href="http://self.logger.info" rel="noreferrer" target="_blank">self.logger.info</a>("Test Case %s Begin" % case_name) + + self.running_case = case_name + # run set_up function for each case + self.set_up() + # run test case + case_obj() + + self._suite_result.test_case_passed() + + <a href="http://self.logger.info" rel="noreferrer" target="_blank">self.logger.info</a>("Test Case %s Result PASSED:" % case_name) + + except VerifyError as v: + case_result = False + self._suite_result.test_case_failed(str(v)) + self.logger.error("Test Case %s Result FAILED: " % (case_name) + str(v)) + except KeyboardInterrupt: + self._suite_result.test_case_blocked("Skipped") + self.logger.error("Test Case %s SKIPPED: " % (case_name)) + self.tear_down() + raise KeyboardInterrupt("Stop DTS") + except SSHTimeoutError as e: + case_result = False + self._suite_result.test_case_failed(str(e)) + self.logger.error("Test Case %s Result FAILED: " % (case_name) + str(e)) + self.logger.error("%s" % (e.get_output())) + except Exception: + case_result = False + trace = traceback.format_exc() + self._suite_result.test_case_failed(trace) + self.logger.error("Test Case %s Result ERROR: " % (case_name) + trace) + finally: + self.execute_tear_down() + return case_result + + @convert_exception(SuiteExecutionError) + def execute_test_cases(self): + """ + Execute all test cases in one suite. + """ + # prepare debugger rerun case environment + if self._enable_func: + for case_obj in self._get_functional_cases(): + for i in range(SETTINGS.re_run + 1): + ret = self.execute_test_case(case_obj) + + if ret is False and SETTINGS.re_run: + self.sut_node.get_session_output(timeout=0.5 * (i + 1)) + time.sleep(i + 1) + <a href="http://self.logger.info" rel="noreferrer" target="_blank">self.logger.info</a>( + " Test case %s failed and re-run %d time" + % (case_obj.__name__, i + 1) + ) + else: + break + + def execute_test_case(self, case_obj): + """ + Execute test case or enter into debug mode. + """ + return self._execute_test_case(case_obj) + + def get_result(self): + """ + Return suite test result + """ + return self._suite_result + + @convert_exception(SuiteCleanupError) + def execute_tear_downall(self): + """ + execute suite tear_down_all function + """ + self.tear_down_all() + + self.sut_node.kill_cleanup_dpdk_apps() + + def execute_tear_down(self): + """ + execute suite tear_down function + """ + try: + self.tear_down() + except Exception: + self.logger.error("tear_down failed:\n" + traceback.format_exc()) + self.logger.warning( + "tear down %s failed, might iterfere next case's result!" + % self.running_case + ) + + @staticmethod + def get_testcases(testsuite_module_path: str) -> list[type["TestCase"]]: + def is_testcase(object) -> bool: + try: + if issubclass(object, TestCase) and object != TestCase: + return True + except TypeError: + return False + return False + + try: + testcase_module = importlib.import_module(testsuite_module_path) + except ModuleNotFoundError as e: + raise TestSuiteNotFound( + f"Testsuite '{testsuite_module_path}' not found." + ) from e + return [ + testcase_class + for _, testcase_class in inspect.getmembers(testcase_module, is_testcase) + ] -- 2.30.2 </blockquote></div></div>