# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2023 PANTHEON.tech s.r.o. # Copyright(c) 2023 University of New Hampshire """ Generic result container and reporters """ import os.path from collections.abc import MutableSequence from enum import Enum, auto from .config import ( OS, Architecture, BuildTargetConfiguration, BuildTargetInfo, Compiler, CPUType, NodeConfiguration, NodeInfo, ) from .exception import DTSError, ErrorSeverity from .logger import DTSLOG from .settings import SETTINGS class Result(Enum): """ An Enum defining the possible states that a setup, a teardown or a test case may end up in. """ PASS = auto() FAIL = auto() ERROR = auto() SKIP = auto() def __bool__(self) -> bool: return self is self.PASS class FixtureResult(object): """ A record that stored the result of a setup or a teardown. The default is FAIL because immediately after creating the object the setup of the corresponding stage will be executed, which also guarantees the execution of teardown. """ result: Result error: Exception | None = None def __init__( self, result: Result = Result.FAIL, error: Exception | None = None, ): self.result = result self.error = error def __bool__(self) -> bool: return bool(self.result) class Statistics(dict): """ A helper class used to store the number of test cases by its result along a few other basic information. Using a dict provides a convenient way to format the data. """ def __init__(self, dpdk_version: str | None): super(Statistics, self).__init__() for result in Result: self[result.name] = 0 self["PASS RATE"] = 0.0 self["DPDK VERSION"] = dpdk_version def __iadd__(self, other: Result) -> "Statistics": """ Add a Result to the final count. """ self[other.name] += 1 self["PASS RATE"] = ( float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result) ) return self def __str__(self) -> str: """ Provide a string representation of the data. """ stats_str = "" for key, value in self.items(): stats_str += f"{key:<12} = {value}\n" # according to docs, we should use \n when writing to text files # on all platforms return stats_str class BaseResult(object): """ The Base class for all results. Stores the results of the setup and teardown portions of the corresponding stage and a list of results from each inner stage in _inner_results. """ setup_result: FixtureResult teardown_result: FixtureResult _inner_results: MutableSequence["BaseResult"] def __init__(self): self.setup_result = FixtureResult() self.teardown_result = FixtureResult() self._inner_results = [] def update_setup(self, result: Result, error: Exception | None = None) -> None: self.setup_result.result = result self.setup_result.error = error def update_teardown(self, result: Result, error: Exception | None = None) -> None: self.teardown_result.result = result self.teardown_result.error = error def _get_setup_teardown_errors(self) -> list[Exception]: errors = [] if self.setup_result.error: errors.append(self.setup_result.error) if self.teardown_result.error: errors.append(self.teardown_result.error) return errors def _get_inner_errors(self) -> list[Exception]: return [ error for inner_result in self._inner_results for error in inner_result.get_errors() ] def get_errors(self) -> list[Exception]: return self._get_setup_teardown_errors() + self._get_inner_errors() def add_stats(self, statistics: Statistics) -> None: for inner_result in self._inner_results: inner_result.add_stats(statistics) class TestCaseResult(BaseResult, FixtureResult): """ The test case specific result. Stores the result of the actual test case. Also stores the test case name. """ test_case_name: str def __init__(self, test_case_name: str): super(TestCaseResult, self).__init__() self.test_case_name = test_case_name def update(self, result: Result, error: Exception | None = None) -> None: self.result = result self.error = error def _get_inner_errors(self) -> list[Exception]: if self.error: return [self.error] return [] def add_stats(self, statistics: Statistics) -> None: statistics += self.result def __bool__(self) -> bool: return bool(self.setup_result) and bool(self.teardown_result) and bool(self.result) class TestSuiteResult(BaseResult): """ The test suite specific result. The _inner_results list stores results of test cases in a given test suite. Also stores the test suite name. """ suite_name: str def __init__(self, suite_name: str): super(TestSuiteResult, self).__init__() self.suite_name = suite_name def add_test_case(self, test_case_name: str) -> TestCaseResult: test_case_result = TestCaseResult(test_case_name) self._inner_results.append(test_case_result) return test_case_result class BuildTargetResult(BaseResult): """ The build target specific result. The _inner_results list stores results of test suites in a given build target. Also stores build target specifics, such as compiler used to build DPDK. """ arch: Architecture os: OS cpu: CPUType compiler: Compiler compiler_version: str | None dpdk_version: str | None def __init__(self, build_target: BuildTargetConfiguration): super(BuildTargetResult, self).__init__() self.arch = build_target.arch self.os = build_target.os self.cpu = build_target.cpu self.compiler = build_target.compiler self.compiler_version = None self.dpdk_version = None def add_build_target_info(self, versions: BuildTargetInfo) -> None: self.compiler_version = versions.compiler_version self.dpdk_version = versions.dpdk_version def add_test_suite(self, test_suite_name: str) -> TestSuiteResult: test_suite_result = TestSuiteResult(test_suite_name) self._inner_results.append(test_suite_result) return test_suite_result class ExecutionResult(BaseResult): """ The execution specific result. The _inner_results list stores results of build targets in a given execution. Also stores the SUT node configuration. """ sut_node: NodeConfiguration sut_os_name: str sut_os_version: str sut_kernel_version: str def __init__(self, sut_node: NodeConfiguration): super(ExecutionResult, self).__init__() self.sut_node = sut_node def add_build_target(self, build_target: BuildTargetConfiguration) -> BuildTargetResult: build_target_result = BuildTargetResult(build_target) self._inner_results.append(build_target_result) return build_target_result def add_sut_info(self, sut_info: NodeInfo): self.sut_os_name = sut_info.os_name self.sut_os_version = sut_info.os_version self.sut_kernel_version = sut_info.kernel_version class DTSResult(BaseResult): """ Stores environment information and test results from a DTS run, which are: * Execution level information, such as SUT and TG hardware. * Build target level information, such as compiler, target OS and cpu. * Test suite results. * All errors that are caught and recorded during DTS execution. The information is stored in nested objects. The class is capable of computing the return code used to exit DTS with from the stored error. It also provides a brief statistical summary of passed/failed test cases. """ dpdk_version: str | None _logger: DTSLOG _errors: list[Exception] _return_code: ErrorSeverity _stats_result: Statistics | None _stats_filename: str def __init__(self, logger: DTSLOG): super(DTSResult, self).__init__() self.dpdk_version = None self._logger = logger self._errors = [] self._return_code = ErrorSeverity.NO_ERR self._stats_result = None self._stats_filename = os.path.join(SETTINGS.output_dir, "statistics.txt") def add_execution(self, sut_node: NodeConfiguration) -> ExecutionResult: execution_result = ExecutionResult(sut_node) self._inner_results.append(execution_result) return execution_result def add_error(self, error) -> None: self._errors.append(error) def process(self) -> None: """ Process the data after a DTS run. The data is added to nested objects during runtime and this parent object is not updated at that time. This requires us to process the nested data after it's all been gathered. The processing gathers all errors and the result statistics of test cases. """ self._errors += self.get_errors() if self._errors and self._logger: self._logger.debug("Summary of errors:") for error in self._errors: self._logger.debug(repr(error)) self._stats_result = Statistics(self.dpdk_version) self.add_stats(self._stats_result) with open(self._stats_filename, "w+") as stats_file: stats_file.write(str(self._stats_result)) def get_return_code(self) -> int: """ Go through all stored Exceptions and return the highest error code found. """ for error in self._errors: error_return_code = ErrorSeverity.GENERIC_ERR if isinstance(error, DTSError): error_return_code = error.severity if error_return_code > self._return_code: self._return_code = error_return_code return int(self._return_code)