@@ -28,6 +28,7 @@
from dataclasses import dataclass
from enum import Enum, auto
from types import MethodType
+from typing import Union
from .config import (
OS,
@@ -129,58 +130,6 @@ def __bool__(self) -> bool:
return bool(self.result)
-class Statistics(dict):
- """How many test cases ended in which result state along some other basic information.
-
- Subclassing :class:`dict` provides a convenient way to format the data.
-
- The data are stored in the following keys:
-
- * **PASS RATE** (:class:`int`) -- The FAIL/PASS ratio of all test cases.
- * **DPDK VERSION** (:class:`str`) -- The tested DPDK version.
- """
-
- def __init__(self, dpdk_version: str | None):
- """Extend the constructor with keys in which the data are stored.
-
- Args:
- dpdk_version: The version of tested DPDK.
- """
- super(Statistics, self).__init__()
- for result in Result:
- self[result.name] = 0
- self["PASS RATE"] = 0.0
- self["DPDK VERSION"] = dpdk_version
-
- def __iadd__(self, other: Result) -> "Statistics":
- """Add a Result to the final count.
-
- Example:
- stats: Statistics = Statistics() # empty Statistics
- stats += Result.PASS # add a Result to `stats`
-
- Args:
- other: The Result to add to this statistics object.
-
- Returns:
- The modified statistics object.
- """
- self[other.name] += 1
- self["PASS RATE"] = (
- float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result)
- )
- return self
-
- def __str__(self) -> str:
- """Each line contains the formatted key = value pair."""
- stats_str = ""
- for key, value in self.items():
- stats_str += f"{key:<12} = {value}\n"
- # according to docs, we should use \n when writing to text files
- # on all platforms
- return stats_str
-
-
class BaseResult(object):
"""Common data and behavior of DTS results.
@@ -245,7 +194,7 @@ def get_errors(self) -> list[Exception]:
"""
return self._get_setup_teardown_errors() + self._get_inner_errors()
- def add_stats(self, statistics: Statistics) -> None:
+ def add_stats(self, statistics: "Statistics") -> None:
"""Collate stats from the whole result hierarchy.
Args:
@@ -255,91 +204,149 @@ def add_stats(self, statistics: Statistics) -> None:
inner_result.add_stats(statistics)
-class TestCaseResult(BaseResult, FixtureResult):
- r"""The test case specific result.
+class DTSResult(BaseResult):
+ """Stores environment information and test results from a DTS run.
- Stores the result of the actual test case. This is done by adding an extra superclass
- in :class:`FixtureResult`. The setup and teardown results are :class:`FixtureResult`\s and
- the class is itself a record of the test case.
+ * Execution level information, such as testbed and the test suite list,
+ * Build target level information, such as compiler, target OS and cpu,
+ * Test suite and test case results,
+ * All errors that are caught and recorded during DTS execution.
+
+ The information is stored hierarchically. This is the first level of the hierarchy
+ and as such is where the data form the whole hierarchy is collated or processed.
+
+ The internal list stores the results of all executions.
Attributes:
- test_case_name: The test case name.
+ dpdk_version: The DPDK version to record.
"""
- test_case_name: str
+ dpdk_version: str | None
+ _logger: DTSLOG
+ _errors: list[Exception]
+ _return_code: ErrorSeverity
+ _stats_result: Union["Statistics", None]
+ _stats_filename: str
- def __init__(self, test_case_name: str):
- """Extend the constructor with `test_case_name`.
+ def __init__(self, logger: DTSLOG):
+ """Extend the constructor with top-level specifics.
Args:
- test_case_name: The test case's name.
+ logger: The logger instance the whole result will use.
"""
- super(TestCaseResult, self).__init__()
- self.test_case_name = test_case_name
+ super(DTSResult, self).__init__()
+ self.dpdk_version = None
+ self._logger = logger
+ self._errors = []
+ self._return_code = ErrorSeverity.NO_ERR
+ self._stats_result = None
+ self._stats_filename = os.path.join(SETTINGS.output_dir, "statistics.txt")
- def update(self, result: Result, error: Exception | None = None) -> None:
- """Update the test case result.
+ def add_execution(self, sut_node: NodeConfiguration) -> "ExecutionResult":
+ """Add and return the inner result (execution).
- This updates the result of the test case itself and doesn't affect
- the results of the setup and teardown steps in any way.
+ Args:
+ sut_node: The SUT node's test run configuration.
+
+ Returns:
+ The execution's result.
+ """
+ execution_result = ExecutionResult(sut_node)
+ self._inner_results.append(execution_result)
+ return execution_result
+
+ def add_error(self, error: Exception) -> None:
+ """Record an error that occurred outside any execution.
Args:
- result: The result of the test case.
- error: The error that occurred in case of a failure.
+ error: The exception to record.
"""
- self.result = result
- self.error = error
+ self._errors.append(error)
- def _get_inner_errors(self) -> list[Exception]:
- if self.error:
- return [self.error]
- return []
+ def process(self) -> None:
+ """Process the data after a whole DTS run.
- def add_stats(self, statistics: Statistics) -> None:
- r"""Add the test case result to statistics.
+ The data is added to inner objects during runtime and this object is not updated
+ at that time. This requires us to process the inner data after it's all been gathered.
- The base method goes through the hierarchy recursively and this method is here to stop
- the recursion, as the :class:`TestCaseResult`\s are the leaves of the hierarchy tree.
+ The processing gathers all errors and the statistics of test case results.
+ """
+ self._errors += self.get_errors()
+ if self._errors and self._logger:
+ self._logger.debug("Summary of errors:")
+ for error in self._errors:
+ self._logger.debug(repr(error))
- Args:
- statistics: The :class:`Statistics` object where the stats will be added.
+ self._stats_result = Statistics(self.dpdk_version)
+ self.add_stats(self._stats_result)
+ with open(self._stats_filename, "w+") as stats_file:
+ stats_file.write(str(self._stats_result))
+
+ def get_return_code(self) -> int:
+ """Go through all stored Exceptions and return the final DTS error code.
+
+ Returns:
+ The highest error code found.
"""
- statistics += self.result
+ for error in self._errors:
+ error_return_code = ErrorSeverity.GENERIC_ERR
+ if isinstance(error, DTSError):
+ error_return_code = error.severity
- def __bool__(self) -> bool:
- """The test case passed only if setup, teardown and the test case itself passed."""
- return bool(self.setup_result) and bool(self.teardown_result) and bool(self.result)
+ if error_return_code > self._return_code:
+ self._return_code = error_return_code
+ return int(self._return_code)
-class TestSuiteResult(BaseResult):
- """The test suite specific result.
- The internal list stores the results of all test cases in a given test suite.
+class ExecutionResult(BaseResult):
+ """The execution specific result.
+
+ The internal list stores the results of all build targets in a given execution.
Attributes:
- suite_name: The test suite name.
+ sut_node: The SUT node used in the execution.
+ sut_os_name: The operating system of the SUT node.
+ sut_os_version: The operating system version of the SUT node.
+ sut_kernel_version: The operating system kernel version of the SUT node.
"""
- suite_name: str
+ sut_node: NodeConfiguration
+ sut_os_name: str
+ sut_os_version: str
+ sut_kernel_version: str
- def __init__(self, suite_name: str):
- """Extend the constructor with `suite_name`.
+ def __init__(self, sut_node: NodeConfiguration):
+ """Extend the constructor with the `sut_node`'s config.
Args:
- suite_name: The test suite's name.
+ sut_node: The SUT node's test run configuration used in the execution.
"""
- super(TestSuiteResult, self).__init__()
- self.suite_name = suite_name
+ super(ExecutionResult, self).__init__()
+ self.sut_node = sut_node
- def add_test_case(self, test_case_name: str) -> TestCaseResult:
- """Add and return the inner result (test case).
+ def add_build_target(self, build_target: BuildTargetConfiguration) -> "BuildTargetResult":
+ """Add and return the inner result (build target).
+
+ Args:
+ build_target: The build target's test run configuration.
Returns:
- The test case's result.
+ The build target's result.
"""
- test_case_result = TestCaseResult(test_case_name)
- self._inner_results.append(test_case_result)
- return test_case_result
+ build_target_result = BuildTargetResult(build_target)
+ self._inner_results.append(build_target_result)
+ return build_target_result
+
+ def add_sut_info(self, sut_info: NodeInfo) -> None:
+ """Add SUT information gathered at runtime.
+
+ Args:
+ sut_info: The additional SUT node information.
+ """
+ self.sut_os_name = sut_info.os_name
+ self.sut_os_version = sut_info.os_version
+ self.sut_kernel_version = sut_info.kernel_version
class BuildTargetResult(BaseResult):
@@ -386,7 +393,7 @@ def add_build_target_info(self, versions: BuildTargetInfo) -> None:
self.compiler_version = versions.compiler_version
self.dpdk_version = versions.dpdk_version
- def add_test_suite(self, test_suite_name: str) -> TestSuiteResult:
+ def add_test_suite(self, test_suite_name: str) -> "TestSuiteResult":
"""Add and return the inner result (test suite).
Returns:
@@ -397,146 +404,140 @@ def add_test_suite(self, test_suite_name: str) -> TestSuiteResult:
return test_suite_result
-class ExecutionResult(BaseResult):
- """The execution specific result.
+class TestSuiteResult(BaseResult):
+ """The test suite specific result.
- The internal list stores the results of all build targets in a given execution.
+ The internal list stores the results of all test cases in a given test suite.
Attributes:
- sut_node: The SUT node used in the execution.
- sut_os_name: The operating system of the SUT node.
- sut_os_version: The operating system version of the SUT node.
- sut_kernel_version: The operating system kernel version of the SUT node.
+ suite_name: The test suite name.
"""
- sut_node: NodeConfiguration
- sut_os_name: str
- sut_os_version: str
- sut_kernel_version: str
+ suite_name: str
- def __init__(self, sut_node: NodeConfiguration):
- """Extend the constructor with the `sut_node`'s config.
+ def __init__(self, suite_name: str):
+ """Extend the constructor with `suite_name`.
Args:
- sut_node: The SUT node's test run configuration used in the execution.
+ suite_name: The test suite's name.
"""
- super(ExecutionResult, self).__init__()
- self.sut_node = sut_node
-
- def add_build_target(self, build_target: BuildTargetConfiguration) -> BuildTargetResult:
- """Add and return the inner result (build target).
+ super(TestSuiteResult, self).__init__()
+ self.suite_name = suite_name
- Args:
- build_target: The build target's test run configuration.
+ def add_test_case(self, test_case_name: str) -> "TestCaseResult":
+ """Add and return the inner result (test case).
Returns:
- The build target's result.
- """
- build_target_result = BuildTargetResult(build_target)
- self._inner_results.append(build_target_result)
- return build_target_result
-
- def add_sut_info(self, sut_info: NodeInfo) -> None:
- """Add SUT information gathered at runtime.
-
- Args:
- sut_info: The additional SUT node information.
+ The test case's result.
"""
- self.sut_os_name = sut_info.os_name
- self.sut_os_version = sut_info.os_version
- self.sut_kernel_version = sut_info.kernel_version
+ test_case_result = TestCaseResult(test_case_name)
+ self._inner_results.append(test_case_result)
+ return test_case_result
-class DTSResult(BaseResult):
- """Stores environment information and test results from a DTS run.
-
- * Execution level information, such as testbed and the test suite list,
- * Build target level information, such as compiler, target OS and cpu,
- * Test suite and test case results,
- * All errors that are caught and recorded during DTS execution.
-
- The information is stored hierarchically. This is the first level of the hierarchy
- and as such is where the data form the whole hierarchy is collated or processed.
+class TestCaseResult(BaseResult, FixtureResult):
+ r"""The test case specific result.
- The internal list stores the results of all executions.
+ Stores the result of the actual test case. This is done by adding an extra superclass
+ in :class:`FixtureResult`. The setup and teardown results are :class:`FixtureResult`\s and
+ the class is itself a record of the test case.
Attributes:
- dpdk_version: The DPDK version to record.
+ test_case_name: The test case name.
"""
- dpdk_version: str | None
- _logger: DTSLOG
- _errors: list[Exception]
- _return_code: ErrorSeverity
- _stats_result: Statistics | None
- _stats_filename: str
+ test_case_name: str
- def __init__(self, logger: DTSLOG):
- """Extend the constructor with top-level specifics.
+ def __init__(self, test_case_name: str):
+ """Extend the constructor with `test_case_name`.
Args:
- logger: The logger instance the whole result will use.
+ test_case_name: The test case's name.
"""
- super(DTSResult, self).__init__()
- self.dpdk_version = None
- self._logger = logger
- self._errors = []
- self._return_code = ErrorSeverity.NO_ERR
- self._stats_result = None
- self._stats_filename = os.path.join(SETTINGS.output_dir, "statistics.txt")
+ super(TestCaseResult, self).__init__()
+ self.test_case_name = test_case_name
- def add_execution(self, sut_node: NodeConfiguration) -> ExecutionResult:
- """Add and return the inner result (execution).
+ def update(self, result: Result, error: Exception | None = None) -> None:
+ """Update the test case result.
- Args:
- sut_node: The SUT node's test run configuration.
+ This updates the result of the test case itself and doesn't affect
+ the results of the setup and teardown steps in any way.
- Returns:
- The execution's result.
+ Args:
+ result: The result of the test case.
+ error: The error that occurred in case of a failure.
"""
- execution_result = ExecutionResult(sut_node)
- self._inner_results.append(execution_result)
- return execution_result
+ self.result = result
+ self.error = error
- def add_error(self, error: Exception) -> None:
- """Record an error that occurred outside any execution.
+ def _get_inner_errors(self) -> list[Exception]:
+ if self.error:
+ return [self.error]
+ return []
+
+ def add_stats(self, statistics: "Statistics") -> None:
+ r"""Add the test case result to statistics.
+
+ The base method goes through the hierarchy recursively and this method is here to stop
+ the recursion, as the :class:`TestCaseResult`\s are the leaves of the hierarchy tree.
Args:
- error: The exception to record.
+ statistics: The :class:`Statistics` object where the stats will be added.
"""
- self._errors.append(error)
+ statistics += self.result
- def process(self) -> None:
- """Process the data after a whole DTS run.
+ def __bool__(self) -> bool:
+ """The test case passed only if setup, teardown and the test case itself passed."""
+ return bool(self.setup_result) and bool(self.teardown_result) and bool(self.result)
- The data is added to inner objects during runtime and this object is not updated
- at that time. This requires us to process the inner data after it's all been gathered.
- The processing gathers all errors and the statistics of test case results.
+class Statistics(dict):
+ """How many test cases ended in which result state along some other basic information.
+
+ Subclassing :class:`dict` provides a convenient way to format the data.
+
+ The data are stored in the following keys:
+
+ * **PASS RATE** (:class:`int`) -- The FAIL/PASS ratio of all test cases.
+ * **DPDK VERSION** (:class:`str`) -- The tested DPDK version.
+ """
+
+ def __init__(self, dpdk_version: str | None):
+ """Extend the constructor with keys in which the data are stored.
+
+ Args:
+ dpdk_version: The version of tested DPDK.
"""
- self._errors += self.get_errors()
- if self._errors and self._logger:
- self._logger.debug("Summary of errors:")
- for error in self._errors:
- self._logger.debug(repr(error))
+ super(Statistics, self).__init__()
+ for result in Result:
+ self[result.name] = 0
+ self["PASS RATE"] = 0.0
+ self["DPDK VERSION"] = dpdk_version
- self._stats_result = Statistics(self.dpdk_version)
- self.add_stats(self._stats_result)
- with open(self._stats_filename, "w+") as stats_file:
- stats_file.write(str(self._stats_result))
+ def __iadd__(self, other: Result) -> "Statistics":
+ """Add a Result to the final count.
- def get_return_code(self) -> int:
- """Go through all stored Exceptions and return the final DTS error code.
+ Example:
+ stats: Statistics = Statistics() # empty Statistics
+ stats += Result.PASS # add a Result to `stats`
+
+ Args:
+ other: The Result to add to this statistics object.
Returns:
- The highest error code found.
+ The modified statistics object.
"""
- for error in self._errors:
- error_return_code = ErrorSeverity.GENERIC_ERR
- if isinstance(error, DTSError):
- error_return_code = error.severity
-
- if error_return_code > self._return_code:
- self._return_code = error_return_code
+ self[other.name] += 1
+ self["PASS RATE"] = (
+ float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result)
+ )
+ return self
- return int(self._return_code)
+ def __str__(self) -> str:
+ """Each line contains the formatted key = value pair."""
+ stats_str = ""
+ for key, value in self.items():
+ stats_str += f"{key:<12} = {value}\n"
+ # according to docs, we should use \n when writing to text files
+ # on all platforms
+ return stats_str