[RFC,v2,08/10] dts: add testsuite class

  This is the base class that all test suites inherit from. The base class
implements methods common to all test suites. The derived test suites
implement tests and any particular setup needed for the suite or tests.

Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
---
 dts/conf.yaml                              |   4 +
 dts/framework/config/__init__.py           |  33 ++-
 dts/framework/config/conf_yaml_schema.json |  49 ++++
 dts/framework/dts.py                       |  29 +++
 dts/framework/exception.py                 |  65 ++++++
 dts/framework/settings.py                  |  25 +++
 dts/framework/test_case.py                 | 246 +++++++++++++++++++++
 7 files changed, 450 insertions(+), 1 deletion(-)
 create mode 100644 dts/framework/test_case.py
  

On Mon, Nov 14, 2022 at 11:54 AM Juraj Linkeš <juraj.linkes@pantheon.tech>
wrote:

> This is the base class that all test suites inherit from. The base class
> implements methods common to all test suites. The derived test suites
> implement tests and any particular setup needed for the suite or tests.
>
> Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
> ---
>  dts/conf.yaml                              |   4 +
>  dts/framework/config/__init__.py           |  33 ++-
>  dts/framework/config/conf_yaml_schema.json |  49 ++++
>  dts/framework/dts.py                       |  29 +++
>  dts/framework/exception.py                 |  65 ++++++
>  dts/framework/settings.py                  |  25 +++
>  dts/framework/test_case.py                 | 246 +++++++++++++++++++++
>  7 files changed, 450 insertions(+), 1 deletion(-)
>  create mode 100644 dts/framework/test_case.py
>
> diff --git a/dts/conf.yaml b/dts/conf.yaml
> index 976888a88e..0b0f2c59b0 100644
> --- a/dts/conf.yaml
> +++ b/dts/conf.yaml
> @@ -7,6 +7,10 @@ executions:
>          os: linux
>          cpu: native
>          compiler: gcc
> +    perf: false
> +    func: true
> +    test_suites:
> +      - hello_world
>      system_under_test: "SUT 1"
>  nodes:
>    - name: "SUT 1"
> diff --git a/dts/framework/config/__init__.py
> b/dts/framework/config/__init__.py
> index 344d697a69..8874b10030 100644
> --- a/dts/framework/config/__init__.py
> +++ b/dts/framework/config/__init__.py
> @@ -11,7 +11,7 @@
>  import pathlib
>  from dataclasses import dataclass
>  from enum import Enum, auto, unique
> -from typing import Any, Iterable
> +from typing import Any, Iterable, TypedDict
>
>  import warlock  # type: ignore
>  import yaml
> @@ -186,9 +186,34 @@ def from_dict(d: dict) -> "BuildTargetConfiguration":
>          )
>
>
> +class TestSuiteConfigDict(TypedDict):
> +    suite: str
> +    cases: list[str]
> +
> +
> +@dataclass(slots=True, frozen=True)
> +class TestSuiteConfig:
> +    test_suite: str
> +    test_cases: list[str]
> +
> +    @staticmethod
> +    def from_dict(
> +        entry: str | TestSuiteConfigDict,
> +    ) -> "TestSuiteConfig":
> +        if isinstance(entry, str):
> +            return TestSuiteConfig(test_suite=entry, test_cases=[])
> +        elif isinstance(entry, dict):
> +            return TestSuiteConfig(test_suite=entry["suite"],
> test_cases=entry["cases"])
> +        else:
> +            raise TypeError(f"{type(entry)} is not valid for a test suite
> config.")
> +
> +
>  @dataclass(slots=True, frozen=True)
>  class ExecutionConfiguration:
>      build_targets: list[BuildTargetConfiguration]
> +    perf: bool
> +    func: bool
> +    test_suites: list[TestSuiteConfig]
>      system_under_test: NodeConfiguration
>
>      @staticmethod
> @@ -196,11 +221,17 @@ def from_dict(d: dict, node_map: dict) ->
> "ExecutionConfiguration":
>          build_targets: list[BuildTargetConfiguration] = list(
>              map(BuildTargetConfiguration.from_dict, d["build_targets"])
>          )
> +        test_suites: list[TestSuiteConfig] = list(
> +            map(TestSuiteConfig.from_dict, d["test_suites"])
> +        )
>          sut_name = d["system_under_test"]
>          assert sut_name in node_map, f"Unknown SUT {sut_name} in
> execution {d}"
>
>          return ExecutionConfiguration(
>              build_targets=build_targets,
> +            perf=d["perf"],
> +            func=d["func"],
> +            test_suites=test_suites,
>              system_under_test=node_map[sut_name],
>          )
>
> diff --git a/dts/framework/config/conf_yaml_schema.json
> b/dts/framework/config/conf_yaml_schema.json
> index c59d3e30e6..e37ced65fe 100644
> --- a/dts/framework/config/conf_yaml_schema.json
> +++ b/dts/framework/config/conf_yaml_schema.json
> @@ -63,6 +63,31 @@
>          }
>        },
>        "additionalProperties": false
> +    },
> +    "test_suite": {
> +      "type": "string",
> +      "enum": [
> +        "hello_world"
> +      ]
> +    },
> +    "test_target": {
> +      "type": "object",
> +      "properties": {
> +        "suite": {
> +          "$ref": "#/definitions/test_suite"
> +        },
> +        "cases": {
> +          "type": "array",
> +          "items": {
> +            "type": "string"
> +          },
> +          "minimum": 1
> +        }
> +      },
> +      "required": [
> +        "suite"
> +      ],
> +      "additionalProperties": false
>      }
>    },
>    "type": "object",
> @@ -130,6 +155,27 @@
>              },
>              "minimum": 1
>            },
> +          "perf": {
> +            "type": "boolean",
> +            "description": "Enable performance testing"
> +          },
> +          "func": {
> +            "type": "boolean",
> +            "description": "Enable functional testing"
> +          },
> +          "test_suites": {
> +            "type": "array",
> +            "items": {
> +              "oneOf": [
> +                {
> +                  "$ref": "#/definitions/test_suite"
> +                },
> +                {
> +                  "$ref": "#/definitions/test_target"
> +                }
> +              ]
> +            }
> +          },
>            "system_under_test": {
>              "$ref": "#/definitions/node_name"
>            }
> @@ -137,6 +183,9 @@
>          "additionalProperties": false,
>          "required": [
>            "build_targets",
> +          "perf",
> +          "func",
> +          "test_suites",
>            "system_under_test"
>          ]
>        },
> diff --git a/dts/framework/dts.py b/dts/framework/dts.py
> index a7c243a5c3..ba3f4b4168 100644
> --- a/dts/framework/dts.py
> +++ b/dts/framework/dts.py
> @@ -15,6 +15,7 @@
>  from .logger import DTSLOG, getLogger
>  from .settings import SETTINGS
>  from .stats_reporter import TestStats
> +from .test_case import TestCase
>  from .test_result import Result
>  from .utils import check_dts_python_version
>
> @@ -129,6 +130,34 @@ def run_suite(
>      Use the given build_target to run the test suite with possibly only a
> subset
>      of tests. If no subset is specified, run all tests.
>      """
> +    for test_suite_config in execution.test_suites:
> +        result.test_suite = test_suite_config.test_suite
> +        full_suite_path =
> f"tests.TestSuite_{test_suite_config.test_suite}"
> +        testcase_classes = TestCase.get_testcases(full_suite_path)
> +        dts_logger.debug(
> +            f"Found testcase classes '{testcase_classes}' in
> '{full_suite_path}'"
> +        )
> +        for testcase_class in testcase_classes:
> +            testcase = testcase_class(
> +                sut_node, test_suite_config.test_suite, build_target,
> execution
> +            )
> +
> +            testcase.init_log()
> +            testcase.set_requested_cases(SETTINGS.test_cases)
> +            testcase.set_requested_cases(test_suite_config.test_cases)
> +
> +            dts_logger.info(f"Running test suite
> '{testcase_class.__name__}'")
> +            try:
> +                testcase.execute_setup_all()
> +                testcase.execute_test_cases()
> +                dts_logger.info(
> +                    f"Finished running test suite
> '{testcase_class.__name__}'"
> +                )
> +                result.copy_suite(testcase.get_result())
> +                test_stats.save(result)  # this was originally after
> teardown
> +
> +            finally:
>

You should probably move the "finished" log message down here, so that it
always runs.


> +                testcase.execute_tear_downall()
>
>
>  def quit_execution(nodes: Iterable[Node], return_code: ReturnCode) ->
> None:
> diff --git a/dts/framework/exception.py b/dts/framework/exception.py
> index 93d99432ae..a35eeff640 100644
> --- a/dts/framework/exception.py
> +++ b/dts/framework/exception.py
> @@ -29,6 +29,10 @@ class ReturnCode(IntEnum):
>      DPDK_BUILD_ERR = 10
>      NODE_SETUP_ERR = 20
>      NODE_CLEANUP_ERR = 21
> +    SUITE_SETUP_ERR = 30
> +    SUITE_EXECUTION_ERR = 31
> +    TESTCASE_VERIFY_ERR = 32
> +    SUITE_CLEANUP_ERR = 33
>
>
>  class DTSError(Exception):
> @@ -153,6 +157,67 @@ def __init__(self):
>          )
>
>
> +class TestSuiteNotFound(DTSError):
> +    """
> +    Raised when a configured test suite cannot be imported.
> +    """
> +
> +    return_code: ClassVar[ReturnCode] = ReturnCode.SUITE_SETUP_ERR
> +
> +
> +class SuiteSetupError(DTSError):
> +    """
> +    Raised when an error occurs during suite setup.
> +    """
> +
> +    return_code: ClassVar[ReturnCode] = ReturnCode.SUITE_SETUP_ERR
> +
> +    def __init__(self):
> +        super(SuiteSetupError, self).__init__("An error occurred during
> suite setup.")
> +
> +
> +class SuiteExecutionError(DTSError):
> +    """
> +    Raised when an error occurs during suite execution.
> +    """
> +
> +    return_code: ClassVar[ReturnCode] = ReturnCode.SUITE_EXECUTION_ERR
> +
> +    def __init__(self):
> +        super(SuiteExecutionError, self).__init__(
> +            "An error occurred during suite execution."
> +        )
> +
> +
> +class VerifyError(DTSError):
> +    """
> +    To be used within the test cases to verify if a command output
> +    is as it was expected.
> +    """
> +
> +    value: str
> +    return_code: ClassVar[ReturnCode] = ReturnCode.TESTCASE_VERIFY_ERR
> +
> +    def __init__(self, value: str):
> +        self.value = value
> +
> +    def __str__(self) -> str:
> +        return repr(self.value)
> +
> +
> +class SuiteCleanupError(DTSError):
> +    """
> +    Raised when an error occurs during suite cleanup.
> +    """
> +
> +    return_code: ClassVar[ReturnCode] = ReturnCode.SUITE_CLEANUP_ERR
> +
> +    def __init__(self):
> +        super(SuiteCleanupError, self).__init__(
> +            "An error occurred during suite cleanup."
> +        )
> +
> +
>  def convert_exception(exception: type[DTSError]) -> Callable[...,
> Callable[..., None]]:
>      """
>      When a non-DTS exception is raised while executing the decorated
> function,
> diff --git a/dts/framework/settings.py b/dts/framework/settings.py
> index e2bf3d2ce4..069f28ce81 100644
> --- a/dts/framework/settings.py
> +++ b/dts/framework/settings.py
> @@ -64,6 +64,8 @@ class _Settings:
>      skip_setup: bool
>      dpdk_ref: Path
>      compile_timeout: float
> +    test_cases: list
> +    re_run: int
>
>
>  def _get_parser() -> argparse.ArgumentParser:
> @@ -138,6 +140,25 @@ def _get_parser() -> argparse.ArgumentParser:
>          help="[DTS_COMPILE_TIMEOUT] The timeout for compiling DPDK.",
>      )
>
> +    parser.add_argument(
> +        "--test-cases",
> +        action=_env_arg("DTS_TESTCASES"),
> +        default="",
> +        required=False,
> +        help="[DTS_TESTCASES] Comma-separated list of testcases to
> execute",
> +    )
> +
> +    parser.add_argument(
> +        "--re-run",
> +        "--re_run",
> +        action=_env_arg("DTS_RERUN"),
> +        default=0,
> +        type=int,
> +        required=False,
> +        help="[DTS_RERUN] Re-run tests the specified amount of times if a
> test failure "
> +        "occurs",
> +    )
> +
>      return parser
>
>
> @@ -151,6 +172,10 @@ def _get_settings() -> _Settings:
>          skip_setup=(parsed_args.skip_setup == "Y"),
>          dpdk_ref=parsed_args.dpdk_ref,
>          compile_timeout=parsed_args.compile_timeout,
> +        test_cases=parsed_args.test_cases.split(",")
> +        if parsed_args.test_cases != ""
> +        else [],
> +        re_run=parsed_args.re_run,
>      )
>
>
> diff --git a/dts/framework/test_case.py b/dts/framework/test_case.py
> new file mode 100644
> index 0000000000..0479f795bb
> --- /dev/null
> +++ b/dts/framework/test_case.py
> @@ -0,0 +1,246 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2010-2014 Intel Corporation
> +# Copyright(c) 2022 PANTHEON.tech s.r.o.
> +
> +"""
> +A base class for creating DTS test cases.
> +"""
> +
> +import importlib
> +import inspect
> +import re
> +import time
> +import traceback
> +
> +from .exception import (
> +    SSHTimeoutError,
> +    SuiteCleanupError,
> +    SuiteExecutionError,
> +    SuiteSetupError,
> +    TestSuiteNotFound,
> +    VerifyError,
> +    convert_exception,
> +)
> +from .logger import getLogger
> +from .settings import SETTINGS
> +from .test_result import Result
> +
> +
> +class TestCase(object):
> +    def __init__(self, sut_node, suitename, target, execution):
> +        self.sut_node = sut_node
> +        self.suite_name = suitename
> +        self.target = target
> +
> +        # local variable
> +        self._requested_tests = []
> +        self._subtitle = None
> +
> +        # result object for save suite result
> +        self._suite_result = Result()
> +        self._suite_result.sut = self.sut_node.config.hostname
> +        self._suite_result.target = target
> +        self._suite_result.test_suite = self.suite_name
> +        if self._suite_result is None:
> +            raise ValueError("Result object should not None")
> +
> +        self._enable_func = execution.func
> +
> +        # command history
> +        self.setup_history = list()
> +        self.test_history = list()
> +
> +    def init_log(self):
> +        # get log handler
> +        class_name = self.__class__.__name__
> +        self.logger = getLogger(class_name)
> +
> +    def set_up_all(self):
> +        pass
> +
> +    def set_up(self):
> +        pass
> +
> +    def tear_down(self):
> +        pass
> +
> +    def tear_down_all(self):
> +        pass
> +
> +    def verify(self, passed, description):
> +        if not passed:
> +            raise VerifyError(description)
> +
> +    def _get_functional_cases(self):
> +        """
> +        Get all functional test cases.
> +        """
> +        return self._get_test_cases(r"test_(?!perf_)")
> +
> +    def _has_it_been_requested(self, test_case, test_name_regex):
> +        """
> +        Check whether test case has been requested for validation.
> +        """
> +        name_matches = re.match(test_name_regex, test_case.__name__)
> +        if self._requested_tests:
> +            return name_matches and test_case.__name__ in
> self._requested_tests
> +
> +        return name_matches
> +
> +    def set_requested_cases(self, case_list):
> +        """
> +        Pass down input cases list for check
> +        """
> +        self._requested_tests += case_list
> +
> +    def _get_test_cases(self, test_name_regex):
> +        """
> +        Return case list which name matched regex.
> +        """
> +        self.logger.debug(f"Searching for testcases in {self.__class__}")
> +        for test_case_name in dir(self):
> +            test_case = getattr(self, test_case_name)
> +            if callable(test_case) and self._has_it_been_requested(
> +                test_case, test_name_regex
> +            ):
> +                yield test_case
> +
> +    @convert_exception(SuiteSetupError)
> +    def execute_setup_all(self):
> +        """
> +        Execute suite setup_all function before cases.
> +        """
> +        try:
> +            self.set_up_all()
> +            return True
> +        except Exception as v:
> +            self.logger.error("set_up_all failed:\n" +
> traceback.format_exc())
> +            # record all cases blocked
> +            if self._enable_func:
> +                for case_obj in self._get_functional_cases():
> +                    self._suite_result.test_case = case_obj.__name__
> +                    self._suite_result.test_case_blocked(
> +                        "set_up_all failed: {}".format(str(v))
> +                    )
> +            return False
> +
> +    def _execute_test_case(self, case_obj):
> +        """
> +        Execute specified test case in specified suite. If any exception
> occurred in
> +        validation process, save the result and tear down this case.
> +        """
> +        case_name = case_obj.__name__
> +        self._suite_result.test_case = case_obj.__name__
> +
> +        case_result = True
> +        try:
> +            self.logger.info("Test Case %s Begin" % case_name)
> +
> +            self.running_case = case_name
> +            # run set_up function for each case
> +            self.set_up()
> +            # run test case
> +            case_obj()
> +
> +            self._suite_result.test_case_passed()
> +
> +            self.logger.info("Test Case %s Result PASSED:" % case_name)
> +
> +        except VerifyError as v:
> +            case_result = False
> +            self._suite_result.test_case_failed(str(v))
> +            self.logger.error("Test Case %s Result FAILED: " %
> (case_name) + str(v))
> +        except KeyboardInterrupt:
> +            self._suite_result.test_case_blocked("Skipped")
> +            self.logger.error("Test Case %s SKIPPED: " % (case_name))
> +            self.tear_down()
> +            raise KeyboardInterrupt("Stop DTS")
> +        except SSHTimeoutError as e:
> +            case_result = False
> +            self._suite_result.test_case_failed(str(e))
> +            self.logger.error("Test Case %s Result FAILED: " %
> (case_name) + str(e))
> +            self.logger.error("%s" % (e.get_output()))
> +        except Exception:
> +            case_result = False
> +            trace = traceback.format_exc()
> +            self._suite_result.test_case_failed(trace)
> +            self.logger.error("Test Case %s Result ERROR: " % (case_name)
> + trace)
> +        finally:
> +            self.execute_tear_down()
> +            return case_result
> +
> +    @convert_exception(SuiteExecutionError)
> +    def execute_test_cases(self):
> +        """
> +        Execute all test cases in one suite.
> +        """
> +        # prepare debugger rerun case environment
> +        if self._enable_func:
> +            for case_obj in self._get_functional_cases():
> +                for i in range(SETTINGS.re_run + 1):
> +                    ret = self.execute_test_case(case_obj)
> +
> +                    if ret is False and SETTINGS.re_run:
> +                        self.sut_node.get_session_output(timeout=0.5 * (i
> + 1))
> +                        time.sleep(i + 1)
> +                        self.logger.info(
> +                            " Test case %s failed and re-run %d time"
> +                            % (case_obj.__name__, i + 1)
> +                        )
> +                    else:
> +                        break
> +
> +    def execute_test_case(self, case_obj):
> +        """
> +        Execute test case or enter into debug mode.
> +        """
> +        return self._execute_test_case(case_obj)
> +
> +    def get_result(self):
> +        """
> +        Return suite test result
> +        """
> +        return self._suite_result
> +
> +    @convert_exception(SuiteCleanupError)
> +    def execute_tear_downall(self):
> +        """
> +        execute suite tear_down_all function
> +        """
> +        self.tear_down_all()
> +
> +        self.sut_node.kill_cleanup_dpdk_apps()
> +
> +    def execute_tear_down(self):
> +        """
> +        execute suite tear_down function
> +        """
> +        try:
> +            self.tear_down()
> +        except Exception:
> +            self.logger.error("tear_down failed:\n" +
> traceback.format_exc())
> +            self.logger.warning(
> +                "tear down %s failed, might iterfere next case's result!"
> +                % self.running_case
> +            )
> +
> +    @staticmethod
> +    def get_testcases(testsuite_module_path: str) ->
> list[type["TestCase"]]:
> +        def is_testcase(object) -> bool:
> +            try:
> +                if issubclass(object, TestCase) and object != TestCase:
> +                    return True
> +            except TypeError:
> +                return False
> +            return False
> +
> +        try:
> +            testcase_module =
> importlib.import_module(testsuite_module_path)
> +        except ModuleNotFoundError as e:
> +            raise TestSuiteNotFound(
> +                f"Testsuite '{testsuite_module_path}' not found."
> +            ) from e
> +        return [
> +            testcase_class
> +            for _, testcase_class in inspect.getmembers(testcase_module,
> is_testcase)
> +        ]
> --
> 2.30.2
>
>