@@ -4,38 +4,179 @@
# Copyright(c) 2022 University of New Hampshire
#
+import os
import sys
+import traceback
from typing import Iterable, Optional
import framework.logger as logger
from .config import CONFIGURATION
+from .exception import VerifyFailure
from .logger import getLogger
from .node import Node
-from .settings import SETTINGS
-from .utils import check_dts_python_version
+from .settings import SETTINGS, DTSRuntimeError, DTSRuntimeErrors
+from .stats_reporter import StatsReporter
+from .sut_node import SutNode
+from .test_case import TestCase
+from .test_result import Result
+from .tg_node import TrafficGeneratorNode
+from .utils import check_dts_python_version, get_subclasses
+requested_tests: Optional[list[str]] = None
+result: Optional[Result] = None
+stats_report: Optional[StatsReporter] = None
log_handler: Optional[logger.DTSLOG] = None
+def dts_nodes_init():
+ """
+ Create dts SUT/TG instance and initialize them.
+ """
+ sut_nodes = []
+ tg_node = None
+ for node_config in CONFIGURATION.nodes:
+ if hasattr(node_config, 'memory_channels'):
+ sut_nodes.append(SutNode(node_config))
+ else:
+ tg_node = TrafficGeneratorNode(node_config)
+ tg_node.set_re_run(SETTINGS.re_run if SETTINGS.re_run > 0 else 0)
+
+ return sut_nodes, tg_node
+
+
+def dts_run_prerequisites(nodes):
+ """
+ Run dts prerequisites function.
+ """
+ # TODO nodes config contains both sut and tg nodes
+ try:
+ for node in nodes:
+ node.prerequisites()
+ except Exception as ex:
+ log_handler.error("NODE PREREQ EXCEPTION " + traceback.format_exc())
+ result.add_failed_sut(node, str(ex))
+ if isinstance(node, TrafficGeneratorNode):
+ DTSRuntimeError = DTSRuntimeErrors.TG_SETUP_ERR
+ else:
+ DTSRuntimeError = DTSRuntimeErrors.SUT_SETUP_ERR
+ return False
+ return True
+
+
+def dts_run_target(sut_nodes, tg_node, targets, test_suites):
+ """
+ Run each target in execution targets.
+ """
+ for target in targets:
+ target = str(target)
+ log_handler.info("\nTARGET " + target)
+ result.target = target
+
+ try:
+ for sut_node in sut_nodes:
+ sut_node.set_target(target)
+ except AssertionError as ex:
+ DTSRuntimeError = DTSRuntimeErrors.DPDK_BUILD_ERR
+ log_handler.error(" TARGET ERROR: " + str(ex))
+ result.add_failed_target(result.sut, target, str(ex))
+ continue
+ except Exception as ex:
+ DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR
+ log_handler.error(" !!! DEBUG IT: " + traceback.format_exc())
+ result.add_failed_target(result.sut, target, str(ex))
+ continue
+
+ dts_run_suite(sut_nodes, tg_node, test_suites, target)
+
+
+def dts_run_suite(sut_nodes, tg_node, test_suites, target):
+ """
+ Run each suite in test suite list.
+ """
+ for suite_name in test_suites:
+ try:
+ # check whether config the test cases
+ append_requested_case_list = None
+ if ":" in suite_name:
+ case_list = suite_name[suite_name.find(":") + 1 :]
+ append_requested_case_list = case_list.split("\\")
+ suite_name = suite_name[: suite_name.find(":")]
+ result.test_suite = suite_name
+ _suite_full_name = "TestSuite_" + suite_name
+ suite_module = __import__(
+ "tests." + _suite_full_name, fromlist=[_suite_full_name]
+ )
+ for test_classname, test_class in get_subclasses(suite_module, TestCase):
+
+ suite_obj = test_class(sut_nodes, tg_node, target, suite_name)
+ suite_obj.init_log()
+ suite_obj.set_requested_cases(requested_tests)
+ suite_obj.set_requested_cases(append_requested_case_list)
+
+ log_handler.info("\nTEST SUITE : " + test_classname)
+
+ if suite_obj.execute_setup_all():
+ suite_obj.execute_test_cases()
+
+ # save suite cases result
+ result.copy_suite(suite_obj.get_result())
+
+ log_handler.info("\nTEST SUITE ENDED: " + test_classname)
+ except VerifyFailure:
+ DTSRuntimeError = DTSRuntimeErrors.SUITE_EXECUTE_ERR
+ log_handler.error(" !!! DEBUG IT: " + traceback.format_exc())
+ except KeyboardInterrupt:
+ # stop/save result/skip execution
+ log_handler.error(" !!! STOPPING DTS")
+ break
+ except Exception as e:
+ DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR
+ log_handler.error(str(e))
+ finally:
+ try:
+ suite_obj.execute_tear_downall()
+ except Exception as e:
+ DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR
+ log_handler.error(str(e))
+ try:
+ stats_report.save(result)
+ except Exception as e:
+ DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR
+ log_handler.error(str(e))
+
+
def run_all() -> None:
"""
Main process of DTS, it will run all test suites in the config file.
"""
global log_handler
+ global result
+ global stats_report
+ global requested_tests
# check the python version of the server that run dts
check_dts_python_version()
+ # prepare the output folder
+ if not os.path.exists(SETTINGS.output_dir):
+ os.mkdir(SETTINGS.output_dir)
+
# init log_handler handler
if SETTINGS.verbose is True:
logger.set_verbose()
log_handler = getLogger("dts")
- nodes = {}
- # This try/finally block means "Run the try block, if there is an exception,
+ # run designated test cases
+ requested_tests = SETTINGS.test_cases
+
+ # report objects
+ stats_report = StatsReporter(SETTINGS.output_dir + "/statistics.txt")
+ result = Result()
+
+ # This try/finally block means "Run the try block and if there is an exception,
# run the finally block before passing it upward. If there is not an exception,
# run the finally block after the try block is finished." This helps avoid the
# problem of python's interpreter exit context, which essentially prevents you
@@ -45,26 +186,35 @@ def run_all() -> None:
# An except block SHOULD NOT be added to this. A failure at this level should
# deliver a full stack trace for debugging, since the only place that exceptions
# should be caught and handled is in the testing code.
+ nodes = []
try:
# for all Execution sections
for execution in CONFIGURATION.executions:
- sut_config = execution.system_under_test
- if sut_config.name not in nodes:
- nodes[sut_config.name] = Node(sut_config)
+ sut_nodes, tg_node = dts_nodes_init()
+ nodes.extend(sut_nodes)
+ nodes.append(tg_node)
+
+ # Run SUT prerequisites
+ if dts_run_prerequisites(nodes) is False:
+ continue
+ result.dpdk_version = sut_nodes[0].dpdk_version
+ dts_run_target(
+ sut_nodes, tg_node, execution.target_descriptions, execution.test_suites
+ )
finally:
- quit_execution(nodes.values())
+ quit_execution(nodes)
-def quit_execution(sut_nodes: Iterable[Node]) -> None:
+def quit_execution(nodes: Iterable[Node]) -> None:
"""
Close session to SUT and TG before quit.
Return exit status when failure occurred.
"""
- for sut_node in sut_nodes:
+ for node in nodes:
# close all session
- sut_node.node_exit()
+ node.node_exit()
if log_handler is not None:
log_handler.info("DTS ended")
- sys.exit(0)
+ sys.exit(DTSRuntimeError)
new file mode 100644
@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2010-2014 Intel Corporation
+# Copyright(c) 2022 PANTHEON.tech s.r.o.
+#
+
+"""
+Simple text file statistics generator
+"""
+
+
+class StatsReporter(object):
+ """
+ Generates a small statistics file containing the number of passing,
+ failing and blocked tests. It makes use of a Result instance as input.
+ """
+
+ def __init__(self, filename):
+ self.filename = filename
+
+ def __add_stat(self, test_result):
+ if test_result is not None:
+ if test_result[0] == "PASSED":
+ self.passed += 1
+ if test_result[0] == "FAILED":
+ self.failed += 1
+ if test_result[0] == "BLOCKED":
+ self.blocked += 1
+ self.total += 1
+
+ def __count_stats(self):
+ for sut in self.result.all_suts():
+ for target in self.result.all_targets(sut):
+ for suite in self.result.all_test_suites(sut, target):
+ for case in self.result.all_test_cases(sut, target, suite):
+ test_result = self.result.result_for(sut, target, suite, case)
+ if len(test_result):
+ self.__add_stat(test_result)
+
+ def __write_stats(self):
+ sut_nodes = self.result.all_suts()
+ if len(sut_nodes) == 1:
+ self.stats_file.write(
+ "dpdk_version = {}\n".format(
+ self.result.current_dpdk_version(sut_nodes[0])
+ )
+ )
+ else:
+ for sut in sut_nodes:
+ dpdk_version = self.result.current_dpdk_version(sut)
+ self.stats_file.write(
+ "{}.dpdk_version = {}\n".format(sut, dpdk_version)
+ )
+ self.__count_stats()
+ self.stats_file.write("Passed = %d\n" % self.passed)
+ self.stats_file.write("Failed = %d\n" % self.failed)
+ self.stats_file.write("Blocked = %d\n" % self.blocked)
+ rate = 0
+ if self.total > 0:
+ rate = self.passed * 100.0 / self.total
+ self.stats_file.write("Pass rate = %.1f\n" % rate)
+
+ def save(self, result):
+ self.passed = 0
+ self.failed = 0
+ self.blocked = 0
+ self.total = 0
+ self.stats_file = open(self.filename, "w+")
+ self.result = result
+ self.__write_stats()
+ self.stats_file.close()