[RFC,v2,07/10] dts: add simple stats report
Checks
Commit Message
Provide a summary of testcase passed/failed/blocked counts.
Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
---
dts/framework/dts.py | 3 ++
dts/framework/stats_reporter.py | 65 +++++++++++++++++++++++++++++++++
2 files changed, 68 insertions(+)
create mode 100644 dts/framework/stats_reporter.py
Comments
You are missing type annotations throughout this.
On Mon, Nov 14, 2022 at 11:54 AM Juraj Linkeš <juraj.linkes@pantheon.tech>
wrote:
> Provide a summary of testcase passed/failed/blocked counts.
>
> Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
> ---
> dts/framework/dts.py | 3 ++
> dts/framework/stats_reporter.py | 65 +++++++++++++++++++++++++++++++++
> 2 files changed, 68 insertions(+)
> create mode 100644 dts/framework/stats_reporter.py
>
> diff --git a/dts/framework/dts.py b/dts/framework/dts.py
> index d606f8de2e..a7c243a5c3 100644
> --- a/dts/framework/dts.py
> +++ b/dts/framework/dts.py
> @@ -14,11 +14,13 @@
> from .exception import DTSError, ReturnCode
> from .logger import DTSLOG, getLogger
> from .settings import SETTINGS
> +from .stats_reporter import TestStats
> from .test_result import Result
> from .utils import check_dts_python_version
>
> dts_logger: DTSLOG = getLogger("dts")
> result: Result = Result()
> +test_stats: TestStats = TestStats(SETTINGS.output_dir + "/statistics.txt")
>
>
> def run_all() -> None:
> @@ -29,6 +31,7 @@ def run_all() -> None:
> return_code = ReturnCode.NO_ERR
> global dts_logger
> global result
> + global test_stats
>
> # check the python version of the server that run dts
> check_dts_python_version()
> diff --git a/dts/framework/stats_reporter.py
> b/dts/framework/stats_reporter.py
> new file mode 100644
> index 0000000000..a2735d0a1d
> --- /dev/null
> +++ b/dts/framework/stats_reporter.py
> @@ -0,0 +1,65 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2010-2014 Intel Corporation
> +# Copyright(c) 2022 PANTHEON.tech s.r.o.
> +
> +"""
> +Simple text file statistics generator
> +"""
> +
> +
> +class TestStats(object):
> + """
> + Generates a small statistics file containing the number of passing,
> + failing and blocked tests. It makes use of a Result instance as input.
> + """
> +
> + def __init__(self, filename):
> + self.filename = filename
> +
> + def __add_stat(self, test_result):
I think that this should probably be an option of an enum that gets matched
over. ex:
match test_result:
case None:
pass
case PASSED:
self.passed += 1
case FAILED:
self.failed += 1
case BLOCKED:
self.blocked += 1
case unknown:
# log this and throw an error.
> + if test_result is not None:
> + if test_result[0] == "PASSED":
> + self.passed += 1
> + if test_result[0] == "FAILED":
> + self.failed += 1
> + if test_result[0] == "BLOCKED":
> + self.blocked += 1
> + self.total += 1
> +
> + def __count_stats(self):
> + for sut in self.result.all_suts():
> + for target in self.result.all_targets(sut):
> + for suite in self.result.all_test_suites(sut, target):
> + for case in self.result.all_test_cases(sut, target,
> suite):
> + test_result = self.result.result_for(sut, target,
> suite, case)
> + if len(test_result):
> + self.__add_stat(test_result)
> +
> + def __write_stats(self):
> + sut_nodes = self.result.all_suts()
> + if len(sut_nodes) == 1:
> + self.stats_file.write(
> + f"dpdk_version =
> {self.result.current_dpdk_version(sut_nodes[0])}\n"
> + )
> + else:
> + for sut in sut_nodes:
> + dpdk_version = self.result.current_dpdk_version(sut)
> + self.stats_file.write(f"{sut}.dpdk_version =
> {dpdk_version}\n")
> + self.__count_stats()
> + self.stats_file.write(f"Passed = {self.passed}\n")
> + self.stats_file.write(f"Failed = {self.failed}\n")
> + self.stats_file.write(f"Blocked = {self.blocked}\n")
> + rate = 0
> + if self.total > 0:
> + rate = self.passed * 100.0 / self.total
> + self.stats_file.write(f"Pass rate = {rate:.1f}\n")
> +
> + def save(self, result):
> + self.passed = 0
> + self.failed = 0
> + self.blocked = 0
> + self.total = 0
> + self.stats_file = open(self.filename, "w+")
> + self.result = result
> + self.__write_stats()
> + self.stats_file.close()
> --
> 2.30.2
>
>
@@ -14,11 +14,13 @@
from .exception import DTSError, ReturnCode
from .logger import DTSLOG, getLogger
from .settings import SETTINGS
+from .stats_reporter import TestStats
from .test_result import Result
from .utils import check_dts_python_version
dts_logger: DTSLOG = getLogger("dts")
result: Result = Result()
+test_stats: TestStats = TestStats(SETTINGS.output_dir + "/statistics.txt")
def run_all() -> None:
@@ -29,6 +31,7 @@ def run_all() -> None:
return_code = ReturnCode.NO_ERR
global dts_logger
global result
+ global test_stats
# check the python version of the server that run dts
check_dts_python_version()
new file mode 100644
@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2010-2014 Intel Corporation
+# Copyright(c) 2022 PANTHEON.tech s.r.o.
+
+"""
+Simple text file statistics generator
+"""
+
+
+class TestStats(object):
+ """
+ Generates a small statistics file containing the number of passing,
+ failing and blocked tests. It makes use of a Result instance as input.
+ """
+
+ def __init__(self, filename):
+ self.filename = filename
+
+ def __add_stat(self, test_result):
+ if test_result is not None:
+ if test_result[0] == "PASSED":
+ self.passed += 1
+ if test_result[0] == "FAILED":
+ self.failed += 1
+ if test_result[0] == "BLOCKED":
+ self.blocked += 1
+ self.total += 1
+
+ def __count_stats(self):
+ for sut in self.result.all_suts():
+ for target in self.result.all_targets(sut):
+ for suite in self.result.all_test_suites(sut, target):
+ for case in self.result.all_test_cases(sut, target, suite):
+ test_result = self.result.result_for(sut, target, suite, case)
+ if len(test_result):
+ self.__add_stat(test_result)
+
+ def __write_stats(self):
+ sut_nodes = self.result.all_suts()
+ if len(sut_nodes) == 1:
+ self.stats_file.write(
+ f"dpdk_version = {self.result.current_dpdk_version(sut_nodes[0])}\n"
+ )
+ else:
+ for sut in sut_nodes:
+ dpdk_version = self.result.current_dpdk_version(sut)
+ self.stats_file.write(f"{sut}.dpdk_version = {dpdk_version}\n")
+ self.__count_stats()
+ self.stats_file.write(f"Passed = {self.passed}\n")
+ self.stats_file.write(f"Failed = {self.failed}\n")
+ self.stats_file.write(f"Blocked = {self.blocked}\n")
+ rate = 0
+ if self.total > 0:
+ rate = self.passed * 100.0 / self.total
+ self.stats_file.write(f"Pass rate = {rate:.1f}\n")
+
+ def save(self, result):
+ self.passed = 0
+ self.failed = 0
+ self.blocked = 0
+ self.total = 0
+ self.stats_file = open(self.filename, "w+")
+ self.result = result
+ self.__write_stats()
+ self.stats_file.close()