[v3,4/6] tests/nic_single_core_perf: revise delta to ratio
Commit Message
the delta used to defines the difference between real pps and expected
pps, the case fails if it exceed the absolute number specified by
'accepted_tolerance' in conf/nic_single_core_perf.cfg, the deault is
1Mpps, which is not friendly when the expected is only a few of million,
so updated the accepted_tolerance to a percentage, considering the CI
intergation, so the type is float.
Here are main changes:
* the meaning of accepted_tolerance
before: threshold for (real-mpps - expected-mpps)
after: threshold for (real-mpps - expected-mpps)/expected-mpps
* Pass criteria
before:
real-mpps - expected-mpps > -accepted_tolerance
after:
(real-mpps - expected-mpps)/expected-mpps > -accepted_tolerance
* rst table
before:
... +---------------+-----------------+--------+
... | Expected-Mpps | Real - Expected | Status |
... +===============+=================+========+
... | 9.900 | -0.040 | PASS |
... +---------------+-----------------+--------+
after:
... +---------------+------------+--------+
... | Expected-Mpps | Fluc Ratio | Status |
... +===============+============+========+
... | 9.900 | -0.004 | PASS |
... +---------------+------------+--------+
* delta in JSON:
before: real-mpps - expected-mpps
after: (real-mpps - expected-mpps)/expected-mpps
Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
---
conf/nic_single_core_perf.cfg | 10 +++++---
tests/TestSuite_nic_single_core_perf.py | 32 ++++++++++++-------------
2 files changed, 22 insertions(+), 20 deletions(-)
@@ -11,8 +11,12 @@
# - test_duration is how many seconds each combination performance will
# be recorded.
#
-# - accepted_tolerance defines the accepted tolerance between test
-# results and expected numbers.
+# - accepted_tolerance defines the accepted tolerance for the
+# fluctuation ratio of performance, it is a float(e.g. 0.05).
+# Assume:
+# F = (Real-Mpps - Expected-Mpps)/Expected-Mpps
+# T = accepted_tolerance
+# If F < -T, the case is PASS, else it is FAIL
#
# - expected_throughput is a dictionary defining expected throughput
# numbers based on NIC, and the pattern is
@@ -38,7 +42,7 @@ test_parameters = {'1C/1T': {64: [512, 2048]},
rx_desc_16byte = 'y'
test_duration = 60
throughput_stat_sample_interval = 2
-accepted_tolerance = 1
+accepted_tolerance = 0.05
expected_throughput = {
'fortville_spirit': {
'1C/1T': {64: {512: 0.00, 2048: 0.00}},
@@ -116,12 +116,12 @@ class TestNicSingleCorePerf(TestCase):
# {'$framesize':{"$nb_desc": 'throughput'}
self.throughput = {}
- # Accepted tolerance in Mpps
- self.gap = self.get_suite_cfg()['accepted_tolerance']
+ # Accepted tolerance is ratio
+ self.gap = self.get_suite_cfg().get('accepted_tolerance', 0.1)
# header to print test result table
self.table_header = ['Fwd_core', 'Frame Size', 'TXD/RXD', 'Real-Mpps', 'Rate',
- 'Expected-Mpps', 'Real - Expected', 'Status']
+ 'Expected-Mpps', 'Fluc Ratio', 'Status']
self.test_result = {}
def flows(self):
@@ -332,12 +332,16 @@ class TestNicSingleCorePerf(TestCase):
ret_data[header[3]] = "{:.3f}".format(_real)
ret_data[header[4]] = "{:.3f}%".format(_real * 100 / wirespeed)
ret_data[header[5]] = "{:.3f}".format(_exp)
- delta = _real - _exp
- ret_data[header[6]] = "{:.3f}".format(delta)
- if delta > -self.gap:
- ret_data[header[7]] = 'PASS'
+ delta = (_real - _exp)/_exp
+ if _exp != 0:
+ ret_data[header[6]] = "{:.3f}".format(delta)
+ if delta > -self.gap:
+ ret_data[header[7]] = 'PASS'
+ else:
+ ret_data[header[7]] = 'FAIL'
else:
- ret_data[header[7]] = 'FAIL'
+ ret_data[header[6]] = "N/A"
+ ret_data[header[7]] = 'PASS'
ret_datas[frame_size][nb_desc] = deepcopy(ret_data)
self.test_result[fwd_config] = deepcopy(ret_datas)
@@ -376,15 +380,9 @@ class TestNicSingleCorePerf(TestCase):
row_dict0 = dict()
row_dict0['performance'] = list()
row_dict0['parameters'] = list()
- result_throughput = float(row_in['Real-Mpps'])
- expected_throughput = float(row_in['Expected-Mpps'])
- # delta value and accepted tolerance in percentage
- delta = result_throughput - expected_throughput
- if delta > -self.gap:
- row_dict0['status'] = 'PASS'
- else:
- row_dict0['status'] = 'FAIL'
- row_dict1 = dict(name="Throughput", value=result_throughput, unit="Mpps", delta=delta)
+ row_dict0['status'] = row_in['Status']
+ row_dict1 = dict(name="Throughput", value=row_in['Real-Mpps'], unit="Mpps",
+ delta=row_in['Fluc Ratio'])
row_dict2 = dict(name="Txd/Rxd", value=row_in["TXD/RXD"], unit="descriptor")
row_dict3 = dict(name="frame_size", value=row_in["Frame Size"], unit="bytes")
row_dict4 = dict(name="Fwd_core", value=row_in["Fwd_core"])