@@ -19,7 +19,7 @@ import time
import framework.utils as utils
from framework.pktgen import PacketGeneratorHelper
-from framework.settings import HEADER_SIZE
+from framework.settings import HEADER_SIZE, UPDATE_EXPECTED, load_global_setting
from framework.test_case import TestCase
from framework.virt_common import VM
@@ -73,6 +73,8 @@ class TestVirtioPVPRegression(TestCase):
self.base_dir = self.dut.base_dir.replace("~", "/root")
self.app_testpmd_path = self.dut.apps_name["test-pmd"]
self.testpmd_name = self.app_testpmd_path.split("/")[-1]
+ self.qemu_num = [item["version"] for item in self.qemu_list]
+ self.gap = self.get_suite_cfg()["accepted_tolerance"]
def set_up(self):
"""
@@ -94,8 +96,10 @@ class TestVirtioPVPRegression(TestCase):
"QemuVersion",
"FrameSize(B)",
"Throughput(Mpps)",
+ "Expectput(Mpps)",
"LineRate(%)",
"Queue Number",
+ "Status",
"Cycle",
]
@@ -340,24 +344,17 @@ class TestVirtioPVPRegression(TestCase):
self.vhost.send_expect("start", "testpmd> ", 60)
- @property
- def check_value(self):
- check_dict = dict.fromkeys(self.frame_sizes)
- linerate = {
- 64: 0.08,
- 128: 0.10,
- 256: 0.17,
- 512: 0.18,
- 1024: 0.40,
- 1280: 0.45,
- 1518: 0.50,
- }
- for size in self.frame_sizes:
- speed = self.wirespeed(self.nic, size, self.number_of_ports)
- check_dict[size] = round(speed * linerate[size], 2)
- return check_dict
-
- def send_verify(self, case_info, qemu_version, tag):
+ def handle_expected(self):
+ """
+ Update expected numbers to configurate file: $DTS_CFG_FOLDER/$suite_name.cfg
+ """
+ if load_global_setting(UPDATE_EXPECTED) == "yes":
+ for self.qemu_version in self.qemu_num:
+ self.expected_throughput[self.qemu_version] = self.throughput[
+ self.case_info
+ ][self.qemu_version]
+
+ def send_verify(self, case_info, qemu_version, tag, i):
for frame_size in self.frame_sizes:
info = "Running test %s, and %d frame size." % (
self.running_case,
@@ -397,11 +394,19 @@ class TestVirtioPVPRegression(TestCase):
)
Mpps = pps / 1000000.0
pct = Mpps * 100 / float(self.wirespeed(self.nic, frame_size, 1))
- self.verify(
- Mpps > self.check_value[frame_size],
- "%s of frame size %d speed verify failed, expect %s, result %s"
- % (self.running_case, frame_size, self.check_value[frame_size], Mpps),
- )
+ try:
+ exepect = self.expected_throughput[self.qemu_num[i]][tag][frame_size]
+
+ except:
+ exepect = 0
+ self.logger.info(
+ 'The first test of %s failed to match the expected result.\
+ Please use "--update-expect" to update the expected result.'
+ % self.qemu_num[i]
+ )
+
+ self.status = "PASS" if Mpps > exepect * (1 - self.gap * 0.01) else "FAILED"
+
# check each queue has data
self.check_packets_of_each_queue(frame_size)
# update print table info
@@ -410,12 +415,36 @@ class TestVirtioPVPRegression(TestCase):
qemu_version,
frame_size,
str(Mpps),
+ str(exepect),
str(pct),
self.queues_number,
+ self.status,
tag,
]
self.result_table_add(data_row)
+ def handle_throughput(self):
+ self.throughput = {}
+
+ for outer in self.get_test_result_table:
+ for item in outer[1:]:
+ key1, key2, key3, key4, value = (
+ item[0],
+ item[1],
+ item[-1],
+ int(item[2]),
+ float(item[3]),
+ )
+ if key1 not in self.throughput:
+ self.throughput[key1] = {}
+ if key2 not in self.throughput[key1]:
+ self.throughput[key1][key2] = {}
+ if key3 not in self.throughput[key1][key2]:
+ self.throughput[key1][key2][key3] = {}
+ self.throughput[key1][key2][key3][key4] = value
+
+ print(self.throughput)
+
def close_testpmd_and_qemu(self):
"""
stop testpmd in vhost and qemu
@@ -435,6 +464,7 @@ class TestVirtioPVPRegression(TestCase):
modem = 1, start vm as virtio 1.0
virtio_path = mergeable/normal/vector_rx
"""
+ self.get_test_result_table = []
for i in range(len(self.qemu_list)):
self.result_table_create(self.header_row)
path = self.qemu_list[i]["path"]
@@ -451,93 +481,148 @@ class TestVirtioPVPRegression(TestCase):
self.start_testpmd_in_vm(virtio_path)
self.logger.info("now testing the qemu path of %s" % path)
time.sleep(5)
- self.send_verify(case_info, version, "before reconnect")
+ self.send_verify(case_info, version, "before reconnect", i)
- self.logger.info("now reconnect from vhost")
- self.dut.send_expect("killall -s INT %s" % self.testpmd_name, "# ")
- self.start_testpmd_as_vhost()
- self.send_verify(case_info, version, "reconnect from vhost")
+ if "virtio-1.1" not in self.case_info:
+ self.logger.info("now reconnect from vhost")
+ self.dut.send_expect("killall -s INT %s" % self.testpmd_name, "# ")
+ self.start_testpmd_as_vhost()
+ self.send_verify(case_info, version, "reconnect from vhost", i)
self.logger.info("now reconnect from vm")
self.dut.send_expect("killall -s INT qemu-system-x86_64", "# ")
self.start_vm(path, version, modem, virtio_path, packed=packed)
self.start_testpmd_in_vm(virtio_path)
- self.send_verify(case_info, version, "reconnect from vm")
+ self.send_verify(case_info, version, "reconnect from vm", i)
+ self.get_test_result_table.append(self.result_table_getrows())
self.result_table_print()
+ self.verify(
+ "FAIL" not in self.status,
+ "Excessive gap between test results and expectations",
+ )
self.close_testpmd_and_qemu()
def test_perf_pvp_regression_with_mergeable_path(self):
"""
Test Case 1: pvp test with virtio 0.95 mergeable path
"""
- case_info = "virtio-0.95 mergeable"
+ self.case_info = "virtio-0.95 mergeable"
modem = 0
virtio_path = "mergeable"
- self.pvp_regression_run(case_info, modem, virtio_path)
+ self.test_target = self.running_case
+ self.expected_throughput = self.get_suite_cfg()[
+ "expected_throughput_virtio-0.95"
+ ][self.test_target][self.case_info]
+ self.pvp_regression_run(self.case_info, modem, virtio_path)
+ self.handle_throughput()
+ self.handle_expected()
def test_perf_pvp_regression_modern_mergeable_path(self):
"""
Test Case 4: pvp test with virtio 1.0 mergeable path
"""
- case_info = "virtio-1.0 mergeable"
+ self.case_info = "virtio-1.0 mergeable"
modem = 1
virtio_path = "mergeable"
- self.pvp_regression_run(case_info, modem, virtio_path)
+ self.test_target = self.running_case
+ self.expected_throughput = self.get_suite_cfg()[
+ "expected_throughput_virtio-1.0"
+ ][self.test_target][self.case_info]
+ self.pvp_regression_run(self.case_info, modem, virtio_path)
+ self.handle_throughput()
+ self.handle_expected()
def test_perf_pvp_regression_non_mergeable_path(self):
"""
Test Case 2: pvp test with virtio 0.95 non-mergeable path
"""
- case_info = "virtio-0.95 normal"
+ self.case_info = "virtio-0.95 normal"
modem = 0
virtio_path = "normal"
- self.pvp_regression_run(case_info, modem, virtio_path)
+ self.test_target = self.running_case
+ self.expected_throughput = self.get_suite_cfg()[
+ "expected_throughput_virtio-0.95"
+ ][self.test_target][self.case_info]
+ self.pvp_regression_run(self.case_info, modem, virtio_path)
+ self.handle_throughput()
+ self.handle_expected()
def test_perf_pvp_regression_modern_non_mergeable_path(self):
"""
Test Case 5: pvp test with virtio 1.0 non-mergeable path
"""
- case_info = "virtio-1.0 normal"
+
+ self.case_info = "virtio-1.0 normal"
modem = 1
virtio_path = "normal"
- self.pvp_regression_run(case_info, modem, virtio_path)
+ self.test_target = self.running_case
+ self.expected_throughput = self.get_suite_cfg()[
+ "expected_throughput_virtio-1.0"
+ ][self.test_target][self.case_info]
+ self.pvp_regression_run(self.case_info, modem, virtio_path)
+ self.handle_throughput()
+ self.handle_expected()
def test_perf_pvp_regression_vector_rx_path(self):
"""
Test Case 3: pvp test with virtio 0.95 vrctor_rx path
"""
- case_info = "virtio-0.95 vector_rx"
+ self.case_info = "virtio-0.95 vector_rx"
modem = 0
virtio_path = "vector_rx"
- self.pvp_regression_run(case_info, modem, virtio_path)
+ self.test_target = self.running_case
+ self.expected_throughput = self.get_suite_cfg()[
+ "expected_throughput_virtio-0.95"
+ ][self.test_target][self.case_info]
+ self.pvp_regression_run(self.case_info, modem, virtio_path)
+ self.handle_throughput()
+ self.handle_expected()
def test_perf_pvp_regression_modern_vector_rx_path(self):
"""
Test Case 6: pvp test with virtio 1.0 vrctor_rx path
"""
- case_info = "virtio-1.0 normal"
+ self.case_info = "virtio-1.0 vector_rx"
modem = 1
virtio_path = "vector_rx"
- self.pvp_regression_run(case_info, modem, virtio_path)
+ self.test_target = self.running_case
+ self.expected_throughput = self.get_suite_cfg()[
+ "expected_throughput_virtio-1.0"
+ ][self.test_target][self.case_info]
+ self.pvp_regression_run(self.case_info, modem, virtio_path)
+ self.handle_throughput()
+ self.handle_expected()
def test_perf_pvp_with_virtio11_mergeable_path(self):
"""
Test Case 7: pvp test with virtio 1.1 mergeable path
"""
- case_info = "virtio-1.1 mergeable"
+ self.case_info = "virtio-1.1 mergeable"
modem = 1
virtio_path = "mergeable"
- self.pvp_regression_run(case_info, modem, virtio_path, packed=True)
+ self.test_target = self.running_case
+ self.expected_throughput = self.get_suite_cfg()[
+ "expected_throughput_virtio-1.1"
+ ][self.test_target][self.case_info]
+ self.pvp_regression_run(self.case_info, modem, virtio_path, packed=True)
+ self.handle_throughput()
+ self.handle_expected()
def test_perf_pvp_with_virtio11_non_mergeable_path(self):
"""
Test Case 8: pvp test with virtio 1.1 non-mergeable path
"""
- case_info = "virtio-1.1 normal"
+ self.case_info = "virtio-1.1 normal"
modem = 1
virtio_path = "normal"
- self.pvp_regression_run(case_info, modem, virtio_path, packed=True)
+ self.test_target = self.running_case
+ self.expected_throughput = self.get_suite_cfg()[
+ "expected_throughput_virtio-1.1"
+ ][self.test_target][self.case_info]
+ self.pvp_regression_run(self.case_info, modem, virtio_path, packed=True)
+ self.handle_throughput()
+ self.handle_expected()
def tear_down(self):
"""