Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/129014/?format=api
{ "id": 129014, "url": "http://patches.dpdk.org/api/patches/129014/?format=api", "web_url": "http://patches.dpdk.org/project/dts/patch/20230627153128.627541-1-dukaix.yuan@intel.com/", "project": { "id": 3, "url": "http://patches.dpdk.org/api/projects/3/?format=api", "name": "DTS", "link_name": "dts", "list_id": "dts.dpdk.org", "list_email": "dts@dpdk.org", "web_url": "", "scm_url": "git://dpdk.org/tools/dts", "webscm_url": "http://git.dpdk.org/tools/dts/", "list_archive_url": "https://inbox.dpdk.org/dts", "list_archive_url_format": "https://inbox.dpdk.org/dts/{}", "commit_url_format": "" }, "msgid": "<20230627153128.627541-1-dukaix.yuan@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dts/20230627153128.627541-1-dukaix.yuan@intel.com", "date": "2023-06-27T15:31:28", "name": "[V1] tests/virtio_pvp_regression: Add the functionality of reading expected results from configuration files", "commit_ref": null, "pull_url": null, "state": "new", "archived": false, "hash": "4a47bb8fbdad71821c98293d7a6344334e8d9276", "submitter": { "id": 2823, "url": "http://patches.dpdk.org/api/people/2823/?format=api", "name": "Yuan, DukaiX", "email": "dukaix.yuan@intel.com" }, "delegate": null, "mbox": "http://patches.dpdk.org/project/dts/patch/20230627153128.627541-1-dukaix.yuan@intel.com/mbox/", "series": [ { "id": 28659, "url": "http://patches.dpdk.org/api/series/28659/?format=api", "web_url": "http://patches.dpdk.org/project/dts/list/?series=28659", "date": "2023-06-27T15:31:28", "name": "[V1] tests/virtio_pvp_regression: Add the functionality of reading expected results from configuration files", "version": 1, "mbox": "http://patches.dpdk.org/series/28659/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/129014/comments/", "check": "warning", "checks": "http://patches.dpdk.org/api/patches/129014/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dts-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 5B7C342D6E;\n\tTue, 27 Jun 2023 09:54:44 +0200 (CEST)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 35AB14113F;\n\tTue, 27 Jun 2023 09:54:44 +0200 (CEST)", "from mga01.intel.com (mga01.intel.com [192.55.52.88])\n by mails.dpdk.org (Postfix) with ESMTP id BCF9240F18\n for <dts@dpdk.org>; Tue, 27 Jun 2023 09:54:42 +0200 (CEST)", "from orsmga008.jf.intel.com ([10.7.209.65])\n by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 27 Jun 2023 00:54:41 -0700", "from unknown (HELO localhost.localdomain) ([10.239.252.44])\n by orsmga008-auth.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 27 Jun 2023 00:54:39 -0700" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple;\n d=intel.com; i=@intel.com; q=dns/txt; s=Intel;\n t=1687852482; x=1719388482;\n h=from:to:cc:subject:date:message-id:mime-version:\n content-transfer-encoding;\n bh=gUQl5t2TuHrGCoqvLLcR1+cq/u/fbjCsneCBnsHh2Nw=;\n b=UP/OGdlAni0b9yr5zMMHMD5OoGGM6T7ovQeVqb0tEeocj518qSD0X2CC\n pO12Rru45Zkzzmg/kut+lVblGf9MNFOOx7Sye7u/NateJHhdSfhvYRhdi\n iU/wuNtMvcBtelaj9xrWOWMCJm4VZaJqsdbJJzeg5S3698OVhuuUtfT4W\n TqAVYzRUJ5Q5hfIlJOMi573fE7oPDgKkCzoiy3UW3s55+KLkLzliAcXGT\n JpFPcOKZ046MvZ/7YCGwY1sRsIPEXR0XK3P29YMcct7aq9TBf3lZb5cz1\n PRNLybBIt1rgGsavB6oIt3nk2lEfebQ2FUDF99U/7rgNi9AAwzNgglt9p w==;", "X-IronPort-AV": [ "E=McAfee;i=\"6600,9927,10753\"; a=\"391994763\"", "E=Sophos;i=\"6.01,161,1684825200\"; d=\"scan'208\";a=\"391994763\"", "E=McAfee;i=\"6600,9927,10753\"; a=\"746106748\"", "E=Sophos;i=\"6.01,161,1684825200\"; d=\"scan'208\";a=\"746106748\"" ], "X-ExtLoop1": "1", "From": "Dukai Yuan <dukaix.yuan@intel.com>", "To": "dts@dpdk.org", "Cc": "Dukai Yuan <dukaix.yuan@intel.com>", "Subject": "[dts][PATCH V1] tests/virtio_pvp_regression: Add the functionality of\n reading expected results from configuration files", "Date": "Tue, 27 Jun 2023 15:31:28 +0000", "Message-Id": "<20230627153128.627541-1-dukaix.yuan@intel.com>", "X-Mailer": "git-send-email 2.31.1", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "dts@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "test suite reviews and discussions <dts.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dts>,\n <mailto:dts-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dts/>", "List-Post": "<mailto:dts@dpdk.org>", "List-Help": "<mailto:dts-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dts>,\n <mailto:dts-request@dpdk.org?subject=subscribe>", "Errors-To": "dts-bounces@dpdk.org" }, "content": "Previously, expected values in the code were fixed for verification. \nNow, we have made changes to read expected values from a configuration file. \nAdditionally, test results can also be saved to the configuration file as new expected values. \nThis optimization can improve the accuracy and maintainability of testing.\n\nSigned-off-by: Dukai Yuan <dukaix.yuan@intel.com>\n---\n tests/TestSuite_virtio_pvp_regression.py | 177 +++++++++++++++++------\n 1 file changed, 131 insertions(+), 46 deletions(-)", "diff": "diff --git a/tests/TestSuite_virtio_pvp_regression.py b/tests/TestSuite_virtio_pvp_regression.py\nindex a2db5f87..a28ebc67 100644\n--- a/tests/TestSuite_virtio_pvp_regression.py\n+++ b/tests/TestSuite_virtio_pvp_regression.py\n@@ -19,7 +19,7 @@ import time\n \n import framework.utils as utils\n from framework.pktgen import PacketGeneratorHelper\n-from framework.settings import HEADER_SIZE\n+from framework.settings import HEADER_SIZE, UPDATE_EXPECTED, load_global_setting\n from framework.test_case import TestCase\n from framework.virt_common import VM\n \n@@ -73,6 +73,8 @@ class TestVirtioPVPRegression(TestCase):\n self.base_dir = self.dut.base_dir.replace(\"~\", \"/root\")\n self.app_testpmd_path = self.dut.apps_name[\"test-pmd\"]\n self.testpmd_name = self.app_testpmd_path.split(\"/\")[-1]\n+ self.qemu_num = [item[\"version\"] for item in self.qemu_list]\n+ self.gap = self.get_suite_cfg()[\"accepted_tolerance\"]\n \n def set_up(self):\n \"\"\"\n@@ -94,8 +96,10 @@ class TestVirtioPVPRegression(TestCase):\n \"QemuVersion\",\n \"FrameSize(B)\",\n \"Throughput(Mpps)\",\n+ \"Expectput(Mpps)\",\n \"LineRate(%)\",\n \"Queue Number\",\n+ \"Status\",\n \"Cycle\",\n ]\n \n@@ -340,24 +344,17 @@ class TestVirtioPVPRegression(TestCase):\n \n self.vhost.send_expect(\"start\", \"testpmd> \", 60)\n \n- @property\n- def check_value(self):\n- check_dict = dict.fromkeys(self.frame_sizes)\n- linerate = {\n- 64: 0.08,\n- 128: 0.10,\n- 256: 0.17,\n- 512: 0.18,\n- 1024: 0.40,\n- 1280: 0.45,\n- 1518: 0.50,\n- }\n- for size in self.frame_sizes:\n- speed = self.wirespeed(self.nic, size, self.number_of_ports)\n- check_dict[size] = round(speed * linerate[size], 2)\n- return check_dict\n-\n- def send_verify(self, case_info, qemu_version, tag):\n+ def handle_expected(self):\n+ \"\"\"\n+ Update expected numbers to configurate file: $DTS_CFG_FOLDER/$suite_name.cfg\n+ \"\"\"\n+ if load_global_setting(UPDATE_EXPECTED) == \"yes\":\n+ for self.qemu_version in self.qemu_num:\n+ self.expected_throughput[self.qemu_version] = self.throughput[\n+ self.case_info\n+ ][self.qemu_version]\n+\n+ def send_verify(self, case_info, qemu_version, tag, i):\n for frame_size in self.frame_sizes:\n info = \"Running test %s, and %d frame size.\" % (\n self.running_case,\n@@ -397,11 +394,19 @@ class TestVirtioPVPRegression(TestCase):\n )\n Mpps = pps / 1000000.0\n pct = Mpps * 100 / float(self.wirespeed(self.nic, frame_size, 1))\n- self.verify(\n- Mpps > self.check_value[frame_size],\n- \"%s of frame size %d speed verify failed, expect %s, result %s\"\n- % (self.running_case, frame_size, self.check_value[frame_size], Mpps),\n- )\n+ try:\n+ exepect = self.expected_throughput[self.qemu_num[i]][tag][frame_size]\n+\n+ except:\n+ exepect = 0\n+ self.logger.info(\n+ 'The first test of %s failed to match the expected result.\\\n+ Please use \"--update-expect\" to update the expected result.'\n+ % self.qemu_num[i]\n+ )\n+\n+ self.status = \"PASS\" if Mpps > exepect * (1 - self.gap * 0.01) else \"FAILED\"\n+\n # check each queue has data\n self.check_packets_of_each_queue(frame_size)\n # update print table info\n@@ -410,12 +415,36 @@ class TestVirtioPVPRegression(TestCase):\n qemu_version,\n frame_size,\n str(Mpps),\n+ str(exepect),\n str(pct),\n self.queues_number,\n+ self.status,\n tag,\n ]\n self.result_table_add(data_row)\n \n+ def handle_throughput(self):\n+ self.throughput = {}\n+\n+ for outer in self.get_test_result_table:\n+ for item in outer[1:]:\n+ key1, key2, key3, key4, value = (\n+ item[0],\n+ item[1],\n+ item[-1],\n+ int(item[2]),\n+ float(item[3]),\n+ )\n+ if key1 not in self.throughput:\n+ self.throughput[key1] = {}\n+ if key2 not in self.throughput[key1]:\n+ self.throughput[key1][key2] = {}\n+ if key3 not in self.throughput[key1][key2]:\n+ self.throughput[key1][key2][key3] = {}\n+ self.throughput[key1][key2][key3][key4] = value\n+\n+ print(self.throughput)\n+\n def close_testpmd_and_qemu(self):\n \"\"\"\n stop testpmd in vhost and qemu\n@@ -435,6 +464,7 @@ class TestVirtioPVPRegression(TestCase):\n modem = 1, start vm as virtio 1.0\n virtio_path = mergeable/normal/vector_rx\n \"\"\"\n+ self.get_test_result_table = []\n for i in range(len(self.qemu_list)):\n self.result_table_create(self.header_row)\n path = self.qemu_list[i][\"path\"]\n@@ -451,93 +481,148 @@ class TestVirtioPVPRegression(TestCase):\n self.start_testpmd_in_vm(virtio_path)\n self.logger.info(\"now testing the qemu path of %s\" % path)\n time.sleep(5)\n- self.send_verify(case_info, version, \"before reconnect\")\n+ self.send_verify(case_info, version, \"before reconnect\", i)\n \n- self.logger.info(\"now reconnect from vhost\")\n- self.dut.send_expect(\"killall -s INT %s\" % self.testpmd_name, \"# \")\n- self.start_testpmd_as_vhost()\n- self.send_verify(case_info, version, \"reconnect from vhost\")\n+ if \"virtio-1.1\" not in self.case_info:\n+ self.logger.info(\"now reconnect from vhost\")\n+ self.dut.send_expect(\"killall -s INT %s\" % self.testpmd_name, \"# \")\n+ self.start_testpmd_as_vhost()\n+ self.send_verify(case_info, version, \"reconnect from vhost\", i)\n \n self.logger.info(\"now reconnect from vm\")\n self.dut.send_expect(\"killall -s INT qemu-system-x86_64\", \"# \")\n self.start_vm(path, version, modem, virtio_path, packed=packed)\n self.start_testpmd_in_vm(virtio_path)\n- self.send_verify(case_info, version, \"reconnect from vm\")\n+ self.send_verify(case_info, version, \"reconnect from vm\", i)\n \n+ self.get_test_result_table.append(self.result_table_getrows())\n self.result_table_print()\n+ self.verify(\n+ \"FAIL\" not in self.status,\n+ \"Excessive gap between test results and expectations\",\n+ )\n self.close_testpmd_and_qemu()\n \n def test_perf_pvp_regression_with_mergeable_path(self):\n \"\"\"\n Test Case 1: pvp test with virtio 0.95 mergeable path\n \"\"\"\n- case_info = \"virtio-0.95 mergeable\"\n+ self.case_info = \"virtio-0.95 mergeable\"\n modem = 0\n virtio_path = \"mergeable\"\n- self.pvp_regression_run(case_info, modem, virtio_path)\n+ self.test_target = self.running_case\n+ self.expected_throughput = self.get_suite_cfg()[\n+ \"expected_throughput_virtio-0.95\"\n+ ][self.test_target][self.case_info]\n+ self.pvp_regression_run(self.case_info, modem, virtio_path)\n+ self.handle_throughput()\n+ self.handle_expected()\n \n def test_perf_pvp_regression_modern_mergeable_path(self):\n \"\"\"\n Test Case 4: pvp test with virtio 1.0 mergeable path\n \"\"\"\n- case_info = \"virtio-1.0 mergeable\"\n+ self.case_info = \"virtio-1.0 mergeable\"\n modem = 1\n virtio_path = \"mergeable\"\n- self.pvp_regression_run(case_info, modem, virtio_path)\n+ self.test_target = self.running_case\n+ self.expected_throughput = self.get_suite_cfg()[\n+ \"expected_throughput_virtio-1.0\"\n+ ][self.test_target][self.case_info]\n+ self.pvp_regression_run(self.case_info, modem, virtio_path)\n+ self.handle_throughput()\n+ self.handle_expected()\n \n def test_perf_pvp_regression_non_mergeable_path(self):\n \"\"\"\n Test Case 2: pvp test with virtio 0.95 non-mergeable path\n \"\"\"\n- case_info = \"virtio-0.95 normal\"\n+ self.case_info = \"virtio-0.95 normal\"\n modem = 0\n virtio_path = \"normal\"\n- self.pvp_regression_run(case_info, modem, virtio_path)\n+ self.test_target = self.running_case\n+ self.expected_throughput = self.get_suite_cfg()[\n+ \"expected_throughput_virtio-0.95\"\n+ ][self.test_target][self.case_info]\n+ self.pvp_regression_run(self.case_info, modem, virtio_path)\n+ self.handle_throughput()\n+ self.handle_expected()\n \n def test_perf_pvp_regression_modern_non_mergeable_path(self):\n \"\"\"\n Test Case 5: pvp test with virtio 1.0 non-mergeable path\n \"\"\"\n- case_info = \"virtio-1.0 normal\"\n+\n+ self.case_info = \"virtio-1.0 normal\"\n modem = 1\n virtio_path = \"normal\"\n- self.pvp_regression_run(case_info, modem, virtio_path)\n+ self.test_target = self.running_case\n+ self.expected_throughput = self.get_suite_cfg()[\n+ \"expected_throughput_virtio-1.0\"\n+ ][self.test_target][self.case_info]\n+ self.pvp_regression_run(self.case_info, modem, virtio_path)\n+ self.handle_throughput()\n+ self.handle_expected()\n \n def test_perf_pvp_regression_vector_rx_path(self):\n \"\"\"\n Test Case 3: pvp test with virtio 0.95 vrctor_rx path\n \"\"\"\n- case_info = \"virtio-0.95 vector_rx\"\n+ self.case_info = \"virtio-0.95 vector_rx\"\n modem = 0\n virtio_path = \"vector_rx\"\n- self.pvp_regression_run(case_info, modem, virtio_path)\n+ self.test_target = self.running_case\n+ self.expected_throughput = self.get_suite_cfg()[\n+ \"expected_throughput_virtio-0.95\"\n+ ][self.test_target][self.case_info]\n+ self.pvp_regression_run(self.case_info, modem, virtio_path)\n+ self.handle_throughput()\n+ self.handle_expected()\n \n def test_perf_pvp_regression_modern_vector_rx_path(self):\n \"\"\"\n Test Case 6: pvp test with virtio 1.0 vrctor_rx path\n \"\"\"\n- case_info = \"virtio-1.0 normal\"\n+ self.case_info = \"virtio-1.0 vector_rx\"\n modem = 1\n virtio_path = \"vector_rx\"\n- self.pvp_regression_run(case_info, modem, virtio_path)\n+ self.test_target = self.running_case\n+ self.expected_throughput = self.get_suite_cfg()[\n+ \"expected_throughput_virtio-1.0\"\n+ ][self.test_target][self.case_info]\n+ self.pvp_regression_run(self.case_info, modem, virtio_path)\n+ self.handle_throughput()\n+ self.handle_expected()\n \n def test_perf_pvp_with_virtio11_mergeable_path(self):\n \"\"\"\n Test Case 7: pvp test with virtio 1.1 mergeable path\n \"\"\"\n- case_info = \"virtio-1.1 mergeable\"\n+ self.case_info = \"virtio-1.1 mergeable\"\n modem = 1\n virtio_path = \"mergeable\"\n- self.pvp_regression_run(case_info, modem, virtio_path, packed=True)\n+ self.test_target = self.running_case\n+ self.expected_throughput = self.get_suite_cfg()[\n+ \"expected_throughput_virtio-1.1\"\n+ ][self.test_target][self.case_info]\n+ self.pvp_regression_run(self.case_info, modem, virtio_path, packed=True)\n+ self.handle_throughput()\n+ self.handle_expected()\n \n def test_perf_pvp_with_virtio11_non_mergeable_path(self):\n \"\"\"\n Test Case 8: pvp test with virtio 1.1 non-mergeable path\n \"\"\"\n- case_info = \"virtio-1.1 normal\"\n+ self.case_info = \"virtio-1.1 normal\"\n modem = 1\n virtio_path = \"normal\"\n- self.pvp_regression_run(case_info, modem, virtio_path, packed=True)\n+ self.test_target = self.running_case\n+ self.expected_throughput = self.get_suite_cfg()[\n+ \"expected_throughput_virtio-1.1\"\n+ ][self.test_target][self.case_info]\n+ self.pvp_regression_run(self.case_info, modem, virtio_path, packed=True)\n+ self.handle_throughput()\n+ self.handle_expected()\n \n def tear_down(self):\n \"\"\"\n", "prefixes": [ "V1" ] }