Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/90156/?format=api
{ "id": 90156, "url": "https://patches.dpdk.org/api/patches/90156/?format=api", "web_url": "https://patches.dpdk.org/project/dts/patch/20210331101659.506185-5-lijuan.tu@intel.com/", "project": { "id": 3, "url": "https://patches.dpdk.org/api/projects/3/?format=api", "name": "DTS", "link_name": "dts", "list_id": "dts.dpdk.org", "list_email": "dts@dpdk.org", "web_url": "", "scm_url": "git://dpdk.org/tools/dts", "webscm_url": "http://git.dpdk.org/tools/dts/", "list_archive_url": "https://inbox.dpdk.org/dts", "list_archive_url_format": "https://inbox.dpdk.org/dts/{}", "commit_url_format": "" }, "msgid": "<20210331101659.506185-5-lijuan.tu@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dts/20210331101659.506185-5-lijuan.tu@intel.com", "date": "2021-03-31T10:16:57", "name": "[v3,4/6] tests/nic_single_core_perf: revise delta to ratio", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": false, "hash": "303745a381592d3860e7f8a938008dc36068b52a", "submitter": { "id": 1098, "url": "https://patches.dpdk.org/api/people/1098/?format=api", "name": "Tu, Lijuan", "email": "lijuan.tu@intel.com" }, "delegate": null, "mbox": "https://patches.dpdk.org/project/dts/patch/20210331101659.506185-5-lijuan.tu@intel.com/mbox/", "series": [ { "id": 15990, "url": "https://patches.dpdk.org/api/series/15990/?format=api", "web_url": "https://patches.dpdk.org/project/dts/list/?series=15990", "date": "2021-03-31T10:16:54", "name": "revise tests/nic_signle_core_perf", "version": 3, "mbox": "https://patches.dpdk.org/series/15990/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/90156/comments/", "check": "pending", "checks": "https://patches.dpdk.org/api/patches/90156/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dts-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 5DF2FA0547;\n\tWed, 31 Mar 2021 04:18:04 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id A5F9D140DFD;\n\tWed, 31 Mar 2021 04:18:03 +0200 (CEST)", "from mga02.intel.com (mga02.intel.com [134.134.136.20])\n by mails.dpdk.org (Postfix) with ESMTP id 97AD840141\n for <dts@dpdk.org>; Wed, 31 Mar 2021 04:18:02 +0200 (CEST)", "from fmsmga002.fm.intel.com ([10.253.24.26])\n by orsmga101.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 30 Mar 2021 19:18:01 -0700", "from dpdk-icelake-perf.sh.intel.com ([10.67.118.199])\n by fmsmga002.fm.intel.com with ESMTP; 30 Mar 2021 19:18:00 -0700" ], "IronPort-SDR": [ "\n 86sZlUHjfTrwyc874P1rV8sxnFPSZXCKlFxe4L1d2Rf1ME+q/bOcjlaSZYJgo3e5nusQvRcaZo\n E/96uZd2wDgg==", "\n Cj0L1rr2Nt1n3hYtg5JxaNn+01USzSM3djZhq2fYO6VRqM3nWE4UTxmZgwXDKJjYvMrRmzSXOD\n f3ypZcReizTQ==" ], "X-IronPort-AV": [ "E=McAfee;i=\"6000,8403,9939\"; a=\"179027933\"", "E=Sophos;i=\"5.81,291,1610438400\"; d=\"scan'208\";a=\"179027933\"", "E=Sophos;i=\"5.81,291,1610438400\"; d=\"scan'208\";a=\"445432415\"" ], "X-ExtLoop1": "1", "From": "Lijuan Tu <lijuan.tu@intel.com>", "To": "dliu@iol.unh.edu,\n\tohilyard@iol.unh.edu,\n\talialnu@nvidia.com", "Cc": "dts@dpdk.org,\n\tLijuan Tu <lijuan.tu@intel.com>", "Date": "Wed, 31 Mar 2021 10:16:57 +0000", "Message-Id": "<20210331101659.506185-5-lijuan.tu@intel.com>", "X-Mailer": "git-send-email 2.25.1", "In-Reply-To": "<20210331101659.506185-1-lijuan.tu@intel.com>", "References": "<20210331101659.506185-1-lijuan.tu@intel.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[dts] [v3 4/6] tests/nic_single_core_perf: revise delta to ratio", "X-BeenThere": "dts@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "test suite reviews and discussions <dts.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dts>,\n <mailto:dts-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dts/>", "List-Post": "<mailto:dts@dpdk.org>", "List-Help": "<mailto:dts-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dts>,\n <mailto:dts-request@dpdk.org?subject=subscribe>", "Errors-To": "dts-bounces@dpdk.org", "Sender": "\"dts\" <dts-bounces@dpdk.org>" }, "content": "the delta used to defines the difference between real pps and expected\npps, the case fails if it exceed the absolute number specified by\n'accepted_tolerance' in conf/nic_single_core_perf.cfg, the deault is\n1Mpps, which is not friendly when the expected is only a few of million,\nso updated the accepted_tolerance to a percentage, considering the CI\nintergation, so the type is float.\n\nHere are main changes:\n* the meaning of accepted_tolerance\n before: threshold for (real-mpps - expected-mpps)\n after: threshold for (real-mpps - expected-mpps)/expected-mpps\n* Pass criteria\n before:\n real-mpps - expected-mpps > -accepted_tolerance\n after:\n (real-mpps - expected-mpps)/expected-mpps > -accepted_tolerance\n* rst table\n before:\n ... +---------------+-----------------+--------+\n ... | Expected-Mpps | Real - Expected | Status |\n ... +===============+=================+========+\n ... | 9.900 | -0.040 | PASS |\n ... +---------------+-----------------+--------+\n\n after:\n ... +---------------+------------+--------+\n ... | Expected-Mpps | Fluc Ratio | Status |\n ... +===============+============+========+\n ... | 9.900 | -0.004 | PASS |\n ... +---------------+------------+--------+\n* delta in JSON:\n before: real-mpps - expected-mpps\n after: (real-mpps - expected-mpps)/expected-mpps\n\nSigned-off-by: Lijuan Tu <lijuan.tu@intel.com>\n---\n conf/nic_single_core_perf.cfg | 10 +++++---\n tests/TestSuite_nic_single_core_perf.py | 32 ++++++++++++-------------\n 2 files changed, 22 insertions(+), 20 deletions(-)", "diff": "diff --git a/conf/nic_single_core_perf.cfg b/conf/nic_single_core_perf.cfg\nindex e6b0a5af..72b8c533 100644\n--- a/conf/nic_single_core_perf.cfg\n+++ b/conf/nic_single_core_perf.cfg\n@@ -11,8 +11,12 @@\n # - test_duration is how many seconds each combination performance will\n # be recorded.\n #\n-# - accepted_tolerance defines the accepted tolerance between test\n-# results and expected numbers.\n+# - accepted_tolerance defines the accepted tolerance for the\n+# fluctuation ratio of performance, it is a float(e.g. 0.05).\n+# Assume:\n+# F = (Real-Mpps - Expected-Mpps)/Expected-Mpps\n+# T = accepted_tolerance\n+# If F < -T, the case is PASS, else it is FAIL\n #\n # - expected_throughput is a dictionary defining expected throughput\n # numbers based on NIC, and the pattern is\n@@ -38,7 +42,7 @@ test_parameters = {'1C/1T': {64: [512, 2048]},\n rx_desc_16byte = 'y'\n test_duration = 60\n throughput_stat_sample_interval = 2\n-accepted_tolerance = 1\n+accepted_tolerance = 0.05\n expected_throughput = {\n 'fortville_spirit': {\n '1C/1T': {64: {512: 0.00, 2048: 0.00}},\ndiff --git a/tests/TestSuite_nic_single_core_perf.py b/tests/TestSuite_nic_single_core_perf.py\nindex 095b508a..dd35caac 100644\n--- a/tests/TestSuite_nic_single_core_perf.py\n+++ b/tests/TestSuite_nic_single_core_perf.py\n@@ -116,12 +116,12 @@ class TestNicSingleCorePerf(TestCase):\n # {'$framesize':{\"$nb_desc\": 'throughput'}\n self.throughput = {}\n \n- # Accepted tolerance in Mpps\n- self.gap = self.get_suite_cfg()['accepted_tolerance']\n+ # Accepted tolerance is ratio\n+ self.gap = self.get_suite_cfg().get('accepted_tolerance', 0.1)\n \n # header to print test result table\n self.table_header = ['Fwd_core', 'Frame Size', 'TXD/RXD', 'Real-Mpps', 'Rate',\n- 'Expected-Mpps', 'Real - Expected', 'Status']\n+ 'Expected-Mpps', 'Fluc Ratio', 'Status']\n self.test_result = {}\n \n def flows(self):\n@@ -332,12 +332,16 @@ class TestNicSingleCorePerf(TestCase):\n ret_data[header[3]] = \"{:.3f}\".format(_real)\n ret_data[header[4]] = \"{:.3f}%\".format(_real * 100 / wirespeed)\n ret_data[header[5]] = \"{:.3f}\".format(_exp)\n- delta = _real - _exp\n- ret_data[header[6]] = \"{:.3f}\".format(delta)\n- if delta > -self.gap:\n- ret_data[header[7]] = 'PASS'\n+ delta = (_real - _exp)/_exp\n+ if _exp != 0:\n+ ret_data[header[6]] = \"{:.3f}\".format(delta)\n+ if delta > -self.gap:\n+ ret_data[header[7]] = 'PASS'\n+ else:\n+ ret_data[header[7]] = 'FAIL'\n else:\n- ret_data[header[7]] = 'FAIL'\n+ ret_data[header[6]] = \"N/A\"\n+ ret_data[header[7]] = 'PASS'\n \n ret_datas[frame_size][nb_desc] = deepcopy(ret_data)\n self.test_result[fwd_config] = deepcopy(ret_datas)\n@@ -376,15 +380,9 @@ class TestNicSingleCorePerf(TestCase):\n row_dict0 = dict()\n row_dict0['performance'] = list()\n row_dict0['parameters'] = list()\n- result_throughput = float(row_in['Real-Mpps'])\n- expected_throughput = float(row_in['Expected-Mpps'])\n- # delta value and accepted tolerance in percentage\n- delta = result_throughput - expected_throughput\n- if delta > -self.gap:\n- row_dict0['status'] = 'PASS'\n- else:\n- row_dict0['status'] = 'FAIL'\n- row_dict1 = dict(name=\"Throughput\", value=result_throughput, unit=\"Mpps\", delta=delta)\n+ row_dict0['status'] = row_in['Status']\n+ row_dict1 = dict(name=\"Throughput\", value=row_in['Real-Mpps'], unit=\"Mpps\",\n+ delta=row_in['Fluc Ratio'])\n row_dict2 = dict(name=\"Txd/Rxd\", value=row_in[\"TXD/RXD\"], unit=\"descriptor\")\n row_dict3 = dict(name=\"frame_size\", value=row_in[\"Frame Size\"], unit=\"bytes\")\n row_dict4 = dict(name=\"Fwd_core\", value=row_in[\"Fwd_core\"])\n", "prefixes": [ "v3", "4/6" ] }