Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/120590/?format=api
{ "id": 120590, "url": "https://patches.dpdk.org/api/patches/120590/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/20221208192918.25022-10-syalavarthi@marvell.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20221208192918.25022-10-syalavarthi@marvell.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20221208192918.25022-10-syalavarthi@marvell.com", "date": "2022-12-08T19:29:16", "name": "[v3,10/12] app/mldev: enable support for inference validation", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "452ff7022383be7b10e5c952bdcc6a7f8de71b10", "submitter": { "id": 2480, "url": "https://patches.dpdk.org/api/people/2480/?format=api", "name": "Srikanth Yalavarthi", "email": "syalavarthi@marvell.com" }, "delegate": { "id": 1, "url": "https://patches.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/20221208192918.25022-10-syalavarthi@marvell.com/mbox/", "series": [ { "id": 26046, "url": "https://patches.dpdk.org/api/series/26046/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=26046", "date": "2022-12-08T19:29:07", "name": "[v3,01/12] app/mldev: implement test framework for mldev", "version": 3, "mbox": "https://patches.dpdk.org/series/26046/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/120590/comments/", "check": "success", "checks": "https://patches.dpdk.org/api/patches/120590/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 54D4BA00C2;\n\tThu, 8 Dec 2022 20:30:35 +0100 (CET)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 1456F42D46;\n\tThu, 8 Dec 2022 20:29:56 +0100 (CET)", "from mx0b-0016f401.pphosted.com (mx0a-0016f401.pphosted.com\n [67.231.148.174])\n by mails.dpdk.org (Postfix) with ESMTP id 712DA42D21\n for <dev@dpdk.org>; Thu, 8 Dec 2022 20:29:49 +0100 (CET)", "from pps.filterd (m0045849.ppops.net [127.0.0.1])\n by mx0a-0016f401.pphosted.com (8.17.1.19/8.17.1.19) with ESMTP id\n 2B8J8KBo001352 for <dev@dpdk.org>; Thu, 8 Dec 2022 11:29:48 -0800", "from dc5-exch01.marvell.com ([199.233.59.181])\n by mx0a-0016f401.pphosted.com (PPS) with ESMTPS id 3mb22svjv3-1\n (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT)\n for <dev@dpdk.org>; Thu, 08 Dec 2022 11:29:48 -0800", "from DC5-EXCH02.marvell.com (10.69.176.39) by DC5-EXCH01.marvell.com\n (10.69.176.38) with Microsoft SMTP Server (TLS) id 15.0.1497.2;\n Thu, 8 Dec 2022 11:29:46 -0800", "from maili.marvell.com (10.69.176.80) by DC5-EXCH02.marvell.com\n (10.69.176.39) with Microsoft SMTP Server id 15.0.1497.18 via Frontend\n Transport; Thu, 8 Dec 2022 11:29:46 -0800", "from ml-host-33.caveonetworks.com (unknown [10.110.143.233])\n by maili.marvell.com (Postfix) with ESMTP id 95E423F7058;\n Thu, 8 Dec 2022 11:29:46 -0800 (PST)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com;\n h=from : to : cc :\n subject : date : message-id : in-reply-to : references : mime-version :\n content-type; s=pfpt0220; bh=kegt+iJjQjXnc0jOxF1LvmI1bRao+Zt+/Vycwwm53oY=;\n b=BtT7kYzQbkRuZ5/TGWAL0tBusyLGsBxOKe83UoVbUUQj/02s2PWVJqAm8IAPcmSMtjwR\n la1Wviv4IL6goSTlmkUuN+dfGhNTgxnR1Gt8JKdRzNfkrNig9nQJ9vv11D2OtKUWBPMs\n +mjNZQ/ggQlag9zqnv7vVhYKANjdeR/87W/RYWQn7te/2NYpK5C63nPOH4n57lvQDSpz\n NpUvjyEjkZwY8yGi2V83W0oJSNNC87rvOb5liwygfpSaBQ3dOSyP2Lb/mQ6LIeHVjcyj\n CX3mJk0X85aW4lXREkIqCP5tCZJIqWQ17qPEHbPdfrhMd1IhsRtZGQvnl8gpsDqz/ful 2A==", "From": "Srikanth Yalavarthi <syalavarthi@marvell.com>", "To": "Srikanth Yalavarthi <syalavarthi@marvell.com>", "CC": "<dev@dpdk.org>, <sshankarnara@marvell.com>, <jerinj@marvell.com>,\n <aprabhu@marvell.com>", "Subject": "[PATCH v3 10/12] app/mldev: enable support for inference validation", "Date": "Thu, 8 Dec 2022 11:29:16 -0800", "Message-ID": "<20221208192918.25022-10-syalavarthi@marvell.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20221208192918.25022-1-syalavarthi@marvell.com>", "References": "<20221129082109.6809-1-syalavarthi@marvell.com>\n <20221208192918.25022-1-syalavarthi@marvell.com>", "MIME-Version": "1.0", "Content-Type": "text/plain", "X-Proofpoint-ORIG-GUID": "kvCg3ZoGN4b_To7vtrvmgnxdL6HC19Fq", "X-Proofpoint-GUID": "kvCg3ZoGN4b_To7vtrvmgnxdL6HC19Fq", "X-Proofpoint-Virus-Version": "vendor=baseguard\n engine=ICAP:2.0.205,Aquarius:18.0.923,Hydra:6.0.545,FMLib:17.11.122.1\n definitions=2022-12-08_11,2022-12-08_01,2022-06-22_01", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "Enabled support to validate inference output with reference\noutput provided by the user. Validation would be successful\nonly when the inference outputs are within the 'tolerance'\nspecified through command line option \"--tolerance\".\n\nSigned-off-by: Srikanth Yalavarthi <syalavarthi@marvell.com>\n---\n app/test-mldev/meson.build | 2 +-\n app/test-mldev/ml_options.c | 36 +++-\n app/test-mldev/ml_options.h | 3 +\n app/test-mldev/test_inference_common.c | 218 ++++++++++++++++++++++++-\n app/test-mldev/test_inference_common.h | 1 +\n app/test-mldev/test_model_common.h | 1 +\n 6 files changed, 250 insertions(+), 11 deletions(-)", "diff": "diff --git a/app/test-mldev/meson.build b/app/test-mldev/meson.build\nindex 41d22fb22c..15db534dc2 100644\n--- a/app/test-mldev/meson.build\n+++ b/app/test-mldev/meson.build\n@@ -21,4 +21,4 @@ sources = files(\n 'test_inference_interleave.c',\n )\n \n-deps += ['mldev']\n+deps += ['mldev', 'hash']\ndiff --git a/app/test-mldev/ml_options.c b/app/test-mldev/ml_options.c\nindex 331ec1704c..092303903f 100644\n--- a/app/test-mldev/ml_options.c\n+++ b/app/test-mldev/ml_options.c\n@@ -5,6 +5,7 @@\n #include <errno.h>\n #include <getopt.h>\n #include <linux/limits.h>\n+#include <math.h>\n #include <stdbool.h>\n #include <stdio.h>\n #include <stdlib.h>\n@@ -34,6 +35,7 @@ ml_options_default(struct ml_options *opt)\n \topt->queue_pairs = 1;\n \topt->queue_size = 1;\n \topt->batches = 0;\n+\topt->tolerance = 0.0;\n \topt->debug = false;\n }\n \n@@ -139,6 +141,13 @@ ml_parse_filelist(struct ml_options *opt, const char *arg)\n \t}\n \tstrlcpy(opt->filelist[opt->nb_filelist].output, token, PATH_MAX);\n \n+\t/* reference - optional */\n+\ttoken = strtok(NULL, delim);\n+\tif (token != NULL)\n+\t\tstrlcpy(opt->filelist[opt->nb_filelist].reference, token, PATH_MAX);\n+\telse\n+\t\tmemset(opt->filelist[opt->nb_filelist].reference, 0, PATH_MAX);\n+\n \topt->nb_filelist++;\n \n \tif (opt->nb_filelist == 0) {\n@@ -183,6 +192,14 @@ ml_parse_batches(struct ml_options *opt, const char *arg)\n \treturn parser_read_uint16(&opt->batches, arg);\n }\n \n+static int\n+ml_parse_tolerance(struct ml_options *opt, const char *arg)\n+{\n+\topt->tolerance = fabs(atof(arg));\n+\n+\treturn 0;\n+}\n+\n static void\n ml_dump_test_options(const char *testname)\n {\n@@ -199,12 +216,13 @@ ml_dump_test_options(const char *testname)\n \n \tif ((strcmp(testname, \"inference_ordered\") == 0) ||\n \t (strcmp(testname, \"inference_interleave\") == 0)) {\n-\t\tprintf(\"\\t\\t--filelist : comma separated list of model, input and output\\n\"\n+\t\tprintf(\"\\t\\t--filelist : comma separated list of model, input, output and reference\\n\"\n \t\t \"\\t\\t--repetitions : number of inference repetitions\\n\"\n \t\t \"\\t\\t--burst_size : inference burst size\\n\"\n \t\t \"\\t\\t--queue_pairs : number of queue pairs to create\\n\"\n \t\t \"\\t\\t--queue_size : size fo queue-pair\\n\"\n-\t\t \"\\t\\t--batches : number of batches of input\\n\");\n+\t\t \"\\t\\t--batches : number of batches of input\\n\"\n+\t\t \"\\t\\t--tolerance : maximum tolerance (%%) for output validation\\n\");\n \t\tprintf(\"\\n\");\n \t}\n }\n@@ -224,12 +242,13 @@ print_usage(char *program)\n \tml_test_dump_names(ml_dump_test_options);\n }\n \n-static struct option lgopts[] = {\n-\t{ML_TEST, 1, 0, 0},\t {ML_DEVICE_ID, 1, 0, 0}, {ML_SOCKET_ID, 1, 0, 0},\n-\t{ML_MODELS, 1, 0, 0},\t {ML_FILELIST, 1, 0, 0}, {ML_REPETITIONS, 1, 0, 0},\n-\t{ML_BURST_SIZE, 1, 0, 0}, {ML_QUEUE_PAIRS, 1, 0, 0}, {ML_QUEUE_SIZE, 1, 0, 0},\n-\t{ML_BATCHES, 1, 0, 0},\t {ML_DEBUG, 0, 0, 0},\t {ML_HELP, 0, 0, 0},\n-\t{NULL, 0, 0, 0}};\n+static struct option lgopts[] = {{ML_TEST, 1, 0, 0},\t {ML_DEVICE_ID, 1, 0, 0},\n+\t\t\t\t {ML_SOCKET_ID, 1, 0, 0}, {ML_MODELS, 1, 0, 0},\n+\t\t\t\t {ML_FILELIST, 1, 0, 0}, {ML_REPETITIONS, 1, 0, 0},\n+\t\t\t\t {ML_BURST_SIZE, 1, 0, 0}, {ML_QUEUE_PAIRS, 1, 0, 0},\n+\t\t\t\t {ML_QUEUE_SIZE, 1, 0, 0}, {ML_BATCHES, 1, 0, 0},\n+\t\t\t\t {ML_TOLERANCE, 1, 0, 0}, {ML_DEBUG, 0, 0, 0},\n+\t\t\t\t {ML_HELP, 0, 0, 0},\t {NULL, 0, 0, 0}};\n \n static int\n ml_opts_parse_long(int opt_idx, struct ml_options *opt)\n@@ -242,6 +261,7 @@ ml_opts_parse_long(int opt_idx, struct ml_options *opt)\n \t\t{ML_FILELIST, ml_parse_filelist}, {ML_REPETITIONS, ml_parse_repetitions},\n \t\t{ML_BURST_SIZE, ml_parse_burst_size}, {ML_QUEUE_PAIRS, ml_parse_queue_pairs},\n \t\t{ML_QUEUE_SIZE, ml_parse_queue_size}, {ML_BATCHES, ml_parse_batches},\n+\t\t{ML_TOLERANCE, ml_parse_tolerance},\n \t};\n \n \tfor (i = 0; i < RTE_DIM(parsermap); i++) {\ndiff --git a/app/test-mldev/ml_options.h b/app/test-mldev/ml_options.h\nindex d23e842895..79ac54de98 100644\n--- a/app/test-mldev/ml_options.h\n+++ b/app/test-mldev/ml_options.h\n@@ -23,6 +23,7 @@\n #define ML_QUEUE_PAIRS (\"queue_pairs\")\n #define ML_QUEUE_SIZE (\"queue_size\")\n #define ML_BATCHES (\"batches\")\n+#define ML_TOLERANCE (\"tolerance\")\n #define ML_DEBUG (\"debug\")\n #define ML_HELP\t (\"help\")\n \n@@ -30,6 +31,7 @@ struct ml_filelist {\n \tchar model[PATH_MAX];\n \tchar input[PATH_MAX];\n \tchar output[PATH_MAX];\n+\tchar reference[PATH_MAX];\n };\n \n struct ml_options {\n@@ -43,6 +45,7 @@ struct ml_options {\n \tuint16_t queue_pairs;\n \tuint16_t queue_size;\n \tuint16_t batches;\n+\tfloat tolerance;\n \tbool debug;\n };\n \ndiff --git a/app/test-mldev/test_inference_common.c b/app/test-mldev/test_inference_common.c\nindex 4e29f6c7eb..008cee1023 100644\n--- a/app/test-mldev/test_inference_common.c\n+++ b/app/test-mldev/test_inference_common.c\n@@ -3,12 +3,15 @@\n */\n \n #include <errno.h>\n+#include <linux/limits.h>\n #include <stdbool.h>\n #include <stdint.h>\n #include <stdio.h>\n+#include <string.h>\n #include <unistd.h>\n \n #include <rte_common.h>\n+#include <rte_hash_crc.h>\n #include <rte_launch.h>\n #include <rte_lcore.h>\n #include <rte_malloc.h>\n@@ -21,6 +24,27 @@\n #include \"test_common.h\"\n #include \"test_inference_common.h\"\n \n+#define ML_TEST_READ_TYPE(buffer, type) (*((type *)buffer))\n+\n+#define ML_TEST_CHECK_OUTPUT(output, reference, tolerance) \\\n+\t(((float)output - (float)reference) <= (((float)reference * tolerance) / 100.0))\n+\n+#define ML_OPEN_WRITE_GET_ERR(name, buffer, size, err) \\\n+\tdo { \\\n+\t\tFILE *fp = fopen(name, \"w+\"); \\\n+\t\tif (fp == NULL) { \\\n+\t\t\tml_err(\"Unable to create file: %s, error: %s\", name, strerror(errno)); \\\n+\t\t\terr = true; \\\n+\t\t} else { \\\n+\t\t\tif (fwrite(buffer, 1, size, fp) != size) { \\\n+\t\t\t\tml_err(\"Error writing output, file: %s, error: %s\", name, \\\n+\t\t\t\t strerror(errno)); \\\n+\t\t\t\terr = true; \\\n+\t\t\t} \\\n+\t\t\tfclose(fp); \\\n+\t\t} \\\n+\t} while (0)\n+\n /* Enqueue inference requests with burst size equal to 1 */\n static int\n ml_enqueue_single(void *arg)\n@@ -362,6 +386,7 @@ test_inference_opt_dump(struct ml_options *opt)\n \tml_dump(\"burst_size\", \"%u\", opt->burst_size);\n \tml_dump(\"queue_pairs\", \"%u\", opt->queue_pairs);\n \tml_dump(\"queue_size\", \"%u\", opt->queue_size);\n+\tml_dump(\"tolerance\", \"%-7.3f\", opt->tolerance);\n \n \tif (opt->batches == 0)\n \t\tml_dump(\"batches\", \"%u (default)\", opt->batches);\n@@ -373,6 +398,8 @@ test_inference_opt_dump(struct ml_options *opt)\n \t\tml_dump_list(\"model\", i, opt->filelist[i].model);\n \t\tml_dump_list(\"input\", i, opt->filelist[i].input);\n \t\tml_dump_list(\"output\", i, opt->filelist[i].output);\n+\t\tif (strcmp(opt->filelist[i].reference, \"\\0\") != 0)\n+\t\t\tml_dump_list(\"reference\", i, opt->filelist[i].reference);\n \t}\n \tml_dump_end;\n }\n@@ -397,6 +424,7 @@ test_inference_setup(struct ml_test *test, struct ml_options *opt)\n \tt = ml_test_priv(test);\n \n \tt->nb_used = 0;\n+\tt->nb_valid = 0;\n \tt->cmn.result = ML_TEST_FAILED;\n \tt->cmn.opt = opt;\n \n@@ -572,6 +600,9 @@ ml_inference_iomem_setup(struct ml_test *test, struct ml_options *opt, int16_t f\n \n \t/* allocate buffer for user data */\n \tmz_size = t->model[fid].inp_dsize + t->model[fid].out_dsize;\n+\tif (strcmp(opt->filelist[fid].reference, \"\\0\") != 0)\n+\t\tmz_size += t->model[fid].out_dsize;\n+\n \tsprintf(mz_name, \"ml_user_data_%d\", fid);\n \tmz = rte_memzone_reserve(mz_name, mz_size, opt->socket_id, 0);\n \tif (mz == NULL) {\n@@ -582,6 +613,10 @@ ml_inference_iomem_setup(struct ml_test *test, struct ml_options *opt, int16_t f\n \n \tt->model[fid].input = mz->addr;\n \tt->model[fid].output = t->model[fid].input + t->model[fid].inp_dsize;\n+\tif (strcmp(opt->filelist[fid].reference, \"\\0\") != 0)\n+\t\tt->model[fid].reference = t->model[fid].output + t->model[fid].out_dsize;\n+\telse\n+\t\tt->model[fid].reference = NULL;\n \n \t/* load input file */\n \tfp = fopen(opt->filelist[fid].input, \"r\");\n@@ -610,6 +645,27 @@ ml_inference_iomem_setup(struct ml_test *test, struct ml_options *opt, int16_t f\n \t}\n \tfclose(fp);\n \n+\t/* load reference file */\n+\tif (t->model[fid].reference != NULL) {\n+\t\tfp = fopen(opt->filelist[fid].reference, \"r\");\n+\t\tif (fp == NULL) {\n+\t\t\tml_err(\"Failed to open reference file : %s\\n\",\n+\t\t\t opt->filelist[fid].reference);\n+\t\t\tret = -errno;\n+\t\t\tgoto error;\n+\t\t}\n+\n+\t\tif (fread(t->model[fid].reference, 1, t->model[fid].out_dsize, fp) !=\n+\t\t t->model[fid].out_dsize) {\n+\t\t\tml_err(\"Failed to read reference file : %s\\n\",\n+\t\t\t opt->filelist[fid].reference);\n+\t\t\tret = -errno;\n+\t\t\tfclose(fp);\n+\t\t\tgoto error;\n+\t\t}\n+\t\tfclose(fp);\n+\t}\n+\n \t/* create mempool for quantized input and output buffers. ml_request_initialize is\n \t * used as a callback for object creation.\n \t */\n@@ -694,6 +750,121 @@ ml_inference_mem_destroy(struct ml_test *test, struct ml_options *opt)\n \t\trte_mempool_free(t->op_pool);\n }\n \n+static bool\n+ml_inference_validation(struct ml_test *test, struct ml_request *req)\n+{\n+\tstruct test_inference *t = ml_test_priv((struct ml_test *)test);\n+\tstruct ml_model *model;\n+\tuint32_t nb_elements;\n+\tuint8_t *reference;\n+\tuint8_t *output;\n+\tbool match;\n+\tuint32_t i;\n+\tuint32_t j;\n+\n+\tmodel = &t->model[req->fid];\n+\n+\t/* compare crc when tolerance is 0 */\n+\tif (t->cmn.opt->tolerance == 0.0) {\n+\t\tmatch = (rte_hash_crc(model->output, model->out_dsize, 0) ==\n+\t\t\t rte_hash_crc(model->reference, model->out_dsize, 0));\n+\t} else {\n+\t\toutput = model->output;\n+\t\treference = model->reference;\n+\n+\t\ti = 0;\n+next_output:\n+\t\tnb_elements =\n+\t\t\tmodel->info.output_info[i].shape.w * model->info.output_info[i].shape.x *\n+\t\t\tmodel->info.output_info[i].shape.y * model->info.output_info[i].shape.z;\n+\t\tj = 0;\n+next_element:\n+\t\tmatch = false;\n+\t\tswitch (model->info.output_info[i].dtype) {\n+\t\tcase RTE_ML_IO_TYPE_INT8:\n+\t\t\tif (ML_TEST_CHECK_OUTPUT(ML_TEST_READ_TYPE(output, int8_t),\n+\t\t\t\t\t\t ML_TEST_READ_TYPE(reference, int8_t),\n+\t\t\t\t\t\t t->cmn.opt->tolerance))\n+\t\t\t\tmatch = true;\n+\n+\t\t\toutput += sizeof(int8_t);\n+\t\t\treference += sizeof(int8_t);\n+\t\t\tbreak;\n+\t\tcase RTE_ML_IO_TYPE_UINT8:\n+\t\t\tif (ML_TEST_CHECK_OUTPUT(ML_TEST_READ_TYPE(output, uint8_t),\n+\t\t\t\t\t\t ML_TEST_READ_TYPE(reference, uint8_t),\n+\t\t\t\t\t\t t->cmn.opt->tolerance))\n+\t\t\t\tmatch = true;\n+\n+\t\t\toutput += sizeof(float);\n+\t\t\treference += sizeof(float);\n+\t\t\tbreak;\n+\t\tcase RTE_ML_IO_TYPE_INT16:\n+\t\t\tif (ML_TEST_CHECK_OUTPUT(ML_TEST_READ_TYPE(output, int16_t),\n+\t\t\t\t\t\t ML_TEST_READ_TYPE(reference, int16_t),\n+\t\t\t\t\t\t t->cmn.opt->tolerance))\n+\t\t\t\tmatch = true;\n+\n+\t\t\toutput += sizeof(int16_t);\n+\t\t\treference += sizeof(int16_t);\n+\t\t\tbreak;\n+\t\tcase RTE_ML_IO_TYPE_UINT16:\n+\t\t\tif (ML_TEST_CHECK_OUTPUT(ML_TEST_READ_TYPE(output, uint16_t),\n+\t\t\t\t\t\t ML_TEST_READ_TYPE(reference, uint16_t),\n+\t\t\t\t\t\t t->cmn.opt->tolerance))\n+\t\t\t\tmatch = true;\n+\n+\t\t\toutput += sizeof(uint16_t);\n+\t\t\treference += sizeof(uint16_t);\n+\t\t\tbreak;\n+\t\tcase RTE_ML_IO_TYPE_INT32:\n+\t\t\tif (ML_TEST_CHECK_OUTPUT(ML_TEST_READ_TYPE(output, int32_t),\n+\t\t\t\t\t\t ML_TEST_READ_TYPE(reference, int32_t),\n+\t\t\t\t\t\t t->cmn.opt->tolerance))\n+\t\t\t\tmatch = true;\n+\n+\t\t\toutput += sizeof(int32_t);\n+\t\t\treference += sizeof(int32_t);\n+\t\t\tbreak;\n+\t\tcase RTE_ML_IO_TYPE_UINT32:\n+\t\t\tif (ML_TEST_CHECK_OUTPUT(ML_TEST_READ_TYPE(output, uint32_t),\n+\t\t\t\t\t\t ML_TEST_READ_TYPE(reference, uint32_t),\n+\t\t\t\t\t\t t->cmn.opt->tolerance))\n+\t\t\t\tmatch = true;\n+\n+\t\t\toutput += sizeof(uint32_t);\n+\t\t\treference += sizeof(uint32_t);\n+\t\t\tbreak;\n+\t\tcase RTE_ML_IO_TYPE_FP32:\n+\t\t\tif (ML_TEST_CHECK_OUTPUT(ML_TEST_READ_TYPE(output, float),\n+\t\t\t\t\t\t ML_TEST_READ_TYPE(reference, float),\n+\t\t\t\t\t\t t->cmn.opt->tolerance))\n+\t\t\t\tmatch = true;\n+\n+\t\t\toutput += sizeof(float);\n+\t\t\treference += sizeof(float);\n+\t\t\tbreak;\n+\t\tdefault: /* other types, fp8, fp16, bfloat16 */\n+\t\t\tmatch = true;\n+\t\t}\n+\n+\t\tif (!match)\n+\t\t\tgoto done;\n+\t\tj++;\n+\t\tif (j < nb_elements)\n+\t\t\tgoto next_element;\n+\n+\t\ti++;\n+\t\tif (i < model->info.nb_outputs)\n+\t\t\tgoto next_output;\n+\t}\n+done:\n+\tif (match)\n+\t\tt->nb_valid++;\n+\n+\treturn match;\n+}\n+\n /* Callback for mempool object iteration. This call would dequantize output data. */\n static void\n ml_request_finish(struct rte_mempool *mp, void *opaque, void *obj, unsigned int obj_idx)\n@@ -701,9 +872,10 @@ ml_request_finish(struct rte_mempool *mp, void *opaque, void *obj, unsigned int\n \tstruct test_inference *t = ml_test_priv((struct ml_test *)opaque);\n \tstruct ml_request *req = (struct ml_request *)obj;\n \tstruct ml_model *model = &t->model[req->fid];\n+\tchar str[PATH_MAX];\n+\tbool error = false;\n \n \tRTE_SET_USED(mp);\n-\tRTE_SET_USED(obj_idx);\n \n \tif (req->niters == 0)\n \t\treturn;\n@@ -711,6 +883,48 @@ ml_request_finish(struct rte_mempool *mp, void *opaque, void *obj, unsigned int\n \tt->nb_used++;\n \trte_ml_io_dequantize(t->cmn.opt->dev_id, model->id, t->model[req->fid].nb_batches,\n \t\t\t req->output, model->output);\n+\n+\tif (model->reference == NULL) {\n+\t\tt->nb_valid++;\n+\t\tgoto dump_output_pass;\n+\t}\n+\n+\tif (!ml_inference_validation(opaque, req))\n+\t\tgoto dump_output_fail;\n+\telse\n+\t\tgoto dump_output_pass;\n+\n+dump_output_pass:\n+\tif (obj_idx == 0) {\n+\t\t/* write quantized output */\n+\t\tsnprintf(str, PATH_MAX, \"%s.q\", t->cmn.opt->filelist[req->fid].output);\n+\t\tML_OPEN_WRITE_GET_ERR(str, req->output, model->out_qsize, error);\n+\t\tif (error)\n+\t\t\treturn;\n+\n+\t\t/* write dequantized output */\n+\t\tsnprintf(str, PATH_MAX, \"%s\", t->cmn.opt->filelist[req->fid].output);\n+\t\tML_OPEN_WRITE_GET_ERR(str, model->output, model->out_dsize, error);\n+\t\tif (error)\n+\t\t\treturn;\n+\t}\n+\n+\treturn;\n+\n+dump_output_fail:\n+\tif (t->cmn.opt->debug) {\n+\t\t/* dump quantized output buffer */\n+\t\tsnprintf(str, PATH_MAX, \"%s.q.%d\", t->cmn.opt->filelist[req->fid].output, obj_idx);\n+\t\tML_OPEN_WRITE_GET_ERR(str, req->output, model->out_qsize, error);\n+\t\tif (error)\n+\t\t\treturn;\n+\n+\t\t/* dump dequantized output buffer */\n+\t\tsnprintf(str, PATH_MAX, \"%s.%d\", t->cmn.opt->filelist[req->fid].output, obj_idx);\n+\t\tML_OPEN_WRITE_GET_ERR(str, model->output, model->out_dsize, error);\n+\t\tif (error)\n+\t\t\treturn;\n+\t}\n }\n \n int\n@@ -722,7 +936,7 @@ ml_inference_result(struct ml_test *test, struct ml_options *opt, int16_t fid)\n \n \trte_mempool_obj_iter(t->model[fid].io_pool, ml_request_finish, test);\n \n-\tif (t->nb_used > 0)\n+\tif (t->nb_used == t->nb_valid)\n \t\tt->cmn.result = ML_TEST_SUCCESS;\n \telse\n \t\tt->cmn.result = ML_TEST_FAILED;\ndiff --git a/app/test-mldev/test_inference_common.h b/app/test-mldev/test_inference_common.h\nindex 1bac2dcfa0..3f2b042360 100644\n--- a/app/test-mldev/test_inference_common.h\n+++ b/app/test-mldev/test_inference_common.h\n@@ -43,6 +43,7 @@ struct test_inference {\n \tstruct rte_mempool *op_pool;\n \n \tuint64_t nb_used;\n+\tuint64_t nb_valid;\n \tint16_t fid;\n \n \tint (*enqueue)(void *arg);\ndiff --git a/app/test-mldev/test_model_common.h b/app/test-mldev/test_model_common.h\nindex dfbf568f0b..ce12cbfecc 100644\n--- a/app/test-mldev/test_model_common.h\n+++ b/app/test-mldev/test_model_common.h\n@@ -31,6 +31,7 @@ struct ml_model {\n \n \tuint8_t *input;\n \tuint8_t *output;\n+\tuint8_t *reference;\n \n \tstruct rte_mempool *io_pool;\n \tuint32_t nb_batches;\n", "prefixes": [ "v3", "10/12" ] }