Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/125204/?format=api
{ "id": 125204, "url": "https://patches.dpdk.org/api/patches/125204/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/20230316211434.13409-6-syalavarthi@marvell.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20230316211434.13409-6-syalavarthi@marvell.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20230316211434.13409-6-syalavarthi@marvell.com", "date": "2023-03-16T21:14:28", "name": "[v7,05/11] app/mldev: add ordered inference test case", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "c0e40d8a56545a09709f2c54ef3574510201be5f", "submitter": { "id": 2480, "url": "https://patches.dpdk.org/api/people/2480/?format=api", "name": "Srikanth Yalavarthi", "email": "syalavarthi@marvell.com" }, "delegate": { "id": 1, "url": "https://patches.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/20230316211434.13409-6-syalavarthi@marvell.com/mbox/", "series": [ { "id": 27428, "url": "https://patches.dpdk.org/api/series/27428/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=27428", "date": "2023-03-16T21:14:23", "name": "Implementation of mldev test application", "version": 7, "mbox": "https://patches.dpdk.org/series/27428/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/125204/comments/", "check": "warning", "checks": "https://patches.dpdk.org/api/patches/125204/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 8D4C441EB5;\n\tThu, 16 Mar 2023 22:15:31 +0100 (CET)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id B832542F8A;\n\tThu, 16 Mar 2023 22:14:51 +0100 (CET)", "from mx0b-0016f401.pphosted.com (mx0a-0016f401.pphosted.com\n [67.231.148.174])\n by mails.dpdk.org (Postfix) with ESMTP id 6952042DB3\n for <dev@dpdk.org>; Thu, 16 Mar 2023 22:14:44 +0100 (CET)", "from pps.filterd (m0045849.ppops.net [127.0.0.1])\n by mx0a-0016f401.pphosted.com (8.17.1.19/8.17.1.19) with ESMTP id\n 32GKcQtc004129 for <dev@dpdk.org>; Thu, 16 Mar 2023 14:14:43 -0700", "from dc5-exch01.marvell.com ([199.233.59.181])\n by mx0a-0016f401.pphosted.com (PPS) with ESMTPS id 3pbxq2ub11-3\n (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT)\n for <dev@dpdk.org>; Thu, 16 Mar 2023 14:14:43 -0700", "from DC5-EXCH02.marvell.com (10.69.176.39) by DC5-EXCH01.marvell.com\n (10.69.176.38) with Microsoft SMTP Server (TLS) id 15.0.1497.42;\n Thu, 16 Mar 2023 14:14:40 -0700", "from maili.marvell.com (10.69.176.80) by DC5-EXCH02.marvell.com\n (10.69.176.39) with Microsoft SMTP Server id 15.0.1497.42 via Frontend\n Transport; Thu, 16 Mar 2023 14:14:40 -0700", "from ml-host-33.caveonetworks.com (unknown [10.110.143.233])\n by maili.marvell.com (Postfix) with ESMTP id 592B15B6939;\n Thu, 16 Mar 2023 14:14:40 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com;\n h=from : to : cc :\n subject : date : message-id : in-reply-to : references : mime-version :\n content-type; s=pfpt0220; bh=GMfi9qTVpYCNU97aF56wqw965+1hTsoylaos8z8XdDY=;\n b=bl2d1OkC5dMXNyktKfwSmxf1u6AGZ9MBATkfpvXIxGg3PU84opEShwkzG2xMAuKkLcA4\n e9Q+ozaEyuo5obHwtgp5oB2L76W9p+WYac+RVHv6+Bk0JrW+R1qYjAnWl+CWIA52EnMr\n KwZ0qoec/+UN/JZDJ65rHTD9f0X7iuqefvt5aNXY4FVmuM7m8u0ot1jx2D7nZ5nFk/Hy\n wLgjPPZQZn1PYH2u635bG4+yWjN1bDqW1pA9PWBqm9V/uHnuaGKzZetxSbxmOv1AHshC\n 94RPnR7HFbuLp29Gj1Rqq6BzE1TaGxqY+QiSDqgLJmHKq51kGUQMX6zy7w3I8D1ug7vu OA==", "From": "Srikanth Yalavarthi <syalavarthi@marvell.com>", "To": "Srikanth Yalavarthi <syalavarthi@marvell.com>", "CC": "<dev@dpdk.org>, <sshankarnara@marvell.com>, <jerinj@marvell.com>,\n <aprabhu@marvell.com>, <ptakkar@marvell.com>, <pshukla@marvell.com>", "Subject": "[PATCH v7 05/11] app/mldev: add ordered inference test case", "Date": "Thu, 16 Mar 2023 14:14:28 -0700", "Message-ID": "<20230316211434.13409-6-syalavarthi@marvell.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20230316211434.13409-1-syalavarthi@marvell.com>", "References": "<20221129070746.20396-1-syalavarthi@marvell.com>\n <20230316211434.13409-1-syalavarthi@marvell.com>", "MIME-Version": "1.0", "Content-Type": "text/plain", "X-Proofpoint-GUID": "iEj_kOL1UDtaSB9Fjxhs7RpChWPl2C6D", "X-Proofpoint-ORIG-GUID": "iEj_kOL1UDtaSB9Fjxhs7RpChWPl2C6D", "X-Proofpoint-Virus-Version": "vendor=baseguard\n engine=ICAP:2.0.254,Aquarius:18.0.942,Hydra:6.0.573,FMLib:17.11.170.22\n definitions=2023-03-16_14,2023-03-16_02,2023-02-09_01", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "Added an ordered test case to execute inferences with single\nor multiple models. In this test case inference requests for\na model are enqueued after completion of all requests for\nthe previous model. Test supports inference repetitions.\n\nOperations sequence when testing with N models and R reps,\n\n(load -> start -> (enqueue + dequeue) x R -> stop -> unload) x N\n\nTest case can be executed by selecting \"inference_ordered\" test\nand repetitions can be specified through \"--repetitions\" argument.\n\nSigned-off-by: Srikanth Yalavarthi <syalavarthi@marvell.com>\nAcked-by: Anup Prabhu <aprabhu@marvell.com>\n---\n app/test-mldev/meson.build | 2 +\n app/test-mldev/ml_options.c | 65 ++\n app/test-mldev/ml_options.h | 17 +-\n app/test-mldev/test_inference_common.c | 567 ++++++++++++++++++\n app/test-mldev/test_inference_common.h | 61 ++\n app/test-mldev/test_inference_ordered.c | 115 ++++\n app/test-mldev/test_model_common.h | 10 +\n .../tools/img/mldev_inference_ordered.svg | 528 ++++++++++++++++\n doc/guides/tools/testmldev.rst | 88 ++-\n 9 files changed, 1445 insertions(+), 8 deletions(-)\n create mode 100644 app/test-mldev/test_inference_common.c\n create mode 100644 app/test-mldev/test_inference_common.h\n create mode 100644 app/test-mldev/test_inference_ordered.c\n create mode 100644 doc/guides/tools/img/mldev_inference_ordered.svg", "diff": "diff --git a/app/test-mldev/meson.build b/app/test-mldev/meson.build\nindex b09e1ccc8a..475d76d126 100644\n--- a/app/test-mldev/meson.build\n+++ b/app/test-mldev/meson.build\n@@ -16,6 +16,8 @@ sources = files(\n 'test_device_ops.c',\n 'test_model_common.c',\n 'test_model_ops.c',\n+ 'test_inference_common.c',\n+ 'test_inference_ordered.c',\n )\n \n deps += ['mldev']\ndiff --git a/app/test-mldev/ml_options.c b/app/test-mldev/ml_options.c\nindex 8ffbab7f75..7b56bca90e 100644\n--- a/app/test-mldev/ml_options.c\n+++ b/app/test-mldev/ml_options.c\n@@ -23,6 +23,7 @@ ml_options_default(struct ml_options *opt)\n \topt->dev_id = 0;\n \topt->socket_id = SOCKET_ID_ANY;\n \topt->nb_filelist = 0;\n+\topt->repetitions = 1;\n \topt->debug = false;\n }\n \n@@ -90,6 +91,60 @@ ml_parse_models(struct ml_options *opt, const char *arg)\n \treturn ret;\n }\n \n+static int\n+ml_parse_filelist(struct ml_options *opt, const char *arg)\n+{\n+\tconst char *delim = \",\";\n+\tchar filelist[PATH_MAX];\n+\tchar *token;\n+\n+\tif (opt->nb_filelist >= ML_TEST_MAX_MODELS) {\n+\t\tml_err(\"Exceeded filelist count, max = %d\\n\", ML_TEST_MAX_MODELS);\n+\t\treturn -1;\n+\t}\n+\n+\tstrlcpy(filelist, arg, PATH_MAX);\n+\n+\t/* model */\n+\ttoken = strtok(filelist, delim);\n+\tif (token == NULL) {\n+\t\tml_err(\"Invalid filelist, model not specified = %s\\n\", arg);\n+\t\treturn -EINVAL;\n+\t}\n+\tstrlcpy(opt->filelist[opt->nb_filelist].model, token, PATH_MAX);\n+\n+\t/* input */\n+\ttoken = strtok(NULL, delim);\n+\tif (token == NULL) {\n+\t\tml_err(\"Invalid filelist, input not specified = %s\\n\", arg);\n+\t\treturn -EINVAL;\n+\t}\n+\tstrlcpy(opt->filelist[opt->nb_filelist].input, token, PATH_MAX);\n+\n+\t/* output */\n+\ttoken = strtok(NULL, delim);\n+\tif (token == NULL) {\n+\t\tml_err(\"Invalid filelist, output not specified = %s\\n\", arg);\n+\t\treturn -EINVAL;\n+\t}\n+\tstrlcpy(opt->filelist[opt->nb_filelist].output, token, PATH_MAX);\n+\n+\topt->nb_filelist++;\n+\n+\tif (opt->nb_filelist == 0) {\n+\t\tml_err(\"Empty filelist. Need at least one filelist entry for the test.\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int\n+ml_parse_repetitions(struct ml_options *opt, const char *arg)\n+{\n+\treturn parser_read_uint64(&opt->repetitions, arg);\n+}\n+\n static void\n ml_dump_test_options(const char *testname)\n {\n@@ -100,6 +155,12 @@ ml_dump_test_options(const char *testname)\n \t\tprintf(\"\\t\\t--models : comma separated list of models\\n\");\n \t\tprintf(\"\\n\");\n \t}\n+\n+\tif (strcmp(testname, \"inference_ordered\") == 0) {\n+\t\tprintf(\"\\t\\t--filelist : comma separated list of model, input and output\\n\"\n+\t\t \"\\t\\t--repetitions : number of inference repetitions\\n\");\n+\t\tprintf(\"\\n\");\n+\t}\n }\n \n static void\n@@ -122,6 +183,8 @@ static struct option lgopts[] = {\n \t{ML_DEVICE_ID, 1, 0, 0},\n \t{ML_SOCKET_ID, 1, 0, 0},\n \t{ML_MODELS, 1, 0, 0},\n+\t{ML_FILELIST, 1, 0, 0},\n+\t{ML_REPETITIONS, 1, 0, 0},\n \t{ML_DEBUG, 0, 0, 0},\n \t{ML_HELP, 0, 0, 0},\n \t{NULL, 0, 0, 0}};\n@@ -136,6 +199,8 @@ ml_opts_parse_long(int opt_idx, struct ml_options *opt)\n \t\t{ML_DEVICE_ID, ml_parse_dev_id},\n \t\t{ML_SOCKET_ID, ml_parse_socket_id},\n \t\t{ML_MODELS, ml_parse_models},\n+\t\t{ML_FILELIST, ml_parse_filelist},\n+\t\t{ML_REPETITIONS, ml_parse_repetitions},\n \t};\n \n \tfor (i = 0; i < RTE_DIM(parsermap); i++) {\ndiff --git a/app/test-mldev/ml_options.h b/app/test-mldev/ml_options.h\nindex 61e938d2e2..6a13f97a30 100644\n--- a/app/test-mldev/ml_options.h\n+++ b/app/test-mldev/ml_options.h\n@@ -12,15 +12,19 @@\n #define ML_TEST_MAX_MODELS 8\n \n /* Options names */\n-#define ML_TEST\t (\"test\")\n-#define ML_DEVICE_ID (\"dev_id\")\n-#define ML_SOCKET_ID (\"socket_id\")\n-#define ML_MODELS (\"models\")\n-#define ML_DEBUG (\"debug\")\n-#define ML_HELP\t (\"help\")\n+#define ML_TEST\t (\"test\")\n+#define ML_DEVICE_ID (\"dev_id\")\n+#define ML_SOCKET_ID (\"socket_id\")\n+#define ML_MODELS (\"models\")\n+#define ML_FILELIST (\"filelist\")\n+#define ML_REPETITIONS (\"repetitions\")\n+#define ML_DEBUG (\"debug\")\n+#define ML_HELP\t (\"help\")\n \n struct ml_filelist {\n \tchar model[PATH_MAX];\n+\tchar input[PATH_MAX];\n+\tchar output[PATH_MAX];\n };\n \n struct ml_options {\n@@ -29,6 +33,7 @@ struct ml_options {\n \tint socket_id;\n \tstruct ml_filelist filelist[ML_TEST_MAX_MODELS];\n \tuint8_t nb_filelist;\n+\tuint64_t repetitions;\n \tbool debug;\n };\n \ndiff --git a/app/test-mldev/test_inference_common.c b/app/test-mldev/test_inference_common.c\nnew file mode 100644\nindex 0000000000..6a6999d524\n--- /dev/null\n+++ b/app/test-mldev/test_inference_common.c\n@@ -0,0 +1,567 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright (c) 2022 Marvell.\n+ */\n+\n+#include <errno.h>\n+#include <unistd.h>\n+\n+#include <rte_common.h>\n+#include <rte_launch.h>\n+#include <rte_lcore.h>\n+#include <rte_malloc.h>\n+#include <rte_memzone.h>\n+#include <rte_mldev.h>\n+\n+#include \"ml_common.h\"\n+#include \"test_inference_common.h\"\n+\n+/* Enqueue inference requests with burst size equal to 1 */\n+static int\n+ml_enqueue_single(void *arg)\n+{\n+\tstruct test_inference *t = ml_test_priv((struct ml_test *)arg);\n+\tstruct ml_request *req = NULL;\n+\tstruct rte_ml_op *op = NULL;\n+\tstruct ml_core_args *args;\n+\tuint64_t model_enq = 0;\n+\tuint32_t burst_enq;\n+\tuint32_t lcore_id;\n+\tuint16_t fid;\n+\tint ret;\n+\n+\tlcore_id = rte_lcore_id();\n+\targs = &t->args[lcore_id];\n+\tmodel_enq = 0;\n+\n+\tif (args->nb_reqs == 0)\n+\t\treturn 0;\n+\n+next_rep:\n+\tfid = args->start_fid;\n+\n+next_model:\n+\tret = rte_mempool_get(t->op_pool, (void **)&op);\n+\tif (ret != 0)\n+\t\tgoto next_model;\n+\n+retry:\n+\tret = rte_mempool_get(t->model[fid].io_pool, (void **)&req);\n+\tif (ret != 0)\n+\t\tgoto retry;\n+\n+\top->model_id = t->model[fid].id;\n+\top->nb_batches = t->model[fid].info.batch_size;\n+\top->mempool = t->op_pool;\n+\n+\top->input.addr = req->input;\n+\top->input.length = t->model[fid].inp_qsize;\n+\top->input.next = NULL;\n+\n+\top->output.addr = req->output;\n+\top->output.length = t->model[fid].out_qsize;\n+\top->output.next = NULL;\n+\n+\top->user_ptr = req;\n+\treq->niters++;\n+\treq->fid = fid;\n+\n+enqueue_req:\n+\tburst_enq = rte_ml_enqueue_burst(t->cmn.opt->dev_id, 0, &op, 1);\n+\tif (burst_enq == 0)\n+\t\tgoto enqueue_req;\n+\n+\tfid++;\n+\tif (likely(fid <= args->end_fid))\n+\t\tgoto next_model;\n+\n+\tmodel_enq++;\n+\tif (likely(model_enq < args->nb_reqs))\n+\t\tgoto next_rep;\n+\n+\treturn 0;\n+}\n+\n+/* Dequeue inference requests with burst size equal to 1 */\n+static int\n+ml_dequeue_single(void *arg)\n+{\n+\tstruct test_inference *t = ml_test_priv((struct ml_test *)arg);\n+\tstruct rte_ml_op_error error;\n+\tstruct rte_ml_op *op = NULL;\n+\tstruct ml_core_args *args;\n+\tstruct ml_request *req;\n+\tuint64_t total_deq = 0;\n+\tuint8_t nb_filelist;\n+\tuint32_t burst_deq;\n+\tuint32_t lcore_id;\n+\n+\tlcore_id = rte_lcore_id();\n+\targs = &t->args[lcore_id];\n+\tnb_filelist = args->end_fid - args->start_fid + 1;\n+\n+\tif (args->nb_reqs == 0)\n+\t\treturn 0;\n+\n+dequeue_req:\n+\tburst_deq = rte_ml_dequeue_burst(t->cmn.opt->dev_id, 0, &op, 1);\n+\n+\tif (likely(burst_deq == 1)) {\n+\t\ttotal_deq += burst_deq;\n+\t\tif (unlikely(op->status == RTE_ML_OP_STATUS_ERROR)) {\n+\t\t\trte_ml_op_error_get(t->cmn.opt->dev_id, op, &error);\n+\t\t\tml_err(\"error_code = 0x%\" PRIx64 \", error_message = %s\\n\", error.errcode,\n+\t\t\t error.message);\n+\t\t\tt->error_count[lcore_id]++;\n+\t\t}\n+\t\treq = (struct ml_request *)op->user_ptr;\n+\t\trte_mempool_put(t->model[req->fid].io_pool, req);\n+\t\trte_mempool_put(t->op_pool, op);\n+\t}\n+\n+\tif (likely(total_deq < args->nb_reqs * nb_filelist))\n+\t\tgoto dequeue_req;\n+\n+\treturn 0;\n+}\n+\n+bool\n+test_inference_cap_check(struct ml_options *opt)\n+{\n+\tstruct rte_ml_dev_info dev_info;\n+\n+\tif (!ml_test_cap_check(opt))\n+\t\treturn false;\n+\n+\trte_ml_dev_info_get(opt->dev_id, &dev_info);\n+\tif (opt->nb_filelist > dev_info.max_models) {\n+\t\tml_err(\"Insufficient capabilities: Filelist count exceeded device limit, count = %u (max limit = %u)\",\n+\t\t opt->nb_filelist, dev_info.max_models);\n+\t\treturn false;\n+\t}\n+\n+\treturn true;\n+}\n+\n+int\n+test_inference_opt_check(struct ml_options *opt)\n+{\n+\tuint32_t i;\n+\tint ret;\n+\n+\t/* check common opts */\n+\tret = ml_test_opt_check(opt);\n+\tif (ret != 0)\n+\t\treturn ret;\n+\n+\t/* check file availability */\n+\tfor (i = 0; i < opt->nb_filelist; i++) {\n+\t\tif (access(opt->filelist[i].model, F_OK) == -1) {\n+\t\t\tml_err(\"Model file not accessible: id = %u, file = %s\", i,\n+\t\t\t opt->filelist[i].model);\n+\t\t\treturn -ENOENT;\n+\t\t}\n+\n+\t\tif (access(opt->filelist[i].input, F_OK) == -1) {\n+\t\t\tml_err(\"Input file not accessible: id = %u, file = %s\", i,\n+\t\t\t opt->filelist[i].input);\n+\t\t\treturn -ENOENT;\n+\t\t}\n+\t}\n+\n+\tif (opt->repetitions == 0) {\n+\t\tml_err(\"Invalid option, repetitions = %\" PRIu64 \"\\n\", opt->repetitions);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/* check number of available lcores. */\n+\tif (rte_lcore_count() < 3) {\n+\t\tml_err(\"Insufficient lcores = %u\\n\", rte_lcore_count());\n+\t\tml_err(\"Minimum lcores required to create %u queue-pairs = %u\\n\", 1, 3);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+void\n+test_inference_opt_dump(struct ml_options *opt)\n+{\n+\tuint32_t i;\n+\n+\t/* dump common opts */\n+\tml_test_opt_dump(opt);\n+\n+\t/* dump test opts */\n+\tml_dump(\"repetitions\", \"%\" PRIu64, opt->repetitions);\n+\n+\tml_dump_begin(\"filelist\");\n+\tfor (i = 0; i < opt->nb_filelist; i++) {\n+\t\tml_dump_list(\"model\", i, opt->filelist[i].model);\n+\t\tml_dump_list(\"input\", i, opt->filelist[i].input);\n+\t\tml_dump_list(\"output\", i, opt->filelist[i].output);\n+\t}\n+\tml_dump_end;\n+}\n+\n+int\n+test_inference_setup(struct ml_test *test, struct ml_options *opt)\n+{\n+\tstruct test_inference *t;\n+\tvoid *test_inference;\n+\tint ret = 0;\n+\tuint32_t i;\n+\n+\ttest_inference = rte_zmalloc_socket(test->name, sizeof(struct test_inference),\n+\t\t\t\t\t RTE_CACHE_LINE_SIZE, opt->socket_id);\n+\tif (test_inference == NULL) {\n+\t\tml_err(\"failed to allocate memory for test_model\");\n+\t\tret = -ENOMEM;\n+\t\tgoto error;\n+\t}\n+\ttest->test_priv = test_inference;\n+\tt = ml_test_priv(test);\n+\n+\tt->nb_used = 0;\n+\tt->cmn.result = ML_TEST_FAILED;\n+\tt->cmn.opt = opt;\n+\tmemset(t->error_count, 0, RTE_MAX_LCORE * sizeof(uint64_t));\n+\n+\t/* get device info */\n+\tret = rte_ml_dev_info_get(opt->dev_id, &t->cmn.dev_info);\n+\tif (ret < 0) {\n+\t\tml_err(\"failed to get device info\");\n+\t\tgoto error;\n+\t}\n+\n+\tt->enqueue = ml_enqueue_single;\n+\tt->dequeue = ml_dequeue_single;\n+\n+\t/* set model initial state */\n+\tfor (i = 0; i < opt->nb_filelist; i++)\n+\t\tt->model[i].state = MODEL_INITIAL;\n+\n+\treturn 0;\n+\n+error:\n+\tif (test_inference != NULL)\n+\t\trte_free(test_inference);\n+\n+\treturn ret;\n+}\n+\n+void\n+test_inference_destroy(struct ml_test *test, struct ml_options *opt)\n+{\n+\tstruct test_inference *t;\n+\n+\tRTE_SET_USED(opt);\n+\n+\tt = ml_test_priv(test);\n+\tif (t != NULL)\n+\t\trte_free(t);\n+}\n+\n+int\n+ml_inference_mldev_setup(struct ml_test *test, struct ml_options *opt)\n+{\n+\tstruct rte_ml_dev_qp_conf qp_conf;\n+\tstruct test_inference *t;\n+\tint ret;\n+\n+\tt = ml_test_priv(test);\n+\n+\tret = ml_test_device_configure(test, opt);\n+\tif (ret != 0)\n+\t\treturn ret;\n+\n+\t/* setup queue pairs */\n+\tqp_conf.nb_desc = t->cmn.dev_info.max_desc;\n+\tqp_conf.cb = NULL;\n+\n+\tret = rte_ml_dev_queue_pair_setup(opt->dev_id, 0, &qp_conf, opt->socket_id);\n+\tif (ret != 0) {\n+\t\tml_err(\"Failed to setup ml device queue-pair, dev_id = %d, qp_id = %u\\n\",\n+\t\t opt->dev_id, 0);\n+\t\tgoto error;\n+\t}\n+\n+\tret = ml_test_device_start(test, opt);\n+\tif (ret != 0)\n+\t\tgoto error;\n+\n+\treturn 0;\n+\n+error:\n+\tml_test_device_close(test, opt);\n+\n+\treturn ret;\n+}\n+\n+int\n+ml_inference_mldev_destroy(struct ml_test *test, struct ml_options *opt)\n+{\n+\tint ret;\n+\n+\tret = ml_test_device_stop(test, opt);\n+\tif (ret != 0)\n+\t\tgoto error;\n+\n+\tret = ml_test_device_close(test, opt);\n+\tif (ret != 0)\n+\t\treturn ret;\n+\n+\treturn 0;\n+\n+error:\n+\tml_test_device_close(test, opt);\n+\n+\treturn ret;\n+}\n+\n+/* Callback for IO pool create. This function would compute the fields of ml_request\n+ * structure and prepare the quantized input data.\n+ */\n+static void\n+ml_request_initialize(struct rte_mempool *mp, void *opaque, void *obj, unsigned int obj_idx)\n+{\n+\tstruct test_inference *t = ml_test_priv((struct ml_test *)opaque);\n+\tstruct ml_request *req = (struct ml_request *)obj;\n+\n+\tRTE_SET_USED(mp);\n+\tRTE_SET_USED(obj_idx);\n+\n+\treq->input = (uint8_t *)obj +\n+\t\t RTE_ALIGN_CEIL(sizeof(struct ml_request), t->cmn.dev_info.min_align_size);\n+\treq->output = req->input +\n+\t\t RTE_ALIGN_CEIL(t->model[t->fid].inp_qsize, t->cmn.dev_info.min_align_size);\n+\treq->niters = 0;\n+\n+\t/* quantize data */\n+\trte_ml_io_quantize(t->cmn.opt->dev_id, t->model[t->fid].id,\n+\t\t\t t->model[t->fid].info.batch_size, t->model[t->fid].input, req->input);\n+}\n+\n+int\n+ml_inference_iomem_setup(struct ml_test *test, struct ml_options *opt, uint16_t fid)\n+{\n+\tstruct test_inference *t = ml_test_priv(test);\n+\tchar mz_name[RTE_MEMZONE_NAMESIZE];\n+\tchar mp_name[RTE_MEMPOOL_NAMESIZE];\n+\tconst struct rte_memzone *mz;\n+\tuint64_t nb_buffers;\n+\tuint32_t buff_size;\n+\tuint32_t mz_size;\n+\tuint32_t fsize;\n+\tFILE *fp;\n+\tint ret;\n+\n+\t/* get input buffer size */\n+\tret = rte_ml_io_input_size_get(opt->dev_id, t->model[fid].id, t->model[fid].info.batch_size,\n+\t\t\t\t &t->model[fid].inp_qsize, &t->model[fid].inp_dsize);\n+\tif (ret != 0) {\n+\t\tml_err(\"Failed to get input size, model : %s\\n\", opt->filelist[fid].model);\n+\t\treturn ret;\n+\t}\n+\n+\t/* get output buffer size */\n+\tret = rte_ml_io_output_size_get(opt->dev_id, t->model[fid].id,\n+\t\t\t\t\tt->model[fid].info.batch_size, &t->model[fid].out_qsize,\n+\t\t\t\t\t&t->model[fid].out_dsize);\n+\tif (ret != 0) {\n+\t\tml_err(\"Failed to get input size, model : %s\\n\", opt->filelist[fid].model);\n+\t\treturn ret;\n+\t}\n+\n+\t/* allocate buffer for user data */\n+\tmz_size = t->model[fid].inp_dsize + t->model[fid].out_dsize;\n+\tsprintf(mz_name, \"ml_user_data_%d\", fid);\n+\tmz = rte_memzone_reserve(mz_name, mz_size, opt->socket_id, 0);\n+\tif (mz == NULL) {\n+\t\tml_err(\"Memzone allocation failed for ml_user_data\\n\");\n+\t\tret = -ENOMEM;\n+\t\tgoto error;\n+\t}\n+\n+\tt->model[fid].input = mz->addr;\n+\tt->model[fid].output = t->model[fid].input + t->model[fid].inp_dsize;\n+\n+\t/* load input file */\n+\tfp = fopen(opt->filelist[fid].input, \"r\");\n+\tif (fp == NULL) {\n+\t\tml_err(\"Failed to open input file : %s\\n\", opt->filelist[fid].input);\n+\t\tret = -errno;\n+\t\tgoto error;\n+\t}\n+\n+\tfseek(fp, 0, SEEK_END);\n+\tfsize = ftell(fp);\n+\tfseek(fp, 0, SEEK_SET);\n+\tif (fsize != t->model[fid].inp_dsize) {\n+\t\tml_err(\"Invalid input file, size = %u (expected size = %\" PRIu64 \")\\n\", fsize,\n+\t\t t->model[fid].inp_dsize);\n+\t\tret = -EINVAL;\n+\t\tfclose(fp);\n+\t\tgoto error;\n+\t}\n+\n+\tif (fread(t->model[fid].input, 1, t->model[fid].inp_dsize, fp) != t->model[fid].inp_dsize) {\n+\t\tml_err(\"Failed to read input file : %s\\n\", opt->filelist[fid].input);\n+\t\tret = -errno;\n+\t\tfclose(fp);\n+\t\tgoto error;\n+\t}\n+\tfclose(fp);\n+\n+\t/* create mempool for quantized input and output buffers. ml_request_initialize is\n+\t * used as a callback for object creation.\n+\t */\n+\tbuff_size = RTE_ALIGN_CEIL(sizeof(struct ml_request), t->cmn.dev_info.min_align_size) +\n+\t\t RTE_ALIGN_CEIL(t->model[fid].inp_qsize, t->cmn.dev_info.min_align_size) +\n+\t\t RTE_ALIGN_CEIL(t->model[fid].out_qsize, t->cmn.dev_info.min_align_size);\n+\tnb_buffers = RTE_MIN((uint64_t)ML_TEST_MAX_POOL_SIZE, opt->repetitions);\n+\n+\tt->fid = fid;\n+\tsprintf(mp_name, \"ml_io_pool_%d\", fid);\n+\tt->model[fid].io_pool = rte_mempool_create(mp_name, nb_buffers, buff_size, 0, 0, NULL, NULL,\n+\t\t\t\t\t\t ml_request_initialize, test, opt->socket_id, 0);\n+\tif (t->model[fid].io_pool == NULL) {\n+\t\tml_err(\"Failed to create io pool : %s\\n\", \"ml_io_pool\");\n+\t\tret = -ENOMEM;\n+\t\tgoto error;\n+\t}\n+\n+\treturn 0;\n+\n+error:\n+\tif (mz != NULL)\n+\t\trte_memzone_free(mz);\n+\n+\tif (t->model[fid].io_pool != NULL) {\n+\t\trte_mempool_free(t->model[fid].io_pool);\n+\t\tt->model[fid].io_pool = NULL;\n+\t}\n+\n+\treturn ret;\n+}\n+\n+void\n+ml_inference_iomem_destroy(struct ml_test *test, struct ml_options *opt, uint16_t fid)\n+{\n+\tchar mz_name[RTE_MEMZONE_NAMESIZE];\n+\tchar mp_name[RTE_MEMPOOL_NAMESIZE];\n+\tconst struct rte_memzone *mz;\n+\tstruct rte_mempool *mp;\n+\n+\tRTE_SET_USED(test);\n+\tRTE_SET_USED(opt);\n+\n+\t/* release user data memzone */\n+\tsprintf(mz_name, \"ml_user_data_%d\", fid);\n+\tmz = rte_memzone_lookup(mz_name);\n+\tif (mz != NULL)\n+\t\trte_memzone_free(mz);\n+\n+\t/* destroy io pool */\n+\tsprintf(mp_name, \"ml_io_pool_%d\", fid);\n+\tmp = rte_mempool_lookup(mp_name);\n+\tif (mp != NULL)\n+\t\trte_mempool_free(mp);\n+}\n+\n+int\n+ml_inference_mem_setup(struct ml_test *test, struct ml_options *opt)\n+{\n+\tstruct test_inference *t = ml_test_priv(test);\n+\n+\t/* create op pool */\n+\tt->op_pool = rte_ml_op_pool_create(\"ml_test_op_pool\", ML_TEST_MAX_POOL_SIZE, 0, 0,\n+\t\t\t\t\t opt->socket_id);\n+\tif (t->op_pool == NULL) {\n+\t\tml_err(\"Failed to create op pool : %s\\n\", \"ml_op_pool\");\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+void\n+ml_inference_mem_destroy(struct ml_test *test, struct ml_options *opt)\n+{\n+\tstruct test_inference *t = ml_test_priv(test);\n+\n+\tRTE_SET_USED(opt);\n+\n+\t/* release op pool */\n+\tif (t->op_pool != NULL)\n+\t\trte_mempool_free(t->op_pool);\n+}\n+\n+/* Callback for mempool object iteration. This call would dequantize output data. */\n+static void\n+ml_request_finish(struct rte_mempool *mp, void *opaque, void *obj, unsigned int obj_idx)\n+{\n+\tstruct test_inference *t = ml_test_priv((struct ml_test *)opaque);\n+\tstruct ml_request *req = (struct ml_request *)obj;\n+\tstruct ml_model *model = &t->model[req->fid];\n+\n+\tRTE_SET_USED(mp);\n+\tRTE_SET_USED(obj_idx);\n+\n+\tif (req->niters == 0)\n+\t\treturn;\n+\n+\tt->nb_used++;\n+\trte_ml_io_dequantize(t->cmn.opt->dev_id, model->id, t->model[req->fid].info.batch_size,\n+\t\t\t req->output, model->output);\n+}\n+\n+int\n+ml_inference_result(struct ml_test *test, struct ml_options *opt, uint16_t fid)\n+{\n+\tstruct test_inference *t = ml_test_priv(test);\n+\tuint64_t error_count = 0;\n+\tuint32_t i;\n+\n+\tRTE_SET_USED(opt);\n+\n+\t/* check for errors */\n+\tfor (i = 0; i < RTE_MAX_LCORE; i++)\n+\t\terror_count += t->error_count[i];\n+\n+\trte_mempool_obj_iter(t->model[fid].io_pool, ml_request_finish, test);\n+\n+\tif ((t->nb_used > 0) && (error_count == 0))\n+\t\tt->cmn.result = ML_TEST_SUCCESS;\n+\telse\n+\t\tt->cmn.result = ML_TEST_FAILED;\n+\n+\treturn t->cmn.result;\n+}\n+\n+int\n+ml_inference_launch_cores(struct ml_test *test, struct ml_options *opt, uint16_t start_fid,\n+\t\t\t uint16_t end_fid)\n+{\n+\tstruct test_inference *t = ml_test_priv(test);\n+\tuint32_t lcore_id;\n+\tuint32_t id = 0;\n+\n+\tRTE_LCORE_FOREACH_WORKER(lcore_id)\n+\t{\n+\t\tif (id == 2)\n+\t\t\tbreak;\n+\n+\t\tt->args[lcore_id].nb_reqs = opt->repetitions;\n+\t\tt->args[lcore_id].start_fid = start_fid;\n+\t\tt->args[lcore_id].end_fid = end_fid;\n+\n+\t\tif (id % 2 == 0)\n+\t\t\trte_eal_remote_launch(t->enqueue, test, lcore_id);\n+\t\telse\n+\t\t\trte_eal_remote_launch(t->dequeue, test, lcore_id);\n+\n+\t\tid++;\n+\t}\n+\n+\treturn 0;\n+}\ndiff --git a/app/test-mldev/test_inference_common.h b/app/test-mldev/test_inference_common.h\nnew file mode 100644\nindex 0000000000..abb20fc9fb\n--- /dev/null\n+++ b/app/test-mldev/test_inference_common.h\n@@ -0,0 +1,61 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright (c) 2022 Marvell.\n+ */\n+\n+#ifndef _ML_TEST_INFERENCE_COMMON_\n+#define _ML_TEST_INFERENCE_COMMON_\n+\n+#include <rte_common.h>\n+#include <rte_mempool.h>\n+#include <rte_mldev.h>\n+\n+#include \"test_model_common.h\"\n+\n+struct ml_request {\n+\tuint8_t *input;\n+\tuint8_t *output;\n+\tuint16_t fid;\n+\tuint64_t niters;\n+};\n+\n+struct ml_core_args {\n+\tuint64_t nb_reqs;\n+\tuint16_t start_fid;\n+\tuint16_t end_fid;\n+};\n+\n+struct test_inference {\n+\t/* common data */\n+\tstruct test_common cmn;\n+\n+\t/* test specific data */\n+\tstruct ml_model model[ML_TEST_MAX_MODELS];\n+\tstruct rte_mempool *op_pool;\n+\n+\tuint64_t nb_used;\n+\tuint16_t fid;\n+\n+\tint (*enqueue)(void *arg);\n+\tint (*dequeue)(void *arg);\n+\n+\tstruct ml_core_args args[RTE_MAX_LCORE];\n+\tuint64_t error_count[RTE_MAX_LCORE];\n+} __rte_cache_aligned;\n+\n+bool test_inference_cap_check(struct ml_options *opt);\n+int test_inference_opt_check(struct ml_options *opt);\n+void test_inference_opt_dump(struct ml_options *opt);\n+int test_inference_setup(struct ml_test *test, struct ml_options *opt);\n+void test_inference_destroy(struct ml_test *test, struct ml_options *opt);\n+\n+int ml_inference_mldev_setup(struct ml_test *test, struct ml_options *opt);\n+int ml_inference_mldev_destroy(struct ml_test *test, struct ml_options *opt);\n+int ml_inference_iomem_setup(struct ml_test *test, struct ml_options *opt, uint16_t fid);\n+void ml_inference_iomem_destroy(struct ml_test *test, struct ml_options *opt, uint16_t fid);\n+int ml_inference_mem_setup(struct ml_test *test, struct ml_options *opt);\n+void ml_inference_mem_destroy(struct ml_test *test, struct ml_options *opt);\n+int ml_inference_result(struct ml_test *test, struct ml_options *opt, uint16_t fid);\n+int ml_inference_launch_cores(struct ml_test *test, struct ml_options *opt, uint16_t start_fid,\n+\t\t\t uint16_t end_fid);\n+\n+#endif /* _ML_TEST_INFERENCE_COMMON_ */\ndiff --git a/app/test-mldev/test_inference_ordered.c b/app/test-mldev/test_inference_ordered.c\nnew file mode 100644\nindex 0000000000..1cd91dc3d3\n--- /dev/null\n+++ b/app/test-mldev/test_inference_ordered.c\n@@ -0,0 +1,115 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright (c) 2022 Marvell.\n+ */\n+\n+#include <rte_common.h>\n+#include <rte_launch.h>\n+\n+#include \"ml_common.h\"\n+#include \"test_inference_common.h\"\n+\n+static int\n+test_inference_ordered_driver(struct ml_test *test, struct ml_options *opt)\n+{\n+\tstruct test_inference *t;\n+\tuint16_t fid = 0;\n+\tint ret = 0;\n+\n+\tt = ml_test_priv(test);\n+\n+\tret = ml_inference_mldev_setup(test, opt);\n+\tif (ret != 0)\n+\t\treturn ret;\n+\n+\tret = ml_inference_mem_setup(test, opt);\n+\tif (ret != 0)\n+\t\treturn ret;\n+\n+next_model:\n+\t/* load model */\n+\tret = ml_model_load(test, opt, &t->model[fid], fid);\n+\tif (ret != 0)\n+\t\tgoto error;\n+\n+\t/* start model */\n+\tret = ml_model_start(test, opt, &t->model[fid], fid);\n+\tif (ret != 0)\n+\t\tgoto error;\n+\n+\tret = ml_inference_iomem_setup(test, opt, fid);\n+\tif (ret != 0)\n+\t\tgoto error;\n+\n+\t/* launch inferences for one model using available queue pairs */\n+\tret = ml_inference_launch_cores(test, opt, fid, fid);\n+\tif (ret != 0) {\n+\t\tml_err(\"failed to launch cores\");\n+\t\tgoto error;\n+\t}\n+\n+\trte_eal_mp_wait_lcore();\n+\n+\tret = ml_inference_result(test, opt, fid);\n+\tif (ret != ML_TEST_SUCCESS)\n+\t\tgoto error;\n+\n+\tml_inference_iomem_destroy(test, opt, fid);\n+\n+\t/* stop model */\n+\tret = ml_model_stop(test, opt, &t->model[fid], fid);\n+\tif (ret != 0)\n+\t\tgoto error;\n+\n+\t/* unload model */\n+\tret = ml_model_unload(test, opt, &t->model[fid], fid);\n+\tif (ret != 0)\n+\t\tgoto error;\n+\n+\tfid++;\n+\tif (fid < opt->nb_filelist)\n+\t\tgoto next_model;\n+\n+\tml_inference_mem_destroy(test, opt);\n+\n+\tret = ml_inference_mldev_destroy(test, opt);\n+\tif (ret != 0)\n+\t\treturn ret;\n+\n+\tt->cmn.result = ML_TEST_SUCCESS;\n+\n+\treturn 0;\n+\n+error:\n+\tml_inference_iomem_destroy(test, opt, fid);\n+\tml_inference_mem_destroy(test, opt);\n+\tml_model_stop(test, opt, &t->model[fid], fid);\n+\tml_model_unload(test, opt, &t->model[fid], fid);\n+\n+\tt->cmn.result = ML_TEST_FAILED;\n+\n+\treturn ret;\n+}\n+\n+static int\n+test_inference_ordered_result(struct ml_test *test, struct ml_options *opt)\n+{\n+\tstruct test_inference *t;\n+\n+\tRTE_SET_USED(opt);\n+\n+\tt = ml_test_priv(test);\n+\n+\treturn t->cmn.result;\n+}\n+\n+static const struct ml_test_ops inference_ordered = {\n+\t.cap_check = test_inference_cap_check,\n+\t.opt_check = test_inference_opt_check,\n+\t.opt_dump = test_inference_opt_dump,\n+\t.test_setup = test_inference_setup,\n+\t.test_destroy = test_inference_destroy,\n+\t.test_driver = test_inference_ordered_driver,\n+\t.test_result = test_inference_ordered_result,\n+};\n+\n+ML_TEST_REGISTER(inference_ordered);\ndiff --git a/app/test-mldev/test_model_common.h b/app/test-mldev/test_model_common.h\nindex 74aec0a797..5ee975109d 100644\n--- a/app/test-mldev/test_model_common.h\n+++ b/app/test-mldev/test_model_common.h\n@@ -20,6 +20,16 @@ struct ml_model {\n \tuint16_t id;\n \tstruct rte_ml_model_info info;\n \tenum model_state state;\n+\n+\tuint64_t inp_dsize;\n+\tuint64_t inp_qsize;\n+\tuint64_t out_dsize;\n+\tuint64_t out_qsize;\n+\n+\tuint8_t *input;\n+\tuint8_t *output;\n+\n+\tstruct rte_mempool *io_pool;\n };\n \n int ml_model_load(struct ml_test *test, struct ml_options *opt, struct ml_model *model,\ndiff --git a/doc/guides/tools/img/mldev_inference_ordered.svg b/doc/guides/tools/img/mldev_inference_ordered.svg\nnew file mode 100644\nindex 0000000000..12fa6acaec\n--- /dev/null\n+++ b/doc/guides/tools/img/mldev_inference_ordered.svg\n@@ -0,0 +1,528 @@\n+<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n+<!-- SPDX-License-Identifier: BSD-3-Clause -->\n+<!-- Copyright (c) 2022 Marvell. -->\n+<!-- Created with Inkscape (http://www.inkscape.org/) -->\n+\n+<svg\n+ width=\"243mm\"\n+ height=\"144mm\"\n+ viewBox=\"0 0 243 144\"\n+ version=\"1.1\"\n+ id=\"svg5\"\n+ inkscape:version=\"1.2.1 (9c6d41e410, 2022-07-14)\"\n+ sodipodi:docname=\"inference_ordered.svg\"\n+ xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n+ xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n+ xmlns:xlink=\"http://www.w3.org/1999/xlink\"\n+ xmlns=\"http://www.w3.org/2000/svg\"\n+ xmlns:svg=\"http://www.w3.org/2000/svg\">\n+ <sodipodi:namedview\n+ id=\"namedview7\"\n+ pagecolor=\"#ffffff\"\n+ bordercolor=\"#000000\"\n+ borderopacity=\"0.25\"\n+ inkscape:showpageshadow=\"2\"\n+ inkscape:pageopacity=\"0.0\"\n+ inkscape:pagecheckerboard=\"0\"\n+ inkscape:deskcolor=\"#d1d1d1\"\n+ inkscape:document-units=\"mm\"\n+ showgrid=\"false\"\n+ inkscape:zoom=\"0.74564394\"\n+ inkscape:cx=\"488.83922\"\n+ inkscape:cy=\"234.69647\"\n+ inkscape:window-width=\"1920\"\n+ inkscape:window-height=\"1017\"\n+ inkscape:window-x=\"1912\"\n+ inkscape:window-y=\"-8\"\n+ inkscape:window-maximized=\"1\"\n+ inkscape:current-layer=\"layer1\" />\n+ <defs\n+ id=\"defs2\">\n+ <marker\n+ style=\"overflow:visible\"\n+ id=\"RoundedArrow\"\n+ refX=\"5\"\n+ refY=\"0\"\n+ orient=\"auto-start-reverse\"\n+ inkscape:stockid=\"RoundedArrow\"\n+ markerWidth=\"6.1347523\"\n+ markerHeight=\"5.9304948\"\n+ viewBox=\"0 0 6.1347524 5.9304951\"\n+ inkscape:isstock=\"true\"\n+ inkscape:collect=\"always\"\n+ preserveAspectRatio=\"xMidYMid\">\n+ <path\n+ transform=\"scale(0.7)\"\n+ d=\"m -0.21114562,-4.1055728 6.42229122,3.21114561 a 1,1 90 0 1 0,1.78885438 L -0.21114562,4.1055728 A 1.236068,1.236068 31.717474 0 1 -2,3 v -6 a 1.236068,1.236068 148.28253 0 1 1.78885438,-1.1055728 z\"\n+ style=\"fill:context-stroke;fill-rule:evenodd;stroke:none\"\n+ id=\"path1367\" />\n+ </marker>\n+ <linearGradient\n+ inkscape:collect=\"always\"\n+ id=\"linearGradient31002\">\n+ <stop\n+ style=\"stop-color:#fff6d5;stop-opacity:1;\"\n+ offset=\"0\"\n+ id=\"stop30998\" />\n+ <stop\n+ style=\"stop-color:#fff6d5;stop-opacity:0;\"\n+ offset=\"1\"\n+ id=\"stop31000\" />\n+ </linearGradient>\n+ <marker\n+ style=\"overflow:visible\"\n+ id=\"TriangleStart\"\n+ refX=\"4\"\n+ refY=\"0\"\n+ orient=\"auto-start-reverse\"\n+ inkscape:stockid=\"TriangleStart\"\n+ markerWidth=\"5.3244081\"\n+ markerHeight=\"6.155385\"\n+ viewBox=\"0 0 5.3244081 6.1553851\"\n+ inkscape:isstock=\"true\"\n+ inkscape:collect=\"always\"\n+ preserveAspectRatio=\"xMidYMid\">\n+ <path\n+ transform=\"scale(0.5)\"\n+ style=\"fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt\"\n+ d=\"M 5.77,0 -2.88,5 V -5 Z\"\n+ id=\"path135\" />\n+ </marker>\n+ <linearGradient\n+ inkscape:collect=\"always\"\n+ xlink:href=\"#linearGradient31002\"\n+ id=\"linearGradient31004\"\n+ x1=\"19.620968\"\n+ y1=\"102.90323\"\n+ x2=\"279.1532\"\n+ y2=\"102.90323\"\n+ gradientUnits=\"userSpaceOnUse\"\n+ gradientTransform=\"matrix(0.89215122,0,0,0.73190287,13.449912,42.668706)\" />\n+ </defs>\n+ <g\n+ inkscape:label=\"Layer 1\"\n+ inkscape:groupmode=\"layer\"\n+ id=\"layer1\">\n+ <g\n+ id=\"g1340\"\n+ transform=\"translate(-25.225796,-45.983871)\">\n+ <rect\n+ style=\"fill:url(#linearGradient31004);fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.404032;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect27876\"\n+ width=\"231.09595\"\n+ height=\"132.45081\"\n+ x=\"31.177822\"\n+ y=\"51.758469\"\n+ ry=\"3.5071263\" />\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00d7fb;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1813\"\n+ width=\"38.408459\"\n+ height=\"45.86002\"\n+ x=\"34.901794\"\n+ y=\"99.14959\"\n+ ry=\"5.2246051\"\n+ inkscape:connector-avoid=\"true\" />\n+ <path\n+ style=\"display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)\"\n+ d=\"m 73.310253,115.94935 36.498807,-11.6509\"\n+ id=\"path1906\"\n+ inkscape:connector-type=\"polyline\"\n+ inkscape:connector-curvature=\"0\"\n+ inkscape:connection-start=\"#rect1813\"\n+ inkscape:connection-end=\"#rect1724-0\" />\n+ <path\n+ style=\"display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)\"\n+ d=\"m 73.310253,122.0796 36.117817,1e-5\"\n+ id=\"path1908\"\n+ inkscape:connector-type=\"polyline\"\n+ inkscape:connector-curvature=\"0\"\n+ inkscape:connection-start=\"#rect1813\"\n+ inkscape:connection-end=\"#rect1724-4-8\" />\n+ <path\n+ style=\"display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)\"\n+ d=\"M 73.310253,128.20983 109.80905,139.8607\"\n+ id=\"path1910\"\n+ inkscape:connector-type=\"polyline\"\n+ inkscape:connector-curvature=\"0\"\n+ inkscape:connection-start=\"#rect1813\"\n+ inkscape:connection-end=\"#rect1724-6-7\" />\n+ <path\n+ style=\"display:inline;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#ff8500;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)\"\n+ d=\"m 141.71839,99.266314 19.42262,-10e-7\"\n+ id=\"path1912\"\n+ inkscape:connector-type=\"polyline\"\n+ inkscape:connector-curvature=\"0\"\n+ inkscape:connection-start=\"#rect1724-0\"\n+ inkscape:connection-end=\"#rect1679\" />\n+ <path\n+ style=\"display:inline;fill:none;fill-rule:evenodd;stroke:#00fb00;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)\"\n+ d=\"m 197.14101,99.266313 19.42259,10e-7\"\n+ id=\"path1914\"\n+ inkscape:connector-type=\"polyline\"\n+ inkscape:connector-curvature=\"0\"\n+ inkscape:connection-start=\"#rect1679\"\n+ inkscape:connection-end=\"#rect1724\" />\n+ <path\n+ style=\"display:inline;fill:none;fill-rule:evenodd;stroke:#ff8500;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)\"\n+ d=\"m 141.71839,122.07961 19.42262,-1e-5\"\n+ id=\"path1916\"\n+ inkscape:connector-type=\"polyline\"\n+ inkscape:connector-curvature=\"0\"\n+ inkscape:connection-start=\"#rect1724-4-8\"\n+ inkscape:connection-end=\"#rect1679-4\" />\n+ <path\n+ style=\"display:inline;fill:none;fill-rule:evenodd;stroke:#00fb00;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)\"\n+ d=\"m 197.14101,122.0796 19.42259,1e-5\"\n+ id=\"path1918\"\n+ inkscape:connector-type=\"polyline\"\n+ inkscape:connector-curvature=\"0\"\n+ inkscape:connection-start=\"#rect1679-4\"\n+ inkscape:connection-end=\"#rect1724-4\" />\n+ <path\n+ style=\"display:inline;fill:none;fill-rule:evenodd;stroke:#ff8500;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)\"\n+ d=\"m 141.71839,144.89282 19.42262,0\"\n+ id=\"path1920\"\n+ inkscape:connector-type=\"polyline\"\n+ inkscape:connector-curvature=\"0\"\n+ inkscape:connection-start=\"#rect1724-6-7\"\n+ inkscape:connection-end=\"#rect1679-8\" />\n+ <path\n+ style=\"display:inline;fill:none;fill-rule:evenodd;stroke:#00fb00;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)\"\n+ d=\"m 197.14101,144.89282 19.42259,0\"\n+ id=\"path1922\"\n+ inkscape:connector-type=\"polyline\"\n+ inkscape:connector-curvature=\"0\"\n+ inkscape:connection-start=\"#rect1679-8\"\n+ inkscape:connection-end=\"#rect1724-6\" />\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:8.46667px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-121.09793\"\n+ y=\"54.031597\"\n+ id=\"text4093\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan4091\"\n+ style=\"font-size:8.46667px;stroke-width:0.75\"\n+ x=\"-121.09793\"\n+ y=\"54.031597\">Model X</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:6.35px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-118.63563\"\n+ y=\"179.13635\"\n+ id=\"text4097\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan4095\"\n+ style=\"font-size:6.35px;stroke-width:0.75\"\n+ x=\"-118.63563\"\n+ y=\"179.13635\">Queue</tspan><tspan\n+ sodipodi:role=\"line\"\n+ style=\"font-size:6.35px;stroke-width:0.75\"\n+ x=\"-126.57313\"\n+ y=\"179.13635\"\n+ id=\"tspan4099\">Pair 1</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:6.35px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-141.44887\"\n+ y=\"179.13635\"\n+ id=\"text4097-5\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan4095-6\"\n+ style=\"font-size:6.35px;stroke-width:0.75\"\n+ x=\"-141.44887\"\n+ y=\"179.13635\">Queue</tspan><tspan\n+ sodipodi:role=\"line\"\n+ style=\"font-size:6.35px;stroke-width:0.75\"\n+ x=\"-149.38637\"\n+ y=\"179.13635\"\n+ id=\"tspan4099-4\">Pair 2</tspan></text>\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.354849;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:0.709699, 0.709699;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1924\"\n+ width=\"44.145252\"\n+ height=\"72.532341\"\n+ x=\"157.06865\"\n+ y=\"85.813438\"\n+ ry=\"4.31247\" />\n+ <g\n+ id=\"g1224\">\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1679\"\n+ width=\"36\"\n+ height=\"18\"\n+ x=\"161.14101\"\n+ y=\"90.266312\"\n+ ry=\"3\" />\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1679-8\"\n+ width=\"36\"\n+ height=\"18\"\n+ x=\"161.14101\"\n+ y=\"135.89282\"\n+ ry=\"3\" />\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1679-4\"\n+ width=\"36\"\n+ height=\"18\"\n+ x=\"161.14101\"\n+ y=\"113.07959\"\n+ ry=\"3\" />\n+ </g>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:6.35px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-95.820801\"\n+ y=\"179.13635\"\n+ id=\"text4097-8\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan4095-4\"\n+ style=\"font-size:6.35px;stroke-width:0.75\"\n+ x=\"-95.820801\"\n+ y=\"179.13635\">Queue</tspan><tspan\n+ sodipodi:role=\"line\"\n+ style=\"font-size:6.35px;stroke-width:0.75\"\n+ x=\"-103.7583\"\n+ y=\"179.13635\"\n+ id=\"tspan4099-5\">Pair 0</tspan></text>\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.317648;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1811\"\n+ width=\"44.196934\"\n+ height=\"16.731901\"\n+ x=\"157.04254\"\n+ y=\"56.49292\"\n+ ry=\"2.761292\" />\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:3.5859px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.317649;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:0.952945, 0.317649;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-60.009941\"\n+ y=\"186.38451\"\n+ id=\"text4156\"\n+ transform=\"matrix(0,-1.040508,0.96106903,0,0,0)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan4154\"\n+ style=\"font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:Arial;-inkscape-font-specification:'Arial Bold';stroke-width:0.317648\"\n+ x=\"-60.009941\"\n+ y=\"186.38451\">Machine Learning</tspan><tspan\n+ sodipodi:role=\"line\"\n+ style=\"font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:Arial;-inkscape-font-specification:'Arial Bold';stroke-width:0.317648\"\n+ x=\"-64.492317\"\n+ y=\"186.38451\"\n+ id=\"tspan4158\">Hardware Engine</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-99.705231\"\n+ y=\"125.91087\"\n+ id=\"text3708\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan3706\"\n+ style=\"font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75\"\n+ x=\"-99.705231\"\n+ y=\"125.91087\">lcore 1</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-145.3221\"\n+ y=\"125.50572\"\n+ id=\"text3708-8\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan3706-7\"\n+ style=\"font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75\"\n+ x=\"-145.3221\"\n+ y=\"125.50572\">lcore 5</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-122.51577\"\n+ y=\"125.52089\"\n+ id=\"text3708-5\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan3706-87\"\n+ style=\"font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75\"\n+ x=\"-122.51577\"\n+ y=\"125.52089\">lcore 3</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-162.06549\"\n+ y=\"125.4589\"\n+ id=\"text4542\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan4540\"\n+ style=\"font-size:5.64444px;stroke-width:0.75\"\n+ x=\"-162.06549\"\n+ y=\"125.4589\">Enqueue Workers</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-99.705231\"\n+ y=\"232.67706\"\n+ id=\"text3708-9\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan3706-9\"\n+ style=\"font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75\"\n+ x=\"-99.705231\"\n+ y=\"232.67706\">lcore 2</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-122.51025\"\n+ y=\"232.66466\"\n+ id=\"text3708-7\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan3706-8\"\n+ style=\"font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75\"\n+ x=\"-122.51025\"\n+ y=\"232.66466\">lcore 4</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-145.33035\"\n+ y=\"232.65778\"\n+ id=\"text3708-78\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan3706-0\"\n+ style=\"font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75\"\n+ x=\"-145.33035\"\n+ y=\"232.65778\">lcore 6</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-162.06549\"\n+ y=\"232.59988\"\n+ id=\"text4542-3\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan4540-7\"\n+ style=\"font-size:5.64444px;stroke-width:0.75\"\n+ x=\"-162.06549\"\n+ y=\"232.59988\">Dequeue Workers</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:6.35px;font-family:Arial;-inkscape-font-specification:'Arial Bold';text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-177.01665\"\n+ y=\"220.07283\"\n+ id=\"text5181\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan5179\"\n+ style=\"font-size:6.35px;stroke-width:0.75\"\n+ x=\"-177.01665\"\n+ y=\"220.07283\">test: inference_ordered</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-weight:bold;font-size:4.23333px;font-family:Arial;-inkscape-font-specification:'Arial Bold';text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:none;fill-rule:evenodd;stroke:#00d7fb;stroke-width:0.499999;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-178.63324\"\n+ y=\"98.67057\"\n+ id=\"text15571\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan15569\"\n+ style=\"font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.23333px;font-family:Arial;-inkscape-font-specification:Arial;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.5;stroke-opacity:1\"\n+ x=\"-178.63324\"\n+ y=\"98.67057\">nb_worker_threads = 2 * MIN(nb_queue_pairs, (lcore_count - 1) / 2)</tspan></text>\n+ <text\n+ xml:space=\"preserve\"\n+ style=\"font-weight:bold;font-size:4.23333px;font-family:Arial;-inkscape-font-specification:'Arial Bold';text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:none;fill-rule:evenodd;stroke:#00d7fb;stroke-width:0.499999;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ x=\"-171.18085\"\n+ y=\"89.26754\"\n+ id=\"text15571-3\"\n+ transform=\"rotate(-90)\"><tspan\n+ sodipodi:role=\"line\"\n+ id=\"tspan15569-9\"\n+ style=\"font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.23333px;font-family:Arial;-inkscape-font-specification:Arial;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.5;stroke-opacity:1\"\n+ x=\"-171.18085\"\n+ y=\"89.26754\">inferences_per_queue_pair = repetitions / nb_queue_pairs</tspan></text>\n+ <path\n+ style=\"display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.75;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-start:url(#RoundedArrow);marker-end:url(#RoundedArrow)\"\n+ d=\"m 179.14101,85.813438 0,-12.588618\"\n+ id=\"path31090\"\n+ inkscape:connector-type=\"polyline\"\n+ inkscape:connector-curvature=\"0\"\n+ inkscape:connection-start=\"#rect1924\" />\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00fb00;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1724\"\n+ width=\"32.290321\"\n+ height=\"11.709678\"\n+ x=\"216.5636\"\n+ y=\"93.411476\"\n+ ry=\"2\" />\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00fb00;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1724-4\"\n+ width=\"32.290321\"\n+ height=\"11.709678\"\n+ x=\"216.5636\"\n+ y=\"116.22478\"\n+ ry=\"2\" />\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00fb00;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1724-6\"\n+ width=\"32.290321\"\n+ height=\"11.709678\"\n+ x=\"216.5636\"\n+ y=\"139.03798\"\n+ ry=\"2\" />\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.354849;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:0.709699, 0.709699;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1924-6\"\n+ width=\"44.145252\"\n+ height=\"72.532341\"\n+ x=\"210.6364\"\n+ y=\"85.813477\"\n+ ry=\"4.31247\" />\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#ff8500;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1724-0\"\n+ width=\"32.290321\"\n+ height=\"11.709678\"\n+ x=\"109.42807\"\n+ y=\"93.411476\"\n+ ry=\"2\" />\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#ff8500;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1724-6-7\"\n+ width=\"32.290321\"\n+ height=\"11.709678\"\n+ x=\"109.42807\"\n+ y=\"139.03798\"\n+ ry=\"2\"\n+ inkscape:connector-avoid=\"true\" />\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#ff8500;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1724-4-8\"\n+ width=\"32.290321\"\n+ height=\"11.709678\"\n+ x=\"109.42807\"\n+ y=\"116.22478\"\n+ ry=\"2\"\n+ inkscape:connector-avoid=\"true\" />\n+ <rect\n+ style=\"fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.354849;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:0.709699, 0.709699;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers\"\n+ id=\"rect1924-6-3\"\n+ width=\"44.145252\"\n+ height=\"72.532341\"\n+ x=\"103.50092\"\n+ y=\"85.813477\"\n+ ry=\"4.31247\" />\n+ </g>\n+ </g>\n+</svg>\ndiff --git a/doc/guides/tools/testmldev.rst b/doc/guides/tools/testmldev.rst\nindex b8a2a16ca2..164fbca64f 100644\n--- a/doc/guides/tools/testmldev.rst\n+++ b/doc/guides/tools/testmldev.rst\n@@ -44,8 +44,8 @@ The following are the command-line options supported by the test application.\n \n * ``--test <name>``\n \n- Name of the test to execute. ML tests are divided into two groups, Device and Model\n- tests. Test name should be one of the following supported tests.\n+ Name of the test to execute. ML tests are divided into three groups, Device, Model\n+ and Inference tests. Test name should be one of the following supported tests.\n \n **ML Device Tests** ::\n \n@@ -55,6 +55,10 @@ The following are the command-line options supported by the test application.\n \n model_ops\n \n+ **ML Inference Tests** ::\n+\n+ inference_ordered\n+\n * ``--dev_id <n>``\n \n Set the device id of the ML device to be used for the test. Default value is `0`.\n@@ -69,6 +73,23 @@ The following are the command-line options supported by the test application.\n ``model_list`` in comma separated form (i.e. ``--models model_A.bin,model_B.bin``).\n Maximum number of models supported by the test is ``8``.\n \n+* ``--filelist <file_list>``\n+\n+ Set the list of model, input, output and reference files to be used for the tests.\n+ Application expects the ``file_list`` to be in comma separated form\n+ (i.e. ``--filelist <model,input,output>[,reference]``).\n+\n+ Multiple filelist entries can be specified when running the tests with multiple models.\n+ Both quantized and dequantized outputs are written to the disk. Dequantized output file\n+ would have the name specified by the user through ``--filelist`` option. A suffix ``.q``\n+ is appended to quantized output filename. Maximum number of filelist entries supported\n+ by the test is ``8``.\n+\n+* ``--repetitions <n>``\n+\n+ Set the number of inference repetitions to be executed in the test per each model. Default\n+ value is `1`.\n+\n * ``--debug``\n \n Enable the tests to run in debug mode.\n@@ -196,6 +217,69 @@ Command to run model_ops test:\n --test=model_ops --models model_1.bin,model_2.bin,model_3.bin, model_4.bin\n \n \n+ML Inference Tests\n+------------------\n+\n+Inference tests are a set of tests to validate end-to-end inference execution on ML device.\n+These tests executes the full sequence of operations required to run inferences with one or\n+multiple models.\n+\n+Application Options\n+~~~~~~~~~~~~~~~~~~~\n+\n+Supported command line options for inference tests are following::\n+\n+ --debug\n+ --test\n+ --dev_id\n+ --socket_id\n+ --filelist\n+ --repetitions\n+\n+\n+List of files to be used for the inference tests can be specified through the option\n+``--filelist <file_list>`` as a comma separated list. A filelist entry would be of the format\n+``--filelist <model_file,input_file,output_file>[,reference_file]`` and is used to specify the\n+list of files required to test with a single model. Multiple filelist entries are supported by\n+the test, one entry per model. Maximum number of file entries supported by the test is `8`.\n+\n+.. Note::\n+\n+ * The ``--filelist <file_list>`` is a mandatory option for running inference tests.\n+ * Options not supported by the tests are ignored if specified.\n+\n+\n+INFERENCE_ORDERED Test\n+~~~~~~~~~~~~~~~~~~~~~~\n+\n+This is a functional test for validating the end-to-end inference execution on ML device. This\n+test configures ML device and queue pairs as per the queue-pair related options (queue_pairs and\n+queue_size) specified by the user. Upon successful configuration of the device and queue pairs,\n+the first model specified through the filelist is loaded to the device and inferences are enqueued\n+by a pool of worker threads to the ML device. Total number of inferences enqueued for the model\n+are equal to the repetitions specified. A dedicated pool of worker threads would dequeue the\n+inferences from the device. The model is unloaded upon completion of all inferences for the model.\n+The test would continue loading and executing inference requests for all models specified\n+through ``filelist`` option in an ordered manner.\n+\n+.. _figure_mldev_inference_ordered:\n+\n+.. figure:: img/mldev_inference_ordered.*\n+\n+ Execution of inference_ordered on single model.\n+\n+\n+Example\n+^^^^^^^\n+\n+Example command to run inference_ordered test:\n+\n+.. code-block:: console\n+\n+ sudo <build_dir>/app/dpdk-test-mldev -c 0xf -a <PCI_ID> -- \\\n+ --test=inference_ordered --filelist model.bin,input.bin,output.bin\n+\n+\n Debug mode\n ----------\n \n", "prefixes": [ "v7", "05/11" ] }