diff mbox series

[v2,09/12] app/mldev: enable support for inference batches

Message ID 20221129082109.6809-9-syalavarthi@marvell.com (mailing list archive)
State Superseded, archived
Delegated to: Thomas Monjalon
Headers
Series [v2,01/12] app/mldev: implement test framework for mldev |

Checks

Context Check Description
ci/checkpatch success coding style OK

Commit Message

Srikanth Yalavarthi Nov. 29, 2022, 8:21 a.m. UTC 
  Enabled support to execute multiple batches of inferences
per each enqueue request. Input and reference for the test
should be appropriately provided for multi-batch run. Number
of batches can be specified through "--batches" option.

Signed-off-by: Srikanth Yalavarthi <syalavarthi@marvell.com>
---
 app/test-mldev/ml_options.c            | 15 ++++++++++++---
 app/test-mldev/ml_options.h            |  2 ++
 app/test-mldev/test_inference_common.c | 22 +++++++++++++---------
 app/test-mldev/test_model_common.c     |  6 ++++++
 app/test-mldev/test_model_common.h     |  1 +
 5 files changed, 34 insertions(+), 12 deletions(-)
diff mbox series

Patch

diff --git a/app/test-mldev/ml_options.c b/app/test-mldev/ml_options.c
index d5182a1701..331ec1704c 100644
--- a/app/test-mldev/ml_options.c
+++ b/app/test-mldev/ml_options.c
@@ -33,6 +33,7 @@  ml_options_default(struct ml_options *opt)
 	opt->burst_size = 1;
 	opt->queue_pairs = 1;
 	opt->queue_size = 1;
+	opt->batches = 0;
 	opt->debug = false;
 }
 
@@ -176,6 +177,12 @@  ml_parse_queue_size(struct ml_options *opt, const char *arg)
 	return parser_read_uint16(&opt->queue_size, arg);
 }
 
+static int
+ml_parse_batches(struct ml_options *opt, const char *arg)
+{
+	return parser_read_uint16(&opt->batches, arg);
+}
+
 static void
 ml_dump_test_options(const char *testname)
 {
@@ -196,7 +203,8 @@  ml_dump_test_options(const char *testname)
 		       "\t\t--repetitions      : number of inference repetitions\n"
 		       "\t\t--burst_size       : inference burst size\n"
 		       "\t\t--queue_pairs      : number of queue pairs to create\n"
-		       "\t\t--queue_size       : size fo queue-pair\n");
+		       "\t\t--queue_size       : size fo queue-pair\n"
+		       "\t\t--batches          : number of batches of input\n");
 		printf("\n");
 	}
 }
@@ -220,7 +228,8 @@  static struct option lgopts[] = {
 	{ML_TEST, 1, 0, 0},	  {ML_DEVICE_ID, 1, 0, 0},   {ML_SOCKET_ID, 1, 0, 0},
 	{ML_MODELS, 1, 0, 0},	  {ML_FILELIST, 1, 0, 0},    {ML_REPETITIONS, 1, 0, 0},
 	{ML_BURST_SIZE, 1, 0, 0}, {ML_QUEUE_PAIRS, 1, 0, 0}, {ML_QUEUE_SIZE, 1, 0, 0},
-	{ML_DEBUG, 0, 0, 0},	  {ML_HELP, 0, 0, 0},	     {NULL, 0, 0, 0}};
+	{ML_BATCHES, 1, 0, 0},	  {ML_DEBUG, 0, 0, 0},	     {ML_HELP, 0, 0, 0},
+	{NULL, 0, 0, 0}};
 
 static int
 ml_opts_parse_long(int opt_idx, struct ml_options *opt)
@@ -232,7 +241,7 @@  ml_opts_parse_long(int opt_idx, struct ml_options *opt)
 		{ML_SOCKET_ID, ml_parse_socket_id},   {ML_MODELS, ml_parse_models},
 		{ML_FILELIST, ml_parse_filelist},     {ML_REPETITIONS, ml_parse_repetitions},
 		{ML_BURST_SIZE, ml_parse_burst_size}, {ML_QUEUE_PAIRS, ml_parse_queue_pairs},
-		{ML_QUEUE_SIZE, ml_parse_queue_size},
+		{ML_QUEUE_SIZE, ml_parse_queue_size}, {ML_BATCHES, ml_parse_batches},
 	};
 
 	for (i = 0; i < RTE_DIM(parsermap); i++) {
diff --git a/app/test-mldev/ml_options.h b/app/test-mldev/ml_options.h
index 6bfef1b979..d23e842895 100644
--- a/app/test-mldev/ml_options.h
+++ b/app/test-mldev/ml_options.h
@@ -22,6 +22,7 @@ 
 #define ML_BURST_SIZE  ("burst_size")
 #define ML_QUEUE_PAIRS ("queue_pairs")
 #define ML_QUEUE_SIZE  ("queue_size")
+#define ML_BATCHES     ("batches")
 #define ML_DEBUG       ("debug")
 #define ML_HELP	       ("help")
 
@@ -41,6 +42,7 @@  struct ml_options {
 	uint16_t burst_size;
 	uint16_t queue_pairs;
 	uint16_t queue_size;
+	uint16_t batches;
 	bool debug;
 };
 
diff --git a/app/test-mldev/test_inference_common.c b/app/test-mldev/test_inference_common.c
index d680d68f3d..4e29f6c7eb 100644
--- a/app/test-mldev/test_inference_common.c
+++ b/app/test-mldev/test_inference_common.c
@@ -56,7 +56,7 @@  ml_enqueue_single(void *arg)
 		goto retry;
 
 	op->model_id = t->model[fid].id;
-	op->nb_batches = t->model[fid].info.batch_size;
+	op->nb_batches = t->model[fid].nb_batches;
 	op->mempool = t->op_pool;
 
 	op->input.addr = req->input;
@@ -168,7 +168,7 @@  ml_enqueue_burst(void *arg)
 
 	for (i = 0; i < ops_count; i++) {
 		args->enq_ops[i]->model_id = t->model[fid].id;
-		args->enq_ops[i]->nb_batches = t->model[fid].info.batch_size;
+		args->enq_ops[i]->nb_batches = t->model[fid].nb_batches;
 		args->enq_ops[i]->mempool = t->op_pool;
 
 		args->enq_ops[i]->input.addr = args->reqs[i]->input;
@@ -363,6 +363,11 @@  test_inference_opt_dump(struct ml_options *opt)
 	ml_dump("queue_pairs", "%u", opt->queue_pairs);
 	ml_dump("queue_size", "%u", opt->queue_size);
 
+	if (opt->batches == 0)
+		ml_dump("batches", "%u (default)", opt->batches);
+	else
+		ml_dump("batches", "%u", opt->batches);
+
 	ml_dump_begin("filelist");
 	for (i = 0; i < opt->nb_filelist; i++) {
 		ml_dump_list("model", i, opt->filelist[i].model);
@@ -531,8 +536,8 @@  ml_request_initialize(struct rte_mempool *mp, void *opaque, void *obj, unsigned
 	req->niters = 0;
 
 	/* quantize data */
-	rte_ml_io_quantize(t->cmn.opt->dev_id, t->model[t->fid].id,
-			   t->model[t->fid].info.batch_size, t->model[t->fid].input, req->input);
+	rte_ml_io_quantize(t->cmn.opt->dev_id, t->model[t->fid].id, t->model[t->fid].nb_batches,
+			   t->model[t->fid].input, req->input);
 }
 
 int
@@ -550,7 +555,7 @@  ml_inference_iomem_setup(struct ml_test *test, struct ml_options *opt, int16_t f
 	int ret;
 
 	/* get input buffer size */
-	ret = rte_ml_io_input_size_get(opt->dev_id, t->model[fid].id, t->model[fid].info.batch_size,
+	ret = rte_ml_io_input_size_get(opt->dev_id, t->model[fid].id, t->model[fid].nb_batches,
 				       &t->model[fid].inp_qsize, &t->model[fid].inp_dsize);
 	if (ret != 0) {
 		ml_err("Failed to get input size, model : %s\n", opt->filelist[fid].model);
@@ -558,9 +563,8 @@  ml_inference_iomem_setup(struct ml_test *test, struct ml_options *opt, int16_t f
 	}
 
 	/* get output buffer size */
-	ret = rte_ml_io_output_size_get(opt->dev_id, t->model[fid].id,
-					t->model[fid].info.batch_size, &t->model[fid].out_qsize,
-					&t->model[fid].out_dsize);
+	ret = rte_ml_io_output_size_get(opt->dev_id, t->model[fid].id, t->model[fid].nb_batches,
+					&t->model[fid].out_qsize, &t->model[fid].out_dsize);
 	if (ret != 0) {
 		ml_err("Failed to get input size, model : %s\n", opt->filelist[fid].model);
 		return ret;
@@ -705,7 +709,7 @@  ml_request_finish(struct rte_mempool *mp, void *opaque, void *obj, unsigned int
 		return;
 
 	t->nb_used++;
-	rte_ml_io_dequantize(t->cmn.opt->dev_id, model->id, t->model[req->fid].info.batch_size,
+	rte_ml_io_dequantize(t->cmn.opt->dev_id, model->id, t->model[req->fid].nb_batches,
 			     req->output, model->output);
 }
 
diff --git a/app/test-mldev/test_model_common.c b/app/test-mldev/test_model_common.c
index 5368be17fe..51260c0789 100644
--- a/app/test-mldev/test_model_common.c
+++ b/app/test-mldev/test_model_common.c
@@ -75,6 +75,12 @@  ml_model_load(struct ml_test *test, struct ml_options *opt, struct ml_model *mod
 		return ret;
 	}
 
+	/* Update number of batches */
+	if (opt->batches == 0)
+		model->nb_batches = model->info.batch_size;
+	else
+		model->nb_batches = opt->batches;
+
 	model->state = MODEL_LOADED;
 
 	return 0;
diff --git a/app/test-mldev/test_model_common.h b/app/test-mldev/test_model_common.h
index c45ae80853..dfbf568f0b 100644
--- a/app/test-mldev/test_model_common.h
+++ b/app/test-mldev/test_model_common.h
@@ -33,6 +33,7 @@  struct ml_model {
 	uint8_t *output;
 
 	struct rte_mempool *io_pool;
+	uint32_t nb_batches;
 };
 
 int ml_model_load(struct ml_test *test, struct ml_options *opt, struct ml_model *model,