@@ -1856,170 +1856,6 @@ cn10k_ml_model_params_update(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_mode
return 0;
}
-int
-cn10k_ml_io_quantize(struct rte_ml_dev *dev, uint16_t model_id, struct rte_ml_buff_seg **dbuffer,
- struct rte_ml_buff_seg **qbuffer)
-{
- struct cnxk_ml_model *model;
- uint8_t model_input_type;
- uint8_t *lcl_dbuffer;
- uint8_t *lcl_qbuffer;
- uint8_t input_type;
- float qscale;
- uint32_t i;
- uint32_t j;
- int ret;
-
- model = dev->data->models[model_id];
-
- if (model == NULL) {
- plt_err("Invalid model_id = %u", model_id);
- return -EINVAL;
- }
-
- lcl_dbuffer = dbuffer[0]->addr;
- lcl_qbuffer = qbuffer[0]->addr;
-
- for (i = 0; i < model->layer[0].glow.metadata.model.num_input; i++) {
- if (i < MRVL_ML_NUM_INPUT_OUTPUT_1) {
- input_type = model->layer[0].glow.metadata.input1[i].input_type;
- model_input_type = model->layer[0].glow.metadata.input1[i].model_input_type;
- qscale = model->layer[0].glow.metadata.input1[i].qscale;
- } else {
- j = i - MRVL_ML_NUM_INPUT_OUTPUT_1;
- input_type = model->layer[0].glow.metadata.input2[j].input_type;
- model_input_type = model->layer[0].glow.metadata.input2[j].model_input_type;
- qscale = model->layer[0].glow.metadata.input2[j].qscale;
- }
-
- if (input_type == model_input_type) {
- rte_memcpy(lcl_qbuffer, lcl_dbuffer, model->layer[0].info.input[i].sz_d);
- } else {
- switch (model->layer[0].glow.metadata.input1[i].model_input_type) {
- case RTE_ML_IO_TYPE_INT8:
- ret = rte_ml_io_float32_to_int8(
- qscale, model->layer[0].info.input[i].nb_elements,
- lcl_dbuffer, lcl_qbuffer);
- break;
- case RTE_ML_IO_TYPE_UINT8:
- ret = rte_ml_io_float32_to_uint8(
- qscale, model->layer[0].info.input[i].nb_elements,
- lcl_dbuffer, lcl_qbuffer);
- break;
- case RTE_ML_IO_TYPE_INT16:
- ret = rte_ml_io_float32_to_int16(
- qscale, model->layer[0].info.input[i].nb_elements,
- lcl_dbuffer, lcl_qbuffer);
- break;
- case RTE_ML_IO_TYPE_UINT16:
- ret = rte_ml_io_float32_to_uint16(
- qscale, model->layer[0].info.input[i].nb_elements,
- lcl_dbuffer, lcl_qbuffer);
- break;
- case RTE_ML_IO_TYPE_FP16:
- ret = rte_ml_io_float32_to_float16(
- model->layer[0].info.input[i].nb_elements, lcl_dbuffer,
- lcl_qbuffer);
- break;
- default:
- plt_err("Unsupported model_input_type[%u] : %u", i,
- model->layer[0].glow.metadata.input1[i].model_input_type);
- ret = -ENOTSUP;
- }
- if (ret < 0)
- return ret;
- }
-
- lcl_dbuffer += model->layer[0].info.input[i].sz_d;
- lcl_qbuffer += model->layer[0].info.input[i].sz_q;
- }
-
- return 0;
-}
-
-int
-cn10k_ml_io_dequantize(struct rte_ml_dev *dev, uint16_t model_id, struct rte_ml_buff_seg **qbuffer,
- struct rte_ml_buff_seg **dbuffer)
-{
- struct cnxk_ml_model *model;
- uint8_t model_output_type;
- uint8_t *lcl_qbuffer;
- uint8_t *lcl_dbuffer;
- uint8_t output_type;
- float dscale;
- uint32_t i;
- uint32_t j;
- int ret;
-
- model = dev->data->models[model_id];
-
- if (model == NULL) {
- plt_err("Invalid model_id = %u", model_id);
- return -EINVAL;
- }
-
- lcl_dbuffer = dbuffer[0]->addr;
- lcl_qbuffer = qbuffer[0]->addr;
-
- for (i = 0; i < model->layer[0].glow.metadata.model.num_output; i++) {
- if (i < MRVL_ML_NUM_INPUT_OUTPUT_1) {
- output_type = model->layer[0].glow.metadata.output1[i].output_type;
- model_output_type =
- model->layer[0].glow.metadata.output1[i].model_output_type;
- dscale = model->layer[0].glow.metadata.output1[i].dscale;
- } else {
- j = i - MRVL_ML_NUM_INPUT_OUTPUT_1;
- output_type = model->layer[0].glow.metadata.output2[j].output_type;
- model_output_type =
- model->layer[0].glow.metadata.output2[j].model_output_type;
- dscale = model->layer[0].glow.metadata.output2[j].dscale;
- }
-
- if (output_type == model_output_type) {
- rte_memcpy(lcl_dbuffer, lcl_qbuffer, model->layer[0].info.output[i].sz_q);
- } else {
- switch (model->layer[0].glow.metadata.output1[i].model_output_type) {
- case RTE_ML_IO_TYPE_INT8:
- ret = rte_ml_io_int8_to_float32(
- dscale, model->layer[0].info.output[i].nb_elements,
- lcl_qbuffer, lcl_dbuffer);
- break;
- case RTE_ML_IO_TYPE_UINT8:
- ret = rte_ml_io_uint8_to_float32(
- dscale, model->layer[0].info.output[i].nb_elements,
- lcl_qbuffer, lcl_dbuffer);
- break;
- case RTE_ML_IO_TYPE_INT16:
- ret = rte_ml_io_int16_to_float32(
- dscale, model->layer[0].info.output[i].nb_elements,
- lcl_qbuffer, lcl_dbuffer);
- break;
- case RTE_ML_IO_TYPE_UINT16:
- ret = rte_ml_io_uint16_to_float32(
- dscale, model->layer[0].info.output[i].nb_elements,
- lcl_qbuffer, lcl_dbuffer);
- break;
- case RTE_ML_IO_TYPE_FP16:
- ret = rte_ml_io_float16_to_float32(
- model->layer[0].info.output[i].nb_elements, lcl_qbuffer,
- lcl_dbuffer);
- break;
- default:
- plt_err("Unsupported model_output_type[%u] : %u", i,
- model->layer[0].glow.metadata.output1[i].model_output_type);
- ret = -ENOTSUP;
- }
- if (ret < 0)
- return ret;
- }
-
- lcl_qbuffer += model->layer[0].info.output[i].sz_q;
- lcl_dbuffer += model->layer[0].info.output[i].sz_d;
- }
-
- return 0;
-}
-
static __rte_always_inline void
queue_index_advance(uint64_t *index, uint64_t nb_desc)
{
@@ -320,13 +320,6 @@ int cn10k_ml_model_stop(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *mo
int cn10k_ml_model_params_update(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model,
void *buffer);
-/* I/O ops */
-int cn10k_ml_io_quantize(struct rte_ml_dev *dev, uint16_t model_id,
- struct rte_ml_buff_seg **dbuffer, struct rte_ml_buff_seg **qbuffer);
-
-int cn10k_ml_io_dequantize(struct rte_ml_dev *dev, uint16_t model_id,
- struct rte_ml_buff_seg **qbuffer, struct rte_ml_buff_seg **dbuffer);
-
/* Fast-path ops */
__rte_hot uint16_t cn10k_ml_enqueue_burst(struct rte_ml_dev *dev, uint16_t qp_id,
struct rte_ml_op **ops, uint16_t nb_ops);
new file mode 100644
@@ -0,0 +1,95 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 Marvell.
+ */
+
+#include <rte_mldev.h>
+
+#include <mldev_utils.h>
+
+#include <roc_api.h>
+
+#include "cnxk_ml_io.h"
+
+inline int
+cnxk_ml_io_quantize_single(struct cnxk_ml_io *input, uint8_t *dbuffer, uint8_t *qbuffer)
+{
+ enum rte_ml_io_type qtype;
+ enum rte_ml_io_type dtype;
+ uint32_t nb_elements;
+ float qscale;
+ int ret = 0;
+
+ dtype = input->dtype;
+ qtype = input->qtype;
+ qscale = input->scale;
+ nb_elements = input->nb_elements;
+
+ if (dtype == qtype) {
+ rte_memcpy(qbuffer, dbuffer, input->sz_d);
+ } else {
+ switch (qtype) {
+ case RTE_ML_IO_TYPE_INT8:
+ ret = rte_ml_io_float32_to_int8(qscale, nb_elements, dbuffer, qbuffer);
+ break;
+ case RTE_ML_IO_TYPE_UINT8:
+ ret = rte_ml_io_float32_to_uint8(qscale, nb_elements, dbuffer, qbuffer);
+ break;
+ case RTE_ML_IO_TYPE_INT16:
+ ret = rte_ml_io_float32_to_int16(qscale, nb_elements, dbuffer, qbuffer);
+ break;
+ case RTE_ML_IO_TYPE_UINT16:
+ ret = rte_ml_io_float32_to_uint16(qscale, nb_elements, dbuffer, qbuffer);
+ break;
+ case RTE_ML_IO_TYPE_FP16:
+ ret = rte_ml_io_float32_to_float16(nb_elements, dbuffer, qbuffer);
+ break;
+ default:
+ plt_err("Unsupported qtype : %u", qtype);
+ ret = -ENOTSUP;
+ }
+ }
+
+ return ret;
+}
+
+inline int
+cnxk_ml_io_dequantize_single(struct cnxk_ml_io *output, uint8_t *qbuffer, uint8_t *dbuffer)
+{
+ enum rte_ml_io_type qtype;
+ enum rte_ml_io_type dtype;
+ uint32_t nb_elements;
+ float dscale;
+ int ret = 0;
+
+ dtype = output->dtype;
+ qtype = output->qtype;
+ dscale = output->scale;
+ nb_elements = output->nb_elements;
+
+ if (dtype == qtype) {
+ rte_memcpy(dbuffer, qbuffer, output->sz_q);
+ } else {
+ switch (qtype) {
+ case RTE_ML_IO_TYPE_INT8:
+ ret = rte_ml_io_int8_to_float32(dscale, nb_elements, qbuffer, dbuffer);
+ break;
+ case RTE_ML_IO_TYPE_UINT8:
+ ret = rte_ml_io_uint8_to_float32(dscale, nb_elements, qbuffer, dbuffer);
+ break;
+ case RTE_ML_IO_TYPE_INT16:
+ ret = rte_ml_io_int16_to_float32(dscale, nb_elements, qbuffer, dbuffer);
+ break;
+ case RTE_ML_IO_TYPE_UINT16:
+ ret = rte_ml_io_uint16_to_float32(dscale, nb_elements, qbuffer, dbuffer);
+ break;
+ case RTE_ML_IO_TYPE_FP16:
+ ret = rte_ml_io_float16_to_float32(nb_elements, qbuffer, dbuffer);
+ break;
+ default:
+ plt_err("Unsupported qtype: %u", qtype);
+ ret = -ENOTSUP;
+ }
+ }
+
+ return ret;
+}
@@ -76,4 +76,7 @@ struct cnxk_ml_io_info {
uint32_t total_output_sz_d;
};
+int cnxk_ml_io_quantize_single(struct cnxk_ml_io *input, uint8_t *dbuffer, uint8_t *qbuffer);
+int cnxk_ml_io_dequantize_single(struct cnxk_ml_io *output, uint8_t *qbuffer, uint8_t *dbuffer);
+
#endif /* _CNXK_ML_IO_H_ */
@@ -5,6 +5,8 @@
#include <rte_mldev.h>
#include <rte_mldev_pmd.h>
+#include <mldev_utils.h>
+
#include "cnxk_ml_dev.h"
#include "cnxk_ml_io.h"
#include "cnxk_ml_model.h"
@@ -708,6 +710,78 @@ cnxk_ml_model_params_update(struct rte_ml_dev *dev, uint16_t model_id, void *buf
return cn10k_ml_model_params_update(cnxk_mldev, model, buffer);
}
+static int
+cnxk_ml_io_quantize(struct rte_ml_dev *dev, uint16_t model_id, struct rte_ml_buff_seg **dbuffer,
+ struct rte_ml_buff_seg **qbuffer)
+{
+ struct cnxk_ml_io_info *info = NULL;
+ struct cnxk_ml_model *model;
+ uint8_t *lcl_dbuffer;
+ uint8_t *lcl_qbuffer;
+ uint32_t i;
+ int ret;
+
+ if ((dev == NULL) || (dbuffer == NULL) || (qbuffer == NULL))
+ return -EINVAL;
+
+ model = dev->data->models[model_id];
+ if (model == NULL) {
+ plt_err("Invalid model_id = %u", model_id);
+ return -EINVAL;
+ }
+
+ info = &model->layer[0].info;
+
+ lcl_dbuffer = dbuffer[0]->addr;
+ lcl_qbuffer = qbuffer[0]->addr;
+ for (i = 0; i < info->nb_inputs; i++) {
+ ret = cnxk_ml_io_quantize_single(&info->input[i], lcl_dbuffer, lcl_qbuffer);
+ if (ret < 0)
+ return ret;
+
+ lcl_dbuffer += info->input[i].sz_d;
+ lcl_qbuffer += info->input[i].sz_q;
+ }
+
+ return 0;
+}
+
+static int
+cnxk_ml_io_dequantize(struct rte_ml_dev *dev, uint16_t model_id, struct rte_ml_buff_seg **qbuffer,
+ struct rte_ml_buff_seg **dbuffer)
+{
+ struct cnxk_ml_io_info *info = NULL;
+ struct cnxk_ml_model *model;
+ uint8_t *lcl_qbuffer;
+ uint8_t *lcl_dbuffer;
+ uint32_t i;
+ int ret;
+
+ if ((dev == NULL) || (qbuffer == NULL) || (dbuffer == NULL))
+ return -EINVAL;
+
+ model = dev->data->models[model_id];
+ if (model == NULL) {
+ plt_err("Invalid model_id = %u", model_id);
+ return -EINVAL;
+ }
+
+ info = &model->layer[model->nb_layers - 1].info;
+
+ lcl_qbuffer = qbuffer[0]->addr;
+ lcl_dbuffer = dbuffer[0]->addr;
+ for (i = 0; i < info->nb_outputs; i++) {
+ ret = cnxk_ml_io_dequantize_single(&info->output[i], lcl_qbuffer, lcl_dbuffer);
+ if (ret < 0)
+ return ret;
+
+ lcl_qbuffer += info->output[i].sz_q;
+ lcl_dbuffer += info->output[i].sz_d;
+ }
+
+ return 0;
+}
+
struct rte_ml_dev_ops cnxk_ml_ops = {
/* Device control ops */
.dev_info_get = cnxk_ml_dev_info_get,
@@ -739,6 +813,6 @@ struct rte_ml_dev_ops cnxk_ml_ops = {
.model_params_update = cnxk_ml_model_params_update,
/* I/O ops */
- .io_quantize = cn10k_ml_io_quantize,
- .io_dequantize = cn10k_ml_io_dequantize,
+ .io_quantize = cnxk_ml_io_quantize,
+ .io_dequantize = cnxk_ml_io_dequantize,
};
@@ -25,6 +25,7 @@ sources = files(
'cn10k_ml_model.c',
'cn10k_ml_ocm.c',
'cnxk_ml_dev.c',
+ 'cnxk_ml_io.c',
'cnxk_ml_model.c',
'cnxk_ml_ops.c',
)