@@ -4,6 +4,7 @@
#include "qat_device.h"
#include "qat_qp.h"
+#include "qat_logs.h"
#include "adf_transport_access_macros.h"
#include "qat_dev_gens.h"
@@ -4,6 +4,7 @@
#include "qat_device.h"
#include "qat_qp.h"
+#include "qat_logs.h"
#include "adf_transport_access_macros.h"
#include "qat_dev_gens.h"
@@ -4,8 +4,10 @@
#include "qat_device.h"
#include "qat_qp.h"
+#include "qat_logs.h"
#include "adf_transport_access_macros.h"
#include "qat_dev_gens.h"
+#include "icp_qat_hw.h"
#include <stdint.h>
@@ -7,6 +7,7 @@
#include "qat_device.h"
#include "qat_qp.h"
+#include "qat_logs.h"
#include "adf_transport_access_macros_gen4vf.h"
#include "adf_pf2vf_msg.h"
#include "qat_pf2vf.h"
@@ -2,16 +2,11 @@
* Copyright(c) 2018-2022 Intel Corporation
*/
-#include <rte_string_fns.h>
#include <rte_devargs.h>
#include <ctype.h>
#include "qat_device.h"
-#include "adf_transport_access_macros.h"
-#include "qat_sym.h"
-#include "qat_comp_pmd.h"
-#include "adf_pf2vf_msg.h"
-#include "qat_pf2vf.h"
+#include "qat_logs.h"
#define NOT_NULL(arg, func, msg, ...) \
do { \
@@ -32,14 +27,30 @@ const char *cmd_arg_strings[] = {
[QAT_CMD_SLICE_MAP_POS + 1] = NULL,
};
+#define MAX_NO_OF_SERVICES 16
+
/* Hardware device information per generation */
struct qat_gen_hw_data qat_gen_config[QAT_N_GENS];
struct qat_dev_hw_spec_funcs *qat_dev_hw_spec[QAT_N_GENS];
+int qat_dev_services_no;
+struct qat_dev_service qat_dev_service[MAX_NO_OF_SERVICES];
/* per-process array of device data */
struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES];
static int qat_nb_pci_devices;
+int
+get_service_pos(const char *name)
+{
+ int i;
+
+ for (i = 0; i < qat_dev_services_no; i++) {
+ if (strcmp(name, qat_dev_service[i].name) == 0)
+ return i;
+ }
+ return -1;
+}
+
/*
* The set of PCI devices this driver supports
*/
@@ -270,7 +281,8 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev)
return NULL;
}
- qat_dev_size = sizeof(struct qat_pci_device) + extra_size;
+ qat_dev_size = sizeof(struct qat_pci_device) + sizeof(void *) *
+ qat_dev_services_no + extra_size;
qat_dev_mz = rte_memzone_reserve(name, qat_dev_size,
rte_socket_id(), 0);
@@ -355,6 +367,7 @@ qat_pci_device_release(struct rte_pci_device *pci_dev)
struct qat_pci_device *qat_dev;
char name[QAT_DEV_NAME_MAX_LEN];
int busy = 0;
+ int i;
if (pci_dev == NULL)
return -EINVAL;
@@ -369,20 +382,12 @@ qat_pci_device_release(struct rte_pci_device *pci_dev)
/* Check that there are no service devs still on pci device */
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
- if (qat_dev->sym_dev != NULL) {
- QAT_LOG(DEBUG, "QAT sym device %s is busy",
- name);
- busy = 1;
- }
- if (qat_dev->asym_dev != NULL) {
- QAT_LOG(DEBUG, "QAT asym device %s is busy",
- name);
- busy = 1;
- }
- if (qat_dev->comp_dev != NULL) {
- QAT_LOG(DEBUG, "QAT comp device %s is busy",
- name);
- busy = 1;
+ for (i = 0; i < qat_dev_services_no; i++) {
+ if (qat_dev->pmd[i] != NULL) {
+ QAT_LOG(DEBUG, "QAT %s device %s is busy",
+ qat_dev_service[i].name, name);
+ busy = 1;
+ }
}
if (busy)
return -EBUSY;
@@ -400,16 +405,19 @@ static int
qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev,
struct rte_pci_device *pci_dev)
{
- qat_sym_dev_destroy(qat_pci_dev);
- qat_comp_dev_destroy(qat_pci_dev);
- qat_asym_dev_destroy(qat_pci_dev);
+ int i;
+
+ for (i = 0; i < qat_dev_services_no; i++) {
+ if (qat_dev_service[i].dev_create)
+ qat_dev_service[i].dev_destroy(qat_pci_dev);
+ }
return qat_pci_device_release(pci_dev);
}
static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
struct rte_pci_device *pci_dev)
{
- int sym_ret = 0, asym_ret = 0, comp_ret = 0;
+ int i, ret = 0;
int num_pmds_created = 0;
struct qat_pci_device *qat_pci_dev;
@@ -422,30 +430,18 @@ static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
if (qat_pci_dev == NULL)
return -ENODEV;
- sym_ret = qat_sym_dev_create(qat_pci_dev);
- if (sym_ret == 0) {
- num_pmds_created++;
+ for (i = 0; i < qat_dev_services_no; i++) {
+ if (qat_dev_service[i].dev_create) {
+ ret = qat_dev_service[i].dev_create(qat_pci_dev);
+ if (ret == 0)
+ num_pmds_created++;
+ else {
+ QAT_LOG(WARNING, "Failed to create %s PMD on device %s",
+ qat_dev_service[i].name,
+ qat_pci_dev->name);
+ }
+ }
}
- else
- QAT_LOG(WARNING,
- "Failed to create QAT SYM PMD on device %s",
- qat_pci_dev->name);
-
- comp_ret = qat_comp_dev_create(qat_pci_dev);
- if (comp_ret == 0)
- num_pmds_created++;
- else
- QAT_LOG(WARNING,
- "Failed to create QAT COMP PMD on device %s",
- qat_pci_dev->name);
-
- asym_ret = qat_asym_dev_create(qat_pci_dev);
- if (asym_ret == 0)
- num_pmds_created++;
- else
- QAT_LOG(WARNING,
- "Failed to create QAT ASYM PMD on device %s",
- qat_pci_dev->name);
if (num_pmds_created == 0)
qat_pci_dev_destroy(qat_pci_dev, pci_dev);
@@ -475,42 +471,6 @@ static struct rte_pci_driver rte_qat_pmd = {
.remove = qat_pci_remove
};
-__rte_weak int
-qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
-{
- return 0;
-}
-
-__rte_weak int
-qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
-{
- return 0;
-}
-
-__rte_weak int
-qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
-{
- return 0;
-}
-
-__rte_weak int
-qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
-{
- return 0;
-}
-
-__rte_weak int
-qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
-{
- return 0;
-}
-
-__rte_weak int
-qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
-{
- return 0;
-}
-
RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd);
RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map);
RTE_PMD_REGISTER_KMOD_DEP(QAT_PCI_NAME, "* igb_uio | uio_pci_generic | vfio-pci");
@@ -7,10 +7,7 @@
#include <bus_pci_driver.h>
#include "qat_common.h"
-#include "qat_logs.h"
#include "qat_qp.h"
-#include "adf_transport_access_macros.h"
-#include "icp_qat_hw.h"
#define QAT_DETACHED (0)
#define QAT_ATTACHED (1)
@@ -63,6 +60,18 @@ struct qat_dev_cmd_param {
uint32_t val;
};
+struct qat_dev_service {
+ const char *name;
+ int (*dev_create)(struct qat_pci_device *qat_pci_dev);
+ int (*dev_destroy)(struct qat_pci_device *qat_pci_dev);
+};
+
+extern int qat_dev_services_no;
+extern struct qat_dev_service qat_dev_service[];
+
+int
+get_service_pos(const char *name);
+
struct qat_device_info {
const struct rte_memzone *mz;
/**< mz to store the qat_pci_device so it can be
@@ -90,9 +99,6 @@ struct qat_device_info {
extern struct qat_device_info qat_pci_devs[];
-struct qat_cryptodev_private;
-struct qat_comp_dev_private;
-
/*
* This struct holds all the data about a QAT pci device
* including data about all services it supports.
@@ -117,30 +123,17 @@ struct qat_pci_device {
struct qat_qp *qps_in_use[QAT_MAX_SERVICES][ADF_MAX_QPS_ON_ANY_SERVICE];
/**< links to qps set up for each service, index same as on API */
-
- /* Data relating to symmetric crypto service */
- struct qat_cryptodev_private *sym_dev;
- /**< link back to cryptodev private data */
-
int qat_sym_driver_id;
/**< Symmetric driver id used by this device */
-
- /* Data relating to asymmetric crypto service */
- struct qat_cryptodev_private *asym_dev;
- /**< link back to cryptodev private data */
-
int qat_asym_driver_id;
/**< Symmetric driver id used by this device */
-
- /* Data relating to compression service */
- struct qat_comp_dev_private *comp_dev;
- /**< link back to compressdev private data */
void *misc_bar_io_addr;
/**< Address of misc bar */
void *dev_private;
/**< Per generation specific information */
struct qat_dev_cmd_param cmd_line_param[QAT_CMD_SLICE_MAP_POS + 1];
/**< Command line parameters values */
+ void *pmd[];
};
struct qat_gen_hw_data {
@@ -163,23 +156,4 @@ extern struct qat_gen_hw_data qat_gen_config[];
struct qat_pci_device *
qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev);
-/* declaration needed for weak functions */
-int
-qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused);
-
-int
-qat_asym_dev_create(struct qat_pci_device *qat_pci_dev);
-
-int
-qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused);
-
-int
-qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused);
-
-int
-qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused);
-
-int
-qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused);
-
#endif /* _QAT_DEVICE_H_ */
@@ -4,6 +4,7 @@
#include "qat_pf2vf.h"
#include "adf_pf2vf_msg.h"
+#include "qat_logs.h"
#include <rte_cycles.h>
@@ -19,8 +19,6 @@
#include "qat_device.h"
#include "qat_qp.h"
#include "qat_sym.h"
-#include "qat_asym.h"
-#include "qat_comp.h"
#define QAT_CQ_MAX_DEQ_RETRIES 10
@@ -8,6 +8,7 @@
#include "qat_comp_pmd.h"
#include "qat_comp.h"
#include "qat_comp_pmd_gens.h"
+#include "qat_logs.h"
#define QAT_NUM_INTERM_BUFS_GEN1 12
@@ -4,6 +4,7 @@
#include "qat_comp_pmd.h"
#include "qat_comp_pmd_gens.h"
+#include "qat_logs.h"
#define QAT_NUM_INTERM_BUFS_GEN2 20
@@ -4,6 +4,7 @@
#include "qat_comp_pmd.h"
#include "qat_comp_pmd_gens.h"
+#include "qat_logs.h"
#define QAT_NUM_INTERM_BUFS_GEN3 64
@@ -7,6 +7,7 @@
#include "qat_comp_pmd_gens.h"
#include "icp_qat_hw_gen4_comp.h"
#include "icp_qat_hw_gen4_comp_defs.h"
+#include "qat_logs.h"
#define QAT_NUM_INTERM_BUFS_GEN4 0
@@ -6,8 +6,10 @@
#include "qat_comp.h"
#include "qat_comp_pmd.h"
+#include "qat_logs.h"
#define QAT_PMD_COMP_SGL_DEF_SEGMENTS 16
+#define SERVICE_NAME "COMP"
struct qat_comp_gen_dev_ops qat_comp_gen_dev_ops[QAT_N_GENS];
@@ -629,22 +631,25 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
{
uint16_t ret = qat_comp_dequeue_burst(qp, ops, nb_ops);
struct qat_qp *tmp_qp = (struct qat_qp *)qp;
+ int dev_pos = get_service_pos(SERVICE_NAME);
+ struct qat_comp_dev_private *dev =
+ tmp_qp->qat_dev->pmd[dev_pos];
if (ret) {
if ((*ops)->debug_status ==
(uint64_t)ERR_CODE_QAT_COMP_WRONG_FW) {
- tmp_qp->qat_dev->comp_dev->compressdev->enqueue_burst =
+ dev->compressdev->enqueue_burst =
qat_comp_pmd_enq_deq_dummy_op_burst;
- tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
+ dev->compressdev->dequeue_burst =
qat_comp_pmd_enq_deq_dummy_op_burst;
- tmp_qp->qat_dev->comp_dev->compressdev->dev_ops =
+ dev->compressdev->dev_ops =
&compress_qat_dummy_ops;
QAT_LOG(ERR,
"This QAT hardware doesn't support compression operation");
} else {
- tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
+ dev->compressdev->dequeue_burst =
qat_comp_dequeue_burst;
}
}
@@ -662,7 +667,7 @@ static const struct rte_driver compdev_qat_driver = {
.alias = qat_comp_drv_name
};
-int
+static int
qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
{
int i = 0;
@@ -681,6 +686,7 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
const struct qat_comp_gen_dev_ops *qat_comp_gen_ops =
&qat_comp_gen_dev_ops[qat_pci_dev->qat_dev_gen];
uint64_t capa_size;
+ int dev_pos = get_service_pos(SERVICE_NAME);
snprintf(name, RTE_COMPRESSDEV_NAME_MAX_LEN, "%s_%s",
qat_pci_dev->name, "comp");
@@ -765,7 +771,7 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
qat_pci_dev->cmd_line_param[i].val;
i++;
}
- qat_pci_dev->comp_dev = comp_dev;
+ qat_pci_dev->pmd[dev_pos] = comp_dev;
QAT_LOG(DEBUG,
"Created QAT COMP device %s as compressdev instance %d",
@@ -773,26 +779,32 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
return 0;
}
-int
+static int
qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev)
{
- struct qat_comp_dev_private *comp_dev;
+ int dev_pos = get_service_pos(SERVICE_NAME);
+ struct qat_comp_dev_private *dev =
+ qat_pci_dev->pmd[dev_pos];
if (qat_pci_dev == NULL)
return -ENODEV;
- comp_dev = qat_pci_dev->comp_dev;
- if (comp_dev == NULL)
- return 0;
-
if (rte_eal_process_type() == RTE_PROC_PRIMARY)
- rte_memzone_free(qat_pci_dev->comp_dev->capa_mz);
+ rte_memzone_free(dev->capa_mz);
/* clean up any resources used by the device */
- qat_comp_dev_close(comp_dev->compressdev);
+ qat_comp_dev_close(dev->compressdev);
- rte_compressdev_pmd_destroy(comp_dev->compressdev);
- qat_pci_dev->comp_dev = NULL;
+ rte_compressdev_pmd_destroy(dev->compressdev);
+ qat_pci_dev->pmd[dev_pos] = NULL;
return 0;
}
+
+RTE_INIT(qat_comp_dev_register)
+{
+ qat_dev_service[qat_dev_services_no].name = SERVICE_NAME;
+ qat_dev_service[qat_dev_services_no].dev_create = qat_comp_dev_create;
+ qat_dev_service[qat_dev_services_no].dev_destroy = qat_comp_dev_destroy;
+ qat_dev_services_no++;
+}
@@ -106,13 +106,6 @@ const struct rte_memzone *
qat_comp_setup_inter_buffers(struct qat_comp_dev_private *comp_dev,
uint32_t buff_size);
-int
-qat_comp_dev_create(struct qat_pci_device *qat_pci_dev);
-
-int
-qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev);
-
-
static __rte_always_inline unsigned int
qat_comp_get_num_im_bufs_required(enum qat_device_gen gen)
{
@@ -6,6 +6,8 @@
#include <cryptodev_pmd.h>
+#include "adf_transport_access_macros.h"
+#include "icp_qat_hw.h"
#include "qat_device.h"
#include "qat_logs.h"
@@ -16,6 +18,7 @@
#include "qat_ec.h"
#define RSA_MODULUS_2048_BITS 2048
+#define SERVICE_NAME "ASYM"
uint8_t qat_asym_driver_id;
@@ -1414,7 +1417,7 @@ qat_asym_init_op_cookie(void *op_cookie)
}
}
-int
+static int
qat_asym_dev_create(struct qat_pci_device *qat_pci_dev)
{
struct qat_cryptodev_private *internals;
@@ -1432,6 +1435,7 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev)
char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
int i = 0;
uint16_t slice_map = 0;
+ int sym_dev_pos = get_service_pos(SERVICE_NAME);
snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
qat_pci_dev->name, "asym");
@@ -1519,31 +1523,33 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev)
return -1;
}
- qat_pci_dev->asym_dev = internals;
+ qat_pci_dev->pmd[sym_dev_pos] = internals;
internals->service_type = QAT_SERVICE_ASYMMETRIC;
QAT_LOG(DEBUG, "Created QAT ASYM device %s as cryptodev instance %d",
cryptodev->data->name, internals->dev_id);
return 0;
}
-int
+static int
qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev)
{
struct rte_cryptodev *cryptodev;
+ int dev_pos = get_service_pos(SERVICE_NAME);
+ struct qat_cryptodev_private *dev =
+ qat_pci_dev->pmd[dev_pos];
if (qat_pci_dev == NULL)
return -ENODEV;
- if (qat_pci_dev->asym_dev == NULL)
+ if (qat_pci_dev->pmd[dev_pos] == NULL)
return 0;
if (rte_eal_process_type() == RTE_PROC_PRIMARY)
- rte_memzone_free(qat_pci_dev->asym_dev->capa_mz);
+ rte_memzone_free(dev->capa_mz);
/* free crypto device */
- cryptodev = rte_cryptodev_pmd_get_dev(
- qat_pci_dev->asym_dev->dev_id);
+ cryptodev = rte_cryptodev_pmd_get_dev(dev->dev_id);
rte_cryptodev_pmd_destroy(cryptodev);
qat_pci_devs[qat_pci_dev->qat_dev_id].asym_rte_dev.name = NULL;
- qat_pci_dev->asym_dev = NULL;
+ qat_pci_dev->pmd[dev_pos] = NULL;
return 0;
}
@@ -1552,3 +1558,11 @@ static struct cryptodev_driver qat_crypto_drv;
RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
cryptodev_qat_asym_driver,
qat_asym_driver_id);
+
+RTE_INIT(qat_asym_dev_register)
+{
+ qat_dev_service[qat_dev_services_no].name = SERVICE_NAME;
+ qat_dev_service[qat_dev_services_no].dev_create = qat_asym_dev_create;
+ qat_dev_service[qat_dev_services_no].dev_destroy = qat_asym_dev_destroy;
+ qat_dev_services_no++;
+}
@@ -14,8 +14,10 @@
#include "qat_sym.h"
#include "qat_crypto.h"
#include "qat_qp.h"
+#include "qat_device.h"
extern const char *cmd_arg_strings[];
+#define SERVICE_NAME "SYM"
uint8_t qat_sym_driver_id;
int qat_legacy_capa;
@@ -183,7 +185,7 @@ qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
qat_sym_process_response, nb_ops);
}
-int
+static int
qat_sym_dev_create(struct qat_pci_device *qat_pci_dev)
{
int i = 0, ret = 0;
@@ -201,6 +203,7 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev)
struct qat_cryptodev_private *internals;
const struct qat_crypto_gen_dev_ops *gen_dev_ops =
&qat_sym_gen_dev_ops[qat_pci_dev->qat_dev_gen];
+ int dev_pos = get_service_pos(SERVICE_NAME);
snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
qat_pci_dev->name, "sym");
@@ -302,7 +305,7 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev)
goto error;
}
internals->service_type = QAT_SERVICE_SYMMETRIC;
- qat_pci_dev->sym_dev = internals;
+ qat_pci_dev->pmd[dev_pos] = internals;
QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d",
cryptodev->data->name, internals->dev_id);
@@ -318,25 +321,28 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev)
return ret;
}
-int
+static int
qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev)
{
struct rte_cryptodev *cryptodev;
+ int dev_pos = get_service_pos(SERVICE_NAME);
+ struct qat_cryptodev_private *dev =
+ qat_pci_dev->pmd[dev_pos];
if (qat_pci_dev == NULL)
return -ENODEV;
- if (qat_pci_dev->sym_dev == NULL)
+ if (qat_pci_dev->pmd[dev_pos] == NULL)
return 0;
if (rte_eal_process_type() == RTE_PROC_PRIMARY)
- rte_memzone_free(qat_pci_dev->sym_dev->capa_mz);
+ rte_memzone_free(dev->capa_mz);
/* free crypto device */
- cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->dev_id);
+ cryptodev = rte_cryptodev_pmd_get_dev(dev->dev_id);
rte_free(cryptodev->security_ctx);
cryptodev->security_ctx = NULL;
rte_cryptodev_pmd_destroy(cryptodev);
qat_pci_devs[qat_pci_dev->qat_dev_id].sym_rte_dev.name = NULL;
- qat_pci_dev->sym_dev = NULL;
+ qat_pci_dev->pmd[dev_pos] = NULL;
return 0;
}
@@ -393,3 +399,11 @@ static struct cryptodev_driver qat_crypto_drv;
RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
cryptodev_qat_sym_driver,
qat_sym_driver_id);
+
+RTE_INIT(qat_sym_dev_register)
+{
+ qat_dev_service[qat_dev_services_no].name = SERVICE_NAME;
+ qat_dev_service[qat_dev_services_no].dev_create = qat_sym_dev_create;
+ qat_dev_service[qat_dev_services_no].dev_destroy = qat_sym_dev_destroy;
+ qat_dev_services_no++;
+}