@@ -71,7 +71,7 @@ endif
if qat_crypto
foreach f: ['qat_sym_pmd.c', 'qat_sym.c', 'qat_sym_session.c',
- 'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c']
+ 'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c']
sources += files(join_paths(qat_crypto_relpath, f))
endforeach
deps += ['security']
@@ -6,6 +6,21 @@
#include "qat_device.h"
#include "qat_logs.h"
+const char *
+qat_service_get_str(enum qat_service_type type)
+{
+ switch (type) {
+ case QAT_SERVICE_SYMMETRIC:
+ return "sym";
+ case QAT_SERVICE_ASYMMETRIC:
+ return "asym";
+ case QAT_SERVICE_COMPRESSION:
+ return "comp";
+ default:
+ return "invalid";
+ }
+}
+
int
qat_sgl_fill_array(struct rte_mbuf *buf, int64_t offset,
void *list_in, uint32_t data_len,
@@ -91,4 +91,7 @@ void
qat_stats_reset(struct qat_pci_device *dev,
enum qat_service_type service);
+const char *
+qat_service_get_str(enum qat_service_type type);
+
#endif /* _QAT_COMMON_H_ */
@@ -76,8 +76,7 @@ struct qat_device_info {
extern struct qat_device_info qat_pci_devs[];
-struct qat_sym_dev_private;
-struct qat_asym_dev_private;
+struct qat_cryptodev_private;
struct qat_comp_dev_private;
/*
@@ -106,14 +105,14 @@ struct qat_pci_device {
/**< links to qps set up for each service, index same as on API */
/* Data relating to symmetric crypto service */
- struct qat_sym_dev_private *sym_dev;
+ struct qat_cryptodev_private *sym_dev;
/**< link back to cryptodev private data */
int qat_sym_driver_id;
/**< Symmetric driver id used by this device */
/* Data relating to asymmetric crypto service */
- struct qat_asym_dev_private *asym_dev;
+ struct qat_cryptodev_private *asym_dev;
/**< link back to cryptodev private data */
int qat_asym_driver_id;
@@ -6,6 +6,7 @@
#include "qat_logs.h"
+#include "qat_crypto.h"
#include "qat_asym.h"
#include "qat_asym_pmd.h"
#include "qat_sym_capabilities.h"
@@ -18,190 +19,45 @@ static const struct rte_cryptodev_capabilities qat_gen1_asym_capabilities[] = {
RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
};
-static int qat_asym_qp_release(struct rte_cryptodev *dev,
- uint16_t queue_pair_id);
-
-static int qat_asym_dev_config(__rte_unused struct rte_cryptodev *dev,
- __rte_unused struct rte_cryptodev_config *config)
-{
- return 0;
-}
-
-static int qat_asym_dev_start(__rte_unused struct rte_cryptodev *dev)
-{
- return 0;
-}
-
-static void qat_asym_dev_stop(__rte_unused struct rte_cryptodev *dev)
-{
-
-}
-
-static int qat_asym_dev_close(struct rte_cryptodev *dev)
-{
- int i, ret;
-
- for (i = 0; i < dev->data->nb_queue_pairs; i++) {
- ret = qat_asym_qp_release(dev, i);
- if (ret < 0)
- return ret;
- }
-
- return 0;
-}
-
-static void qat_asym_dev_info_get(struct rte_cryptodev *dev,
- struct rte_cryptodev_info *info)
-{
- struct qat_asym_dev_private *internals = dev->data->dev_private;
- struct qat_pci_device *qat_dev = internals->qat_dev;
-
- if (info != NULL) {
- info->max_nb_queue_pairs = qat_qps_per_service(qat_dev,
- QAT_SERVICE_ASYMMETRIC);
- info->feature_flags = dev->feature_flags;
- info->capabilities = internals->qat_dev_capabilities;
- info->driver_id = qat_asym_driver_id;
- /* No limit of number of sessions */
- info->sym.max_nb_sessions = 0;
- }
-}
-
-static void qat_asym_stats_get(struct rte_cryptodev *dev,
- struct rte_cryptodev_stats *stats)
-{
- struct qat_common_stats qat_stats = {0};
- struct qat_asym_dev_private *qat_priv;
-
- if (stats == NULL || dev == NULL) {
- QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev);
- return;
- }
- qat_priv = dev->data->dev_private;
-
- qat_stats_get(qat_priv->qat_dev, &qat_stats, QAT_SERVICE_ASYMMETRIC);
- stats->enqueued_count = qat_stats.enqueued_count;
- stats->dequeued_count = qat_stats.dequeued_count;
- stats->enqueue_err_count = qat_stats.enqueue_err_count;
- stats->dequeue_err_count = qat_stats.dequeue_err_count;
-}
-
-static void qat_asym_stats_reset(struct rte_cryptodev *dev)
+void
+qat_asym_init_op_cookie(void *op_cookie)
{
- struct qat_asym_dev_private *qat_priv;
+ int j;
+ struct qat_asym_op_cookie *cookie = op_cookie;
- if (dev == NULL) {
- QAT_LOG(ERR, "invalid asymmetric cryptodev ptr %p", dev);
- return;
- }
- qat_priv = dev->data->dev_private;
+ cookie->input_addr = rte_mempool_virt2iova(cookie) +
+ offsetof(struct qat_asym_op_cookie,
+ input_params_ptrs);
- qat_stats_reset(qat_priv->qat_dev, QAT_SERVICE_ASYMMETRIC);
-}
-
-static int qat_asym_qp_release(struct rte_cryptodev *dev,
- uint16_t queue_pair_id)
-{
- struct qat_asym_dev_private *qat_private = dev->data->dev_private;
- enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen;
-
- QAT_LOG(DEBUG, "Release asym qp %u on device %d",
- queue_pair_id, dev->data->dev_id);
-
- qat_private->qat_dev->qps_in_use[QAT_SERVICE_ASYMMETRIC][queue_pair_id]
- = NULL;
-
- return qat_qp_release(qat_dev_gen, (struct qat_qp **)
- &(dev->data->queue_pairs[queue_pair_id]));
-}
+ cookie->output_addr = rte_mempool_virt2iova(cookie) +
+ offsetof(struct qat_asym_op_cookie,
+ output_params_ptrs);
-static int qat_asym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
- const struct rte_cryptodev_qp_conf *qp_conf,
- int socket_id)
-{
- struct qat_qp_config qat_qp_conf;
- struct qat_qp *qp;
- int ret = 0;
- uint32_t i;
-
- struct qat_qp **qp_addr =
- (struct qat_qp **)&(dev->data->queue_pairs[qp_id]);
- struct qat_asym_dev_private *qat_private = dev->data->dev_private;
- struct qat_pci_device *qat_dev = qat_private->qat_dev;
- const struct qat_qp_hw_data *asym_hw_qps =
- qat_gen_config[qat_private->qat_dev->qat_dev_gen]
- .qp_hw_data[QAT_SERVICE_ASYMMETRIC];
- const struct qat_qp_hw_data *qp_hw_data = asym_hw_qps + qp_id;
-
- /* If qp is already in use free ring memory and qp metadata. */
- if (*qp_addr != NULL) {
- ret = qat_asym_qp_release(dev, qp_id);
- if (ret < 0)
- return ret;
- }
- if (qp_id >= qat_qps_per_service(qat_dev, QAT_SERVICE_ASYMMETRIC)) {
- QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
- return -EINVAL;
- }
-
- qat_qp_conf.hw = qp_hw_data;
- qat_qp_conf.cookie_size = sizeof(struct qat_asym_op_cookie);
- qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
- qat_qp_conf.socket_id = socket_id;
- qat_qp_conf.service_str = "asym";
-
- ret = qat_qp_setup(qat_private->qat_dev, qp_addr, qp_id, &qat_qp_conf);
- if (ret != 0)
- return ret;
-
- /* store a link to the qp in the qat_pci_device */
- qat_private->qat_dev->qps_in_use[QAT_SERVICE_ASYMMETRIC][qp_id]
- = *qp_addr;
-
- qp = (struct qat_qp *)*qp_addr;
- qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
-
- for (i = 0; i < qp->nb_descriptors; i++) {
- int j;
-
- struct qat_asym_op_cookie __rte_unused *cookie =
- qp->op_cookies[i];
- cookie->input_addr = rte_mempool_virt2iova(cookie) +
+ for (j = 0; j < 8; j++) {
+ cookie->input_params_ptrs[j] =
+ rte_mempool_virt2iova(cookie) +
offsetof(struct qat_asym_op_cookie,
- input_params_ptrs);
-
- cookie->output_addr = rte_mempool_virt2iova(cookie) +
+ input_array[j]);
+ cookie->output_params_ptrs[j] =
+ rte_mempool_virt2iova(cookie) +
offsetof(struct qat_asym_op_cookie,
- output_params_ptrs);
-
- for (j = 0; j < 8; j++) {
- cookie->input_params_ptrs[j] =
- rte_mempool_virt2iova(cookie) +
- offsetof(struct qat_asym_op_cookie,
- input_array[j]);
- cookie->output_params_ptrs[j] =
- rte_mempool_virt2iova(cookie) +
- offsetof(struct qat_asym_op_cookie,
- output_array[j]);
- }
+ output_array[j]);
}
-
- return ret;
}
-struct rte_cryptodev_ops crypto_qat_ops = {
+static struct rte_cryptodev_ops crypto_qat_ops = {
/* Device related operations */
- .dev_configure = qat_asym_dev_config,
- .dev_start = qat_asym_dev_start,
- .dev_stop = qat_asym_dev_stop,
- .dev_close = qat_asym_dev_close,
- .dev_infos_get = qat_asym_dev_info_get,
+ .dev_configure = qat_cryptodev_config,
+ .dev_start = qat_cryptodev_start,
+ .dev_stop = qat_cryptodev_stop,
+ .dev_close = qat_cryptodev_close,
+ .dev_infos_get = qat_cryptodev_info_get,
- .stats_get = qat_asym_stats_get,
- .stats_reset = qat_asym_stats_reset,
- .queue_pair_setup = qat_asym_qp_setup,
- .queue_pair_release = qat_asym_qp_release,
+ .stats_get = qat_cryptodev_stats_get,
+ .stats_reset = qat_cryptodev_stats_reset,
+ .queue_pair_setup = qat_cryptodev_qp_setup,
+ .queue_pair_release = qat_cryptodev_qp_release,
/* Crypto related operations */
.asym_session_get_size = qat_asym_session_get_private_size,
@@ -241,15 +97,14 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
struct qat_device_info *qat_dev_instance =
&qat_pci_devs[qat_pci_dev->qat_dev_id];
struct rte_cryptodev_pmd_init_params init_params = {
- .name = "",
- .socket_id =
- qat_dev_instance->pci_dev->device.numa_node,
- .private_data_size = sizeof(struct qat_asym_dev_private)
+ .name = "",
+ .socket_id = qat_dev_instance->pci_dev->device.numa_node,
+ .private_data_size = sizeof(struct qat_cryptodev_private)
};
char name[RTE_CRYPTODEV_NAME_MAX_LEN];
char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
struct rte_cryptodev *cryptodev;
- struct qat_asym_dev_private *internals;
+ struct qat_cryptodev_private *internals;
if (qat_pci_dev->qat_dev_gen == QAT_GEN4) {
QAT_LOG(ERR, "Asymmetric crypto PMD not supported on QAT 4xxx");
@@ -310,8 +165,9 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
internals = cryptodev->data->dev_private;
internals->qat_dev = qat_pci_dev;
- internals->asym_dev_id = cryptodev->data->dev_id;
+ internals->dev_id = cryptodev->data->dev_id;
internals->qat_dev_capabilities = qat_gen1_asym_capabilities;
+ internals->service_type = QAT_SERVICE_ASYMMETRIC;
internals->capa_mz = rte_memzone_lookup(capa_memz_name);
if (internals->capa_mz == NULL) {
@@ -347,7 +203,7 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
rte_cryptodev_pmd_probing_finish(cryptodev);
QAT_LOG(DEBUG, "Created QAT ASYM device %s as cryptodev instance %d",
- cryptodev->data->name, internals->asym_dev_id);
+ cryptodev->data->name, internals->dev_id);
return 0;
}
@@ -365,7 +221,7 @@ qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev)
/* free crypto device */
cryptodev = rte_cryptodev_pmd_get_dev(
- qat_pci_dev->asym_dev->asym_dev_id);
+ qat_pci_dev->asym_dev->dev_id);
rte_cryptodev_pmd_destroy(cryptodev);
qat_pci_devs[qat_pci_dev->qat_dev_id].asym_rte_dev.name = NULL;
qat_pci_dev->asym_dev = NULL;
@@ -15,21 +15,8 @@
extern uint8_t qat_asym_driver_id;
-/** private data structure for a QAT device.
- * This QAT device is a device offering only asymmetric crypto service,
- * there can be one of these on each qat_pci_device (VF).
- */
-struct qat_asym_dev_private {
- struct qat_pci_device *qat_dev;
- /**< The qat pci device hosting the service */
- uint8_t asym_dev_id;
- /**< Device instance for this rte_cryptodev */
- const struct rte_cryptodev_capabilities *qat_dev_capabilities;
- /* QAT device asymmetric crypto capabilities */
- const struct rte_memzone *capa_mz;
- /* Shared memzone for storing capabilities */
- uint16_t min_enq_burst_threshold;
-};
+void
+qat_asym_init_op_cookie(void *op_cookie);
uint16_t
qat_asym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
@@ -39,16 +26,4 @@ uint16_t
qat_asym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
uint16_t nb_ops);
-int qat_asym_session_configure(struct rte_cryptodev *dev,
- struct rte_crypto_asym_xform *xform,
- struct rte_cryptodev_asym_session *sess,
- struct rte_mempool *mempool);
-
-int
-qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
- struct qat_dev_cmd_param *qat_dev_cmd_param);
-
-int
-qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev);
-
#endif /* _QAT_ASYM_PMD_H_ */
new file mode 100644
@@ -0,0 +1,172 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Intel Corporation
+ */
+
+#include "qat_device.h"
+#include "qat_qp.h"
+#include "qat_crypto.h"
+#include "qat_sym.h"
+#include "qat_asym.h"
+
+int
+qat_cryptodev_config(__rte_unused struct rte_cryptodev *dev,
+ __rte_unused struct rte_cryptodev_config *config)
+{
+ return 0;
+}
+
+int
+qat_cryptodev_start(__rte_unused struct rte_cryptodev *dev)
+{
+ return 0;
+}
+
+void
+qat_cryptodev_stop(__rte_unused struct rte_cryptodev *dev)
+{
+}
+
+int
+qat_cryptodev_close(struct rte_cryptodev *dev)
+{
+ int i, ret;
+
+ for (i = 0; i < dev->data->nb_queue_pairs; i++) {
+ ret = dev->dev_ops->queue_pair_release(dev, i);
+ if (ret < 0)
+ return ret;
+ }
+
+ return 0;
+}
+
+void
+qat_cryptodev_info_get(struct rte_cryptodev *dev,
+ struct rte_cryptodev_info *info)
+{
+ struct qat_cryptodev_private *qat_private = dev->data->dev_private;
+ struct qat_pci_device *qat_dev = qat_private->qat_dev;
+ enum qat_service_type service_type = qat_private->service_type;
+
+ if (info != NULL) {
+ info->max_nb_queue_pairs =
+ qat_qps_per_service(qat_dev, service_type);
+ info->feature_flags = dev->feature_flags;
+ info->capabilities = qat_private->qat_dev_capabilities;
+ info->driver_id = qat_sym_driver_id;
+ /* No limit of number of sessions */
+ info->sym.max_nb_sessions = 0;
+ }
+}
+
+void
+qat_cryptodev_stats_get(struct rte_cryptodev *dev,
+ struct rte_cryptodev_stats *stats)
+{
+ struct qat_common_stats qat_stats = {0};
+ struct qat_cryptodev_private *qat_priv;
+
+ if (stats == NULL || dev == NULL) {
+ QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev);
+ return;
+ }
+ qat_priv = dev->data->dev_private;
+
+ qat_stats_get(qat_priv->qat_dev, &qat_stats, qat_priv->service_type);
+ stats->enqueued_count = qat_stats.enqueued_count;
+ stats->dequeued_count = qat_stats.dequeued_count;
+ stats->enqueue_err_count = qat_stats.enqueue_err_count;
+ stats->dequeue_err_count = qat_stats.dequeue_err_count;
+}
+
+void
+qat_cryptodev_stats_reset(struct rte_cryptodev *dev)
+{
+ struct qat_cryptodev_private *qat_priv;
+
+ if (dev == NULL) {
+ QAT_LOG(ERR, "invalid cryptodev ptr %p", dev);
+ return;
+ }
+ qat_priv = dev->data->dev_private;
+
+ qat_stats_reset(qat_priv->qat_dev, qat_priv->service_type);
+
+}
+
+int
+qat_cryptodev_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
+{
+ struct qat_cryptodev_private *qat_private = dev->data->dev_private;
+ struct qat_pci_device *qat_dev = qat_private->qat_dev;
+ enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
+ enum qat_service_type service_type = qat_private->service_type;
+
+ QAT_LOG(DEBUG, "Release %s qp %u on device %d",
+ qat_service_get_str(service_type),
+ queue_pair_id, dev->data->dev_id);
+
+ qat_private->qat_dev->qps_in_use[service_type][queue_pair_id] = NULL;
+
+ return qat_qp_release(qat_dev_gen, (struct qat_qp **)
+ &(dev->data->queue_pairs[queue_pair_id]));
+}
+
+int
+qat_cryptodev_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+ const struct rte_cryptodev_qp_conf *qp_conf, int socket_id)
+{
+ struct qat_qp **qp_addr =
+ (struct qat_qp **)&(dev->data->queue_pairs[qp_id]);
+ struct qat_cryptodev_private *qat_private = dev->data->dev_private;
+ struct qat_pci_device *qat_dev = qat_private->qat_dev;
+ enum qat_service_type service_type = qat_private->service_type;
+ struct qat_qp_config qat_qp_conf = {0};
+ struct qat_qp *qp;
+ int ret = 0;
+ uint32_t i;
+
+ /* If qp is already in use free ring memory and qp metadata. */
+ if (*qp_addr != NULL) {
+ ret = dev->dev_ops->queue_pair_release(dev, qp_id);
+ if (ret < 0)
+ return -EBUSY;
+ }
+ if (qp_id >= qat_qps_per_service(qat_dev, service_type)) {
+ QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
+ return -EINVAL;
+ }
+
+ qat_qp_conf.hw = qat_qp_get_hw_data(qat_dev, service_type,
+ qp_id);
+ if (qat_qp_conf.hw == NULL) {
+ QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
+ return -EINVAL;
+ }
+
+ qat_qp_conf.cookie_size = service_type == QAT_SERVICE_SYMMETRIC ?
+ sizeof(struct qat_sym_op_cookie) :
+ sizeof(struct qat_asym_op_cookie);
+ qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
+ qat_qp_conf.socket_id = socket_id;
+ qat_qp_conf.service_str = qat_service_get_str(service_type);
+
+ ret = qat_qp_setup(qat_dev, qp_addr, qp_id, &qat_qp_conf);
+ if (ret != 0)
+ return ret;
+
+ /* store a link to the qp in the qat_pci_device */
+ qat_dev->qps_in_use[service_type][qp_id] = *qp_addr;
+
+ qp = (struct qat_qp *)*qp_addr;
+ qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
+
+ for (i = 0; i < qp->nb_descriptors; i++) {
+ if (service_type == QAT_SERVICE_SYMMETRIC)
+ qat_sym_init_op_cookie(qp->op_cookies[i]);
+ else
+ qat_asym_init_op_cookie(qp->op_cookies[i]);
+ }
+
+ return ret;
+}
new file mode 100644
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Intel Corporation
+ */
+
+ #ifndef _QAT_CRYPTO_H_
+ #define _QAT_CRYPTO_H_
+
+#include <rte_cryptodev.h>
+#ifdef RTE_LIB_SECURITY
+#include <rte_security.h>
+#endif
+
+#include "qat_device.h"
+
+extern uint8_t qat_sym_driver_id;
+extern uint8_t qat_asym_driver_id;
+
+/** helper macro to set cryptodev capability range **/
+#define CAP_RNG(n, l, r, i) .n = {.min = l, .max = r, .increment = i}
+
+#define CAP_RNG_ZERO(n) .n = {.min = 0, .max = 0, .increment = 0}
+/** helper macro to set cryptodev capability value **/
+#define CAP_SET(n, v) .n = v
+
+/** private data structure for a QAT device.
+ * there can be one of these on each qat_pci_device (VF).
+ */
+struct qat_cryptodev_private {
+ struct qat_pci_device *qat_dev;
+ /**< The qat pci device hosting the service */
+ uint8_t dev_id;
+ /**< Device instance for this rte_cryptodev */
+ const struct rte_cryptodev_capabilities *qat_dev_capabilities;
+ /* QAT device symmetric crypto capabilities */
+ const struct rte_memzone *capa_mz;
+ /* Shared memzone for storing capabilities */
+ uint16_t min_enq_burst_threshold;
+ uint32_t internal_capabilities; /* see flags QAT_SYM_CAP_xxx */
+ enum qat_service_type service_type;
+};
+
+struct qat_capabilities_info {
+ struct rte_cryptodev_capabilities *data;
+ uint64_t size;
+};
+
+int
+qat_cryptodev_config(struct rte_cryptodev *dev,
+ struct rte_cryptodev_config *config);
+
+int
+qat_cryptodev_start(struct rte_cryptodev *dev);
+
+void
+qat_cryptodev_stop(struct rte_cryptodev *dev);
+
+int
+qat_cryptodev_close(struct rte_cryptodev *dev);
+
+void
+qat_cryptodev_info_get(struct rte_cryptodev *dev,
+ struct rte_cryptodev_info *info);
+
+void
+qat_cryptodev_stats_get(struct rte_cryptodev *dev,
+ struct rte_cryptodev_stats *stats);
+
+void
+qat_cryptodev_stats_reset(struct rte_cryptodev *dev);
+
+int
+qat_cryptodev_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+ const struct rte_cryptodev_qp_conf *qp_conf, int socket_id);
+
+int
+qat_cryptodev_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id);
+
+#endif
@@ -13,6 +13,7 @@
#endif
#include "qat_logs.h"
+#include "qat_crypto.h"
#include "qat_sym.h"
#include "qat_sym_session.h"
#include "qat_sym_pmd.h"
@@ -59,213 +60,19 @@ static const struct rte_security_capability qat_security_capabilities[] = {
};
#endif
-static int qat_sym_qp_release(struct rte_cryptodev *dev,
- uint16_t queue_pair_id);
-
-static int qat_sym_dev_config(__rte_unused struct rte_cryptodev *dev,
- __rte_unused struct rte_cryptodev_config *config)
-{
- return 0;
-}
-
-static int qat_sym_dev_start(__rte_unused struct rte_cryptodev *dev)
-{
- return 0;
-}
-
-static void qat_sym_dev_stop(__rte_unused struct rte_cryptodev *dev)
-{
- return;
-}
-
-static int qat_sym_dev_close(struct rte_cryptodev *dev)
-{
- int i, ret;
-
- for (i = 0; i < dev->data->nb_queue_pairs; i++) {
- ret = qat_sym_qp_release(dev, i);
- if (ret < 0)
- return ret;
- }
-
- return 0;
-}
-
-static void qat_sym_dev_info_get(struct rte_cryptodev *dev,
- struct rte_cryptodev_info *info)
-{
- struct qat_sym_dev_private *internals = dev->data->dev_private;
- struct qat_pci_device *qat_dev = internals->qat_dev;
-
- if (info != NULL) {
- info->max_nb_queue_pairs =
- qat_qps_per_service(qat_dev, QAT_SERVICE_SYMMETRIC);
- info->feature_flags = dev->feature_flags;
- info->capabilities = internals->qat_dev_capabilities;
- info->driver_id = qat_sym_driver_id;
- /* No limit of number of sessions */
- info->sym.max_nb_sessions = 0;
- }
-}
-
-static void qat_sym_stats_get(struct rte_cryptodev *dev,
- struct rte_cryptodev_stats *stats)
-{
- struct qat_common_stats qat_stats = {0};
- struct qat_sym_dev_private *qat_priv;
-
- if (stats == NULL || dev == NULL) {
- QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev);
- return;
- }
- qat_priv = dev->data->dev_private;
-
- qat_stats_get(qat_priv->qat_dev, &qat_stats, QAT_SERVICE_SYMMETRIC);
- stats->enqueued_count = qat_stats.enqueued_count;
- stats->dequeued_count = qat_stats.dequeued_count;
- stats->enqueue_err_count = qat_stats.enqueue_err_count;
- stats->dequeue_err_count = qat_stats.dequeue_err_count;
-}
-
-static void qat_sym_stats_reset(struct rte_cryptodev *dev)
-{
- struct qat_sym_dev_private *qat_priv;
-
- if (dev == NULL) {
- QAT_LOG(ERR, "invalid cryptodev ptr %p", dev);
- return;
- }
- qat_priv = dev->data->dev_private;
-
- qat_stats_reset(qat_priv->qat_dev, QAT_SERVICE_SYMMETRIC);
-
-}
-
-static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
-{
- struct qat_sym_dev_private *qat_private = dev->data->dev_private;
- enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen;
-
- QAT_LOG(DEBUG, "Release sym qp %u on device %d",
- queue_pair_id, dev->data->dev_id);
-
- qat_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][queue_pair_id]
- = NULL;
-
- return qat_qp_release(qat_dev_gen, (struct qat_qp **)
- &(dev->data->queue_pairs[queue_pair_id]));
-}
-
-static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
- const struct rte_cryptodev_qp_conf *qp_conf,
- int socket_id)
-{
- struct qat_qp *qp;
- int ret = 0;
- uint32_t i;
- struct qat_qp_config qat_qp_conf;
- struct qat_qp **qp_addr =
- (struct qat_qp **)&(dev->data->queue_pairs[qp_id]);
- struct qat_sym_dev_private *qat_private = dev->data->dev_private;
- struct qat_pci_device *qat_dev = qat_private->qat_dev;
-
- /* If qp is already in use free ring memory and qp metadata. */
- if (*qp_addr != NULL) {
- ret = qat_sym_qp_release(dev, qp_id);
- if (ret < 0)
- return ret;
- }
- if (qp_id >= qat_qps_per_service(qat_dev, QAT_SERVICE_SYMMETRIC)) {
- QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
- return -EINVAL;
- }
-
- qat_qp_conf.hw = qat_qp_get_hw_data(qat_dev, QAT_SERVICE_SYMMETRIC,
- qp_id);
- if (qat_qp_conf.hw == NULL) {
- QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
- return -EINVAL;
- }
-
- qat_qp_conf.cookie_size = sizeof(struct qat_sym_op_cookie);
- qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
- qat_qp_conf.socket_id = socket_id;
- qat_qp_conf.service_str = "sym";
-
- ret = qat_qp_setup(qat_private->qat_dev, qp_addr, qp_id, &qat_qp_conf);
- if (ret != 0)
- return ret;
-
- /* store a link to the qp in the qat_pci_device */
- qat_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][qp_id]
- = *qp_addr;
-
- qp = (struct qat_qp *)*qp_addr;
- qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
-
- for (i = 0; i < qp->nb_descriptors; i++) {
-
- struct qat_sym_op_cookie *cookie =
- qp->op_cookies[i];
-
- cookie->qat_sgl_src_phys_addr =
- rte_mempool_virt2iova(cookie) +
- offsetof(struct qat_sym_op_cookie,
- qat_sgl_src);
-
- cookie->qat_sgl_dst_phys_addr =
- rte_mempool_virt2iova(cookie) +
- offsetof(struct qat_sym_op_cookie,
- qat_sgl_dst);
-
- cookie->opt.spc_gmac.cd_phys_addr =
- rte_mempool_virt2iova(cookie) +
- offsetof(struct qat_sym_op_cookie,
- opt.spc_gmac.cd_cipher);
-
- }
-
- /* Get fw version from QAT (GEN2), skip if we've got it already */
- if (qp->qat_dev_gen == QAT_GEN2 && !(qat_private->internal_capabilities
- & QAT_SYM_CAP_VALID)) {
- ret = qat_cq_get_fw_version(qp);
-
- if (ret < 0) {
- qat_sym_qp_release(dev, qp_id);
- return ret;
- }
-
- if (ret != 0)
- QAT_LOG(DEBUG, "QAT firmware version: %d.%d.%d",
- (ret >> 24) & 0xff,
- (ret >> 16) & 0xff,
- (ret >> 8) & 0xff);
- else
- QAT_LOG(DEBUG, "unknown QAT firmware version");
-
- /* set capabilities based on the fw version */
- qat_private->internal_capabilities = QAT_SYM_CAP_VALID |
- ((ret >= MIXED_CRYPTO_MIN_FW_VER) ?
- QAT_SYM_CAP_MIXED_CRYPTO : 0);
- ret = 0;
- }
-
- return ret;
-}
-
static struct rte_cryptodev_ops crypto_qat_ops = {
/* Device related operations */
- .dev_configure = qat_sym_dev_config,
- .dev_start = qat_sym_dev_start,
- .dev_stop = qat_sym_dev_stop,
- .dev_close = qat_sym_dev_close,
- .dev_infos_get = qat_sym_dev_info_get,
+ .dev_configure = qat_cryptodev_config,
+ .dev_start = qat_cryptodev_start,
+ .dev_stop = qat_cryptodev_stop,
+ .dev_close = qat_cryptodev_close,
+ .dev_infos_get = qat_cryptodev_info_get,
- .stats_get = qat_sym_stats_get,
- .stats_reset = qat_sym_stats_reset,
- .queue_pair_setup = qat_sym_qp_setup,
- .queue_pair_release = qat_sym_qp_release,
+ .stats_get = qat_cryptodev_stats_get,
+ .stats_reset = qat_cryptodev_stats_reset,
+ .queue_pair_setup = qat_cryptodev_qp_setup,
+ .queue_pair_release = qat_cryptodev_qp_release,
/* Crypto related operations */
.sym_session_get_size = qat_sym_session_get_private_size,
@@ -295,6 +102,27 @@ static struct rte_security_ops security_qat_ops = {
};
#endif
+void
+qat_sym_init_op_cookie(void *op_cookie)
+{
+ struct qat_sym_op_cookie *cookie = op_cookie;
+
+ cookie->qat_sgl_src_phys_addr =
+ rte_mempool_virt2iova(cookie) +
+ offsetof(struct qat_sym_op_cookie,
+ qat_sgl_src);
+
+ cookie->qat_sgl_dst_phys_addr =
+ rte_mempool_virt2iova(cookie) +
+ offsetof(struct qat_sym_op_cookie,
+ qat_sgl_dst);
+
+ cookie->opt.spc_gmac.cd_phys_addr =
+ rte_mempool_virt2iova(cookie) +
+ offsetof(struct qat_sym_op_cookie,
+ opt.spc_gmac.cd_cipher);
+}
+
static uint16_t
qat_sym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
uint16_t nb_ops)
@@ -330,15 +158,14 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
&qat_pci_devs[qat_pci_dev->qat_dev_id];
struct rte_cryptodev_pmd_init_params init_params = {
- .name = "",
- .socket_id =
- qat_dev_instance->pci_dev->device.numa_node,
- .private_data_size = sizeof(struct qat_sym_dev_private)
+ .name = "",
+ .socket_id = qat_dev_instance->pci_dev->device.numa_node,
+ .private_data_size = sizeof(struct qat_cryptodev_private)
};
char name[RTE_CRYPTODEV_NAME_MAX_LEN];
char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
struct rte_cryptodev *cryptodev;
- struct qat_sym_dev_private *internals;
+ struct qat_cryptodev_private *internals;
const struct rte_cryptodev_capabilities *capabilities;
uint64_t capa_size;
@@ -424,8 +251,9 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
internals = cryptodev->data->dev_private;
internals->qat_dev = qat_pci_dev;
+ internals->service_type = QAT_SERVICE_SYMMETRIC;
- internals->sym_dev_id = cryptodev->data->dev_id;
+ internals->dev_id = cryptodev->data->dev_id;
switch (qat_pci_dev->qat_dev_gen) {
case QAT_GEN1:
capabilities = qat_gen1_sym_capabilities;
@@ -480,7 +308,7 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
qat_pci_dev->sym_dev = internals;
QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d",
- cryptodev->data->name, internals->sym_dev_id);
+ cryptodev->data->name, internals->dev_id);
rte_cryptodev_pmd_probing_finish(cryptodev);
@@ -511,7 +339,7 @@ qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev)
rte_memzone_free(qat_pci_dev->sym_dev->capa_mz);
/* free crypto device */
- cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->sym_dev_id);
+ cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->dev_id);
#ifdef RTE_LIB_SECURITY
rte_free(cryptodev->security_ctx);
cryptodev->security_ctx = NULL;
@@ -14,6 +14,7 @@
#endif
#include "qat_sym_capabilities.h"
+#include "qat_crypto.h"
#include "qat_device.h"
/** Intel(R) QAT Symmetric Crypto PMD driver name */
@@ -25,23 +26,6 @@
extern uint8_t qat_sym_driver_id;
-/** private data structure for a QAT device.
- * This QAT device is a device offering only symmetric crypto service,
- * there can be one of these on each qat_pci_device (VF).
- */
-struct qat_sym_dev_private {
- struct qat_pci_device *qat_dev;
- /**< The qat pci device hosting the service */
- uint8_t sym_dev_id;
- /**< Device instance for this rte_cryptodev */
- const struct rte_cryptodev_capabilities *qat_dev_capabilities;
- /* QAT device symmetric crypto capabilities */
- const struct rte_memzone *capa_mz;
- /* Shared memzone for storing capabilities */
- uint16_t min_enq_burst_threshold;
- uint32_t internal_capabilities; /* see flags QAT_SYM_CAP_xxx */
-};
-
int
qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
struct qat_dev_cmd_param *qat_dev_cmd_param);
@@ -49,5 +33,8 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
int
qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev);
+void
+qat_sym_init_op_cookie(void *op_cookie);
+
#endif
#endif /* _QAT_SYM_PMD_H_ */
@@ -131,7 +131,7 @@ bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo,
static int
qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo,
- struct qat_sym_dev_private *internals)
+ struct qat_cryptodev_private *internals)
{
int i = 0;
const struct rte_cryptodev_capabilities *capability;
@@ -152,7 +152,7 @@ qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo,
static int
qat_is_auth_alg_supported(enum rte_crypto_auth_algorithm algo,
- struct qat_sym_dev_private *internals)
+ struct qat_cryptodev_private *internals)
{
int i = 0;
const struct rte_cryptodev_capabilities *capability;
@@ -267,7 +267,7 @@ qat_sym_session_configure_cipher(struct rte_cryptodev *dev,
struct rte_crypto_sym_xform *xform,
struct qat_sym_session *session)
{
- struct qat_sym_dev_private *internals = dev->data->dev_private;
+ struct qat_cryptodev_private *internals = dev->data->dev_private;
struct rte_crypto_cipher_xform *cipher_xform = NULL;
enum qat_device_gen qat_dev_gen =
internals->qat_dev->qat_dev_gen;
@@ -532,7 +532,8 @@ static void
qat_sym_session_handle_mixed(const struct rte_cryptodev *dev,
struct qat_sym_session *session)
{
- const struct qat_sym_dev_private *qat_private = dev->data->dev_private;
+ const struct qat_cryptodev_private *qat_private =
+ dev->data->dev_private;
enum qat_device_gen min_dev_gen = (qat_private->internal_capabilities &
QAT_SYM_CAP_MIXED_CRYPTO) ? QAT_GEN2 : QAT_GEN3;
@@ -564,7 +565,7 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev,
struct rte_crypto_sym_xform *xform, void *session_private)
{
struct qat_sym_session *session = session_private;
- struct qat_sym_dev_private *internals = dev->data->dev_private;
+ struct qat_cryptodev_private *internals = dev->data->dev_private;
enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;
int ret;
int qat_cmd_id;
@@ -707,7 +708,7 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev,
struct qat_sym_session *session)
{
struct rte_crypto_auth_xform *auth_xform = qat_get_auth_xform(xform);
- struct qat_sym_dev_private *internals = dev->data->dev_private;
+ struct qat_cryptodev_private *internals = dev->data->dev_private;
const uint8_t *key_data = auth_xform->key.data;
uint8_t key_length = auth_xform->key.length;
enum qat_device_gen qat_dev_gen =
@@ -875,7 +876,7 @@ qat_sym_session_configure_aead(struct rte_cryptodev *dev,
{
struct rte_crypto_aead_xform *aead_xform = &xform->aead;
enum rte_crypto_auth_operation crypto_operation;
- struct qat_sym_dev_private *internals =
+ struct qat_cryptodev_private *internals =
dev->data->dev_private;
enum qat_device_gen qat_dev_gen =
internals->qat_dev->qat_dev_gen;