@@ -87,6 +87,54 @@ union rte_crypto_sym_ofs {
} ofs;
};
+
+/**
+ * Asynchronous operation job descriptor.
+ * Used by HW crypto devices direct API call that supports such activity
+ **/
+struct rte_crypto_sym_job {
+ union {
+ /**
+ * When RTE_CRYPTO_HW_ENQ_FLAG_IS_SGL bit is set in flags, sgl
+ * field is used as input data. Otherwise data_iova is
+ * used.
+ **/
+ rte_iova_t data_iova;
+ struct rte_crypto_sgl *sgl;
+ };
+ union {
+ /**
+ * Different than cryptodev ops, all ofs and len fields have
+ * the unit of bytes (including Snow3G/Kasumi/Zuc.
+ **/
+ struct {
+ uint32_t cipher_ofs;
+ uint32_t cipher_len;
+ } cipher_only;
+ struct {
+ uint32_t auth_ofs;
+ uint32_t auth_len;
+ rte_iova_t digest_iova;
+ } auth_only;
+ struct {
+ uint32_t aead_ofs;
+ uint32_t aead_len;
+ rte_iova_t tag_iova;
+ uint8_t *aad;
+ rte_iova_t aad_iova;
+ } aead;
+ struct {
+ uint32_t cipher_ofs;
+ uint32_t cipher_len;
+ uint32_t auth_ofs;
+ uint32_t auth_len;
+ rte_iova_t digest_iova;
+ } chain;
+ };
+ uint8_t *iv;
+ rte_iova_t iv_iova;
+};
+
/** Symmetric Cipher Algorithms */
enum rte_crypto_cipher_algorithm {
RTE_CRYPTO_CIPHER_NULL = 1,
@@ -1866,6 +1866,28 @@ rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id,
return dev->dev_ops->sym_cpu_process(dev, sess, ofs, vec);
}
+int
+rte_cryptodev_sym_get_hw_ops(uint8_t dev_id, uint16_t qp_id,
+ struct rte_crypto_hw_ops *hw_ops)
+{
+ struct rte_cryptodev *dev;
+
+ if (!hw_ops)
+ return -EINVAL;
+
+ memset(hw_ops, 0, sizeof(*hw_ops));
+
+ if (!rte_cryptodev_get_qp_status(dev_id, qp_id))
+ return -EINVAL;
+
+ dev = rte_cryptodev_pmd_get_dev(dev_id);
+ if (!(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_HW_DIRECT_API) ||
+ *dev->dev_ops->sym_get_hw_ops == NULL)
+ return -ENOTSUP;
+
+ return dev->dev_ops->sym_get_hw_ops(dev, qp_id, hw_ops);
+}
+
/** Initialise rte_crypto_op mempool element */
static void
rte_crypto_op_init(struct rte_mempool *mempool,
@@ -466,7 +466,8 @@ rte_cryptodev_asym_get_xform_enum(enum rte_crypto_asym_xform_type *xform_enum,
/**< Support symmetric session-less operations */
#define RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA (1ULL << 23)
/**< Support operations on data which is not byte aligned */
-
+#define RTE_CRYPTODEV_FF_SYM_HW_DIRECT_API (1ULL << 24)
+/**< Support hardware accelerator specific raw data as input */
/**
* Get the name of a crypto device feature flag
@@ -737,7 +738,7 @@ rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
* - 1: qp was configured
* - -ENODEV: device was not configured
*/
-int
+__rte_experimental int
rte_cryptodev_get_qp_status(uint8_t dev_id, uint16_t queue_pair_id);
/**
@@ -1348,6 +1349,174 @@ rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id,
struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs ofs,
struct rte_crypto_sym_vec *vec);
+
+/* HW direct symmetric crypto data-path APIs */
+
+/* Bit-masks used for enqueuing job */
+#define RTE_CRYPTO_HW_ENQ_FLAG_START (1ULL << 0)
+/**< Bit-mask to indicate the first job in a burst. With this bit set the
+ * driver may write but not read the drv_data buffer, otherwise the driver
+ * shall read and update the drv_data.
+ */
+#define RTE_CRYPTO_HW_ENQ_FLAG_SET_OPAQUE (1ULL << 1)
+/**< Bit-mask to indicate write opaque pointer into HW crypto descriptor. */
+#define RTE_CRYPTO_HW_ENQ_FLAG_END (1ULL << 2)
+/**< Bit-mask to indicate the last job in a burst. With this bit set the
+ * driver may read but not write the drv_data buffer, and kick the HW to
+ * start processing all jobs written.
+ */
+#define RTE_CRYPTO_HW_ENQ_FLAG_IS_SGL (1ULL << 3)
+/**< Bit-mask to indicate the input job is an SGL buffer */
+
+/* Bit-masks used for dequeuing job */
+#define RTE_CRYPTO_HW_DEQ_FLAG_START (1ULL << 0)
+/**< Bit-mask to indicate the first job to be dequeued. With this bit set the
+ * driver may write but not read the drv_data buffer, otherwise the driver
+ * shall read and update the drv_data.
+ */
+#define RTE_CRYPTO_HW_DEQ_FLAG_EXHAUST (1ULL << 1)
+/**< Bit-mask to indicate dequeuing as many as n jobs in dequeue-many function.
+ * Without this bit once the driver found out the ready-to-dequeue jobs are
+ * not as many as n, it shall stop immediate, leave all processed jobs in the
+ * queue, and return the ready jobs in negative. With this bit set the
+ * function shall continue dequeue all done jobs and return the dequeued
+ * job count in positive.
+ */
+
+/**
+ * Typedef for HW direct data-path enqueue callback function.
+ *
+ * @param qp Queue pair data.
+ * @param sess Cryptodev session.
+ * @param job Job data.
+ * @param opaque Opaque data to be written to queue descriptor
+ * when RTE_CRYPTO_HW_ENQ_SET_OPAQUE is
+ * set.
+ * @param drv_data User created temporary driver data for the
+ * driver to store and update data used between
+ * adjacent enqueues operations.
+ * @param flags Bitmask of RTE_CRYPTO_HW_ENQ_* flags
+ * @return
+ * - On success return 0
+ * - On fail return -1
+ **/
+typedef int (*rte_crypto_hw_enq_cb_fn)(void *qp,
+ struct rte_cryptodev_sym_session *sess,
+ struct rte_crypto_sym_job *job, void *opaque, uint64_t *drv_data,
+ uint64_t flags);
+
+/**
+ * Typedef for HW direct data-path dequeue one job callback function.
+ *
+ * @param qp Queue pair data.
+ * @param drv_data User created temporary driver data for the
+ * driver to store and update data used between
+ * adjacent enqueues operations.
+ * @param flags Bitmask of RTE_CRYPTO_HW_DEQ_* flags
+ * @param status The buffer for the driver to write operation
+ * status.
+ * @return
+ * - On success return the opaque data user write in enqueue (if any) and
+ * - status written as 1 when operation is successful.
+ * - status written as -1 when operation is failed (e.g. bad MAC)
+ * - On fail return NULL and status written as 0 when operation is still
+ * under processing.
+ **/
+typedef void * (*rte_crypto_hw_deq_one_cb_fn)(void *qp, uint64_t *drv_data,
+ uint64_t flags, int *status);
+
+/**
+ * Typedef that the user provided to deal with jobs' status when
+ * dequeue in a bulk.
+ *
+ * @param data User provided data.
+ * @param index Index number of the processed job.
+ * @param is_op_success Driver filled operation status.
+ **/
+typedef void (*rte_crpyto_hw_user_post_deq_cb_fn)(void *data, uint32_t index,
+ uint8_t is_op_success);
+
+/**
+ * Typedef for HW direct data-path dequeue bulk jobs callback function.
+ *
+ * @param qp Queue pair data.
+ * @param drv_data User created temporary driver data for the
+ * driver to store and update data used between
+ * adjacent enqueues operations.
+ * @param user_data User provided data to be passed into cb
+ * function.
+ * @param cb User provided callback functions to deal with
+ * driver returned job status.
+ * @param n The number of expected jobs to be dequeued.
+ * @param flags Bitmask of RTE_CRYPTO_HW_DEQ_* flags
+ * @param n_fail The buffer for driver to write the number of
+ * failed jobs.
+ * @return
+ * - Return the number of dequeued jobs.
+ **/
+typedef uint32_t (*rte_crypto_hw_deq_many_cb_fn)(void *qp, uint64_t *drv_data,
+ void *user_data, rte_crpyto_hw_user_post_deq_cb_fn cb,
+ uint32_t n, uint64_t flags, uint32_t *n_fail);
+/**
+ * Typedef for querying HW the number of processed jobs.
+ *
+ * @param qp Queue pair data.
+ * @param nb_jobs The expected processed job number.
+ * @return
+ * - If the nb_jobs ready, return 1.
+ * - Otherwise return 0.
+ **/
+typedef int (*rte_crypto_hw_query_processed)(void *qp, uint32_t nb_jobs);
+
+/* Struct for user to perform HW specific enqueue/dequeue function calls */
+struct rte_crypto_hw_ops {
+ /* Driver written queue pair data pointer, should NOT be alterred by
+ * the user.
+ */
+ void *qp;
+ /* Function handler to enqueue AEAD job */
+ rte_crypto_hw_enq_cb_fn enqueue_aead;
+ /* Function handler to enqueue cipher only job */
+ rte_crypto_hw_enq_cb_fn enqueue_cipher;
+ /* Function handler to enqueue auth only job */
+ rte_crypto_hw_enq_cb_fn enqueue_auth;
+ /* Function handler to enqueue cipher + hash chaining job */
+ rte_crypto_hw_enq_cb_fn enqueue_chain;
+ /* Function handler to query processed jobs */
+ rte_crypto_hw_query_processed query_processed;
+ /* Function handler to dequeue one job and return opaque data stored */
+ rte_crypto_hw_deq_one_cb_fn dequeue_one;
+ /* Function handler to dequeue many jobs */
+ rte_crypto_hw_deq_many_cb_fn dequeue_many;
+ /* Reserved */
+ void *reserved[8];
+};
+
+/**
+ * Get the symmetric crypto hardware ops function pointers and queue pair data.
+ *
+ * @param dev_id The device identifier.
+ * @param qp_id The index of the queue pair from which to retrieve
+ * processed packets. The value must be in the range
+ * [0, nb_queue_pair - 1] previously supplied to
+ * rte_cryptodev_configure().
+ * @param hw_ops User provided rte_crypto_hw_ops buffer.
+ *
+ * @return
+ * - On success hw_ops will be written the HW crypto device's queue pair data
+ * and function pointers for data enqueue/dequeue.
+ * - On fail hw_ops is cleared and negative integer is returned.
+ */
+__rte_experimental
+int
+rte_cryptodev_sym_get_hw_ops(
+ uint8_t dev_id, uint16_t qp_id,
+ struct rte_crypto_hw_ops *hw_ops);
+int
+rte_cryptodev_sym_get_hw_ops(
+ uint8_t dev_id, uint16_t qp_id,
+ struct rte_crypto_hw_ops *hw_ops);
+
#ifdef __cplusplus
}
#endif
@@ -316,6 +316,10 @@ typedef uint32_t (*cryptodev_sym_cpu_crypto_process_t)
(struct rte_cryptodev *dev, struct rte_cryptodev_sym_session *sess,
union rte_crypto_sym_ofs ofs, struct rte_crypto_sym_vec *vec);
+struct rte_crypto_hw_ops;
+
+typedef int (*cryptodev_sym_hw_get_ops_t)(struct rte_cryptodev *dev,
+ uint16_t qp_id, struct rte_crypto_hw_ops *hw_ops);
/** Crypto device operations function pointer table */
struct rte_cryptodev_ops {
@@ -348,8 +352,12 @@ struct rte_cryptodev_ops {
/**< Clear a Crypto sessions private data. */
cryptodev_asym_free_session_t asym_session_clear;
/**< Clear a Crypto sessions private data. */
- cryptodev_sym_cpu_crypto_process_t sym_cpu_process;
- /**< process input data synchronously (cpu-crypto). */
+ union {
+ cryptodev_sym_cpu_crypto_process_t sym_cpu_process;
+ /**< process input data synchronously (cpu-crypto). */
+ cryptodev_sym_hw_get_ops_t sym_get_hw_ops;
+ /**< Get HW crypto data-path call back functions and data */
+ };
};
@@ -85,6 +85,7 @@ EXPERIMENTAL {
rte_cryptodev_sym_session_set_user_data;
rte_crypto_asym_op_strings;
rte_crypto_asym_xform_strings;
+ rte_cryptodev_get_qp_status;
# added in 20.05
__rte_cryptodev_trace_configure;
@@ -103,4 +104,7 @@ EXPERIMENTAL {
__rte_cryptodev_trace_asym_session_clear;
__rte_cryptodev_trace_dequeue_burst;
__rte_cryptodev_trace_enqueue_burst;
+
+ # added in 20.08
+ rte_cryptodev_sym_get_hw_ops;
};