[v2,3/6] crypto/aesni_mb: add support for DOCSIS protocol

diff --git a/drivers/crypto/aesni_mb/aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/aesni_mb_pmd_private.h
index 347b4b7e0..b4ad27792 100644
--- a/drivers/crypto/aesni_mb/aesni_mb_pmd_private.h
+++ b/drivers/crypto/aesni_mb/aesni_mb_pmd_private.h
@@ -7,6 +7,12 @@ 
 
 #include <intel-ipsec-mb.h>
 
+#if defined(RTE_LIBRTE_SECURITY) && IMB_VERSION_NUM >= IMB_VERSION(0, 54, 0)
+#define AESNI_MB_DOCSIS_SEC_ENABLED 1
+#include <rte_security.h>
+#include <rte_security_driver.h>
+#endif
+
 enum aesni_mb_vector_mode {
 	RTE_AESNI_MB_NOT_SUPPORTED = 0,
 	RTE_AESNI_MB_SSE,
@@ -272,8 +278,19 @@  aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
 		struct aesni_mb_session *sess,
 		const struct rte_crypto_sym_xform *xform);
 
-/** device specific operations function pointer structure */
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+extern int
+aesni_mb_set_docsis_sec_session_parameters(
+		__rte_unused struct rte_cryptodev *dev,
+		struct rte_security_session_conf *conf,
+		void *sess);
+#endif
+
+/** device specific operations function pointer structures */
 extern struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops;
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+extern struct rte_security_ops *rte_aesni_mb_pmd_sec_ops;
+#endif
 
 extern uint32_t
 aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
diff --git a/drivers/crypto/aesni_mb/meson.build b/drivers/crypto/aesni_mb/meson.build
index e557e0103..419b4743f 100644
--- a/drivers/crypto/aesni_mb/meson.build
+++ b/drivers/crypto/aesni_mb/meson.build
@@ -22,4 +22,4 @@  else
 endif
 
 sources = files('rte_aesni_mb_pmd.c', 'rte_aesni_mb_pmd_ops.c')
-deps += ['bus_vdev']
+deps += ['bus_vdev', 'net', 'security']
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
index 2d688f4d3..1fc0c83c3 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
@@ -12,6 +12,7 @@ 
 #include <rte_malloc.h>
 #include <rte_cpuflags.h>
 #include <rte_per_lcore.h>
+#include <rte_ether.h>
 
 #include "aesni_mb_pmd_private.h"
 
@@ -720,6 +721,136 @@  aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
 	return 0;
 }
 
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+/** Check DOCSIS security session configuration is valid */
+static int
+check_docsis_sec_session(struct rte_security_session_conf *conf)
+{
+	struct rte_crypto_sym_xform *crypto_sym = conf->crypto_xform;
+	struct rte_security_docsis_xform *docsis = &conf->docsis;
+
+	/* CRC generate -> Cipher encrypt */
+	if (docsis->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
+
+		if (crypto_sym != NULL &&
+		    crypto_sym->type ==	RTE_CRYPTO_SYM_XFORM_CIPHER &&
+		    crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT &&
+		    crypto_sym->cipher.algo ==
+					RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
+		    (crypto_sym->cipher.key.length == AES_128_BYTES ||
+		     crypto_sym->cipher.key.length == AES_256_BYTES) &&
+		    crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&
+		    crypto_sym->next == NULL &&
+		    docsis->crc_size == RTE_ETHER_CRC_LEN) {
+			return 0;
+		}
+	/* Cipher decrypt -> CRC verify */
+	} else if (docsis->direction == RTE_SECURITY_DOCSIS_UPLINK) {
+
+		if (crypto_sym != NULL &&
+		    crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
+		    crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&
+		    crypto_sym->cipher.algo ==
+					RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
+		    (crypto_sym->cipher.key.length == AES_128_BYTES ||
+		     crypto_sym->cipher.key.length == AES_256_BYTES) &&
+		    crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&
+		    crypto_sym->next == NULL &&
+		    docsis->crc_size == RTE_ETHER_CRC_LEN) {
+			return 0;
+		}
+	}
+
+	return -EINVAL;
+}
+
+/** Set DOCSIS security session auth (CRC) parameters */
+static int
+aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,
+		struct rte_security_docsis_xform *xform)
+{
+	if (xform == NULL) {
+		AESNI_MB_LOG(ERR, "Invalid DOCSIS xform");
+		return -EINVAL;
+	}
+
+	/* Select CRC generate/verify */
+	if (xform->direction == RTE_SECURITY_DOCSIS_UPLINK) {
+		sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
+		sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
+	} else if (xform->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
+		sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
+		sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
+	} else {
+		AESNI_MB_LOG(ERR, "Unsupported DOCSIS direction");
+		return -ENOTSUP;
+	}
+
+	sess->auth.req_digest_len = xform->crc_size;
+	sess->auth.gen_digest_len = xform->crc_size;
+
+	return 0;
+}
+
+/**
+ * Parse DOCSIS security session configuration and set private session
+ * parameters
+ */
+int
+aesni_mb_set_docsis_sec_session_parameters(
+		__rte_unused struct rte_cryptodev *dev,
+		struct rte_security_session_conf *conf,
+		void *sess)
+{
+	struct rte_security_docsis_xform *docsis_xform;
+	struct rte_crypto_sym_xform *cipher_xform;
+	struct aesni_mb_session *aesni_sess = sess;
+	struct aesni_mb_private *internals = dev->data->dev_private;
+	int ret;
+
+	ret = check_docsis_sec_session(conf);
+	if (ret) {
+		AESNI_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
+		return ret;
+	}
+
+	switch (conf->docsis.direction) {
+	case RTE_SECURITY_DOCSIS_UPLINK:
+		aesni_sess->chain_order = CIPHER_HASH;
+		docsis_xform = &conf->docsis;
+		cipher_xform = conf->crypto_xform;
+		break;
+	case RTE_SECURITY_DOCSIS_DOWNLINK:
+		aesni_sess->chain_order = HASH_CIPHER;
+		cipher_xform = conf->crypto_xform;
+		docsis_xform = &conf->docsis;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Default IV length = 0 */
+	aesni_sess->iv.length = 0;
+
+	ret = aesni_mb_set_docsis_sec_session_auth_parameters(aesni_sess,
+			docsis_xform);
+	if (ret != 0) {
+		AESNI_MB_LOG(ERR, "Invalid/unsupported DOCSIS parameters");
+		return -EINVAL;
+	}
+
+	ret = aesni_mb_set_session_cipher_parameters(internals->mb_mgr,
+			aesni_sess, cipher_xform);
+
+	if (ret != 0) {
+		AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+#endif
+
 /**
  * burst enqueue, place crypto operations on ingress queue for processing.
  *
@@ -1158,6 +1289,131 @@  set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
 	return 0;
 }
 
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+/** Get multi buffer session for a security op */
+static inline struct aesni_mb_session *
+get_sec_op_session(struct rte_crypto_op *op,
+		struct rte_crypto_sym_op *crypto_sym)
+{
+	struct aesni_mb_session *sess = NULL;
+
+	if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
+		if (likely(crypto_sym->sec_session != NULL))
+			sess = (struct aesni_mb_session *)
+					get_sec_session_private_data(
+						crypto_sym->sec_session);
+	}
+
+	if (unlikely(sess == NULL))
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+
+	return sess;
+}
+
+/**
+ * Process a crypto operation containing a security op and complete a
+ * JOB_AES_HMAC job structure for submission to the multi buffer library for
+ * processing.
+ */
+static inline int
+set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
+		struct rte_crypto_op *op, uint8_t *digest_idx)
+{
+	struct rte_mbuf *m_src, *m_dst;
+	struct rte_security_op *s_op;
+	struct rte_security_docsis_op *d_op;
+	struct rte_crypto_sym_op *sym;
+	struct aesni_mb_session *session;
+
+	s_op = (struct rte_security_op *)&op->security;
+
+	/* Only DOCSIS protocol operations supported now */
+	if (s_op->type != RTE_SECURITY_OP_TYPE_DOCSIS) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return -1;
+	}
+
+	d_op = &s_op->docsis;
+	sym = &d_op->crypto_sym;
+
+	session = get_sec_op_session(op, sym);
+	if (unlikely(session == NULL)) {
+		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+		return -1;
+	}
+
+	m_src = sym->m_src;
+
+	if (likely(sym->m_dst == NULL || sym->m_dst == m_src)) {
+		/* in-place operation */
+		m_dst = m_src;
+	} else {
+		/* out-of-place operation not supported */
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return -ENOTSUP;
+	}
+
+	/* Set crypto operation */
+	job->chain_order = session->chain_order;
+
+	/* Set cipher parameters */
+	job->cipher_direction = session->cipher.direction;
+	job->cipher_mode = session->cipher.mode;
+
+	job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
+	job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
+	job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode;
+
+	/* Set IV parameters */
+	job->iv_len_in_bytes = session->iv.length;
+	job->iv = (uint8_t *)op + session->iv.offset;
+
+	/* Set authentication parameters */
+	job->hash_alg = session->auth.algo;
+
+	/* Set digest output location */
+	job->auth_tag_output = qp->temp_digests[*digest_idx];
+	*digest_idx = (*digest_idx + 1) % MAX_JOBS;
+
+	/* Set digest length */
+	job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
+
+	/* Set data parameters */
+	job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
+	job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *,
+						sym->cipher.data.offset);
+
+	job->cipher_start_src_offset_in_bytes = sym->cipher.data.offset;
+	job->msg_len_to_cipher_in_bytes = sym->cipher.data.length;
+
+	job->hash_start_src_offset_in_bytes = d_op->crc.offset;
+	job->msg_len_to_hash_in_bytes = d_op->crc.length;
+
+	job->user_data = op;
+
+	return 0;
+}
+
+static inline void
+verify_docsis_sec_crc(JOB_AES_HMAC *job, uint16_t crc_len, uint8_t *status)
+{
+	uint16_t crc_offset;
+	uint8_t *crc;
+
+	if (!job->msg_len_to_hash_in_bytes)
+		return;
+
+	crc_offset = job->hash_start_src_offset_in_bytes +
+			job->msg_len_to_hash_in_bytes -
+			job->cipher_start_src_offset_in_bytes;
+	crc = job->dst + crc_offset;
+
+	/* Verify CRC (at the end of the message) */
+	if (memcmp(job->auth_tag_output, crc, crc_len) != 0)
+		*status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+}
+#endif
+
 static inline void
 verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status)
 {
@@ -1196,9 +1452,27 @@  static inline struct rte_crypto_op *
 post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
 {
 	struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
-	struct aesni_mb_session *sess = get_sym_session_private_data(
-							op->sym->session,
-							cryptodev_driver_id);
+	struct aesni_mb_session *sess = NULL;
+
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+	struct rte_security_op *sec_op = NULL;
+
+	if (unlikely(op->type == RTE_CRYPTO_OP_TYPE_SECURITY)) {
+		/*
+		 * Assuming at this point that if it's a security type op, that
+		 * this is for DOCSIS
+		 */
+		sec_op = (struct rte_security_op *)&op->security;
+		struct rte_crypto_sym_op *crypto_sym =
+						&sec_op->docsis.crypto_sym;
+		sess = get_sec_session_private_data(crypto_sym->sec_session);
+	} else
+#endif
+	{
+		sess = get_sym_session_private_data(op->sym->session,
+						cryptodev_driver_id);
+	}
+
 	if (unlikely(sess == NULL)) {
 		op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 		return op;
@@ -1220,6 +1494,12 @@  post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
 						op->sym->aead.digest.data,
 						sess->auth.req_digest_len,
 						&op->status);
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+				else if (sec_op != NULL)
+					verify_docsis_sec_crc(job,
+						sess->auth.req_digest_len,
+						&op->status);
+#endif
 				else
 					verify_digest(job,
 						op->sym->auth.digest.data,
@@ -1382,7 +1662,14 @@  aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 		if (retval < 0)
 			break;
 
-		retval = set_mb_job_params(job, qp, op, &digest_idx);
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+		if (unlikely(op->type == RTE_CRYPTO_OP_TYPE_SECURITY))
+			retval = set_sec_mb_job_params(job, qp, op,
+						&digest_idx);
+		else
+#endif
+			retval = set_mb_job_params(job, qp, op, &digest_idx);
+
 		if (unlikely(retval != 0)) {
 			qp->stats.dequeue_err_count++;
 			set_job_null_op(job, op);
@@ -1619,6 +1906,9 @@  cryptodev_aesni_mb_create(const char *name,
 	struct aesni_mb_private *internals;
 	enum aesni_mb_vector_mode vector_mode;
 	MB_MGR *mb_mgr;
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+	struct rte_security_ctx *security_instance;
+#endif
 
 	dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
 	if (dev == NULL) {
@@ -1647,7 +1937,23 @@  cryptodev_aesni_mb_create(const char *name,
 			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
 			RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
-			RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
+			RTE_CRYPTODEV_FF_SYM_SESSIONLESS
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+			| RTE_CRYPTODEV_FF_SECURITY
+#endif
+			;
+
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+	security_instance = rte_malloc("aesni_mb_sec",
+				sizeof(struct rte_security_ctx), 0);
+	if (security_instance == NULL)
+		AESNI_MB_LOG(ERR, "rte_security_ctx memory alloc failed\n");
+
+	security_instance->device = (void *)dev;
+	security_instance->ops = rte_aesni_mb_pmd_sec_ops;
+	security_instance->sess_cnt = 0;
+	dev->security_ctx = security_instance;
+#endif
 
 	/* Check CPU for support for AES instruction set */
 	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES))
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
index 8c5e0cd92..4294d416c 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
@@ -7,6 +7,7 @@ 
 #include <rte_string_fns.h>
 #include <rte_common.h>
 #include <rte_malloc.h>
+#include <rte_ether.h>
 #include <rte_cryptodev_pmd.h>
 
 #include "aesni_mb_pmd_private.h"
@@ -499,6 +500,57 @@  static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
 	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 };
 
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+static const struct rte_cryptodev_capabilities
+					aesni_mb_pmd_security_crypto_cap[] = {
+	{	/* AES DOCSIS BPI */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
+				.block_size = 16,
+				.key_size = {
+					.min = 16,
+					.max = 32,
+					.increment = 16
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				}
+			}, }
+		}, }
+	},
+
+	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
+static const struct rte_security_capability aesni_mb_pmd_security_cap[] = {
+	{	/* DOCSIS Uplink */
+		.action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
+		.protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
+		.docsis = {
+			.direction = RTE_SECURITY_DOCSIS_UPLINK,
+			.crc_size = RTE_ETHER_CRC_LEN
+		},
+		.crypto_capabilities = aesni_mb_pmd_security_crypto_cap
+	},
+	{	/* DOCSIS Downlink */
+		.action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
+		.protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
+		.docsis = {
+			.direction = RTE_SECURITY_DOCSIS_DOWNLINK,
+			.crc_size = RTE_ETHER_CRC_LEN
+		},
+		.crypto_capabilities = aesni_mb_pmd_security_crypto_cap
+	},
+	{
+		.action = RTE_SECURITY_ACTION_TYPE_NONE
+	}
+};
+#endif
 
 /** Configure device */
 static int
@@ -810,3 +862,78 @@  struct rte_cryptodev_ops aesni_mb_pmd_ops = {
 };
 
 struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;
+
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+/**
+ * Configure a aesni multi-buffer session from a security session
+ * configuration
+ */
+static int
+aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,
+		struct rte_security_session *sess,
+		struct rte_mempool *mempool)
+{
+	void *sess_private_data;
+	struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
+	int ret;
+
+	if (rte_mempool_get(mempool, &sess_private_data)) {
+		AESNI_MB_LOG(ERR, "Couldn't get object from session mempool");
+		return -ENOMEM;
+	}
+
+	if (conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
+		AESNI_MB_LOG(ERR, "Invalid security protocol");
+		return -EINVAL;
+	}
+
+	ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,
+			sess_private_data);
+
+	if (ret != 0) {
+		AESNI_MB_LOG(ERR, "Failed to configure session parameters");
+
+		/* Return session to mempool */
+		rte_mempool_put(mempool, sess_private_data);
+		return ret;
+	}
+
+	set_sec_session_private_data(sess, sess_private_data);
+
+	return ret;
+}
+
+/** Clear the memory of session so it doesn't leave key material behind */
+static int
+aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,
+		struct rte_security_session *sess)
+{
+	void *sess_priv = get_sec_session_private_data(sess);
+
+	if (sess_priv) {
+		struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
+		memset(sess, 0, sizeof(struct aesni_mb_session));
+		set_sec_session_private_data(sess, NULL);
+		rte_mempool_put(sess_mp, sess_priv);
+	}
+	return 0;
+}
+
+/** Get security capabilities for aesni multi-buffer */
+static const struct rte_security_capability *
+aesni_mb_pmd_sec_capa_get(void *device __rte_unused)
+{
+	return aesni_mb_pmd_security_cap;
+}
+
+static struct rte_security_ops aesni_mb_pmd_sec_ops = {
+		.session_create = aesni_mb_pmd_sec_sess_create,
+		.session_update = NULL,
+		.session_stats_get = NULL,
+		.session_destroy = aesni_mb_pmd_sec_sess_destroy,
+		.set_pkt_metadata = NULL,
+		.capabilities_get = aesni_mb_pmd_sec_capa_get
+};
+
+struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;
+#endif