@@ -14,6 +14,7 @@
#include <rte_cryptodev.h>
#include <cryptodev_pmd.h>
#include <rte_log.h>
+#include <rte_bitmap.h>
#include "ionic_common.h"
#include "ionic_crypto_if.h"
@@ -154,6 +155,32 @@ struct iocpt_admin_q {
uint16_t flags;
};
+#define IOCPT_S_F_INITED BIT(0)
+
+struct iocpt_session_priv {
+ struct iocpt_dev *dev;
+
+ uint32_t index;
+
+ uint16_t iv_offset;
+ uint16_t iv_length;
+ uint16_t digest_length;
+ uint16_t aad_length;
+
+ uint8_t flags;
+ uint8_t op;
+ uint8_t type;
+
+ uint16_t key_len;
+ uint8_t key[IOCPT_SESS_KEY_LEN_MAX_SYMM];
+};
+
+static inline uint32_t
+iocpt_session_size(void)
+{
+ return sizeof(struct iocpt_session_priv);
+}
+
#define IOCPT_DEV_F_INITED BIT(0)
#define IOCPT_DEV_F_UP BIT(1)
#define IOCPT_DEV_F_FW_RESET BIT(2)
@@ -186,6 +213,8 @@ struct iocpt_dev {
struct iocpt_admin_q *adminq;
+ struct rte_bitmap *sess_bm; /* SET bit indicates index is free */
+
uint64_t features;
uint32_t hw_features;
@@ -239,6 +268,10 @@ void iocpt_dev_reset(struct iocpt_dev *dev);
int iocpt_adminq_post_wait(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx);
+int iocpt_session_init(struct iocpt_session_priv *priv);
+int iocpt_session_update(struct iocpt_session_priv *priv);
+void iocpt_session_deinit(struct iocpt_session_priv *priv);
+
struct ionic_doorbell __iomem *iocpt_db_map(struct iocpt_dev *dev,
struct iocpt_queue *q);
@@ -112,6 +112,116 @@ iocpt_q_free(struct iocpt_queue *q)
}
}
+static int
+iocpt_session_write(struct iocpt_session_priv *priv,
+ enum iocpt_sess_control_oper oper)
+{
+ struct iocpt_dev *dev = priv->dev;
+ struct iocpt_admin_ctx ctx = {
+ .pending_work = true,
+ .cmd.sess_control = {
+ .opcode = IOCPT_CMD_SESS_CONTROL,
+ .type = priv->type,
+ .oper = oper,
+ .index = rte_cpu_to_le_32(priv->index),
+ .key_len = rte_cpu_to_le_16(priv->key_len),
+ .key_seg_len = (uint8_t)RTE_MIN(priv->key_len,
+ IOCPT_SESS_KEY_SEG_LEN),
+ },
+ };
+ struct iocpt_sess_control_cmd *cmd = &ctx.cmd.sess_control;
+ uint16_t key_offset;
+ uint8_t key_segs, seg;
+ int err;
+
+ key_segs = ((priv->key_len - 1) >> IOCPT_SESS_KEY_SEG_SHFT) + 1;
+
+ for (seg = 0; seg < key_segs; seg++) {
+ ctx.pending_work = true;
+
+ key_offset = seg * cmd->key_seg_len;
+ memcpy(cmd->key, &priv->key[key_offset],
+ IOCPT_SESS_KEY_SEG_LEN);
+ cmd->key_seg_idx = seg;
+
+ /* Mark final segment */
+ if (seg + 1 == key_segs)
+ cmd->flags |= rte_cpu_to_le_16(IOCPT_SCTL_F_END);
+
+ err = iocpt_adminq_post_wait(dev, &ctx);
+ if (err != 0)
+ return err;
+ }
+
+ return 0;
+}
+
+int
+iocpt_session_init(struct iocpt_session_priv *priv)
+{
+ struct iocpt_dev *dev = priv->dev;
+ uint64_t bm_slab = 0;
+ uint32_t bm_pos = 0;
+ int err = 0;
+
+ rte_spinlock_lock(&dev->adminq_lock);
+
+ if (rte_bitmap_scan(dev->sess_bm, &bm_pos, &bm_slab) > 0) {
+ priv->index = bm_pos + rte_ctz64(bm_slab);
+ rte_bitmap_clear(dev->sess_bm, priv->index);
+ } else
+ err = -ENOSPC;
+
+ rte_spinlock_unlock(&dev->adminq_lock);
+
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "session index space exhausted");
+ return err;
+ }
+
+ err = iocpt_session_write(priv, IOCPT_SESS_INIT);
+ if (err != 0) {
+ rte_spinlock_lock(&dev->adminq_lock);
+ rte_bitmap_set(dev->sess_bm, priv->index);
+ rte_spinlock_unlock(&dev->adminq_lock);
+ return err;
+ }
+
+ priv->flags |= IOCPT_S_F_INITED;
+
+ return 0;
+}
+
+int
+iocpt_session_update(struct iocpt_session_priv *priv)
+{
+ return iocpt_session_write(priv, IOCPT_SESS_UPDATE_KEY);
+}
+
+void
+iocpt_session_deinit(struct iocpt_session_priv *priv)
+{
+ struct iocpt_dev *dev = priv->dev;
+ struct iocpt_admin_ctx ctx = {
+ .pending_work = true,
+ .cmd.sess_control = {
+ .opcode = IOCPT_CMD_SESS_CONTROL,
+ .type = priv->type,
+ .oper = IOCPT_SESS_DISABLE,
+ .index = rte_cpu_to_le_32(priv->index),
+ .key_len = rte_cpu_to_le_16(priv->key_len),
+ },
+ };
+
+ (void)iocpt_adminq_post_wait(dev, &ctx);
+
+ rte_spinlock_lock(&dev->adminq_lock);
+ rte_bitmap_set(dev->sess_bm, priv->index);
+ rte_spinlock_unlock(&dev->adminq_lock);
+
+ priv->flags &= ~IOCPT_S_F_INITED;
+}
+
static const struct rte_memzone *
iocpt_dma_zone_reserve(const char *type_name, uint16_t qid, size_t size,
unsigned int align, int socket_id)
@@ -305,6 +415,8 @@ iocpt_adminq_free(struct iocpt_admin_q *aq)
static int
iocpt_alloc_objs(struct iocpt_dev *dev)
{
+ uint32_t bmsize, i;
+ uint8_t *bm;
int err;
IOCPT_PRINT(DEBUG, "Crypto: %s", dev->name);
@@ -331,8 +443,33 @@ iocpt_alloc_objs(struct iocpt_dev *dev)
dev->info = dev->info_z->addr;
dev->info_pa = dev->info_z->iova;
+ bmsize = rte_bitmap_get_memory_footprint(dev->max_sessions);
+ bm = rte_malloc_socket("iocpt", bmsize,
+ RTE_CACHE_LINE_SIZE, dev->socket_id);
+ if (bm == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate %uB bitmap memory", bmsize);
+ err = -ENOMEM;
+ goto err_free_dmazone;
+ }
+
+ dev->sess_bm = rte_bitmap_init(dev->max_sessions, bm, bmsize);
+ if (dev->sess_bm == NULL) {
+ IOCPT_PRINT(ERR, "Cannot initialize bitmap");
+ err = -EFAULT;
+ goto err_free_bm;
+ }
+ for (i = 0; i < dev->max_sessions; i++)
+ rte_bitmap_set(dev->sess_bm, i);
+
return 0;
+err_free_bm:
+ rte_free(bm);
+err_free_dmazone:
+ rte_memzone_free(dev->info_z);
+ dev->info_z = NULL;
+ dev->info = NULL;
+ dev->info_pa = 0;
err_free_adminq:
iocpt_adminq_free(dev->adminq);
dev->adminq = NULL;
@@ -383,6 +520,12 @@ iocpt_free_objs(struct iocpt_dev *dev)
{
IOCPT_PRINT_CALL();
+ if (dev->sess_bm != NULL) {
+ rte_bitmap_free(dev->sess_bm);
+ rte_free(dev->sess_bm);
+ dev->sess_bm = NULL;
+ }
+
if (dev->adminq != NULL) {
iocpt_adminq_free(dev->adminq);
dev->adminq = NULL;
@@ -48,10 +48,133 @@ iocpt_op_info_get(struct rte_cryptodev *cdev, struct rte_cryptodev_info *info)
info->min_mbuf_tailroom_req = 0;
}
+static unsigned int
+iocpt_op_get_session_size(struct rte_cryptodev *cdev __rte_unused)
+{
+ return iocpt_session_size();
+}
+
+static inline int
+iocpt_is_algo_supported(struct rte_crypto_sym_xform *xform)
+{
+ if (xform->next != NULL) {
+ IOCPT_PRINT(ERR, "chaining not supported");
+ return -ENOTSUP;
+ }
+
+ if (xform->type != RTE_CRYPTO_SYM_XFORM_AEAD) {
+ IOCPT_PRINT(ERR, "xform->type %d not supported", xform->type);
+ return -ENOTSUP;
+ }
+
+ return 0;
+}
+
+static __rte_always_inline int
+iocpt_fill_sess_aead(struct rte_crypto_sym_xform *xform,
+ struct iocpt_session_priv *priv)
+{
+ struct rte_crypto_aead_xform *aead_form = &xform->aead;
+
+ if (aead_form->algo != RTE_CRYPTO_AEAD_AES_GCM) {
+ IOCPT_PRINT(ERR, "Unknown algo");
+ return -EINVAL;
+ }
+ if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
+ priv->op = IOCPT_DESC_OPCODE_GCM_AEAD_ENCRYPT;
+ } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT) {
+ priv->op = IOCPT_DESC_OPCODE_GCM_AEAD_DECRYPT;
+ } else {
+ IOCPT_PRINT(ERR, "Unknown cipher operations");
+ return -1;
+ }
+
+ if (aead_form->key.length < IOCPT_SESS_KEY_LEN_MIN ||
+ aead_form->key.length > IOCPT_SESS_KEY_LEN_MAX_SYMM) {
+ IOCPT_PRINT(ERR, "Invalid cipher keylen %u",
+ aead_form->key.length);
+ return -1;
+ }
+ priv->key_len = aead_form->key.length;
+ memcpy(priv->key, aead_form->key.data, priv->key_len);
+
+ priv->type = IOCPT_SESS_AEAD_AES_GCM;
+ priv->iv_offset = aead_form->iv.offset;
+ priv->iv_length = aead_form->iv.length;
+ priv->digest_length = aead_form->digest_length;
+ priv->aad_length = aead_form->aad_length;
+
+ return 0;
+}
+
+static int
+iocpt_session_cfg(struct iocpt_dev *dev,
+ struct rte_crypto_sym_xform *xform,
+ struct rte_cryptodev_sym_session *sess)
+{
+ struct rte_crypto_sym_xform *chain;
+ struct iocpt_session_priv *priv = NULL;
+
+ if (iocpt_is_algo_supported(xform) < 0)
+ return -ENOTSUP;
+
+ if (unlikely(sess == NULL)) {
+ IOCPT_PRINT(ERR, "invalid session");
+ return -EINVAL;
+ }
+
+ priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
+ priv->dev = dev;
+
+ chain = xform;
+ while (chain) {
+ switch (chain->type) {
+ case RTE_CRYPTO_SYM_XFORM_AEAD:
+ if (iocpt_fill_sess_aead(chain, priv))
+ return -EIO;
+ break;
+ default:
+ IOCPT_PRINT(ERR, "invalid crypto xform type %d",
+ chain->type);
+ return -ENOTSUP;
+ }
+ chain = chain->next;
+ }
+
+ return iocpt_session_init(priv);
+}
+
+static int
+iocpt_op_session_cfg(struct rte_cryptodev *cdev,
+ struct rte_crypto_sym_xform *xform,
+ struct rte_cryptodev_sym_session *sess)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+
+ return iocpt_session_cfg(dev, xform, sess);
+}
+
+static void
+iocpt_session_clear(struct rte_cryptodev_sym_session *sess)
+{
+ iocpt_session_deinit(CRYPTODEV_GET_SYM_SESS_PRIV(sess));
+}
+
+static void
+iocpt_op_session_clear(struct rte_cryptodev *cdev __rte_unused,
+ struct rte_cryptodev_sym_session *sess)
+{
+ iocpt_session_clear(sess);
+}
+
static struct rte_cryptodev_ops iocpt_ops = {
.dev_configure = iocpt_op_config,
.dev_close = iocpt_op_close,
.dev_infos_get = iocpt_op_info_get,
+
+ .sym_session_get_size = iocpt_op_get_session_size,
+ .sym_session_configure = iocpt_op_session_cfg,
+ .sym_session_clear = iocpt_op_session_clear,
};
int