@@ -684,16 +684,12 @@ ipsec_poll_mode_worker(void)
qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in;
qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in;
qconf->inbound.cdev_map = cdev_map_in;
- qconf->inbound.session_pool = socket_ctx[socket_id].session_pool;
- qconf->inbound.session_priv_pool =
- socket_ctx[socket_id].session_priv_pool;
+ qconf->inbound.lcore_id = lcore_id;
qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out;
qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out;
qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out;
qconf->outbound.cdev_map = cdev_map_out;
- qconf->outbound.session_pool = socket_ctx[socket_id].session_pool;
- qconf->outbound.session_priv_pool =
- socket_ctx[socket_id].session_priv_pool;
+ qconf->outbound.lcore_id = lcore_id;
qconf->frag.pool_indir = socket_ctx[socket_id].mbuf_pool_indir;
rc = ipsec_sad_lcore_cache_init(app_sa_prm.cache_sz);
@@ -1458,7 +1454,7 @@ check_all_ports_link_status(uint32_t port_mask)
}
static int32_t
-add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id,
+add_mapping(const char *str, uint16_t cdev_id,
uint16_t qp, struct lcore_params *params,
struct ipsec_ctx *ipsec_ctx,
const struct rte_cryptodev_capabilities *cipher,
@@ -1477,7 +1473,7 @@ add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id,
if (aead)
key.aead_algo = aead->sym.aead.algo;
- ret = rte_hash_lookup(map, &key);
+ ret = rte_hash_lookup(ipsec_ctx->cdev_map, &key);
if (ret != -ENOENT)
return 0;
@@ -1499,7 +1495,7 @@ add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id,
cdev_id, qp, i);
}
- ret = rte_hash_add_key_data(map, &key, (void *)i);
+ ret = rte_hash_add_key_data(ipsec_ctx->cdev_map, &key, (void *)i);
if (ret < 0) {
printf("Faled to insert cdev mapping for (lcore %u, "
"cdev %u, qp %u), errno %d\n",
@@ -1517,20 +1513,19 @@ add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id,
{
int32_t ret = 0;
const struct rte_cryptodev_capabilities *i, *j;
- struct rte_hash *map;
struct lcore_conf *qconf;
struct ipsec_ctx *ipsec_ctx;
const char *str;
qconf = &lcore_conf[params->lcore_id];
- if ((unprotected_port_mask & (1 << params->port_id)) == 0) {
- map = cdev_map_out;
+ if (!is_unprotected_port(params->port_id)) {
ipsec_ctx = &qconf->outbound;
+ ipsec_ctx->cdev_map = cdev_map_out;
str = "Outbound";
} else {
- map = cdev_map_in;
ipsec_ctx = &qconf->inbound;
+ ipsec_ctx->cdev_map = cdev_map_in;
str = "Inbound";
}
@@ -1545,7 +1540,7 @@ add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id,
continue;
if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) {
- ret |= add_mapping(map, str, cdev_id, qp, params,
+ ret |= add_mapping(str, cdev_id, qp, params,
ipsec_ctx, NULL, NULL, i);
continue;
}
@@ -1561,7 +1556,7 @@ add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id,
if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
continue;
- ret |= add_mapping(map, str, cdev_id, qp, params,
+ ret |= add_mapping(str, cdev_id, qp, params,
ipsec_ctx, i, j, NULL);
}
}
@@ -3074,7 +3069,7 @@ main(int32_t argc, char **argv)
if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
(socket_ctx[socket_id].sa_in == NULL) &&
(socket_ctx[socket_id].sa_out == NULL)) {
- sa_init(&socket_ctx[socket_id], socket_id);
+ sa_init(&socket_ctx[socket_id], socket_id, lcore_conf);
sp4_init(&socket_ctx[socket_id], socket_id);
sp6_init(&socket_ctx[socket_id], socket_id);
rt_init(&socket_ctx[socket_id], socket_id);
@@ -55,37 +55,71 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
}
int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
- struct rte_ipsec_session *ips)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx_lcore[],
+ struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+ struct rte_ipsec_session *ips)
{
+ uint16_t cdev_id = RTE_CRYPTO_MAX_DEVS;
struct rte_cryptodev_info cdev_info;
unsigned long cdev_id_qp = 0;
- int32_t ret = 0;
struct cdev_key key = { 0 };
+ struct ipsec_ctx *ipsec_ctx;
+ uint32_t lcore_id;
+ int32_t ret = 0;
- key.lcore_id = (uint8_t)rte_lcore_id();
+ RTE_LCORE_FOREACH(lcore_id) {
+ ipsec_ctx = ipsec_ctx_lcore[lcore_id];
- key.cipher_algo = (uint8_t)sa->cipher_algo;
- key.auth_algo = (uint8_t)sa->auth_algo;
- key.aead_algo = (uint8_t)sa->aead_algo;
+ /* Core is not bound to any cryptodev, skip it */
+ if (ipsec_ctx->cdev_map == NULL)
+ continue;
- ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
- (void **)&cdev_id_qp);
- if (ret < 0) {
- RTE_LOG(ERR, IPSEC,
- "No cryptodev: core %u, cipher_algo %u, "
- "auth_algo %u, aead_algo %u\n",
- key.lcore_id,
- key.cipher_algo,
- key.auth_algo,
- key.aead_algo);
- return -1;
+ /* Looking for cryptodev, which can handle this SA */
+ key.lcore_id = (uint8_t)lcore_id;
+ key.cipher_algo = (uint8_t)sa->cipher_algo;
+ key.auth_algo = (uint8_t)sa->auth_algo;
+ key.aead_algo = (uint8_t)sa->aead_algo;
+
+ ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
+ (void **)&cdev_id_qp);
+ if (ret == -ENOENT)
+ continue;
+ if (ret < 0) {
+ RTE_LOG(ERR, IPSEC,
+ "No cryptodev: core %u, cipher_algo %u, "
+ "auth_algo %u, aead_algo %u\n",
+ key.lcore_id,
+ key.cipher_algo,
+ key.auth_algo,
+ key.aead_algo);
+ return ret;
+ }
+
+ /* Verify that all cores are using same cryptodev for current
+ * algorithm combination, required by SA.
+ * Current cryptodev mapping process will map SA to the first
+ * cryptodev that matches requirements, so it's a double check,
+ * not an additional restriction.
+ */
+ if (cdev_id == RTE_CRYPTO_MAX_DEVS)
+ cdev_id = ipsec_ctx->tbl[cdev_id_qp].id;
+ else if (cdev_id != ipsec_ctx->tbl[cdev_id_qp].id) {
+ RTE_LOG(ERR, IPSEC,
+ "SA mapping to multiple cryptodevs is "
+ "not supported!");
+ return -EINVAL;
+ }
+
+ /* Store per core queue pair information */
+ sa->cqp[lcore_id] = &ipsec_ctx->tbl[cdev_id_qp];
+ }
+ if (cdev_id == RTE_CRYPTO_MAX_DEVS) {
+ RTE_LOG(WARNING, IPSEC, "No cores found to handle SA\n");
+ return 0;
}
- RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
- "%u qp %u\n", sa->spi,
- ipsec_ctx->tbl[cdev_id_qp].id,
- ipsec_ctx->tbl[cdev_id_qp].qp);
+ RTE_LOG(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
+ "%u\n", sa->spi, cdev_id);
if (ips->type != RTE_SECURITY_ACTION_TYPE_NONE &&
ips->type != RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) {
@@ -111,14 +145,14 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
if (ips->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
struct rte_security_ctx *ctx = (struct rte_security_ctx *)
rte_cryptodev_get_sec_ctx(
- ipsec_ctx->tbl[cdev_id_qp].id);
+ cdev_id);
/* Set IPsec parameters in conf */
set_ipsec_conf(sa, &(sess_conf.ipsec));
ips->security.ses = rte_security_session_create(ctx,
- &sess_conf, ipsec_ctx->session_pool,
- ipsec_ctx->session_priv_pool);
+ &sess_conf, skt_ctx->session_pool,
+ skt_ctx->session_priv_pool);
if (ips->security.ses == NULL) {
RTE_LOG(ERR, IPSEC,
"SEC Session init failed: err: %d\n", ret);
@@ -130,8 +164,6 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
return -1;
}
} else {
- uint16_t cdev_id = ipsec_ctx->tbl[cdev_id_qp].id;
-
if (ips->type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) {
struct rte_cryptodev_info info;
@@ -143,16 +175,14 @@ create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
}
ips->crypto.dev_id = cdev_id;
ips->crypto.ses = rte_cryptodev_sym_session_create(
- ipsec_ctx->session_pool);
+ skt_ctx->session_pool);
rte_cryptodev_sym_session_init(cdev_id,
ips->crypto.ses, sa->xforms,
- ipsec_ctx->session_priv_pool);
+ skt_ctx->session_priv_pool);
rte_cryptodev_info_get(cdev_id, &cdev_info);
}
- sa->cdev_id_qp = cdev_id_qp;
-
return 0;
}
@@ -621,8 +651,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
rte_prefetch0(&priv->sym_cop);
- if ((unlikely(ips->security.ses == NULL)) &&
- create_lookaside_session(ipsec_ctx, sa, ips)) {
+ if (unlikely(ips->security.ses == NULL)) {
free_pkts(&pkts[i], 1);
continue;
}
@@ -656,8 +685,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
rte_prefetch0(&priv->sym_cop);
- if ((unlikely(ips->crypto.ses == NULL)) &&
- create_lookaside_session(ipsec_ctx, sa, ips)) {
+ if (unlikely(ips->crypto.ses == NULL)) {
free_pkts(&pkts[i], 1);
continue;
}
@@ -704,8 +732,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
continue;
}
- RTE_ASSERT(sa->cdev_id_qp < ipsec_ctx->nb_qps);
- enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);
+ enqueue_cop(sa->cqp[ipsec_ctx->lcore_id], &priv->cop);
}
}
@@ -112,7 +112,7 @@ ipsec_mask_saptr(void *ptr)
struct ipsec_sa {
struct rte_ipsec_session sessions[IPSEC_SESSION_MAX];
uint32_t spi;
- uint32_t cdev_id_qp;
+ struct cdev_qp *cqp[RTE_MAX_LCORE];
uint64_t seq;
uint32_t salt;
uint32_t fallback_sessions;
@@ -230,12 +230,11 @@ struct ipsec_ctx {
uint16_t nb_qps;
uint16_t last_qp;
struct cdev_qp tbl[MAX_QP_PER_LCORE];
- struct rte_mempool *session_pool;
- struct rte_mempool *session_priv_pool;
struct rte_mbuf *ol_pkts[MAX_PKT_BURST] __rte_aligned(sizeof(void *));
uint16_t ol_pkts_cnt;
uint64_t ipv4_offloads;
uint64_t ipv6_offloads;
+ uint32_t lcore_id;
};
struct cdev_key {
@@ -424,7 +423,8 @@ int
sa_spi_present(struct sa_ctx *sa_ctx, uint32_t spi, int inbound);
void
-sa_init(struct socket_ctx *ctx, int32_t socket_id);
+sa_init(struct socket_ctx *ctx, int32_t socket_id,
+ struct lcore_conf *lcore_conf);
void
rt_init(struct socket_ctx *ctx, int32_t socket_id);
@@ -440,8 +440,9 @@ void
enqueue_cop_burst(struct cdev_qp *cqp);
int
-create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
- struct rte_ipsec_session *ips);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx[],
+ struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
+ struct rte_ipsec_session *ips);
int
create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
@@ -73,32 +73,18 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
}
static inline int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
- struct ipsec_sa *sa)
+check_ipsec_session(const struct rte_ipsec_session *ss)
{
- int32_t rc;
-
- /* setup crypto section */
if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE ||
ss->type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) {
- RTE_ASSERT(ss->crypto.ses == NULL);
- rc = create_lookaside_session(ctx, sa, ss);
- if (rc != 0)
- return rc;
- /* setup session action type */
+ if (ss->crypto.ses == NULL)
+ return -ENOENT;
} else if (ss->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
- RTE_ASSERT(ss->security.ses == NULL);
- rc = create_lookaside_session(ctx, sa, ss);
- if (rc != 0)
- return rc;
+ if (ss->security.ses == NULL)
+ return -ENOENT;
} else
RTE_ASSERT(0);
-
- rc = rte_ipsec_session_prepare(ss);
- if (rc != 0)
- memset(ss, 0, sizeof(*ss));
-
- return rc;
+ return 0;
}
/*
@@ -183,7 +169,7 @@ ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa,
uint32_t j, k;
struct ipsec_mbuf_metadata *priv;
- cqp = &ctx->tbl[sa->cdev_id_qp];
+ cqp = sa->cqp[ctx->lcore_id];
/* for that app each mbuf has it's own crypto op */
for (j = 0; j != cnt; j++) {
@@ -274,9 +260,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
ipsec_get_fallback_session(sa) :
ipsec_get_primary_session(sa);
- /* no valid HW session for that SA, try to create one */
- if (sa == NULL || (ips->crypto.ses == NULL &&
- fill_ipsec_session(ips, ctx, sa) != 0))
+ /* no valid HW session for that SA */
+ if (sa == NULL || unlikely(check_ipsec_session(ips) != 0))
k = 0;
/* process packets inline */
@@ -892,15 +892,11 @@ ipsec_wrkr_non_burst_int_port_app_mode(struct eh_event_link_info *links,
lconf.inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in;
lconf.inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in;
lconf.inbound.sa_ctx = socket_ctx[socket_id].sa_in;
- lconf.inbound.session_pool = socket_ctx[socket_id].session_pool;
- lconf.inbound.session_priv_pool =
- socket_ctx[socket_id].session_priv_pool;
+ lconf.inbound.lcore_id = lcore_id;
lconf.outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out;
lconf.outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out;
lconf.outbound.sa_ctx = socket_ctx[socket_id].sa_out;
- lconf.outbound.session_pool = socket_ctx[socket_id].session_pool;
- lconf.outbound.session_priv_pool =
- socket_ctx[socket_id].session_priv_pool;
+ lconf.outbound.lcore_id = lcore_id;
RTE_LOG(INFO, IPSEC,
"Launching event mode worker (non-burst - Tx internal port - "
@@ -1227,7 +1227,8 @@ sa_add_address_inline_crypto(struct ipsec_sa *sa)
static int
sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
uint32_t nb_entries, uint32_t inbound,
- struct socket_ctx *skt_ctx)
+ struct socket_ctx *skt_ctx,
+ struct ipsec_ctx *ips_ctx[])
{
struct ipsec_sa *sa;
uint32_t i, idx;
@@ -1398,6 +1399,13 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
"create_inline_session() failed\n");
return -EINVAL;
}
+ } else {
+ rc = create_lookaside_session(ips_ctx, skt_ctx, sa, ips);
+ if (rc != 0) {
+ RTE_LOG(ERR, IPSEC_ESP,
+ "create_lookaside_session() failed\n");
+ return -EINVAL;
+ }
}
if (sa->fdir_flag && inbound) {
@@ -1414,16 +1422,18 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
static inline int
sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
- uint32_t nb_entries, struct socket_ctx *skt_ctx)
+ uint32_t nb_entries, struct socket_ctx *skt_ctx,
+ struct ipsec_ctx *ips_ctx[])
{
- return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
+ return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx, ips_ctx);
}
static inline int
sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
- uint32_t nb_entries, struct socket_ctx *skt_ctx)
+ uint32_t nb_entries, struct socket_ctx *skt_ctx,
+ struct ipsec_ctx *ips_ctx[])
{
- return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
+ return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx, ips_ctx);
}
/*
@@ -1497,14 +1507,9 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa)
ss->sa = sa;
- if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
- ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
- if (ss->security.ses != NULL) {
- rc = rte_ipsec_session_prepare(ss);
- if (rc != 0)
- memset(ss, 0, sizeof(*ss));
- }
- }
+ rc = rte_ipsec_session_prepare(ss);
+ if (rc != 0)
+ memset(ss, 0, sizeof(*ss));
return rc;
}
@@ -1643,10 +1648,13 @@ sa_spi_present(struct sa_ctx *sa_ctx, uint32_t spi, int inbound)
}
void
-sa_init(struct socket_ctx *ctx, int32_t socket_id)
+sa_init(struct socket_ctx *ctx, int32_t socket_id,
+ struct lcore_conf *lcore_conf)
{
int32_t rc;
const char *name;
+ uint32_t lcore_id;
+ struct ipsec_ctx *ipsec_ctx[RTE_MAX_LCORE];
if (ctx == NULL)
rte_exit(EXIT_FAILURE, "NULL context.\n");
@@ -1671,8 +1679,9 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
&sa_in_cnt);
if (rc != 0)
rte_exit(EXIT_FAILURE, "failed to init SAD\n");
-
- sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
+ RTE_LCORE_FOREACH(lcore_id)
+ ipsec_ctx[lcore_id] = &lcore_conf[lcore_id].inbound;
+ sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx, ipsec_ctx);
if (app_sa_prm.enable != 0) {
rc = ipsec_satbl_init(ctx->sa_in, nb_sa_in,
@@ -1692,7 +1701,9 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
"context %s in socket %d\n", rte_errno,
name, socket_id);
- sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
+ RTE_LCORE_FOREACH(lcore_id)
+ ipsec_ctx[lcore_id] = &lcore_conf[lcore_id].outbound;
+ sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx, ipsec_ctx);
if (app_sa_prm.enable != 0) {
rc = ipsec_satbl_init(ctx->sa_out, nb_sa_out,