Hi Fan,
Comments for inbound part inline.
As I can see majority of my v1 comments still are not addressed.
Please check.
Konstantin
>
> This patch updates the ipsec library to handle the newly introduced
> RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO action.
>
> Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com>
> ---
> lib/librte_ipsec/crypto.h | 24 +++
> lib/librte_ipsec/esp_inb.c | 200 ++++++++++++++++++++++--
> lib/librte_ipsec/esp_outb.c | 369 +++++++++++++++++++++++++++++++++++++++++---
> lib/librte_ipsec/sa.c | 53 ++++++-
> lib/librte_ipsec/sa.h | 29 ++++
> lib/librte_ipsec/ses.c | 4 +-
> 6 files changed, 643 insertions(+), 36 deletions(-)
>
> diff --git a/lib/librte_ipsec/crypto.h b/lib/librte_ipsec/crypto.h
> index f8fbf8d4f..901c8c7de 100644
> --- a/lib/librte_ipsec/crypto.h
> +++ b/lib/librte_ipsec/crypto.h
> @@ -179,4 +179,28 @@ lksd_none_cop_prepare(struct rte_crypto_op *cop,
> __rte_crypto_sym_op_attach_sym_session(sop, cs);
> }
>
> +typedef void* (*_set_icv_f)(void *val, struct rte_mbuf *ml, uint32_t icv_off);
> +
> +static inline void *
> +set_icv_va_pa(void *val, struct rte_mbuf *ml, uint32_t icv_off)
> +{
> + union sym_op_data *icv = val;
> +
> + icv->va = rte_pktmbuf_mtod_offset(ml, void *, icv_off);
> + icv->pa = rte_pktmbuf_iova_offset(ml, icv_off);
> +
> + return icv->va;
> +}
> +
> +static inline void *
> +set_icv_va(__rte_unused void *val, __rte_unused struct rte_mbuf *ml,
> + __rte_unused uint32_t icv_off)
> +{
> + void **icv_va = val;
> +
> + *icv_va = rte_pktmbuf_mtod_offset(ml, void *, icv_off);
> +
> + return *icv_va;
> +}
> +
> #endif /* _CRYPTO_H_ */
> diff --git a/lib/librte_ipsec/esp_inb.c b/lib/librte_ipsec/esp_inb.c
> index 8e3ecbc64..c4476e819 100644
> --- a/lib/librte_ipsec/esp_inb.c
> +++ b/lib/librte_ipsec/esp_inb.c
> @@ -105,6 +105,78 @@ inb_cop_prepare(struct rte_crypto_op *cop,
> }
> }
>
> +static inline int
> +inb_cpu_crypto_proc_prepare(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb,
> + uint32_t pofs, uint32_t plen,
> + struct rte_security_vec *buf, struct iovec *cur_vec,
> + void *iv)
> +{
> + struct rte_mbuf *ms;
> + struct iovec *vec = cur_vec;
> + struct aead_gcm_iv *gcm;
> + struct aesctr_cnt_blk *ctr;
> + uint64_t *ivp;
> + uint32_t algo;
> + uint32_t left;
> + uint32_t off = 0, n_seg = 0;
Same comment as for v1:
Please separate variable definition and value assignment.
It makes it hard to read, plus we don't do that in the rest of the library,
so better to follow rest of the code style.
> +
> + ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *,
> + pofs + sizeof(struct rte_esp_hdr));
> + algo = sa->algo_type;
> +
> + switch (algo) {
> + case ALGO_TYPE_AES_GCM:
> + gcm = (struct aead_gcm_iv *)iv;
> + aead_gcm_iv_fill(gcm, ivp[0], sa->salt);
> + off = sa->ctp.cipher.offset + pofs;
> + left = plen - sa->ctp.cipher.length;
> + break;
> + case ALGO_TYPE_AES_CBC:
> + case ALGO_TYPE_3DES_CBC:
> + copy_iv(iv, ivp, sa->iv_len);
> + off = sa->ctp.auth.offset + pofs;
> + left = plen - sa->ctp.auth.length;
> + break;
> + case ALGO_TYPE_AES_CTR:
> + copy_iv(iv, ivp, sa->iv_len);
> + off = sa->ctp.auth.offset + pofs;
> + left = plen - sa->ctp.auth.length;
> + ctr = (struct aesctr_cnt_blk *)iv;
> + aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);
> + break;
> + case ALGO_TYPE_NULL:
> + left = plen - sa->ctp.cipher.length;
> + break;
> + default:
> + return -EINVAL;
How we can endup here?
If we have an unknown algorithm, shouldn't we fail at init stage?
> + }
> +
> + ms = mbuf_get_seg_ofs(mb, &off);
> + if (!ms)
> + return -1;
Same comment as for v1:
inb_pkt_prepare() should already check that we have a valid packet.
I don't think there is a need to check for any failure here.
Another thing, our esp header will be in the first segment for sure,
so do we need get_seg_ofs() here at all?
> +
> + while (n_seg < RTE_LIBRTE_IP_FRAG_MAX_FRAG && left && ms) {
> + uint32_t len = RTE_MIN(left, ms->data_len - off);
Again, same comments as for v1:
- I don't think this is right, we shouldn't impose additional limitations to
the number of segments in the packet.
- Whole construction seems a bit over-complicated here...
Why just not have a separate function that would dill iovec[] from mbuf
And return an error if there is not enough iovec[] entries?
Something like:
static inline int
mbuf_to_iovec(const struct rte_mbuf *mb, uint32_t ofs, uint32_t len, struct iovec vec[], uint32_t num)
{
uint32_t i;
if (mb->nb_seg > num)
return - mb->nb_seg;
vec[0].iov_base = rte_pktmbuf_mtod_offset(mb, void *, off);
vec[0].iov_len = mb->data_len - off;
for (i = 1, ms = mb->next; mb != NULL; ms = ms->next, i++) {
vec[i].iov_base = rte_pktmbuf_mtod(ms);
vec[i].iov_len = ms->data_len;
}
vec[i].iov_len -= mb->pkt_len - len;
return i;
}
Then we can use that function to fill our iovec[] in a loop.
L- ooking at this function, it seems to consist of 2 separate parts:
1. calculates offset and generates iv
2. setup iovec[].
Probably worth to split it into 2 separate functions like that.
Would be much easier to read/understand.
> +
> + vec->iov_base = rte_pktmbuf_mtod_offset(ms, void *, off);
> + vec->iov_len = len;
> +
> + left -= len;
> + vec++;
> + n_seg++;
> + ms = ms->next;
> + off = 0;
> + }
> +
> + if (left)
> + return -1;
> +
> + buf->vec = cur_vec;
> + buf->num = n_seg;
> +
> + return n_seg;
> +}
> +
> /*
> * Helper function for prepare() to deal with situation when
> * ICV is spread by two segments. Tries to move ICV completely into the
> @@ -139,20 +211,21 @@ move_icv(struct rte_mbuf *ml, uint32_t ofs)
> */
> static inline void
> inb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,
> - const union sym_op_data *icv)
> + uint8_t *icv_va, void *aad_buf, uint32_t aad_off)
> {
> struct aead_gcm_aad *aad;
>
> /* insert SQN.hi between ESP trailer and ICV */
> if (sa->sqh_len != 0)
> - insert_sqh(sqn_hi32(sqc), icv->va, sa->icv_len);
> + insert_sqh(sqn_hi32(sqc), icv_va, sa->icv_len);
>
> /*
> * fill AAD fields, if any (aad fields are placed after icv),
> * right now we support only one AEAD algorithm: AES-GCM.
> */
> if (sa->aad_len != 0) {
> - aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len);
> + aad = aad_buf ? aad_buf :
> + (struct aead_gcm_aad *)(icv_va + aad_off);
> aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa));
> }
> }
> @@ -162,13 +235,15 @@ inb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,
> */
> static inline int32_t
> inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn,
> - struct rte_mbuf *mb, uint32_t hlen, union sym_op_data *icv)
> + struct rte_mbuf *mb, uint32_t hlen, _set_icv_f set_icv, void *icv_val,
> + void *aad_buf)
This whole construct with another function pointer , overloaded arguments, etc.,
looks a bit clumsy and overcomplicated.
I think it would be much cleaner and easier to re-arrange the code like that:
1. update inb_pkt_xprepare to take aad buffer pointer and aad len as a parameters:
static inline void
inb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,
const union sym_op_data *icv, void *aad, uint32_t aad_len)
{
/* insert SQN.hi between ESP trailer and ICV */
if (sa->sqh_len != 0)
insert_sqh(sqn_hi32(sqc), icv->va, sa->icv_len);
/*
* fill AAD fields, if any (aad fields are placed after icv),
* right now we support only one AEAD algorithm: AES-GCM.
*/
if (aad_len != 0)
aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa));
}
2. split inb_pkt_prepare() into 2 reusable helper functions:
*
* retrieve and reconstruct SQN, then check it, then
* convert it back into network byte order.
*/
static inline int
inb_get_sqn(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn,
struct rte_mbuf *mb, uint32_t hlen, rte_be64_t *sqc)
{
int32_t rc;
uint64_t sqn;
struct rte_esp_hdr *esph;
esph = rte_pktmbuf_mtod_offset(mb, struct rte_esp_hdr *, hlen);
/*
* retrieve and reconstruct SQN, then check it, then
* convert it back into network byte order.
*/
sqn = rte_be_to_cpu_32(esph->seq);
if (IS_ESN(sa))
sqn = reconstruct_esn(rsn->sqn, sqn, sa->replay.win_sz);
rc = esn_inb_check_sqn(rsn, sa, sqn);
if (rc == 0)
*sqc = rte_cpu_to_be_64(sqn);
return rc;
}
static inline int32_t
inb_prepare(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb,
uint32_t hlen, uint32_t aad_len, union sym_op_data *icv)
{
uint32_t clen, icv_len, icv_ofs, plen;
struct rte_mbuf *ml;
/* start packet manipulation */
plen = mb->pkt_len;
plen = plen - hlen;
/* check that packet has a valid length */
clen = plen - sa->ctp.cipher.length;
if ((int32_t)clen < 0 || (clen & (sa->pad_align - 1)) != 0)
return -EBADMSG;
/* find ICV location */
icv_len = sa->icv_len;
icv_ofs = mb->pkt_len - icv_len;
ml = mbuf_get_seg_ofs(mb, &icv_ofs);
/*
* if ICV is spread by two segments, then try to
* move ICV completely into the last segment.
*/
if (ml->data_len < icv_ofs + icv_len) {
ml = move_icv(ml, icv_ofs);
if (ml == NULL)
return -ENOSPC;
/* new ICV location */
icv_ofs = 0;
}
icv_ofs += sa->sqh_len;
/* we have to allocate space for AAD somewhere,
* right now - just use free trailing space at the last segment.
* Would probably be more convenient to reserve space for AAD
* inside rte_crypto_op itself
* (again for IV space is already reserved inside cop).
*/
if (aad_len + sa->sqh_len > rte_pktmbuf_tailroom(ml))
return -ENOSPC;
icv->va = rte_pktmbuf_mtod_offset(ml, void *, icv_ofs);
icv->pa = rte_pktmbuf_iova_offset(ml, icv_ofs);
/*
* if esn is used then high-order 32 bits are also used in ICV
* calculation but are not transmitted, update packet length
* to be consistent with auth data length and offset, this will
* be subtracted from packet length in post crypto processing
*/
mb->pkt_len += sa->sqh_len;
ml->data_len += sa->sqh_len;
return plen;
}
3. Now inb_pkt_prepare() becomes a simple sequential invocation of these 3 sub-functions
with right parameters:
static inline int32_t
inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn,
struct rte_mbuf *mb, uint32_t hlen, union sym_op_data *icv)
{
int32_t rc;
uint64_t sqn;
rc = inb_get_sqn(sa, rsn, mb, hlen, &sqn);
if (rc != 0)
return rc;
rc = inb_prepare(sa, mb, hlen, sa->aad_len, icv);
if (rc < 0)
return rc;
inb_pkt_xprepare(sa, sqn, icv, icv->va + sa->icv_len, sa->aad_len);
return rc;
}
4. And that would be version of inb_pkt_prepare for cpu_cypto:
static inline int32_t
inb_cpu_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn,
struct rte_mbuf *mb, uint32_t hlen, union sym_op_data *icv, void *aad)
{
int32_t rc;
uint64_t sqn;
rc = inb_get_sqn(sa, rsn, mb, hlen, &sqn);
if (rc != 0)
return rc;
rc = inb_prepare(sa, mb, hlen, 0, icv);
if (rc < 0)
return rc;
inb_pkt_xprepare(sa, sqn, icv, aad, sa->aad_len);
return rc;
}
> {
> int32_t rc;
> uint64_t sqn;
> uint32_t clen, icv_len, icv_ofs, plen;
> struct rte_mbuf *ml;
> struct rte_esp_hdr *esph;
> + void *icv_va;
>
> esph = rte_pktmbuf_mtod_offset(mb, struct rte_esp_hdr *, hlen);
>
> @@ -226,8 +301,8 @@ inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn,
> if (sa->aad_len + sa->sqh_len > rte_pktmbuf_tailroom(ml))
> return -ENOSPC;
>
> - icv->va = rte_pktmbuf_mtod_offset(ml, void *, icv_ofs);
> - icv->pa = rte_pktmbuf_iova_offset(ml, icv_ofs);
> + icv_va = set_icv(icv_val, ml, icv_ofs);
> + inb_pkt_xprepare(sa, sqn, icv_va, aad_buf, sa->icv_len);
>
> /*
> * if esn is used then high-order 32 bits are also used in ICV
> @@ -238,7 +313,6 @@ inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn,
> mb->pkt_len += sa->sqh_len;
> ml->data_len += sa->sqh_len;
>
> - inb_pkt_xprepare(sa, sqn, icv);
> return plen;
> }
>
> @@ -265,7 +339,8 @@ esp_inb_pkt_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
> for (i = 0; i != num; i++) {
>
> hl = mb[i]->l2_len + mb[i]->l3_len;
> - rc = inb_pkt_prepare(sa, rsn, mb[i], hl, &icv);
> + rc = inb_pkt_prepare(sa, rsn, mb[i], hl, set_icv_va_pa,
> + (void *)&icv, NULL);
> if (rc >= 0) {
> lksd_none_cop_prepare(cop[k], cs, mb[i]);
> inb_cop_prepare(cop[k], sa, mb[i], &icv, hl, rc);
> @@ -512,7 +587,6 @@ tun_process(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],
> return k;
> }
>
> -
> /*
> * *process* function for tunnel packets
> */
> @@ -625,6 +699,114 @@ esp_inb_pkt_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],
> return n;
> }
>
> +/*
> + * process packets using sync crypto engine
> + */
> +static uint16_t
> +esp_inb_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss,
> + struct rte_mbuf *mb[], uint16_t num,
> + esp_inb_process_t process)
> +{
> + int32_t rc;
> + uint32_t i, hl, n, p;
> + struct rte_ipsec_sa *sa;
> + struct replay_sqn *rsn;
> + void *icv_va;
> + uint32_t sqn[num];
> + uint32_t dr[num];
> + uint8_t sqh_len;
> +
> + /* cpu crypto specific variables */
> + struct rte_security_vec buf[num];
> + struct iovec vec[RTE_LIBRTE_IP_FRAG_MAX_FRAG * num];
Same comment as for v1:
I don't think this is right, we shouldn't impose additional limitations to
the number of segments in the packet.
> + uint32_t vec_idx = 0;
> + uint64_t iv_buf[num][IPSEC_MAX_IV_QWORD];
> + void *iv[num];
> + int status[num];
> + uint8_t *aad_buf[num][sizeof(struct aead_gcm_aad)];
> + void *aad[num];
> + void *digest[num];
> + uint32_t k;
> +
> + sa = ss->sa;
> + rsn = rsn_acquire(sa);
> + sqh_len = sa->sqh_len;
> +
> + k = 0;
> + for (i = 0; i != num; i++) {
> + hl = mb[i]->l2_len + mb[i]->l3_len;
> + rc = inb_pkt_prepare(sa, rsn, mb[i], hl, set_icv_va,
> + (void *)&icv_va, (void *)aad_buf[k]);
Better to have separate function similar to inb_pkt_prepare(), see above.
> + if (rc >= 0) {
> + iv[k] = (void *)iv_buf[k];
> + aad[k] = (void *)aad_buf[k];
> + digest[k] = (void *)icv_va;
> +
> + rc = inb_cpu_crypto_proc_prepare(sa, mb[i], hl,
> + rc, &buf[k], &vec[vec_idx], iv[k]);
> + if (rc < 0) {
> + dr[i - k] = i;
I think with you current aproach you can't do like that.
As your vec[] still can contain some entries from fail mbuf.
So in theoryyou need to cleanup these entries which is quite complicated,
better to avoid such case at all.
I think you need to reorder the code, as described above.
> + continue;
> + }
> +
> + vec_idx += rc;
> + k++;
> + } else
> + dr[i - k] = i;
> + }
> +
> + /* copy not prepared mbufs beyond good ones */
> + if (k != num) {
> + rte_errno = EBADMSG;
> +
> + if (unlikely(k == 0))
> + return 0;
> +
> + move_bad_mbufs(mb, dr, num, num - k);
> + }
> +
> + /* process the packets */
> + n = 0;
> + rc = rte_security_process_cpu_crypto_bulk(ss->security.ctx,
> + ss->security.ses, buf, iv, aad, digest, status, k);
> + /* move failed process packets to dr */
Same comment as for v1:
That just doesn't look right, instead of updating dr[] and move_bad_mbufs(), you need to:
if (rc != 0) {walk through status[] and for failed ones set PKT_RX_SEC_OFFLOAD_FAILED in appropriate mbuf->ol_flags}.
tun_process(), etc. expects PKT_RX_SEC_OFFLOAD_FAILED to be set in mb->ol_flags
for failed packets.
> + for (i = 0; i < k; i++) {
> + if (status[i]) {
> + dr[n++] = i;
> + rte_errno = EBADMSG;
> + }
> + }
> +
> + /* move bad packets to the back */
> + if (n)
> + move_bad_mbufs(mb, dr, k, n);
> +
> + /* process packets */
> + p = process(sa, mb, sqn, dr, k - n, sqh_len);
> +
> + if (p != k - n && p != 0)
> + move_bad_mbufs(mb, dr, k - n, k - n - p);
> +
> + if (p != num)
> + rte_errno = EBADMSG;
> +
> + return p;
> +}
> +
> +uint16_t
> +esp_inb_tun_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss,
> + struct rte_mbuf *mb[], uint16_t num)
> +{
> + return esp_inb_cpu_crypto_pkt_process(ss, mb, num, tun_process);
> +}
> +
> +uint16_t
> +esp_inb_trs_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss,
> + struct rte_mbuf *mb[], uint16_t num)
> +{
> + return esp_inb_cpu_crypto_pkt_process(ss, mb, num, trs_process);
> +}
> +
@@ -179,4 +179,28 @@ lksd_none_cop_prepare(struct rte_crypto_op *cop,
__rte_crypto_sym_op_attach_sym_session(sop, cs);
}
+typedef void* (*_set_icv_f)(void *val, struct rte_mbuf *ml, uint32_t icv_off);
+
+static inline void *
+set_icv_va_pa(void *val, struct rte_mbuf *ml, uint32_t icv_off)
+{
+ union sym_op_data *icv = val;
+
+ icv->va = rte_pktmbuf_mtod_offset(ml, void *, icv_off);
+ icv->pa = rte_pktmbuf_iova_offset(ml, icv_off);
+
+ return icv->va;
+}
+
+static inline void *
+set_icv_va(__rte_unused void *val, __rte_unused struct rte_mbuf *ml,
+ __rte_unused uint32_t icv_off)
+{
+ void **icv_va = val;
+
+ *icv_va = rte_pktmbuf_mtod_offset(ml, void *, icv_off);
+
+ return *icv_va;
+}
+
#endif /* _CRYPTO_H_ */
@@ -105,6 +105,78 @@ inb_cop_prepare(struct rte_crypto_op *cop,
}
}
+static inline int
+inb_cpu_crypto_proc_prepare(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb,
+ uint32_t pofs, uint32_t plen,
+ struct rte_security_vec *buf, struct iovec *cur_vec,
+ void *iv)
+{
+ struct rte_mbuf *ms;
+ struct iovec *vec = cur_vec;
+ struct aead_gcm_iv *gcm;
+ struct aesctr_cnt_blk *ctr;
+ uint64_t *ivp;
+ uint32_t algo;
+ uint32_t left;
+ uint32_t off = 0, n_seg = 0;
+
+ ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *,
+ pofs + sizeof(struct rte_esp_hdr));
+ algo = sa->algo_type;
+
+ switch (algo) {
+ case ALGO_TYPE_AES_GCM:
+ gcm = (struct aead_gcm_iv *)iv;
+ aead_gcm_iv_fill(gcm, ivp[0], sa->salt);
+ off = sa->ctp.cipher.offset + pofs;
+ left = plen - sa->ctp.cipher.length;
+ break;
+ case ALGO_TYPE_AES_CBC:
+ case ALGO_TYPE_3DES_CBC:
+ copy_iv(iv, ivp, sa->iv_len);
+ off = sa->ctp.auth.offset + pofs;
+ left = plen - sa->ctp.auth.length;
+ break;
+ case ALGO_TYPE_AES_CTR:
+ copy_iv(iv, ivp, sa->iv_len);
+ off = sa->ctp.auth.offset + pofs;
+ left = plen - sa->ctp.auth.length;
+ ctr = (struct aesctr_cnt_blk *)iv;
+ aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);
+ break;
+ case ALGO_TYPE_NULL:
+ left = plen - sa->ctp.cipher.length;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ ms = mbuf_get_seg_ofs(mb, &off);
+ if (!ms)
+ return -1;
+
+ while (n_seg < RTE_LIBRTE_IP_FRAG_MAX_FRAG && left && ms) {
+ uint32_t len = RTE_MIN(left, ms->data_len - off);
+
+ vec->iov_base = rte_pktmbuf_mtod_offset(ms, void *, off);
+ vec->iov_len = len;
+
+ left -= len;
+ vec++;
+ n_seg++;
+ ms = ms->next;
+ off = 0;
+ }
+
+ if (left)
+ return -1;
+
+ buf->vec = cur_vec;
+ buf->num = n_seg;
+
+ return n_seg;
+}
+
/*
* Helper function for prepare() to deal with situation when
* ICV is spread by two segments. Tries to move ICV completely into the
@@ -139,20 +211,21 @@ move_icv(struct rte_mbuf *ml, uint32_t ofs)
*/
static inline void
inb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,
- const union sym_op_data *icv)
+ uint8_t *icv_va, void *aad_buf, uint32_t aad_off)
{
struct aead_gcm_aad *aad;
/* insert SQN.hi between ESP trailer and ICV */
if (sa->sqh_len != 0)
- insert_sqh(sqn_hi32(sqc), icv->va, sa->icv_len);
+ insert_sqh(sqn_hi32(sqc), icv_va, sa->icv_len);
/*
* fill AAD fields, if any (aad fields are placed after icv),
* right now we support only one AEAD algorithm: AES-GCM.
*/
if (sa->aad_len != 0) {
- aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len);
+ aad = aad_buf ? aad_buf :
+ (struct aead_gcm_aad *)(icv_va + aad_off);
aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa));
}
}
@@ -162,13 +235,15 @@ inb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,
*/
static inline int32_t
inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn,
- struct rte_mbuf *mb, uint32_t hlen, union sym_op_data *icv)
+ struct rte_mbuf *mb, uint32_t hlen, _set_icv_f set_icv, void *icv_val,
+ void *aad_buf)
{
int32_t rc;
uint64_t sqn;
uint32_t clen, icv_len, icv_ofs, plen;
struct rte_mbuf *ml;
struct rte_esp_hdr *esph;
+ void *icv_va;
esph = rte_pktmbuf_mtod_offset(mb, struct rte_esp_hdr *, hlen);
@@ -226,8 +301,8 @@ inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn,
if (sa->aad_len + sa->sqh_len > rte_pktmbuf_tailroom(ml))
return -ENOSPC;
- icv->va = rte_pktmbuf_mtod_offset(ml, void *, icv_ofs);
- icv->pa = rte_pktmbuf_iova_offset(ml, icv_ofs);
+ icv_va = set_icv(icv_val, ml, icv_ofs);
+ inb_pkt_xprepare(sa, sqn, icv_va, aad_buf, sa->icv_len);
/*
* if esn is used then high-order 32 bits are also used in ICV
@@ -238,7 +313,6 @@ inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn,
mb->pkt_len += sa->sqh_len;
ml->data_len += sa->sqh_len;
- inb_pkt_xprepare(sa, sqn, icv);
return plen;
}
@@ -265,7 +339,8 @@ esp_inb_pkt_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
for (i = 0; i != num; i++) {
hl = mb[i]->l2_len + mb[i]->l3_len;
- rc = inb_pkt_prepare(sa, rsn, mb[i], hl, &icv);
+ rc = inb_pkt_prepare(sa, rsn, mb[i], hl, set_icv_va_pa,
+ (void *)&icv, NULL);
if (rc >= 0) {
lksd_none_cop_prepare(cop[k], cs, mb[i]);
inb_cop_prepare(cop[k], sa, mb[i], &icv, hl, rc);
@@ -512,7 +587,6 @@ tun_process(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],
return k;
}
-
/*
* *process* function for tunnel packets
*/
@@ -625,6 +699,114 @@ esp_inb_pkt_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],
return n;
}
+/*
+ * process packets using sync crypto engine
+ */
+static uint16_t
+esp_inb_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num,
+ esp_inb_process_t process)
+{
+ int32_t rc;
+ uint32_t i, hl, n, p;
+ struct rte_ipsec_sa *sa;
+ struct replay_sqn *rsn;
+ void *icv_va;
+ uint32_t sqn[num];
+ uint32_t dr[num];
+ uint8_t sqh_len;
+
+ /* cpu crypto specific variables */
+ struct rte_security_vec buf[num];
+ struct iovec vec[RTE_LIBRTE_IP_FRAG_MAX_FRAG * num];
+ uint32_t vec_idx = 0;
+ uint64_t iv_buf[num][IPSEC_MAX_IV_QWORD];
+ void *iv[num];
+ int status[num];
+ uint8_t *aad_buf[num][sizeof(struct aead_gcm_aad)];
+ void *aad[num];
+ void *digest[num];
+ uint32_t k;
+
+ sa = ss->sa;
+ rsn = rsn_acquire(sa);
+ sqh_len = sa->sqh_len;
+
+ k = 0;
+ for (i = 0; i != num; i++) {
+ hl = mb[i]->l2_len + mb[i]->l3_len;
+ rc = inb_pkt_prepare(sa, rsn, mb[i], hl, set_icv_va,
+ (void *)&icv_va, (void *)aad_buf[k]);
+ if (rc >= 0) {
+ iv[k] = (void *)iv_buf[k];
+ aad[k] = (void *)aad_buf[k];
+ digest[k] = (void *)icv_va;
+
+ rc = inb_cpu_crypto_proc_prepare(sa, mb[i], hl,
+ rc, &buf[k], &vec[vec_idx], iv[k]);
+ if (rc < 0) {
+ dr[i - k] = i;
+ continue;
+ }
+
+ vec_idx += rc;
+ k++;
+ } else
+ dr[i - k] = i;
+ }
+
+ /* copy not prepared mbufs beyond good ones */
+ if (k != num) {
+ rte_errno = EBADMSG;
+
+ if (unlikely(k == 0))
+ return 0;
+
+ move_bad_mbufs(mb, dr, num, num - k);
+ }
+
+ /* process the packets */
+ n = 0;
+ rc = rte_security_process_cpu_crypto_bulk(ss->security.ctx,
+ ss->security.ses, buf, iv, aad, digest, status, k);
+ /* move failed process packets to dr */
+ for (i = 0; i < k; i++) {
+ if (status[i]) {
+ dr[n++] = i;
+ rte_errno = EBADMSG;
+ }
+ }
+
+ /* move bad packets to the back */
+ if (n)
+ move_bad_mbufs(mb, dr, k, n);
+
+ /* process packets */
+ p = process(sa, mb, sqn, dr, k - n, sqh_len);
+
+ if (p != k - n && p != 0)
+ move_bad_mbufs(mb, dr, k - n, k - n - p);
+
+ if (p != num)
+ rte_errno = EBADMSG;
+
+ return p;
+}
+
+uint16_t
+esp_inb_tun_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num)
+{
+ return esp_inb_cpu_crypto_pkt_process(ss, mb, num, tun_process);
+}
+
+uint16_t
+esp_inb_trs_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num)
+{
+ return esp_inb_cpu_crypto_pkt_process(ss, mb, num, trs_process);
+}
+
/*
* process group of ESP inbound tunnel packets.
*/
@@ -104,7 +104,7 @@ outb_cop_prepare(struct rte_crypto_op *cop,
static inline int32_t
outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,
const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb,
- union sym_op_data *icv, uint8_t sqh_len)
+ _set_icv_f set_icv, void *icv_val, uint8_t sqh_len)
{
uint32_t clen, hlen, l2len, pdlen, pdofs, plen, tlen;
struct rte_mbuf *ml;
@@ -177,8 +177,8 @@ outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,
espt->pad_len = pdlen;
espt->next_proto = sa->proto;
- icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs);
- icv->pa = rte_pktmbuf_iova_offset(ml, pdofs);
+ /* set icv va/pa value(s) */
+ set_icv(icv_val, ml, pdofs);
return clen;
}
@@ -189,14 +189,14 @@ outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,
*/
static inline void
outb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,
- const union sym_op_data *icv)
+ uint8_t *icv_va, void *aad_buf)
{
uint32_t *psqh;
struct aead_gcm_aad *aad;
/* insert SQN.hi between ESP trailer and ICV */
if (sa->sqh_len != 0) {
- psqh = (uint32_t *)(icv->va - sa->sqh_len);
+ psqh = (uint32_t *)(icv_va - sa->sqh_len);
psqh[0] = sqn_hi32(sqc);
}
@@ -205,7 +205,7 @@ outb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,
* right now we support only one AEAD algorithm: AES-GCM .
*/
if (sa->aad_len != 0) {
- aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len);
+ aad = aad_buf;
aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa));
}
}
@@ -242,11 +242,12 @@ esp_outb_tun_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
gen_iv(iv, sqc);
/* try to update the packet itself */
- rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv,
- sa->sqh_len);
+ rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], set_icv_va_pa,
+ (void *)&icv, sa->sqh_len);
/* success, setup crypto op */
if (rc >= 0) {
- outb_pkt_xprepare(sa, sqc, &icv);
+ outb_pkt_xprepare(sa, sqc, icv.va,
+ (void *)(icv.va + sa->icv_len));
lksd_none_cop_prepare(cop[k], cs, mb[i]);
outb_cop_prepare(cop[k], sa, iv, &icv, 0, rc);
k++;
@@ -270,7 +271,7 @@ esp_outb_tun_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
static inline int32_t
outb_trs_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,
const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb,
- uint32_t l2len, uint32_t l3len, union sym_op_data *icv,
+ uint32_t l2len, uint32_t l3len, _set_icv_f set_icv, void *icv_val,
uint8_t sqh_len)
{
uint8_t np;
@@ -340,8 +341,7 @@ outb_trs_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,
espt->pad_len = pdlen;
espt->next_proto = np;
- icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs);
- icv->pa = rte_pktmbuf_iova_offset(ml, pdofs);
+ set_icv(icv_val, ml, pdofs);
return clen;
}
@@ -381,11 +381,12 @@ esp_outb_trs_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
gen_iv(iv, sqc);
/* try to update the packet itself */
- rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i], l2, l3, &icv,
- sa->sqh_len);
+ rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i], l2, l3,
+ set_icv_va_pa, (void *)&icv, sa->sqh_len);
/* success, setup crypto op */
if (rc >= 0) {
- outb_pkt_xprepare(sa, sqc, &icv);
+ outb_pkt_xprepare(sa, sqc, icv.va,
+ (void *)(icv.va + sa->icv_len));
lksd_none_cop_prepare(cop[k], cs, mb[i]);
outb_cop_prepare(cop[k], sa, iv, &icv, l2 + l3, rc);
k++;
@@ -403,6 +404,335 @@ esp_outb_trs_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
return k;
}
+
+static inline int
+outb_cpu_crypto_proc_prepare(struct rte_mbuf *m, const struct rte_ipsec_sa *sa,
+ uint32_t hlen, uint32_t plen,
+ struct rte_security_vec *buf, struct iovec *cur_vec, void *iv)
+{
+ struct rte_mbuf *ms;
+ uint64_t *ivp = iv;
+ struct aead_gcm_iv *gcm;
+ struct aesctr_cnt_blk *ctr;
+ struct iovec *vec = cur_vec;
+ uint32_t left;
+ uint32_t off = 0;
+ uint32_t n_seg = 0;
+ uint32_t algo;
+
+ algo = sa->algo_type;
+
+ switch (algo) {
+ case ALGO_TYPE_AES_GCM:
+ gcm = iv;
+ aead_gcm_iv_fill(gcm, ivp[0], sa->salt);
+ off = sa->ctp.cipher.offset + hlen;
+ left = sa->ctp.cipher.length + plen;
+ break;
+ case ALGO_TYPE_AES_CBC:
+ case ALGO_TYPE_3DES_CBC:
+ off = sa->ctp.auth.offset + hlen;
+ left = sa->ctp.auth.length + plen;
+ break;
+ case ALGO_TYPE_AES_CTR:
+ off = sa->ctp.auth.offset + hlen;
+ left = sa->ctp.auth.length + plen;
+ ctr = iv;
+ aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);
+ break;
+ case ALGO_TYPE_NULL:
+ left = sa->ctp.cipher.length + plen;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ ms = mbuf_get_seg_ofs(m, &off);
+ if (!ms)
+ return -1;
+
+ while (n_seg < m->nb_segs && left && ms) {
+ uint32_t len = RTE_MIN(left, ms->data_len - off);
+
+ vec->iov_base = rte_pktmbuf_mtod_offset(ms, void *, off);
+ vec->iov_len = len;
+
+ left -= len;
+ vec++;
+ n_seg++;
+ ms = ms->next;
+ off = 0;
+ }
+
+ if (left)
+ return -1;
+
+ buf->vec = cur_vec;
+ buf->num = n_seg;
+
+ return n_seg;
+}
+
+static uint16_t
+esp_outb_tun_cpu_crypto_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num)
+{
+ uint64_t sqn;
+ rte_be64_t sqc;
+ struct rte_ipsec_sa *sa;
+ struct rte_security_ctx *ctx;
+ struct rte_security_session *rss;
+ void *icv_va;
+ uint32_t dr[num];
+ uint32_t i, n;
+ int32_t rc;
+
+ /* cpu crypto specific variables */
+ struct rte_security_vec buf[num];
+ struct iovec vec[RTE_LIBRTE_IP_FRAG_MAX_FRAG * num];
+ uint32_t vec_idx = 0;
+ uint64_t iv_buf[num][IPSEC_MAX_IV_QWORD];
+ void *iv[num];
+ int status[num];
+ uint8_t *aad_buf[num][sizeof(struct aead_gcm_aad)];
+ void *aad[num];
+ void *digest[num];
+ uint32_t k;
+
+ sa = ss->sa;
+ ctx = ss->security.ctx;
+ rss = ss->security.ses;
+
+ k = 0;
+ n = num;
+ sqn = esn_outb_update_sqn(sa, &n);
+ if (n != num)
+ rte_errno = EOVERFLOW;
+
+ for (i = 0; i != n; i++) {
+ sqc = rte_cpu_to_be_64(sqn + i);
+ gen_iv(iv_buf[k], sqc);
+
+ /* try to update the packet itself */
+ rc = outb_tun_pkt_prepare(sa, sqc, iv_buf[k], mb[i], set_icv_va,
+ (void *)&icv_va, sa->sqh_len);
+
+ /* success, setup crypto op */
+ if (rc >= 0) {
+ iv[k] = (void *)iv_buf[k];
+ aad[k] = (void *)aad_buf[k];
+ digest[k] = (void *)icv_va;
+
+ outb_pkt_xprepare(sa, sqc, icv_va, aad[k]);
+
+ rc = outb_cpu_crypto_proc_prepare(mb[i], sa,
+ 0, rc, &buf[k], &vec[vec_idx], iv[k]);
+ if (rc < 0) {
+ dr[i - k] = i;
+ rte_errno = -rc;
+ continue;
+ }
+
+ vec_idx += rc;
+ k++;
+ /* failure, put packet into the death-row */
+ } else {
+ dr[i - k] = i;
+ rte_errno = -rc;
+ }
+ }
+
+ /* copy not prepared mbufs beyond good ones */
+ if (k != n && k != 0)
+ move_bad_mbufs(mb, dr, n, n - k);
+
+ if (unlikely(k == 0)) {
+ rte_errno = EBADMSG;
+ return 0;
+ }
+
+ /* process the packets */
+ n = 0;
+ rc = rte_security_process_cpu_crypto_bulk(ctx, rss, buf, iv, aad,
+ digest, status, k);
+ /* move failed process packets to dr */
+ if (rc < 0)
+ for (i = 0; i < n; i++) {
+ if (status[i])
+ dr[n++] = i;
+ }
+
+ if (n)
+ move_bad_mbufs(mb, dr, k, n);
+
+ return k - n;
+}
+
+static uint16_t
+esp_outb_trs_cpu_crypto_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num)
+
+{
+ uint64_t sqn;
+ rte_be64_t sqc;
+ struct rte_ipsec_sa *sa;
+ struct rte_security_ctx *ctx;
+ struct rte_security_session *rss;
+ void *icv_va;
+ uint32_t dr[num];
+ uint32_t i, n;
+ uint32_t l2, l3;
+ int32_t rc;
+
+ /* cpu crypto specific variables */
+ struct rte_security_vec buf[num];
+ struct iovec vec[RTE_LIBRTE_IP_FRAG_MAX_FRAG * num];
+ uint32_t vec_idx = 0;
+ uint64_t iv_buf[num][IPSEC_MAX_IV_QWORD];
+ void *iv[num];
+ int status[num];
+ uint8_t *aad_buf[num][sizeof(struct aead_gcm_aad)];
+ void *aad[num];
+ void *digest[num];
+ uint32_t k;
+
+ sa = ss->sa;
+ ctx = ss->security.ctx;
+ rss = ss->security.ses;
+
+ k = 0;
+ n = num;
+ sqn = esn_outb_update_sqn(sa, &n);
+ if (n != num)
+ rte_errno = EOVERFLOW;
+
+ for (i = 0; i != n; i++) {
+ l2 = mb[i]->l2_len;
+ l3 = mb[i]->l3_len;
+
+ sqc = rte_cpu_to_be_64(sqn + i);
+ gen_iv(iv_buf[k], sqc);
+
+ /* try to update the packet itself */
+ rc = outb_trs_pkt_prepare(sa, sqc, iv_buf[k], mb[i], l2, l3,
+ set_icv_va, (void *)&icv_va, sa->sqh_len);
+
+ /* success, setup crypto op */
+ if (rc >= 0) {
+ iv[k] = (void *)iv_buf[k];
+ aad[k] = (void *)aad_buf[k];
+ digest[k] = (void *)icv_va;
+
+ outb_pkt_xprepare(sa, sqc, icv_va, aad[k]);
+
+ rc = outb_cpu_crypto_proc_prepare(mb[i], sa,
+ l2 + l3, rc, &buf[k], &vec[vec_idx],
+ iv[k]);
+ if (rc < 0) {
+ dr[i - k] = i;
+ rte_errno = -rc;
+ continue;
+ }
+
+ vec_idx += rc;
+ k++;
+ /* failure, put packet into the death-row */
+ } else {
+ dr[i - k] = i;
+ rte_errno = -rc;
+ }
+ }
+
+ /* copy not prepared mbufs beyond good ones */
+ if (k != n && k != 0)
+ move_bad_mbufs(mb, dr, n, n - k);
+
+ /* process the packets */
+ n = 0;
+ rc = rte_security_process_cpu_crypto_bulk(ctx, rss, buf, iv, aad,
+ digest, status, k);
+ /* move failed process packets to dr */
+ if (rc < 0)
+ for (i = 0; i < k; i++) {
+ if (status[i])
+ dr[n++] = i;
+ }
+
+ if (n)
+ move_bad_mbufs(mb, dr, k, n);
+
+ return k - n;
+}
+
+uint16_t
+esp_outb_tun_cpu_crypto_sqh_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num)
+{
+ struct rte_ipsec_sa *sa = ss->sa;
+ uint32_t icv_len;
+ void *icv;
+ uint16_t n;
+ uint16_t i;
+
+ n = esp_outb_tun_cpu_crypto_process(ss, mb, num);
+
+ icv_len = sa->icv_len;
+
+ for (i = 0; i < n; i++) {
+ struct rte_mbuf *ml = rte_pktmbuf_lastseg(mb[i]);
+
+ mb[i]->pkt_len -= sa->sqh_len;
+ ml->data_len -= sa->sqh_len;
+
+ icv = rte_pktmbuf_mtod_offset(ml, void *,
+ ml->data_len - icv_len);
+ remove_sqh(icv, sa->icv_len);
+ }
+
+ return n;
+}
+
+uint16_t
+esp_outb_tun_cpu_crypto_flag_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num)
+{
+ return esp_outb_tun_cpu_crypto_process(ss, mb, num);
+}
+
+uint16_t
+esp_outb_trs_cpu_crypto_sqh_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num)
+{
+ struct rte_ipsec_sa *sa = ss->sa;
+ uint32_t icv_len;
+ void *icv;
+ uint16_t n;
+ uint16_t i;
+
+ n = esp_outb_trs_cpu_crypto_process(ss, mb, num);
+ icv_len = sa->icv_len;
+
+ for (i = 0; i < n; i++) {
+ struct rte_mbuf *ml = rte_pktmbuf_lastseg(mb[i]);
+
+ mb[i]->pkt_len -= sa->sqh_len;
+ ml->data_len -= sa->sqh_len;
+
+ icv = rte_pktmbuf_mtod_offset(ml, void *,
+ ml->data_len - icv_len);
+ remove_sqh(icv, sa->icv_len);
+ }
+
+ return n;
+}
+
+uint16_t
+esp_outb_trs_cpu_crypto_flag_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num)
+{
+ return esp_outb_trs_cpu_crypto_process(ss, mb, num);
+}
+
/*
* process outbound packets for SA with ESN support,
* for algorithms that require SQN.hibits to be implictly included
@@ -410,8 +740,8 @@ esp_outb_trs_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
* In that case we have to move ICV bytes back to their proper place.
*/
uint16_t
-esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
- uint16_t num)
+esp_outb_sqh_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num)
{
uint32_t i, k, icv_len, *icv;
struct rte_mbuf *ml;
@@ -498,7 +828,8 @@ inline_outb_tun_pkt_process(const struct rte_ipsec_session *ss,
gen_iv(iv, sqc);
/* try to update the packet itself */
- rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv, 0);
+ rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], set_icv_va_pa,
+ (void *)&icv, 0);
k += (rc >= 0);
@@ -552,7 +883,7 @@ inline_outb_trs_pkt_process(const struct rte_ipsec_session *ss,
/* try to update the packet itself */
rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i],
- l2, l3, &icv, 0);
+ l2, l3, set_icv_va_pa, (void *)&icv, 0);
k += (rc >= 0);
@@ -544,9 +544,9 @@ lksd_proto_prepare(const struct rte_ipsec_session *ss,
* - inbound/outbound for RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL
* - outbound for RTE_SECURITY_ACTION_TYPE_NONE when ESN is disabled
*/
-static uint16_t
-pkt_flag_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
- uint16_t num)
+uint16_t
+esp_outb_pkt_flag_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num)
{
uint32_t i, k;
uint32_t dr[num];
@@ -599,12 +599,48 @@ lksd_none_pkt_func_select(const struct rte_ipsec_sa *sa,
case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV6):
pf->prepare = esp_outb_tun_prepare;
pf->process = (sa->sqh_len != 0) ?
- esp_outb_sqh_process : pkt_flag_process;
+ esp_outb_sqh_process : esp_outb_pkt_flag_process;
break;
case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TRANS):
pf->prepare = esp_outb_trs_prepare;
pf->process = (sa->sqh_len != 0) ?
- esp_outb_sqh_process : pkt_flag_process;
+ esp_outb_sqh_process : esp_outb_pkt_flag_process;
+ break;
+ default:
+ rc = -ENOTSUP;
+ }
+
+ return rc;
+}
+
+static int
+cpu_crypto_pkt_func_select(const struct rte_ipsec_sa *sa,
+ struct rte_ipsec_sa_pkt_func *pf)
+{
+ int32_t rc;
+
+ static const uint64_t msk = RTE_IPSEC_SATP_DIR_MASK |
+ RTE_IPSEC_SATP_MODE_MASK;
+
+ rc = 0;
+ switch (sa->type & msk) {
+ case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV4):
+ case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV6):
+ pf->process = esp_inb_tun_cpu_crypto_pkt_process;
+ break;
+ case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TRANS):
+ pf->process = esp_inb_trs_cpu_crypto_pkt_process;
+ break;
+ case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV4):
+ case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV6):
+ pf->process = (sa->sqh_len != 0) ?
+ esp_outb_tun_cpu_crypto_sqh_process :
+ esp_outb_tun_cpu_crypto_flag_process;
+ break;
+ case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TRANS):
+ pf->process = (sa->sqh_len != 0) ?
+ esp_outb_trs_cpu_crypto_sqh_process :
+ esp_outb_trs_cpu_crypto_flag_process;
break;
default:
rc = -ENOTSUP;
@@ -672,13 +708,16 @@ ipsec_sa_pkt_func_select(const struct rte_ipsec_session *ss,
case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
if ((sa->type & RTE_IPSEC_SATP_DIR_MASK) ==
RTE_IPSEC_SATP_DIR_IB)
- pf->process = pkt_flag_process;
+ pf->process = esp_outb_pkt_flag_process;
else
pf->process = inline_proto_outb_pkt_process;
break;
case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
pf->prepare = lksd_proto_prepare;
- pf->process = pkt_flag_process;
+ pf->process = esp_outb_pkt_flag_process;
+ break;
+ case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO:
+ rc = cpu_crypto_pkt_func_select(sa, pf);
break;
default:
rc = -ENOTSUP;
@@ -156,6 +156,14 @@ uint16_t
inline_inb_trs_pkt_process(const struct rte_ipsec_session *ss,
struct rte_mbuf *mb[], uint16_t num);
+uint16_t
+esp_inb_tun_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num);
+
+uint16_t
+esp_inb_trs_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num);
+
/* outbound processing */
uint16_t
@@ -170,6 +178,10 @@ uint16_t
esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
uint16_t num);
+uint16_t
+esp_outb_pkt_flag_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num);
+
uint16_t
inline_outb_tun_pkt_process(const struct rte_ipsec_session *ss,
struct rte_mbuf *mb[], uint16_t num);
@@ -182,4 +194,21 @@ uint16_t
inline_proto_outb_pkt_process(const struct rte_ipsec_session *ss,
struct rte_mbuf *mb[], uint16_t num);
+uint16_t
+esp_outb_tun_cpu_crypto_sqh_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num);
+
+uint16_t
+esp_outb_tun_cpu_crypto_flag_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num);
+
+uint16_t
+esp_outb_trs_cpu_crypto_sqh_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num);
+
+uint16_t
+esp_outb_trs_cpu_crypto_flag_process(const struct rte_ipsec_session *ss,
+ struct rte_mbuf *mb[], uint16_t num);
+
+
#endif /* _SA_H_ */
@@ -19,7 +19,9 @@ session_check(struct rte_ipsec_session *ss)
return -EINVAL;
if ((ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
ss->type ==
- RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) &&
+ RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+ ss->type ==
+ RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) &&
ss->security.ctx == NULL)
return -EINVAL;
}