[1/1] net/cnxk: add outb soft expiry notification support

  Patch implements soft expiry notification mechanism in out bound
path by creating required number of ring buffers and a common poll
thread which polls for soft expiry events enqueued by ucode.

Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
---
 drivers/common/cnxk/roc_idev.c            |  15 +++
 drivers/common/cnxk/roc_idev.h            |   2 +
 drivers/common/cnxk/roc_ie_ot.h           |  25 +++-
 drivers/common/cnxk/roc_nix_inl.c         |  71 +++++++++++
 drivers/common/cnxk/roc_nix_inl.h         |  20 ++-
 drivers/common/cnxk/roc_nix_inl_dev.c     | 146 +++++++++++++++++++++-
 drivers/common/cnxk/roc_nix_inl_dev_irq.c |   2 +-
 drivers/common/cnxk/roc_nix_inl_priv.h    |  13 ++
 drivers/common/cnxk/roc_nix_priv.h        |   2 +
 drivers/common/cnxk/roc_platform.h        |   3 +
 drivers/common/cnxk/version.map           |   2 +
 drivers/net/cnxk/cn10k_ethdev_sec.c       |  52 +++++++-
 drivers/net/cnxk/cnxk_ethdev.c            |   4 +
 drivers/net/cnxk/cnxk_ethdev_sec.c        |   1 +
 14 files changed, 350 insertions(+), 8 deletions(-)
  

On Thu, Feb 24, 2022 at 11:41 AM Vamsi Attunuru <vattunuru@marvell.com> wrote:
>
> Patch implements soft expiry notification mechanism in out bound

outbound

> path by creating required number of ring buffers and a common poll
> thread which polls for soft expiry events enqueued by ucode.

ucode -> microcode.

Subject suggestion:
net/cnxk: support outbound soft expiry notification

Please rebase to next-net-mrvl

[for-next-net]dell[dpdk-next-net-mrvl] $ git pw series apply 21843
Failed to apply patch:
Applying: net/cnxk: add outb soft expiry notification support
Using index info to reconstruct a base tree...
M       drivers/common/cnxk/roc_ie_ot.h
M       drivers/common/cnxk/roc_nix_inl.c
M       drivers/common/cnxk/roc_nix_inl.h
M       drivers/common/cnxk/roc_nix_inl_dev.c
M       drivers/common/cnxk/roc_nix_inl_dev_irq.c
M       drivers/common/cnxk/roc_nix_inl_priv.h
M       drivers/common/cnxk/roc_nix_priv.h
M       drivers/common/cnxk/roc_platform.h
M       drivers/common/cnxk/version.map
M       drivers/net/cnxk/cn10k_ethdev_sec.c
M       drivers/net/cnxk/cnxk_ethdev.c
M       drivers/net/cnxk/cnxk_ethdev_sec.c
Falling back to patching base and 3-way merge...
Auto-merging drivers/net/cnxk/cnxk_ethdev_sec.c
CONFLICT (content): Merge conflict in drivers/net/cnxk/cnxk_ethdev_sec.c
Auto-merging drivers/net/cnxk/cnxk_ethdev.c
Auto-merging drivers/net/cnxk/cn10k_ethdev_sec.c
CONFLICT (content): Merge conflict in drivers/net/cnxk/cn10k_ethdev_sec.c
Auto-merging drivers/common/cnxk/version.map
Auto-merging drivers/common/cnxk/roc_platform.h
Auto-merging drivers/common/cnxk/roc_nix_priv.h
Auto-merging drivers/common/cnxk/roc_nix_inl_priv.h
Auto-merging drivers/common/cnxk/roc_nix_inl_dev_irq.c
Auto-merging drivers/common/cnxk/roc_nix_inl_dev.c
CONFLICT (content): Merge conflict in drivers/common/cnxk/roc_nix_inl_dev.c
Auto-merging drivers/common/cnxk/roc_nix_inl.h
CONFLICT (content): Merge conflict in drivers/common/cnxk/roc_nix_inl.h
Auto-merging drivers/common/cnxk/roc_nix_inl.c
Auto-merging drivers/common/cnxk/roc_ie_ot.h
error: Failed to merge in the changes.
hint: Use 'git am --show-current-patch=diff' to see the failed patch
Patch failed at 0001 net/cnxk: add outb soft expiry notification support
When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort"
>
> Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
> ---
>  drivers/common/cnxk/roc_idev.c            |  15 +++
>  drivers/common/cnxk/roc_idev.h            |   2 +
>  drivers/common/cnxk/roc_ie_ot.h           |  25 +++-
>  drivers/common/cnxk/roc_nix_inl.c         |  71 +++++++++++
>  drivers/common/cnxk/roc_nix_inl.h         |  20 ++-
>  drivers/common/cnxk/roc_nix_inl_dev.c     | 146 +++++++++++++++++++++-
>  drivers/common/cnxk/roc_nix_inl_dev_irq.c |   2 +-
>  drivers/common/cnxk/roc_nix_inl_priv.h    |  13 ++
>  drivers/common/cnxk/roc_nix_priv.h        |   2 +
>  drivers/common/cnxk/roc_platform.h        |   3 +
>  drivers/common/cnxk/version.map           |   2 +
>  drivers/net/cnxk/cn10k_ethdev_sec.c       |  52 +++++++-
>  drivers/net/cnxk/cnxk_ethdev.c            |   4 +
>  drivers/net/cnxk/cnxk_ethdev_sec.c        |   1 +
>  14 files changed, 350 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/common/cnxk/roc_idev.c b/drivers/common/cnxk/roc_idev.c
> index b1f38fb5fc..a08c7ce8fd 100644
> --- a/drivers/common/cnxk/roc_idev.c
> +++ b/drivers/common/cnxk/roc_idev.c
> @@ -185,6 +185,21 @@ roc_idev_cpt_get(void)
>         return NULL;
>  }
>
> +uint64_t *
> +roc_nix_inl_outb_ring_base_get(struct roc_nix *roc_nix)
> +{
> +       struct nix *nix = roc_nix_to_nix_priv(roc_nix);
> +       struct idev_cfg *idev = idev_get_cfg();
> +       struct nix_inl_dev *inl_dev;
> +
> +       if (!idev || !idev->nix_inl_dev)
> +               return NULL;
> +
> +       inl_dev = idev->nix_inl_dev;
> +
> +       return (uint64_t *)&inl_dev->sa_soft_exp_ring[nix->outb_se_ring_base];
> +}
> +
>  void
>  roc_idev_cpt_set(struct roc_cpt *cpt)
>  {
> diff --git a/drivers/common/cnxk/roc_idev.h b/drivers/common/cnxk/roc_idev.h
> index 16793c2828..7e0beed495 100644
> --- a/drivers/common/cnxk/roc_idev.h
> +++ b/drivers/common/cnxk/roc_idev.h
> @@ -17,4 +17,6 @@ void __roc_api roc_idev_cpt_set(struct roc_cpt *cpt);
>
>  struct roc_nix *__roc_api roc_idev_npa_nix_get(void);
>
> +uint64_t *__roc_api roc_nix_inl_outb_ring_base_get(struct roc_nix *roc_nix);
> +
>  #endif /* _ROC_IDEV_H_ */
> diff --git a/drivers/common/cnxk/roc_ie_ot.h b/drivers/common/cnxk/roc_ie_ot.h
> index c502c7983f..537eafbe79 100644
> --- a/drivers/common/cnxk/roc_ie_ot.h
> +++ b/drivers/common/cnxk/roc_ie_ot.h
> @@ -176,6 +176,28 @@ enum {
>         (PLT_ALIGN_CEIL(ROC_AR_WIN_SIZE_MAX, BITS_PER_LONG_LONG) /             \
>          BITS_PER_LONG_LONG)
>
> +#define ROC_IPSEC_ERR_RING_MAX_ENTRY 65536
> +
> +union roc_ot_ipsec_err_ring_head {
> +       uint64_t u64;
> +       struct {
> +               uint16_t tail_pos;
> +               uint16_t tail_gen;
> +               uint16_t head_pos;
> +               uint16_t head_gen;
> +       } s;
> +};
> +
> +union roc_ot_ipsec_err_ring_entry {
> +       uint64_t u64;
> +       struct {
> +               uint64_t data0 : 44;
> +               uint64_t data1 : 9;
> +               uint64_t rsvd : 3;
> +               uint64_t comp_code : 8;
> +       } s;
> +};
> +
>  /* Common bit fields between inbound and outbound SA */
>  union roc_ot_ipsec_sa_word2 {
>         struct {
> @@ -428,7 +450,8 @@ struct roc_ot_ipsec_outb_sa {
>                         uint64_t count_mib_pkts : 1;
>                         uint64_t hw_ctx_off : 7;
>
> -                       uint64_t rsvd1 : 32;
> +                       uint64_t ctx_id : 16;
> +                       uint64_t rsvd1 : 16;
>
>                         uint64_t ctx_push_size : 7;
>                         uint64_t rsvd2 : 1;
> diff --git a/drivers/common/cnxk/roc_nix_inl.c b/drivers/common/cnxk/roc_nix_inl.c
> index e8981c4aa4..0d9e5dd5c4 100644
> --- a/drivers/common/cnxk/roc_nix_inl.c
> +++ b/drivers/common/cnxk/roc_nix_inl.c
> @@ -5,6 +5,8 @@
>  #include "roc_api.h"
>  #include "roc_priv.h"
>
> +uint32_t soft_exp_consumer_cnt;
> +
>  PLT_STATIC_ASSERT(ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ ==
>                   1UL << ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ_LOG2);
>  PLT_STATIC_ASSERT(ROC_NIX_INL_ONF_IPSEC_INB_SA_SZ == 512);
> @@ -394,6 +396,34 @@ roc_nix_inl_outb_init(struct roc_nix *roc_nix)
>         nix->nb_cpt_lf = nb_lf;
>         nix->outb_err_sso_pffunc = sso_pffunc;
>         nix->inl_outb_ena = true;
> +       nix->outb_se_ring_cnt =
> +               roc_nix->ipsec_out_max_sa / ROC_IPSEC_ERR_RING_MAX_ENTRY + 1;
> +       nix->outb_se_ring_base =
> +               roc_nix->port_id * ROC_NIX_SOFT_EXP_PER_PORT_MAX_RINGS;
> +
> +       if (inl_dev == NULL) {
> +               nix->outb_se_ring_cnt = 0;
> +               return 0;
> +       }
> +
> +       /* Allocate memory to be used as a ring buffer to poll for
> +        * soft expiry event from ucode
> +        */
> +       for (i = 0; i < nix->outb_se_ring_cnt; i++) {
> +               inl_dev->sa_soft_exp_ring[nix->outb_se_ring_base + i] =
> +                       plt_zmalloc((ROC_IPSEC_ERR_RING_MAX_ENTRY + 1) *
> +                                           sizeof(uint64_t),
> +                                   0);
> +               if (!inl_dev->sa_soft_exp_ring[i]) {
> +                       plt_err("Couldn't allocate memory for soft exp ring");
> +                       while (i--)
> +                               plt_free(inl_dev->sa_soft_exp_ring
> +                                                [nix->outb_se_ring_base + i]);
> +                       rc = -ENOMEM;
> +                       goto lf_fini;
> +               }
> +       }
> +
>         return 0;
>
>  lf_fini:
> @@ -412,7 +442,9 @@ roc_nix_inl_outb_fini(struct roc_nix *roc_nix)
>  {
>         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
>         struct roc_cpt_lf *lf_base = nix->cpt_lf_base;
> +       struct idev_cfg *idev = idev_get_cfg();
>         struct dev *dev = &nix->dev;
> +       struct nix_inl_dev *inl_dev;
>         int i, rc, ret = 0;
>
>         if (!nix->inl_outb_ena)
> @@ -444,6 +476,15 @@ roc_nix_inl_outb_fini(struct roc_nix *roc_nix)
>         plt_free(nix->outb_sa_base);
>         nix->outb_sa_base = NULL;
>
> +       if (idev && idev->nix_inl_dev) {
> +               inl_dev = idev->nix_inl_dev;
> +
> +               for (i = 0; i < ROC_NIX_INL_MAX_SOFT_EXP_RNGS; i++) {
> +                       if (inl_dev->sa_soft_exp_ring[i])
> +                               plt_free(inl_dev->sa_soft_exp_ring[i]);
> +               }
> +       }
> +
>         ret |= rc;
>         return ret;
>  }
> @@ -628,6 +669,36 @@ roc_nix_inb_mode_set(struct roc_nix *roc_nix, bool use_inl_dev)
>         nix->inb_inl_dev = use_inl_dev;
>  }
>
> +int
> +roc_nix_inl_outb_soft_exp_poll_switch(struct roc_nix *roc_nix, bool poll)
> +{
> +       struct nix *nix = roc_nix_to_nix_priv(roc_nix);
> +       struct idev_cfg *idev = idev_get_cfg();
> +       struct nix_inl_dev *inl_dev;
> +       uint16_t ring_idx, i;
> +
> +       if (!idev || !idev->nix_inl_dev)
> +               return 0;
> +
> +       inl_dev = idev->nix_inl_dev;
> +
> +       for (i = 0; i < nix->outb_se_ring_cnt; i++) {
> +               ring_idx = nix->outb_se_ring_base + i;
> +
> +               if (poll)
> +                       plt_bitmap_set(inl_dev->soft_exp_ring_bmap, ring_idx);
> +               else
> +                       plt_bitmap_clear(inl_dev->soft_exp_ring_bmap, ring_idx);
> +       }
> +
> +       if (poll)
> +               soft_exp_consumer_cnt++;
> +       else
> +               soft_exp_consumer_cnt--;
> +
> +       return 0;
> +}
> +
>  bool
>  roc_nix_inb_is_with_inl_dev(struct roc_nix *roc_nix)
>  {
> diff --git a/drivers/common/cnxk/roc_nix_inl.h b/drivers/common/cnxk/roc_nix_inl.h
> index abbeac684a..3c23f611c3 100644
> --- a/drivers/common/cnxk/roc_nix_inl.h
> +++ b/drivers/common/cnxk/roc_nix_inl.h
> @@ -43,6 +43,18 @@
>  /* Alignment of SA Base */
>  #define ROC_NIX_INL_SA_BASE_ALIGN BIT_ULL(16)
>
> +#define ROC_NIX_INL_SA_SOFT_EXP_ERR_MAX_POLL_COUNT 25
> +
> +#define ROC_NIX_SOFT_EXP_ERR_RING_MAX_ENTRY_LOG2 16
> +
> +#define ROC_NIX_SOFT_EXP_PER_PORT_MAX_RINGS 4
> +
> +#define ROC_NIX_MAX_TOTAL_OUTB_IPSEC_SA                                        \
> +       (ROC_IPSEC_ERR_RING_MAX_ENTRY * ROC_NIX_SOFT_EXP_PER_PORT_MAX_RINGS)
> +
> +#define ROC_NIX_INL_MAX_SOFT_EXP_RNGS                                          \
> +       (PLT_MAX_ETHPORTS * ROC_NIX_SOFT_EXP_PER_PORT_MAX_RINGS)
> +
>  static inline struct roc_onf_ipsec_inb_sa *
>  roc_nix_inl_onf_ipsec_inb_sa(uintptr_t base, uint64_t idx)
>  {
> @@ -100,7 +112,8 @@ roc_nix_inl_ot_ipsec_outb_sa_sw_rsvd(void *sa)
>  }
>
>  /* Inline device SSO Work callback */
> -typedef void (*roc_nix_inl_sso_work_cb_t)(uint64_t *gw, void *args);
> +typedef void (*roc_nix_inl_sso_work_cb_t)(uint64_t *gw, void *args,
> +                                         uint32_t soft_exp_event);
>
>  struct roc_nix_inl_dev {
>         /* Input parameters */
> @@ -111,9 +124,10 @@ struct roc_nix_inl_dev {
>         uint16_t channel;
>         uint16_t chan_mask;
>         bool attach_cptlf;
> +       bool set_soft_exp_poll;
>         /* End of input parameters */
>
> -#define ROC_NIX_INL_MEM_SZ (1280)
> +#define ROC_NIX_INL_MEM_SZ (2304)
>         uint8_t reserved[ROC_NIX_INL_MEM_SZ] __plt_cache_aligned;
>  } __plt_cache_aligned;
>
> @@ -157,6 +171,8 @@ uint16_t __roc_api roc_nix_inl_outb_sso_pffunc_get(struct roc_nix *roc_nix);
>  int __roc_api roc_nix_inl_cb_register(roc_nix_inl_sso_work_cb_t cb, void *args);
>  int __roc_api roc_nix_inl_cb_unregister(roc_nix_inl_sso_work_cb_t cb,
>                                         void *args);
> +int __roc_api roc_nix_inl_outb_soft_exp_poll_switch(struct roc_nix *roc_nix,
> +                                                   bool poll);
>  /* NIX Inline/Outbound API */
>  enum roc_nix_inl_sa_sync_op {
>         ROC_NIX_INL_SA_OP_FLUSH,
> diff --git a/drivers/common/cnxk/roc_nix_inl_dev.c b/drivers/common/cnxk/roc_nix_inl_dev.c
> index dd93765a2b..10f237f9b1 100644
> --- a/drivers/common/cnxk/roc_nix_inl_dev.c
> +++ b/drivers/common/cnxk/roc_nix_inl_dev.c
> @@ -13,6 +13,9 @@
>          ROC_NIX_LF_RX_CFG_LEN_IL4 | ROC_NIX_LF_RX_CFG_LEN_IL3 |               \
>          ROC_NIX_LF_RX_CFG_LEN_OL4 | ROC_NIX_LF_RX_CFG_LEN_OL3)
>
> +extern uint32_t soft_exp_consumer_cnt;
> +static bool soft_exp_poll_thread_exit = true;
> +
>  uint16_t
>  nix_inl_dev_pffunc_get(void)
>  {
> @@ -28,10 +31,11 @@ nix_inl_dev_pffunc_get(void)
>  }
>
>  static void
> -nix_inl_selftest_work_cb(uint64_t *gw, void *args)
> +nix_inl_selftest_work_cb(uint64_t *gw, void *args, uint32_t soft_exp_event)
>  {
>         uintptr_t work = gw[1];
>
> +       (void)soft_exp_event;
>         *((uintptr_t *)args + (gw[0] & 0x1)) = work;
>
>         plt_atomic_thread_fence(__ATOMIC_ACQ_REL);
> @@ -518,6 +522,133 @@ nix_inl_lf_detach(struct nix_inl_dev *inl_dev)
>         return mbox_process(dev->mbox);
>  }
>
> +static void
> +inl_outb_soft_exp_poll(struct nix_inl_dev *inl_dev, uint32_t ring_idx)
> +{
> +       union roc_ot_ipsec_err_ring_head head;
> +       struct roc_ot_ipsec_outb_sa *sa;
> +       uint16_t head_l, tail_l;
> +       uint64_t *ring_base;
> +       uint32_t port_id;
> +
> +       port_id = ring_idx / ROC_NIX_SOFT_EXP_PER_PORT_MAX_RINGS;
> +       ring_base = inl_dev->sa_soft_exp_ring[ring_idx];
> +       if (!ring_base) {
> +               plt_err("Invalid soft exp ring base");
> +               return;
> +       }
> +
> +       head.u64 = __atomic_load_n(ring_base, __ATOMIC_ACQUIRE);
> +       head_l = head.s.head_pos;
> +       tail_l = head.s.tail_pos;
> +
> +       while (tail_l != head_l) {
> +               union roc_ot_ipsec_err_ring_entry entry;
> +               int poll_counter = 0;
> +
> +               while (poll_counter++ <
> +                      ROC_NIX_INL_SA_SOFT_EXP_ERR_MAX_POLL_COUNT) {
> +                       plt_delay_us(20);
> +                       entry.u64 = __atomic_load_n(ring_base + tail_l + 1,
> +                                                   __ATOMIC_ACQUIRE);
> +                       if (likely(entry.u64))
> +                               break;
> +               }
> +
> +               entry.u64 = plt_be_to_cpu_64(entry.u64);
> +               sa = (struct roc_ot_ipsec_outb_sa *)(((uint64_t)entry.s.data1
> +                                                     << 51) |
> +                                                    (entry.s.data0 << 7));
> +
> +               if (sa != NULL) {
> +                       uint64_t tmp = ~(uint32_t)0x0;
> +                       inl_dev->work_cb(&tmp, sa, (port_id << 8) | 0x1);
> +                       __atomic_store_n(ring_base + tail_l + 1, 0ULL,
> +                                        __ATOMIC_RELAXED);
> +                       __atomic_add_fetch((uint32_t *)ring_base, 1,
> +                                          __ATOMIC_ACQ_REL);
> +               } else
> +                       plt_err("Invalid SA");
> +
> +               tail_l++;
> +       }
> +}
> +
> +static void *
> +nix_inl_outb_poll_thread(void *args)
> +{
> +       struct nix_inl_dev *inl_dev = args;
> +       uint32_t poll_freq;
> +       uint32_t i;
> +       bool bit;
> +
> +       poll_freq = inl_dev->soft_exp_poll_freq;
> +
> +       while (!soft_exp_poll_thread_exit) {
> +               if (soft_exp_consumer_cnt) {
> +                       for (i = 0; i < ROC_NIX_INL_MAX_SOFT_EXP_RNGS; i++) {
> +                               bit = plt_bitmap_get(
> +                                       inl_dev->soft_exp_ring_bmap, i);
> +                               if (bit)
> +                                       inl_outb_soft_exp_poll(inl_dev, i);
> +                       }
> +               }
> +               plt_delay_us(poll_freq);
> +       }
> +
> +       return 0;
> +}
> +
> +static int
> +nix_inl_outb_poll_thread_setup(struct nix_inl_dev *inl_dev)
> +{
> +       struct plt_bitmap *bmap;
> +       size_t bmap_sz;
> +       uint32_t i;
> +       void *mem;
> +       int rc;
> +
> +       /* Allocate a bitmap that pool thread uses to get the port_id
> +        * that's corresponding to the inl_outb_soft_exp_ring
> +        */
> +       bmap_sz =
> +               plt_bitmap_get_memory_footprint(ROC_NIX_INL_MAX_SOFT_EXP_RNGS);
> +       mem = plt_zmalloc(bmap_sz, PLT_CACHE_LINE_SIZE);
> +       if (mem == NULL) {
> +               plt_err("soft expiry ring bmap alloc failed");
> +               rc = -ENOMEM;
> +               goto exit;
> +       }
> +
> +       bmap = plt_bitmap_init(ROC_NIX_INL_MAX_SOFT_EXP_RNGS, mem, bmap_sz);
> +       if (!bmap) {
> +               plt_err("soft expiry ring bmap init failed");
> +               plt_free(mem);
> +               rc = -ENOMEM;
> +               goto exit;
> +       }
> +
> +       inl_dev->soft_exp_ring_bmap_mem = mem;
> +       inl_dev->soft_exp_ring_bmap = bmap;
> +
> +       for (i = 0; i < ROC_NIX_INL_MAX_SOFT_EXP_RNGS; i++)
> +               plt_bitmap_clear(inl_dev->soft_exp_ring_bmap, i);
> +
> +       soft_exp_consumer_cnt = 0;
> +       soft_exp_poll_thread_exit = false;
> +       inl_dev->soft_exp_poll_freq = 100;
> +       rc = plt_ctrl_thread_create(&inl_dev->soft_exp_poll_thread,
> +                                   "OUTB_SOFT_EXP_POLL_THREAD", NULL,
> +                                   nix_inl_outb_poll_thread, inl_dev);
> +       if (rc) {
> +               plt_bitmap_free(inl_dev->soft_exp_ring_bmap);
> +               plt_free(inl_dev->soft_exp_ring_bmap_mem);
> +       }
> +
> +exit:
> +       return rc;
> +}
> +
>  int
>  roc_nix_inl_dev_init(struct roc_nix_inl_dev *roc_inl_dev)
>  {
> @@ -581,6 +712,12 @@ roc_nix_inl_dev_init(struct roc_nix_inl_dev *roc_inl_dev)
>         if (rc)
>                 goto sso_release;
>
> +       if (roc_inl_dev->set_soft_exp_poll) {
> +               rc = nix_inl_outb_poll_thread_setup(inl_dev);
> +               if (rc)
> +                       goto cpt_release;
> +       }
> +
>         /* Perform selftest if asked for */
>         if (inl_dev->selftest) {
>                 rc = nix_inl_selftest();
> @@ -624,6 +761,13 @@ roc_nix_inl_dev_fini(struct roc_nix_inl_dev *roc_inl_dev)
>         inl_dev = idev->nix_inl_dev;
>         pci_dev = inl_dev->pci_dev;
>
> +       if (roc_inl_dev->set_soft_exp_poll) {
> +               soft_exp_poll_thread_exit = true;
> +               pthread_join(inl_dev->soft_exp_poll_thread, NULL);
> +               plt_bitmap_free(inl_dev->soft_exp_ring_bmap);
> +               plt_free(inl_dev->soft_exp_ring_bmap_mem);
> +       }
> +
>         /* Flush Inbound CTX cache entries */
>         nix_inl_cpt_ctx_cache_sync(inl_dev);
>
> diff --git a/drivers/common/cnxk/roc_nix_inl_dev_irq.c b/drivers/common/cnxk/roc_nix_inl_dev_irq.c
> index 848523b010..0cb9b46b44 100644
> --- a/drivers/common/cnxk/roc_nix_inl_dev_irq.c
> +++ b/drivers/common/cnxk/roc_nix_inl_dev_irq.c
> @@ -29,7 +29,7 @@ nix_inl_sso_work_cb(struct nix_inl_dev *inl_dev)
>         /* Do we have any work? */
>         if (work) {
>                 if (inl_dev->work_cb)
> -                       inl_dev->work_cb(gw.u64, inl_dev->cb_args);
> +                       inl_dev->work_cb(gw.u64, inl_dev->cb_args, false);
>                 else
>                         plt_warn("Undelivered inl dev work gw0: %p gw1: %p",
>                                  (void *)gw.u64[0], (void *)gw.u64[1]);
> diff --git a/drivers/common/cnxk/roc_nix_inl_priv.h b/drivers/common/cnxk/roc_nix_inl_priv.h
> index 2cdab6dc7a..a695163e84 100644
> --- a/drivers/common/cnxk/roc_nix_inl_priv.h
> +++ b/drivers/common/cnxk/roc_nix_inl_priv.h
> @@ -3,6 +3,8 @@
>   */
>  #ifndef _ROC_NIX_INL_PRIV_H_
>  #define _ROC_NIX_INL_PRIV_H_
> +#include <pthread.h>
> +#include <sys/types.h>
>
>  struct nix_inl_dev {
>         /* Base device object */
> @@ -48,6 +50,17 @@ struct nix_inl_dev {
>         /* CPT data */
>         struct roc_cpt_lf cpt_lf;
>
> +       /* OUTB soft expiry poll thread */
> +       pthread_t soft_exp_poll_thread;
> +       uint32_t soft_exp_poll_freq;
> +       void *sa_soft_exp_ring[ROC_NIX_INL_MAX_SOFT_EXP_RNGS];
> +
> +       /* Soft expiry ring bitmap */
> +       struct plt_bitmap *soft_exp_ring_bmap;
> +
> +       /* bitmap memory */
> +       void *soft_exp_ring_bmap_mem;
> +
>         /* Device arguments */
>         uint8_t selftest;
>         uint16_t channel;
> diff --git a/drivers/common/cnxk/roc_nix_priv.h b/drivers/common/cnxk/roc_nix_priv.h
> index deb2a6ba11..959de778f2 100644
> --- a/drivers/common/cnxk/roc_nix_priv.h
> +++ b/drivers/common/cnxk/roc_nix_priv.h
> @@ -178,6 +178,8 @@ struct nix {
>         uint16_t outb_err_sso_pffunc;
>         struct roc_cpt_lf *cpt_lf_base;
>         uint16_t nb_cpt_lf;
> +       uint16_t outb_se_ring_cnt;
> +       uint16_t outb_se_ring_base;
>         /* Mode provided by driver */
>         bool inb_inl_dev;
>
> diff --git a/drivers/common/cnxk/roc_platform.h b/drivers/common/cnxk/roc_platform.h
> index adfb88c68d..1adbdd5084 100644
> --- a/drivers/common/cnxk/roc_platform.h
> +++ b/drivers/common/cnxk/roc_platform.h
> @@ -14,6 +14,7 @@
>  #include <rte_ether.h>
>  #include <rte_interrupts.h>
>  #include <rte_io.h>
> +#include <rte_lcore.h>
>  #include <rte_log.h>
>  #include <rte_malloc.h>
>  #include <rte_memzone.h>
> @@ -56,6 +57,7 @@
>  #define BITMASK_ULL             GENMASK_ULL
>  #define PLT_ALIGN_CEIL          RTE_ALIGN_CEIL
>  #define PLT_INIT                RTE_INIT
> +#define PLT_MAX_ETHPORTS        RTE_MAX_ETHPORTS
>  #define PLT_TAILQ_FOREACH_SAFE  RTE_TAILQ_FOREACH_SAFE
>
>  #ifndef PLT_ETHER_ADDR_LEN
> @@ -114,6 +116,7 @@
>  #define plt_intr_disable            rte_intr_disable
>  #define plt_thread_is_intr          rte_thread_is_intr
>  #define plt_intr_callback_fn        rte_intr_callback_fn
> +#define plt_ctrl_thread_create      rte_ctrl_thread_create
>
>  #define plt_intr_efd_counter_size_get  rte_intr_efd_counter_size_get
>  #define plt_intr_efd_counter_size_set  rte_intr_efd_counter_size_set
> diff --git a/drivers/common/cnxk/version.map b/drivers/common/cnxk/version.map
> index ad1b5e8476..aff99ee7c8 100644
> --- a/drivers/common/cnxk/version.map
> +++ b/drivers/common/cnxk/version.map
> @@ -83,6 +83,7 @@ INTERNAL {
>         roc_hash_sha512_gen;
>         roc_idev_cpt_get;
>         roc_idev_cpt_set;
> +       roc_nix_inl_outb_ring_base_get;
>         roc_idev_lmt_base_addr_get;
>         roc_idev_npa_maxpools_get;
>         roc_idev_npa_maxpools_set;
> @@ -154,6 +155,7 @@ INTERNAL {
>         roc_nix_inl_outb_sa_base_get;
>         roc_nix_inl_outb_sso_pffunc_get;
>         roc_nix_inl_outb_is_enabled;
> +       roc_nix_inl_outb_soft_exp_poll_switch;
>         roc_nix_inl_sa_sync;
>         roc_nix_inl_ctx_write;
>         roc_nix_inl_inb_sa_init;
> diff --git a/drivers/net/cnxk/cn10k_ethdev_sec.c b/drivers/net/cnxk/cn10k_ethdev_sec.c
> index 12cec0a17e..d6a0e7ab64 100644
> --- a/drivers/net/cnxk/cn10k_ethdev_sec.c
> +++ b/drivers/net/cnxk/cn10k_ethdev_sec.c
> @@ -9,6 +9,7 @@
>
>  #include <cn10k_ethdev.h>
>  #include <cnxk_security.h>
> +#include <roc_priv.h>
>
>  static struct rte_cryptodev_capabilities cn10k_eth_sec_crypto_caps[] = {
>         {       /* AES GCM */
> @@ -139,7 +140,7 @@ static const struct rte_security_capability cn10k_eth_sec_capabilities[] = {
>  };
>
>  static void
> -cn10k_eth_sec_sso_work_cb(uint64_t *gw, void *args)
> +cn10k_eth_sec_sso_work_cb(uint64_t *gw, void *args, uint32_t soft_exp_event)
>  {
>         struct rte_eth_event_ipsec_desc desc;
>         struct cn10k_sec_sess_priv sess_priv;
> @@ -172,8 +173,18 @@ cn10k_eth_sec_sso_work_cb(uint64_t *gw, void *args)
>                 }
>                 /* Fall through */
>         default:
> -               plt_err("Unknown event gw[0] = 0x%016lx, gw[1] = 0x%016lx",
> -                       gw[0], gw[1]);
> +               if (soft_exp_event & 0x1) {
> +                       sa = (struct roc_ot_ipsec_outb_sa *)args;
> +                       priv = roc_nix_inl_ot_ipsec_outb_sa_sw_rsvd(sa);
> +                       desc.metadata = (uint64_t)priv->userdata;
> +                       desc.subtype = RTE_ETH_EVENT_IPSEC_SA_TIME_EXPIRY;
> +                       eth_dev = &rte_eth_devices[soft_exp_event >> 8];
> +                       rte_eth_dev_callback_process(eth_dev,
> +                               RTE_ETH_EVENT_IPSEC, &desc);
> +               } else {
> +                       plt_err("Unknown event gw[0] = 0x%016lx, gw[1] = 0x%016lx",
> +                               gw[0], gw[1]);
> +               }
>                 return;
>         }
>
> @@ -225,6 +236,30 @@ cn10k_eth_sec_sso_work_cb(uint64_t *gw, void *args)
>         rte_pktmbuf_free(mbuf);
>  }
>
> +static int
> +cn10k_eth_sec_outb_sa_misc_fill(struct roc_nix *roc_nix,
> +                               struct roc_ot_ipsec_outb_sa *sa, void *sa_cptr,
> +                               struct rte_security_ipsec_xform *ipsec_xfrm,
> +                               uint32_t sa_idx)
> +{
> +       uint64_t *ring_base, ring_addr;
> +
> +       if (ipsec_xfrm->life.bytes_soft_limit |
> +           ipsec_xfrm->life.packets_soft_limit) {
> +               ring_base = roc_nix_inl_outb_ring_base_get(roc_nix);
> +               if (ring_base == NULL)
> +                       return -ENOTSUP;
> +
> +               ring_addr = ring_base[sa_idx >>
> +                                     ROC_NIX_SOFT_EXP_ERR_RING_MAX_ENTRY_LOG2];
> +               sa->ctx.err_ctl.s.mode = ROC_IE_OT_ERR_CTL_MODE_RING;
> +               sa->ctx.err_ctl.s.address = ring_addr >> 3;
> +               sa->w0.s.ctx_id = ((uintptr_t)sa_cptr >> 51) & 0x1ff;
> +       }
> +
> +       return 0;
> +}
> +
>  static int
>  cn10k_eth_sec_session_create(void *device,
>                              struct rte_security_session_conf *conf,
> @@ -386,6 +421,17 @@ cn10k_eth_sec_session_create(void *device,
>                         goto mempool_put;
>                 }
>
> +               /* Fill outbound sa misc params */
> +               rc = cn10k_eth_sec_outb_sa_misc_fill(&dev->nix, outb_sa_dptr,
> +                                                    outb_sa, ipsec, sa_idx);
> +               if (rc) {
> +                       snprintf(tbuf, sizeof(tbuf),
> +                                "Failed to init outb sa misc params, rc=%d",
> +                                rc);
> +                       rc |= cnxk_eth_outb_sa_idx_put(dev, sa_idx);
> +                       goto mempool_put;
> +               }
> +
>                 /* Save userdata */
>                 outb_priv->userdata = conf->userdata;
>                 outb_priv->sa_idx = sa_idx;
> diff --git a/drivers/net/cnxk/cnxk_ethdev.c b/drivers/net/cnxk/cnxk_ethdev.c
> index 27751a6956..f4045169f0 100644
> --- a/drivers/net/cnxk/cnxk_ethdev.c
> +++ b/drivers/net/cnxk/cnxk_ethdev.c
> @@ -1405,6 +1405,8 @@ cnxk_nix_dev_stop(struct rte_eth_dev *eth_dev)
>         /* Disable Rx via NPC */
>         roc_nix_npc_rx_ena_dis(&dev->nix, false);
>
> +       roc_nix_inl_outb_soft_exp_poll_switch(&dev->nix, false);
> +
>         /* Stop rx queues and free up pkts pending */
>         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
>                 rc = dev_ops->rx_queue_stop(eth_dev, i);
> @@ -1503,6 +1505,8 @@ cnxk_nix_dev_start(struct rte_eth_dev *eth_dev)
>
>         cnxk_nix_toggle_flag_link_cfg(dev, false);
>
> +       roc_nix_inl_outb_soft_exp_poll_switch(&dev->nix, true);
> +
>         return 0;
>
>  rx_disable:
> diff --git a/drivers/net/cnxk/cnxk_ethdev_sec.c b/drivers/net/cnxk/cnxk_ethdev_sec.c
> index 3fef0562ea..1f87ef8265 100644
> --- a/drivers/net/cnxk/cnxk_ethdev_sec.c
> +++ b/drivers/net/cnxk/cnxk_ethdev_sec.c
> @@ -278,6 +278,7 @@ cnxk_nix_inl_dev_probe(struct rte_pci_driver *pci_drv,
>         }
>
>         inl_dev->attach_cptlf = true;
> +       inl_dev->set_soft_exp_poll = true;
>         rc = roc_nix_inl_dev_init(inl_dev);
>         if (rc) {
>                 plt_err("Failed to init nix inl device, rc=%d(%s)", rc,
> --
> 2.25.1
>