@@ -30,6 +30,7 @@ LIBABIVER := 1
# all source are stored in SRCS-y
#
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_EVENTDEV) += otx2_worker.c
SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_EVENTDEV) += otx2_evdev.c
SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_EVENTDEV) += otx2_evdev_irq.c
@@ -2,7 +2,8 @@
# Copyright(C) 2019 Marvell International Ltd.
#
-sources = files('otx2_evdev.c',
+sources = files('otx2_worker.c',
+ 'otx2_evdev.c',
'otx2_evdev_irq.c',
)
@@ -82,6 +82,13 @@ enum otx2_sso_lf_type {
SSO_LF_GWS
};
+enum {
+ SSO_SYNC_ORDERED,
+ SSO_SYNC_ATOMIC,
+ SSO_SYNC_UNTAGGED,
+ SSO_SYNC_EMPTY
+};
+
struct otx2_sso_evdev {
OTX2_DEV; /* Base class */
uint8_t max_event_queues;
new file mode 100644
@@ -0,0 +1,5 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2019 Marvell International Ltd.
+ */
+
+#include "otx2_worker.h"
new file mode 100644
@@ -0,0 +1,186 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2019 Marvell International Ltd.
+ */
+
+#ifndef __OTX2_WORKER_H__
+#define __OTX2_WORKER_H__
+
+#include <rte_common.h>
+#include <rte_branch_prediction.h>
+
+#include <otx2_common.h>
+#include "otx2_evdev.h"
+
+/* SSO Operations */
+
+static __rte_always_inline uint16_t
+otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev)
+{
+ uint64_t get_work0;
+ uint64_t get_work1;
+
+ otx2_write64(BIT_ULL(16) | /* wait for work. */
+ 1, /* Use Mask set 0. */
+ ws->getwrk_op);
+
+#ifdef RTE_ARCH_ARM64
+ asm volatile(
+ " ldr %[tag], [%[tag_loc]] \n"
+ " ldr %[wqp], [%[wqp_loc]] \n"
+ " tbz %[tag], 63, done%= \n"
+ " sevl \n"
+ "rty%=: wfe \n"
+ " ldr %[tag], [%[tag_loc]] \n"
+ " ldr %[wqp], [%[wqp_loc]] \n"
+ " tbnz %[tag], 63, rty%= \n"
+ "done%=: prfm pldl1strm, [%[wqp]] \n"
+ " dmb ld \n"
+ : [tag] "=&r" (get_work0), [wqp] "=&r" (get_work1)
+ : [tag_loc] "r" (ws->tag_op),
+ [wqp_loc] "r" (ws->wqp_op)
+ );
+#else
+ get_work0 = otx2_read64(ws->tag_op);
+ while ((BIT_ULL(63)) & get_work0)
+ get_work0 = otx2_read64(ws->tag_op);
+
+ get_work1 = otx2_read64(ws->wqp_op);
+ rte_prefetch_non_temporal((const void *)get_work1);
+#endif
+
+ ws->cur_tt = (get_work0 >> 32) & 0x3;
+ ws->cur_grp = (get_work0 >> 36) & 0x3FF;
+
+ get_work0 = (get_work0 & (0x3ull << 32)) << 6 |
+ (get_work0 & (0x3FFull << 36)) << 4 |
+ (get_work0 & 0xffffffff);
+
+ ev->event = get_work0;
+ ev->u64 = get_work1;
+
+ return !!get_work1;
+}
+
+/* Used in cleaning up workslot. */
+static __rte_always_inline uint16_t
+otx2_ssogws_get_work_empty(struct otx2_ssogws *ws, struct rte_event *ev)
+{
+ uint64_t get_work0;
+ uint64_t get_work1;
+
+#ifdef RTE_ARCH_ARM64
+ asm volatile(
+ " ldr %[tag], [%[tag_loc]] \n"
+ " ldr %[wqp], [%[wqp_loc]] \n"
+ " tbz %[tag], 63, done%= \n"
+ " sevl \n"
+ "rty%=: wfe \n"
+ " ldr %[tag], [%[tag_loc]] \n"
+ " ldr %[wqp], [%[wqp_loc]] \n"
+ " tbnz %[tag], 63, rty%= \n"
+ "done%=: prfm pldl1strm, [%[wqp]] \n"
+ " dmb ld \n"
+ : [tag] "=&r" (get_work0), [wqp] "=&r" (get_work1)
+ : [tag_loc] "r" (ws->tag_op),
+ [wqp_loc] "r" (ws->wqp_op)
+ );
+#else
+ get_work0 = otx2_read64(ws->tag_op);
+ while ((BIT_ULL(63)) & get_work0)
+ get_work0 = otx2_read64(ws->tag_op);
+
+ get_work1 = otx2_read64(ws->wqp_op);
+ rte_prefetch_non_temporal((const void *)get_work1);
+#endif
+
+ ws->cur_tt = (get_work0 >> 32) & 0x3;
+ ws->cur_grp = (get_work0 >> 36) & 0x3FF;
+
+ get_work0 = (get_work0 & (0x3ull << 32)) << 6 |
+ (get_work0 & (0x3FFull << 36)) << 4 |
+ (get_work0 & 0xffffffff);
+
+ ev->event = get_work0;
+ ev->u64 = get_work1;
+
+ return !!get_work1;
+}
+
+static __rte_always_inline void
+otx2_ssogws_add_work(struct otx2_ssogws *ws, const uint64_t event_ptr,
+ const uint32_t tag, const uint8_t new_tt,
+ const uint16_t grp)
+{
+ uint64_t add_work0;
+
+ add_work0 = tag | ((uint64_t)(new_tt) << 32);
+ otx2_store_pair(add_work0, event_ptr, ws->grps_base[grp]);
+}
+
+static __rte_always_inline void
+otx2_ssogws_swtag_desched(struct otx2_ssogws *ws, uint32_t tag, uint8_t new_tt,
+ uint16_t grp)
+{
+ uint64_t val;
+
+ val = tag | ((uint64_t)(new_tt & 0x3) << 32) | ((uint64_t)grp << 34);
+ otx2_write64(val, ws->swtag_desched_op);
+}
+
+static __rte_always_inline void
+otx2_ssogws_swtag_norm(struct otx2_ssogws *ws, uint32_t tag, uint8_t new_tt)
+{
+ uint64_t val;
+
+ val = tag | ((uint64_t)(new_tt & 0x3) << 32);
+ otx2_write64(val, ws->swtag_norm_op);
+}
+
+static __rte_always_inline void
+otx2_ssogws_swtag_untag(struct otx2_ssogws *ws)
+{
+ otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
+ SSOW_LF_GWS_OP_SWTAG_UNTAG);
+ ws->cur_tt = SSO_SYNC_UNTAGGED;
+}
+
+static __rte_always_inline void
+otx2_ssogws_swtag_flush(struct otx2_ssogws *ws)
+{
+ otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
+ SSOW_LF_GWS_OP_SWTAG_FLUSH);
+ ws->cur_tt = SSO_SYNC_EMPTY;
+}
+
+static __rte_always_inline void
+otx2_ssogws_desched(struct otx2_ssogws *ws)
+{
+ otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
+ SSOW_LF_GWS_OP_DESCHED);
+}
+
+static __rte_always_inline void
+otx2_ssogws_swtag_wait(struct otx2_ssogws *ws)
+{
+#ifdef RTE_ARCH_ARM64
+ uint64_t swtp;
+
+ asm volatile (
+ " ldr %[swtb], [%[swtp_loc]] \n"
+ " cbz %[swtb], done%= \n"
+ " sevl \n"
+ "rty%=: wfe \n"
+ " ldr %[swtb], [%[swtp_loc]] \n"
+ " cbnz %[swtb], rty%= \n"
+ "done%=: \n"
+ : [swtb] "=&r" (swtp)
+ : [swtp_loc] "r" (ws->swtp_op)
+ );
+#else
+ /* Wait for the SWTAG/SWTAG_FULL operation */
+ while (otx2_read64(ws->swtp_op))
+ ;
+#endif
+}
+
+#endif