Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/92667/?format=api
{ "id": 92667, "url": "http://patches.dpdk.org/api/patches/92667/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20210503152238.2437-30-pbhagavatula@marvell.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20210503152238.2437-30-pbhagavatula@marvell.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20210503152238.2437-30-pbhagavatula@marvell.com", "date": "2021-05-03T15:22:32", "name": "[v4,29/34] event/cnxk: add timer arm routine", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "18867b90e06cbb4d7038d3162af45070027401c9", "submitter": { "id": 1183, "url": "http://patches.dpdk.org/api/people/1183/?format=api", "name": "Pavan Nikhilesh Bhagavatula", "email": "pbhagavatula@marvell.com" }, "delegate": null, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20210503152238.2437-30-pbhagavatula@marvell.com/mbox/", "series": [ { "id": 16799, "url": "http://patches.dpdk.org/api/series/16799/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=16799", "date": "2021-05-03T15:22:03", "name": "Marvell CNXK Event device Driver", "version": 4, "mbox": "http://patches.dpdk.org/series/16799/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/92667/comments/", "check": "warning", "checks": "http://patches.dpdk.org/api/patches/92667/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id C0007A0562;\n\tMon, 3 May 2021 17:26:22 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 3EC03410DE;\n\tMon, 3 May 2021 17:24:33 +0200 (CEST)", "from mx0b-0016f401.pphosted.com (mx0b-0016f401.pphosted.com\n [67.231.156.173])\n by mails.dpdk.org (Postfix) with ESMTP id 5E58F411B0\n for <dev@dpdk.org>; Mon, 3 May 2021 17:24:31 +0200 (CEST)", "from pps.filterd (m0045851.ppops.net [127.0.0.1])\n by mx0b-0016f401.pphosted.com (8.16.0.43/8.16.0.43) with SMTP id\n 143FAGSa032536 for <dev@dpdk.org>; Mon, 3 May 2021 08:24:30 -0700", "from dc5-exch02.marvell.com ([199.233.59.182])\n by mx0b-0016f401.pphosted.com with ESMTP id 38ad05hequ-1\n (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT)\n for <dev@dpdk.org>; Mon, 03 May 2021 08:24:29 -0700", "from DC5-EXCH02.marvell.com (10.69.176.39) by DC5-EXCH02.marvell.com\n (10.69.176.39) with Microsoft SMTP Server (TLS) id 15.0.1497.2;\n Mon, 3 May 2021 08:24:28 -0700", "from maili.marvell.com (10.69.176.80) by DC5-EXCH02.marvell.com\n (10.69.176.39) with Microsoft SMTP Server id 15.0.1497.2 via Frontend\n Transport; Mon, 3 May 2021 08:24:28 -0700", "from BG-LT7430.marvell.com (BG-LT7430.marvell.com [10.28.177.176])\n by maili.marvell.com (Postfix) with ESMTP id D5E293F703F;\n Mon, 3 May 2021 08:24:25 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com;\n h=from : to : cc :\n subject : date : message-id : in-reply-to : references : mime-version :\n content-transfer-encoding : content-type; s=pfpt0220;\n bh=pxkCf5F3AyDuNwv5E7VaVST6OcKY8OVlei0YXclRvhU=;\n b=C6T0E1OiuzqXlOsmoCV+pFxFJ9XrTa9x1iE9CtyoAitfbdM3VUYuLCs8VUm//oZ4QUZs\n tviZqjVNafIHBOUF4N4baysRpiSyFVxXnLx5cClczjkcZ2X5+aYNirwxjuwMK089smis\n ina7itMZjvzvRC9F+JaX/Ye3mpbgn3tlL7a4gn+HW59/jCCJpvlZT5TD/1f/12o+alA7\n QSGja4rMrUd5VI3keBPmUi3sb0DhEHG5aejf0GXojauBYgpVZxmjiHgHDnmjqTSXmP9e\n x5k/dlDlFzGhWSwgZy7cHFCfcLE5wAvnNED0LA8CB3Ji5b4tg1mQlPiefDYafHknz/N9 IA==", "From": "<pbhagavatula@marvell.com>", "To": "<jerinj@marvell.com>, Pavan Nikhilesh <pbhagavatula@marvell.com>, \"Shijith\n Thotton\" <sthotton@marvell.com>", "CC": "<dev@dpdk.org>", "Date": "Mon, 3 May 2021 20:52:32 +0530", "Message-ID": "<20210503152238.2437-30-pbhagavatula@marvell.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20210503152238.2437-1-pbhagavatula@marvell.com>", "References": "<20210430135336.2749-1-pbhagavatula@marvell.com>\n <20210503152238.2437-1-pbhagavatula@marvell.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Content-Type": "text/plain", "X-Proofpoint-GUID": "qCrSEXtZ1sHHo9A87t4lugknYkCDhmxs", "X-Proofpoint-ORIG-GUID": "qCrSEXtZ1sHHo9A87t4lugknYkCDhmxs", "X-Proofpoint-Virus-Version": "vendor=fsecure engine=2.50.10434:6.0.391, 18.0.761\n definitions=2021-05-03_10:2021-05-03,\n 2021-05-03 signatures=0", "Subject": "[dpdk-dev] [PATCH v4 29/34] event/cnxk: add timer arm routine", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "From: Pavan Nikhilesh <pbhagavatula@marvell.com>\n\nAdd event timer arm routine.\n\nSigned-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>\nSigned-off-by: Shijith Thotton <sthotton@marvell.com>\n---\n drivers/event/cnxk/cnxk_tim_evdev.c | 18 ++\n drivers/event/cnxk/cnxk_tim_evdev.h | 23 ++\n drivers/event/cnxk/cnxk_tim_worker.c | 95 +++++++++\n drivers/event/cnxk/cnxk_tim_worker.h | 300 +++++++++++++++++++++++++++\n 4 files changed, 436 insertions(+)", "diff": "diff --git a/drivers/event/cnxk/cnxk_tim_evdev.c b/drivers/event/cnxk/cnxk_tim_evdev.c\nindex e06fe2f52..ecc952a6a 100644\n--- a/drivers/event/cnxk/cnxk_tim_evdev.c\n+++ b/drivers/event/cnxk/cnxk_tim_evdev.c\n@@ -76,6 +76,21 @@ cnxk_tim_chnk_pool_create(struct cnxk_tim_ring *tim_ring,\n \treturn rc;\n }\n \n+static void\n+cnxk_tim_set_fp_ops(struct cnxk_tim_ring *tim_ring)\n+{\n+\tuint8_t prod_flag = !tim_ring->prod_type_sp;\n+\n+\t/* [DFB/FB] [SP][MP]*/\n+\tconst rte_event_timer_arm_burst_t arm_burst[2][2] = {\n+#define FP(_name, _f2, _f1, flags) [_f2][_f1] = cnxk_tim_arm_burst_##_name,\n+\t\tTIM_ARM_FASTPATH_MODES\n+#undef FP\n+\t};\n+\n+\tcnxk_tim_ops.arm_burst = arm_burst[tim_ring->ena_dfb][prod_flag];\n+}\n+\n static void\n cnxk_tim_ring_info_get(const struct rte_event_timer_adapter *adptr,\n \t\t struct rte_event_timer_adapter_info *adptr_info)\n@@ -173,6 +188,9 @@ cnxk_tim_ring_create(struct rte_event_timer_adapter *adptr)\n \tplt_write64((uint64_t)tim_ring->bkt, tim_ring->base + TIM_LF_RING_BASE);\n \tplt_write64(tim_ring->aura, tim_ring->base + TIM_LF_RING_AURA);\n \n+\t/* Set fastpath ops. */\n+\tcnxk_tim_set_fp_ops(tim_ring);\n+\n \t/* Update SSO xae count. */\n \tcnxk_sso_updt_xae_cnt(cnxk_sso_pmd_priv(dev->event_dev), tim_ring,\n \t\t\t RTE_EVENT_TYPE_TIMER);\ndiff --git a/drivers/event/cnxk/cnxk_tim_evdev.h b/drivers/event/cnxk/cnxk_tim_evdev.h\nindex f6895417a..1f2aad17a 100644\n--- a/drivers/event/cnxk/cnxk_tim_evdev.h\n+++ b/drivers/event/cnxk/cnxk_tim_evdev.h\n@@ -14,6 +14,7 @@\n #include <rte_event_timer_adapter.h>\n #include <rte_malloc.h>\n #include <rte_memzone.h>\n+#include <rte_reciprocal.h>\n \n #include \"roc_api.h\"\n \n@@ -37,6 +38,11 @@\n #define CNXK_TIM_CHNK_SLOTS \"tim_chnk_slots\"\n #define CNXK_TIM_RINGS_LMT \"tim_rings_lmt\"\n \n+#define CNXK_TIM_SP\t 0x1\n+#define CNXK_TIM_MP\t 0x2\n+#define CNXK_TIM_ENA_FB\t 0x10\n+#define CNXK_TIM_ENA_DFB 0x20\n+\n #define TIM_BUCKET_W1_S_CHUNK_REMAINDER (48)\n #define TIM_BUCKET_W1_M_CHUNK_REMAINDER \\\n \t((1ULL << (64 - TIM_BUCKET_W1_S_CHUNK_REMAINDER)) - 1)\n@@ -107,10 +113,14 @@ struct cnxk_tim_ring {\n \tuintptr_t base;\n \tuint16_t nb_chunk_slots;\n \tuint32_t nb_bkts;\n+\tuint64_t last_updt_cyc;\n+\tuint64_t ring_start_cyc;\n \tuint64_t tck_int;\n \tuint64_t tot_int;\n \tstruct cnxk_tim_bkt *bkt;\n \tstruct rte_mempool *chunk_pool;\n+\tstruct rte_reciprocal_u64 fast_div;\n+\tstruct rte_reciprocal_u64 fast_bkt;\n \tuint64_t arm_cnt;\n \tuint8_t prod_type_sp;\n \tuint8_t disable_npa;\n@@ -201,6 +211,19 @@ cnxk_tim_cntfrq(void)\n }\n #endif\n \n+#define TIM_ARM_FASTPATH_MODES \\\n+\tFP(sp, 0, 0, CNXK_TIM_ENA_DFB | CNXK_TIM_SP) \\\n+\tFP(mp, 0, 1, CNXK_TIM_ENA_DFB | CNXK_TIM_MP) \\\n+\tFP(fb_sp, 1, 0, CNXK_TIM_ENA_FB | CNXK_TIM_SP) \\\n+\tFP(fb_mp, 1, 1, CNXK_TIM_ENA_FB | CNXK_TIM_MP)\n+\n+#define FP(_name, _f2, _f1, flags) \\\n+\tuint16_t cnxk_tim_arm_burst_##_name( \\\n+\t\tconst struct rte_event_timer_adapter *adptr, \\\n+\t\tstruct rte_event_timer **tim, const uint16_t nb_timers);\n+TIM_ARM_FASTPATH_MODES\n+#undef FP\n+\n int cnxk_tim_caps_get(const struct rte_eventdev *dev, uint64_t flags,\n \t\t uint32_t *caps,\n \t\t const struct rte_event_timer_adapter_ops **ops);\ndiff --git a/drivers/event/cnxk/cnxk_tim_worker.c b/drivers/event/cnxk/cnxk_tim_worker.c\nindex 49ee85245..268f845c8 100644\n--- a/drivers/event/cnxk/cnxk_tim_worker.c\n+++ b/drivers/event/cnxk/cnxk_tim_worker.c\n@@ -4,3 +4,98 @@\n \n #include \"cnxk_tim_evdev.h\"\n #include \"cnxk_tim_worker.h\"\n+\n+static inline int\n+cnxk_tim_arm_checks(const struct cnxk_tim_ring *const tim_ring,\n+\t\t struct rte_event_timer *const tim)\n+{\n+\tif (unlikely(tim->state)) {\n+\t\ttim->state = RTE_EVENT_TIMER_ERROR;\n+\t\trte_errno = EALREADY;\n+\t\tgoto fail;\n+\t}\n+\n+\tif (unlikely(!tim->timeout_ticks ||\n+\t\t tim->timeout_ticks > tim_ring->nb_bkts)) {\n+\t\ttim->state = tim->timeout_ticks ?\n+\t\t\t\t\t RTE_EVENT_TIMER_ERROR_TOOLATE :\n+\t\t\t\t\t RTE_EVENT_TIMER_ERROR_TOOEARLY;\n+\t\trte_errno = EINVAL;\n+\t\tgoto fail;\n+\t}\n+\n+\treturn 0;\n+\n+fail:\n+\treturn -EINVAL;\n+}\n+\n+static inline void\n+cnxk_tim_format_event(const struct rte_event_timer *const tim,\n+\t\t struct cnxk_tim_ent *const entry)\n+{\n+\tentry->w0 = (tim->ev.event & 0xFFC000000000) >> 6 |\n+\t\t (tim->ev.event & 0xFFFFFFFFF);\n+\tentry->wqe = tim->ev.u64;\n+}\n+\n+static inline void\n+cnxk_tim_sync_start_cyc(struct cnxk_tim_ring *tim_ring)\n+{\n+\tuint64_t cur_cyc = cnxk_tim_cntvct();\n+\tuint32_t real_bkt;\n+\n+\tif (cur_cyc - tim_ring->last_updt_cyc > tim_ring->tot_int) {\n+\t\treal_bkt = plt_read64(tim_ring->base + TIM_LF_RING_REL) >> 44;\n+\t\tcur_cyc = cnxk_tim_cntvct();\n+\n+\t\ttim_ring->ring_start_cyc =\n+\t\t\tcur_cyc - (real_bkt * tim_ring->tck_int);\n+\t\ttim_ring->last_updt_cyc = cur_cyc;\n+\t}\n+}\n+\n+static __rte_always_inline uint16_t\n+cnxk_tim_timer_arm_burst(const struct rte_event_timer_adapter *adptr,\n+\t\t\t struct rte_event_timer **tim, const uint16_t nb_timers,\n+\t\t\t const uint8_t flags)\n+{\n+\tstruct cnxk_tim_ring *tim_ring = adptr->data->adapter_priv;\n+\tstruct cnxk_tim_ent entry;\n+\tuint16_t index;\n+\tint ret;\n+\n+\tcnxk_tim_sync_start_cyc(tim_ring);\n+\tfor (index = 0; index < nb_timers; index++) {\n+\t\tif (cnxk_tim_arm_checks(tim_ring, tim[index]))\n+\t\t\tbreak;\n+\n+\t\tcnxk_tim_format_event(tim[index], &entry);\n+\t\tif (flags & CNXK_TIM_SP)\n+\t\t\tret = cnxk_tim_add_entry_sp(tim_ring,\n+\t\t\t\t\t\t tim[index]->timeout_ticks,\n+\t\t\t\t\t\t tim[index], &entry, flags);\n+\t\tif (flags & CNXK_TIM_MP)\n+\t\t\tret = cnxk_tim_add_entry_mp(tim_ring,\n+\t\t\t\t\t\t tim[index]->timeout_ticks,\n+\t\t\t\t\t\t tim[index], &entry, flags);\n+\n+\t\tif (unlikely(ret)) {\n+\t\t\trte_errno = -ret;\n+\t\t\tbreak;\n+\t\t}\n+\t}\n+\n+\treturn index;\n+}\n+\n+#define FP(_name, _f2, _f1, _flags) \\\n+\tuint16_t __rte_noinline cnxk_tim_arm_burst_##_name( \\\n+\t\tconst struct rte_event_timer_adapter *adptr, \\\n+\t\tstruct rte_event_timer **tim, const uint16_t nb_timers) \\\n+\t{ \\\n+\t\treturn cnxk_tim_timer_arm_burst(adptr, tim, nb_timers, \\\n+\t\t\t\t\t\t_flags); \\\n+\t}\n+TIM_ARM_FASTPATH_MODES\n+#undef FP\ndiff --git a/drivers/event/cnxk/cnxk_tim_worker.h b/drivers/event/cnxk/cnxk_tim_worker.h\nindex d56e67360..de8464e33 100644\n--- a/drivers/event/cnxk/cnxk_tim_worker.h\n+++ b/drivers/event/cnxk/cnxk_tim_worker.h\n@@ -120,4 +120,304 @@ cnxk_tim_bkt_clr_nent(struct cnxk_tim_bkt *bktp)\n \treturn __atomic_and_fetch(&bktp->w1, v, __ATOMIC_ACQ_REL);\n }\n \n+static inline uint64_t\n+cnxk_tim_bkt_fast_mod(uint64_t n, uint64_t d, struct rte_reciprocal_u64 R)\n+{\n+\treturn (n - (d * rte_reciprocal_divide_u64(n, &R)));\n+}\n+\n+static __rte_always_inline void\n+cnxk_tim_get_target_bucket(struct cnxk_tim_ring *const tim_ring,\n+\t\t\t const uint32_t rel_bkt, struct cnxk_tim_bkt **bkt,\n+\t\t\t struct cnxk_tim_bkt **mirr_bkt)\n+{\n+\tconst uint64_t bkt_cyc = cnxk_tim_cntvct() - tim_ring->ring_start_cyc;\n+\tuint64_t bucket =\n+\t\trte_reciprocal_divide_u64(bkt_cyc, &tim_ring->fast_div) +\n+\t\trel_bkt;\n+\tuint64_t mirr_bucket = 0;\n+\n+\tbucket = cnxk_tim_bkt_fast_mod(bucket, tim_ring->nb_bkts,\n+\t\t\t\t tim_ring->fast_bkt);\n+\tmirr_bucket =\n+\t\tcnxk_tim_bkt_fast_mod(bucket + (tim_ring->nb_bkts >> 1),\n+\t\t\t\t tim_ring->nb_bkts, tim_ring->fast_bkt);\n+\t*bkt = &tim_ring->bkt[bucket];\n+\t*mirr_bkt = &tim_ring->bkt[mirr_bucket];\n+}\n+\n+static struct cnxk_tim_ent *\n+cnxk_tim_clr_bkt(struct cnxk_tim_ring *const tim_ring,\n+\t\t struct cnxk_tim_bkt *const bkt)\n+{\n+#define TIM_MAX_OUTSTANDING_OBJ 64\n+\tvoid *pend_chunks[TIM_MAX_OUTSTANDING_OBJ];\n+\tstruct cnxk_tim_ent *chunk;\n+\tstruct cnxk_tim_ent *pnext;\n+\tuint8_t objs = 0;\n+\n+\tchunk = ((struct cnxk_tim_ent *)(uintptr_t)bkt->first_chunk);\n+\tchunk = (struct cnxk_tim_ent *)(uintptr_t)(chunk +\n+\t\t\t\t\t\t tim_ring->nb_chunk_slots)\n+\t\t\t->w0;\n+\twhile (chunk) {\n+\t\tpnext = (struct cnxk_tim_ent *)(uintptr_t)(\n+\t\t\t(chunk + tim_ring->nb_chunk_slots)->w0);\n+\t\tif (objs == TIM_MAX_OUTSTANDING_OBJ) {\n+\t\t\trte_mempool_put_bulk(tim_ring->chunk_pool, pend_chunks,\n+\t\t\t\t\t objs);\n+\t\t\tobjs = 0;\n+\t\t}\n+\t\tpend_chunks[objs++] = chunk;\n+\t\tchunk = pnext;\n+\t}\n+\n+\tif (objs)\n+\t\trte_mempool_put_bulk(tim_ring->chunk_pool, pend_chunks, objs);\n+\n+\treturn (struct cnxk_tim_ent *)(uintptr_t)bkt->first_chunk;\n+}\n+\n+static struct cnxk_tim_ent *\n+cnxk_tim_refill_chunk(struct cnxk_tim_bkt *const bkt,\n+\t\t struct cnxk_tim_bkt *const mirr_bkt,\n+\t\t struct cnxk_tim_ring *const tim_ring)\n+{\n+\tstruct cnxk_tim_ent *chunk;\n+\n+\tif (bkt->nb_entry || !bkt->first_chunk) {\n+\t\tif (unlikely(rte_mempool_get(tim_ring->chunk_pool,\n+\t\t\t\t\t (void **)&chunk)))\n+\t\t\treturn NULL;\n+\t\tif (bkt->nb_entry) {\n+\t\t\t*(uint64_t *)(((struct cnxk_tim_ent *)\n+\t\t\t\t\t mirr_bkt->current_chunk) +\n+\t\t\t\t tim_ring->nb_chunk_slots) =\n+\t\t\t\t(uintptr_t)chunk;\n+\t\t} else {\n+\t\t\tbkt->first_chunk = (uintptr_t)chunk;\n+\t\t}\n+\t} else {\n+\t\tchunk = cnxk_tim_clr_bkt(tim_ring, bkt);\n+\t\tbkt->first_chunk = (uintptr_t)chunk;\n+\t}\n+\t*(uint64_t *)(chunk + tim_ring->nb_chunk_slots) = 0;\n+\n+\treturn chunk;\n+}\n+\n+static struct cnxk_tim_ent *\n+cnxk_tim_insert_chunk(struct cnxk_tim_bkt *const bkt,\n+\t\t struct cnxk_tim_bkt *const mirr_bkt,\n+\t\t struct cnxk_tim_ring *const tim_ring)\n+{\n+\tstruct cnxk_tim_ent *chunk;\n+\n+\tif (unlikely(rte_mempool_get(tim_ring->chunk_pool, (void **)&chunk)))\n+\t\treturn NULL;\n+\n+\t*(uint64_t *)(chunk + tim_ring->nb_chunk_slots) = 0;\n+\tif (bkt->nb_entry) {\n+\t\t*(uint64_t *)(((struct cnxk_tim_ent *)(uintptr_t)\n+\t\t\t\t mirr_bkt->current_chunk) +\n+\t\t\t tim_ring->nb_chunk_slots) = (uintptr_t)chunk;\n+\t} else {\n+\t\tbkt->first_chunk = (uintptr_t)chunk;\n+\t}\n+\treturn chunk;\n+}\n+\n+static __rte_always_inline int\n+cnxk_tim_add_entry_sp(struct cnxk_tim_ring *const tim_ring,\n+\t\t const uint32_t rel_bkt, struct rte_event_timer *const tim,\n+\t\t const struct cnxk_tim_ent *const pent,\n+\t\t const uint8_t flags)\n+{\n+\tstruct cnxk_tim_bkt *mirr_bkt;\n+\tstruct cnxk_tim_ent *chunk;\n+\tstruct cnxk_tim_bkt *bkt;\n+\tuint64_t lock_sema;\n+\tint16_t rem;\n+\n+__retry:\n+\tcnxk_tim_get_target_bucket(tim_ring, rel_bkt, &bkt, &mirr_bkt);\n+\n+\t/* Get Bucket sema*/\n+\tlock_sema = cnxk_tim_bkt_fetch_sema_lock(bkt);\n+\n+\t/* Bucket related checks. */\n+\tif (unlikely(cnxk_tim_bkt_get_hbt(lock_sema))) {\n+\t\tif (cnxk_tim_bkt_get_nent(lock_sema) != 0) {\n+\t\t\tuint64_t hbt_state;\n+#ifdef RTE_ARCH_ARM64\n+\t\t\tasm volatile(PLT_CPU_FEATURE_PREAMBLE\n+\t\t\t\t \"\t\tldxr %[hbt], [%[w1]]\t\\n\"\n+\t\t\t\t \"\t\ttbz %[hbt], 33, dne%=\t\\n\"\n+\t\t\t\t \"\t\tsevl\t\t\t\\n\"\n+\t\t\t\t \"rty%=:\twfe\t\t\t\\n\"\n+\t\t\t\t \"\t\tldxr %[hbt], [%[w1]]\t\\n\"\n+\t\t\t\t \"\t\ttbnz %[hbt], 33, rty%=\t\\n\"\n+\t\t\t\t \"dne%=:\t\t\t\t\\n\"\n+\t\t\t\t : [hbt] \"=&r\"(hbt_state)\n+\t\t\t\t : [w1] \"r\"((&bkt->w1))\n+\t\t\t\t : \"memory\");\n+#else\n+\t\t\tdo {\n+\t\t\t\thbt_state = __atomic_load_n(&bkt->w1,\n+\t\t\t\t\t\t\t __ATOMIC_RELAXED);\n+\t\t\t} while (hbt_state & BIT_ULL(33));\n+#endif\n+\n+\t\t\tif (!(hbt_state & BIT_ULL(34))) {\n+\t\t\t\tcnxk_tim_bkt_dec_lock(bkt);\n+\t\t\t\tgoto __retry;\n+\t\t\t}\n+\t\t}\n+\t}\n+\t/* Insert the work. */\n+\trem = cnxk_tim_bkt_fetch_rem(lock_sema);\n+\n+\tif (!rem) {\n+\t\tif (flags & CNXK_TIM_ENA_FB)\n+\t\t\tchunk = cnxk_tim_refill_chunk(bkt, mirr_bkt, tim_ring);\n+\t\tif (flags & CNXK_TIM_ENA_DFB)\n+\t\t\tchunk = cnxk_tim_insert_chunk(bkt, mirr_bkt, tim_ring);\n+\n+\t\tif (unlikely(chunk == NULL)) {\n+\t\t\tbkt->chunk_remainder = 0;\n+\t\t\ttim->impl_opaque[0] = 0;\n+\t\t\ttim->impl_opaque[1] = 0;\n+\t\t\ttim->state = RTE_EVENT_TIMER_ERROR;\n+\t\t\tcnxk_tim_bkt_dec_lock(bkt);\n+\t\t\treturn -ENOMEM;\n+\t\t}\n+\t\tmirr_bkt->current_chunk = (uintptr_t)chunk;\n+\t\tbkt->chunk_remainder = tim_ring->nb_chunk_slots - 1;\n+\t} else {\n+\t\tchunk = (struct cnxk_tim_ent *)mirr_bkt->current_chunk;\n+\t\tchunk += tim_ring->nb_chunk_slots - rem;\n+\t}\n+\n+\t/* Copy work entry. */\n+\t*chunk = *pent;\n+\n+\ttim->impl_opaque[0] = (uintptr_t)chunk;\n+\ttim->impl_opaque[1] = (uintptr_t)bkt;\n+\t__atomic_store_n(&tim->state, RTE_EVENT_TIMER_ARMED, __ATOMIC_RELEASE);\n+\tcnxk_tim_bkt_inc_nent(bkt);\n+\tcnxk_tim_bkt_dec_lock_relaxed(bkt);\n+\n+\treturn 0;\n+}\n+\n+static __rte_always_inline int\n+cnxk_tim_add_entry_mp(struct cnxk_tim_ring *const tim_ring,\n+\t\t const uint32_t rel_bkt, struct rte_event_timer *const tim,\n+\t\t const struct cnxk_tim_ent *const pent,\n+\t\t const uint8_t flags)\n+{\n+\tstruct cnxk_tim_bkt *mirr_bkt;\n+\tstruct cnxk_tim_ent *chunk;\n+\tstruct cnxk_tim_bkt *bkt;\n+\tuint64_t lock_sema;\n+\tint16_t rem;\n+\n+__retry:\n+\tcnxk_tim_get_target_bucket(tim_ring, rel_bkt, &bkt, &mirr_bkt);\n+\t/* Get Bucket sema*/\n+\tlock_sema = cnxk_tim_bkt_fetch_sema_lock(bkt);\n+\n+\t/* Bucket related checks. */\n+\tif (unlikely(cnxk_tim_bkt_get_hbt(lock_sema))) {\n+\t\tif (cnxk_tim_bkt_get_nent(lock_sema) != 0) {\n+\t\t\tuint64_t hbt_state;\n+#ifdef RTE_ARCH_ARM64\n+\t\t\tasm volatile(PLT_CPU_FEATURE_PREAMBLE\n+\t\t\t\t \"\t\tldxr %[hbt], [%[w1]]\t\\n\"\n+\t\t\t\t \"\t\ttbz %[hbt], 33, dne%=\t\\n\"\n+\t\t\t\t \"\t\tsevl\t\t\t\\n\"\n+\t\t\t\t \"rty%=:\twfe\t\t\t\\n\"\n+\t\t\t\t \"\t\tldxr %[hbt], [%[w1]]\t\\n\"\n+\t\t\t\t \"\t\ttbnz %[hbt], 33, rty%=\t\\n\"\n+\t\t\t\t \"dne%=:\t\t\t\t\\n\"\n+\t\t\t\t : [hbt] \"=&r\"(hbt_state)\n+\t\t\t\t : [w1] \"r\"((&bkt->w1))\n+\t\t\t\t : \"memory\");\n+#else\n+\t\t\tdo {\n+\t\t\t\thbt_state = __atomic_load_n(&bkt->w1,\n+\t\t\t\t\t\t\t __ATOMIC_RELAXED);\n+\t\t\t} while (hbt_state & BIT_ULL(33));\n+#endif\n+\n+\t\t\tif (!(hbt_state & BIT_ULL(34))) {\n+\t\t\t\tcnxk_tim_bkt_dec_lock(bkt);\n+\t\t\t\tgoto __retry;\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\trem = cnxk_tim_bkt_fetch_rem(lock_sema);\n+\tif (rem < 0) {\n+\t\tcnxk_tim_bkt_dec_lock(bkt);\n+#ifdef RTE_ARCH_ARM64\n+\t\tasm volatile(PLT_CPU_FEATURE_PREAMBLE\n+\t\t\t \"\t\tldxr %[rem], [%[crem]]\t\\n\"\n+\t\t\t \"\t\ttbz %[rem], 63, dne%=\t\t\\n\"\n+\t\t\t \"\t\tsevl\t\t\t\t\\n\"\n+\t\t\t \"rty%=:\twfe\t\t\t\t\\n\"\n+\t\t\t \"\t\tldxr %[rem], [%[crem]]\t\\n\"\n+\t\t\t \"\t\ttbnz %[rem], 63, rty%=\t\t\\n\"\n+\t\t\t \"dne%=:\t\t\t\t\t\\n\"\n+\t\t\t : [rem] \"=&r\"(rem)\n+\t\t\t : [crem] \"r\"(&bkt->w1)\n+\t\t\t : \"memory\");\n+#else\n+\t\twhile (__atomic_load_n((int64_t *)&bkt->w1, __ATOMIC_RELAXED) <\n+\t\t 0)\n+\t\t\t;\n+#endif\n+\t\tgoto __retry;\n+\t} else if (!rem) {\n+\t\t/* Only one thread can be here*/\n+\t\tif (flags & CNXK_TIM_ENA_FB)\n+\t\t\tchunk = cnxk_tim_refill_chunk(bkt, mirr_bkt, tim_ring);\n+\t\tif (flags & CNXK_TIM_ENA_DFB)\n+\t\t\tchunk = cnxk_tim_insert_chunk(bkt, mirr_bkt, tim_ring);\n+\n+\t\tif (unlikely(chunk == NULL)) {\n+\t\t\ttim->impl_opaque[0] = 0;\n+\t\t\ttim->impl_opaque[1] = 0;\n+\t\t\ttim->state = RTE_EVENT_TIMER_ERROR;\n+\t\t\tcnxk_tim_bkt_set_rem(bkt, 0);\n+\t\t\tcnxk_tim_bkt_dec_lock(bkt);\n+\t\t\treturn -ENOMEM;\n+\t\t}\n+\t\t*chunk = *pent;\n+\t\tif (cnxk_tim_bkt_fetch_lock(lock_sema)) {\n+\t\t\tdo {\n+\t\t\t\tlock_sema = __atomic_load_n(&bkt->w1,\n+\t\t\t\t\t\t\t __ATOMIC_RELAXED);\n+\t\t\t} while (cnxk_tim_bkt_fetch_lock(lock_sema) - 1);\n+\t\t}\n+\t\trte_atomic_thread_fence(__ATOMIC_ACQUIRE);\n+\t\tmirr_bkt->current_chunk = (uintptr_t)chunk;\n+\t\t__atomic_store_n(&bkt->chunk_remainder,\n+\t\t\t\t tim_ring->nb_chunk_slots - 1,\n+\t\t\t\t __ATOMIC_RELEASE);\n+\t} else {\n+\t\tchunk = (struct cnxk_tim_ent *)mirr_bkt->current_chunk;\n+\t\tchunk += tim_ring->nb_chunk_slots - rem;\n+\t\t*chunk = *pent;\n+\t}\n+\n+\ttim->impl_opaque[0] = (uintptr_t)chunk;\n+\ttim->impl_opaque[1] = (uintptr_t)bkt;\n+\t__atomic_store_n(&tim->state, RTE_EVENT_TIMER_ARMED, __ATOMIC_RELEASE);\n+\tcnxk_tim_bkt_inc_nent(bkt);\n+\tcnxk_tim_bkt_dec_lock_relaxed(bkt);\n+\n+\treturn 0;\n+}\n+\n #endif /* __CNXK_TIM_WORKER_H__ */\n", "prefixes": [ "v4", "29/34" ] }