Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/77521/?format=api
{ "id": 77521, "url": "http://patches.dpdk.org/api/patches/77521/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1599855987-25976-16-git-send-email-timothy.mcdaniel@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1599855987-25976-16-git-send-email-timothy.mcdaniel@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1599855987-25976-16-git-send-email-timothy.mcdaniel@intel.com", "date": "2020-09-11T20:26:20", "name": "[15/22] event/dlb2: add enqueue and its burst variants", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "184cdb5a484ba3bf3d46762e0414eac469d9d8a5", "submitter": { "id": 826, "url": "http://patches.dpdk.org/api/people/826/?format=api", "name": "Timothy McDaniel", "email": "timothy.mcdaniel@intel.com" }, "delegate": { "id": 310, "url": "http://patches.dpdk.org/api/users/310/?format=api", "username": "jerin", "first_name": "Jerin", "last_name": "Jacob", "email": "jerinj@marvell.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1599855987-25976-16-git-send-email-timothy.mcdaniel@intel.com/mbox/", "series": [ { "id": 12164, "url": "http://patches.dpdk.org/api/series/12164/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=12164", "date": "2020-09-11T20:26:05", "name": "Add DLB2 PMD", "version": 1, "mbox": "http://patches.dpdk.org/series/12164/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/77521/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/77521/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 49EFEA04C1;\n\tFri, 11 Sep 2020 22:32:48 +0200 (CEST)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 97DDF1C225;\n\tFri, 11 Sep 2020 22:30:37 +0200 (CEST)", "from mga04.intel.com (mga04.intel.com [192.55.52.120])\n by dpdk.org (Postfix) with ESMTP id 525E61C126\n for <dev@dpdk.org>; Fri, 11 Sep 2020 22:30:05 +0200 (CEST)", "from orsmga005.jf.intel.com ([10.7.209.41])\n by fmsmga104.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 11 Sep 2020 13:30:04 -0700", "from txasoft-yocto.an.intel.com ([10.123.72.192])\n by orsmga005.jf.intel.com with ESMTP; 11 Sep 2020 13:30:04 -0700" ], "IronPort-SDR": [ "\n eWmBz/nq99oetsqMyr/K/myIyMF55+TMkAGAg7+1R49U5+fv1dB/hrY9MU16r+5v18may2gGSs\n bNhNwvjt/zGQ==", "\n LmFOmPc2dbUnnn6yb+fKbqgePvDEkzmKrvSRtaBZ4vcp6gcUOlF6USkDBSEuJI5NgkQD7bcEEX\n Ole6m+klCDDg==" ], "X-IronPort-AV": [ "E=McAfee;i=\"6000,8403,9741\"; a=\"156244385\"", "E=Sophos;i=\"5.76,417,1592895600\"; d=\"scan'208\";a=\"156244385\"", "E=Sophos;i=\"5.76,417,1592895600\"; d=\"scan'208\";a=\"481453636\"" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "From": "Timothy McDaniel <timothy.mcdaniel@intel.com>", "To": "", "Cc": "dev@dpdk.org, erik.g.carrillo@intel.com, gage.eads@intel.com,\n harry.van.haaren@intel.com, jerinj@marvell.com", "Date": "Fri, 11 Sep 2020 15:26:20 -0500", "Message-Id": "<1599855987-25976-16-git-send-email-timothy.mcdaniel@intel.com>", "X-Mailer": "git-send-email 1.7.10", "In-Reply-To": "<1599855987-25976-1-git-send-email-timothy.mcdaniel@intel.com>", "References": "<1599855987-25976-1-git-send-email-timothy.mcdaniel@intel.com>", "Subject": "[dpdk-dev] [PATCH 15/22] event/dlb2: add enqueue and its burst\n\tvariants", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Add support for enqueue and its variants.\n\nSigned-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>\n---\n drivers/event/dlb2/dlb2.c | 592 ++++++++++++++++++++++++++++++++++++++++++++++\n 1 file changed, 592 insertions(+)", "diff": "diff --git a/drivers/event/dlb2/dlb2.c b/drivers/event/dlb2/dlb2.c\nindex ef23def..ac4cf19 100644\n--- a/drivers/event/dlb2/dlb2.c\n+++ b/drivers/event/dlb2/dlb2.c\n@@ -2165,6 +2165,592 @@ dlb2_eventdev_start(struct rte_eventdev *dev)\n \treturn 0;\n }\n \n+static uint8_t cmd_byte_map[DLB2_NUM_PORT_TYPES][DLB2_NUM_HW_SCHED_TYPES] = {\n+\t{\n+\t\t/* Load-balanced cmd bytes */\n+\t\t[RTE_EVENT_OP_NEW] = DLB2_NEW_CMD_BYTE,\n+\t\t[RTE_EVENT_OP_FORWARD] = DLB2_FWD_CMD_BYTE,\n+\t\t[RTE_EVENT_OP_RELEASE] = DLB2_COMP_CMD_BYTE,\n+\t},\n+\t{\n+\t\t/* Directed cmd bytes */\n+\t\t[RTE_EVENT_OP_NEW] = DLB2_NEW_CMD_BYTE,\n+\t\t[RTE_EVENT_OP_FORWARD] = DLB2_NEW_CMD_BYTE,\n+\t\t[RTE_EVENT_OP_RELEASE] = DLB2_NOOP_CMD_BYTE,\n+\t},\n+};\n+\n+static inline uint32_t\n+dlb2_port_credits_get(struct dlb2_port *qm_port,\n+\t\t enum dlb2_hw_queue_types type)\n+{\n+\tuint32_t credits = *qm_port->credit_pool[type];\n+\tuint32_t batch_size = DLB2_SW_CREDIT_BATCH_SZ;\n+\n+\tif (unlikely(credits < batch_size))\n+\t\tbatch_size = credits;\n+\n+\tif (likely(credits &&\n+\t\t __atomic_compare_exchange_n(\n+\t\t\tqm_port->credit_pool[type],\n+\t\t\t&credits, credits - batch_size, false,\n+\t\t\t__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)))\n+\t\treturn batch_size;\n+\telse\n+\t\treturn 0;\n+}\n+\n+static inline void\n+dlb2_replenish_sw_credits(struct dlb2_eventdev *dlb2,\n+\t\t\t struct dlb2_eventdev_port *ev_port)\n+{\n+\tuint16_t quanta = ev_port->credit_update_quanta;\n+\n+\tif (ev_port->inflight_credits >= quanta * 2) {\n+\t\t/* Replenish credits, saving one quanta for enqueues */\n+\t\tuint16_t val = ev_port->inflight_credits - quanta;\n+\n+\t\t__atomic_fetch_sub(&dlb2->inflights, val, __ATOMIC_SEQ_CST);\n+\t\tev_port->inflight_credits -= val;\n+\t}\n+}\n+\n+static inline int\n+dlb2_check_enqueue_sw_credits(struct dlb2_eventdev *dlb2,\n+\t\t\t struct dlb2_eventdev_port *ev_port)\n+{\n+\tuint32_t sw_inflights = __atomic_load_n(&dlb2->inflights,\n+\t\t\t\t\t\t__ATOMIC_SEQ_CST);\n+\tconst int num = 1;\n+\n+\tif (unlikely(ev_port->inflight_max < sw_inflights)) {\n+\t\tDLB2_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);\n+\t\trte_errno = -ENOSPC;\n+\t\treturn 1;\n+\t}\n+\n+\tif (ev_port->inflight_credits < num) {\n+\t\t/* check if event enqueue brings ev_port over max threshold */\n+\t\tuint32_t credit_update_quanta = ev_port->credit_update_quanta;\n+\n+\t\tif (sw_inflights + credit_update_quanta >\n+\t\t\t\tdlb2->new_event_limit) {\n+\t\t\tDLB2_INC_STAT(\n+\t\t\tev_port->stats.traffic.tx_nospc_new_event_limit,\n+\t\t\t1);\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\n+\t\t__atomic_fetch_add(&dlb2->inflights, credit_update_quanta,\n+\t\t\t\t __ATOMIC_SEQ_CST);\n+\t\tev_port->inflight_credits += (credit_update_quanta);\n+\n+\t\tif (ev_port->inflight_credits < num) {\n+\t\t\tDLB2_INC_STAT(\n+\t\t\tev_port->stats.traffic.tx_nospc_inflight_credits,\n+\t\t\t1);\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static inline int\n+dlb2_check_enqueue_hw_ldb_credits(struct dlb2_port *qm_port)\n+{\n+\tif (unlikely(qm_port->cached_ldb_credits == 0)) {\n+\t\tqm_port->cached_ldb_credits =\n+\t\t\tdlb2_port_credits_get(qm_port,\n+\t\t\t\t\t DLB2_LDB_QUEUE);\n+\t\tif (unlikely(qm_port->cached_ldb_credits == 0)) {\n+\t\t\tDLB2_INC_STAT(\n+\t\t\tqm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,\n+\t\t\t1);\n+\t\t\tDLB2_LOG_DBG(\"ldb credits exhausted\\n\");\n+\t\t\treturn 1; /* credits exhausted */\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static inline int\n+dlb2_check_enqueue_hw_dir_credits(struct dlb2_port *qm_port)\n+{\n+\tif (unlikely(qm_port->cached_dir_credits == 0)) {\n+\t\tqm_port->cached_dir_credits =\n+\t\t\tdlb2_port_credits_get(qm_port,\n+\t\t\t\t\t DLB2_DIR_QUEUE);\n+\t\tif (unlikely(qm_port->cached_dir_credits == 0)) {\n+\t\t\tDLB2_INC_STAT(\n+\t\t\tqm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,\n+\t\t\t1);\n+\t\t\tDLB2_LOG_DBG(\"dir credits exhausted\\n\");\n+\t\t\treturn 1; /* credits exhausted */\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static __rte_always_inline void\n+dlb2_pp_write(struct dlb2_enqueue_qe *qe4,\n+\t struct process_local_port_data *port_data)\n+{\n+\tqm_mmio_fns.pp_enqueue_four(qe4, port_data->pp_addr);\n+}\n+\n+static inline int\n+dlb2_consume_qe_immediate(struct dlb2_port *qm_port, int num)\n+{\n+\tstruct process_local_port_data *port_data;\n+\tstruct dlb2_cq_pop_qe *qe;\n+\n+\tRTE_ASSERT(qm_port->config_state == DLB2_CONFIGURED);\n+\n+\tqe = qm_port->consume_qe;\n+\n+\tqe->tokens = num - 1;\n+\n+\t/* No store fence needed since no pointer is being sent, and CQ token\n+\t * pops can be safely reordered with other HCWs.\n+\t */\n+\tport_data = &dlb2_port[qm_port->id][PORT_TYPE(qm_port)];\n+\n+\tdlb2_movntdq_single(qe, port_data->pp_addr);\n+\n+\tDLB2_LOG_DBG(\"dlb2: consume immediate - %d QEs\\n\", num);\n+\n+\tqm_port->owed_tokens = 0;\n+\n+\treturn 0;\n+}\n+\n+static inline void\n+dlb2_hw_do_enqueue(struct dlb2_port *qm_port,\n+\t\t bool do_sfence,\n+\t\t struct process_local_port_data *port_data)\n+{\n+\t/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that\n+\t * application writes complete before enqueueing the QE.\n+\t */\n+\tif (do_sfence)\n+\t\trte_wmb();\n+\n+\tdlb2_pp_write(qm_port->qe4, port_data);\n+}\n+\n+static inline void\n+dlb2_construct_token_pop_qe(struct dlb2_port *qm_port, int idx)\n+{\n+\tstruct dlb2_cq_pop_qe *qe = (void *)qm_port->qe4;\n+\tint num = qm_port->owed_tokens;\n+\n+\tqe[idx].cmd_byte = DLB2_POP_CMD_BYTE;\n+\tqe[idx].tokens = num - 1;\n+\n+\tqm_port->owed_tokens = 0;\n+}\n+\n+static inline void\n+dlb2_event_build_hcws(struct dlb2_port *qm_port,\n+\t\t const struct rte_event ev[],\n+\t\t int num,\n+\t\t uint8_t *sched_type,\n+\t\t uint8_t *queue_id)\n+{\n+\tstruct dlb2_enqueue_qe *qe;\n+\tuint16_t sched_word[4];\n+\t__m128i sse_qe[2];\n+\tint i;\n+\n+\tqe = qm_port->qe4;\n+\n+\tsse_qe[0] = _mm_setzero_si128();\n+\tsse_qe[1] = _mm_setzero_si128();\n+\n+\tswitch (num) {\n+\tcase 4:\n+\t\t/* Construct the metadata portion of two HCWs in one 128b SSE\n+\t\t * register. HCW metadata is constructed in the SSE registers\n+\t\t * like so:\n+\t\t * sse_qe[0][63:0]: qe[0]'s metadata\n+\t\t * sse_qe[0][127:64]: qe[1]'s metadata\n+\t\t * sse_qe[1][63:0]: qe[2]'s metadata\n+\t\t * sse_qe[1][127:64]: qe[3]'s metadata\n+\t\t */\n+\n+\t\t/* Convert the event operation into a command byte and store it\n+\t\t * in the metadata:\n+\t\t * sse_qe[0][63:56] = cmd_byte_map[is_directed][ev[0].op]\n+\t\t * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]\n+\t\t * sse_qe[1][63:56] = cmd_byte_map[is_directed][ev[2].op]\n+\t\t * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]\n+\t\t */\n+#define DLB2_QE_CMD_BYTE 7\n+\t\tsse_qe[0] = _mm_insert_epi8(sse_qe[0],\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[0].op],\n+\t\t\t\tDLB2_QE_CMD_BYTE);\n+\t\tsse_qe[0] = _mm_insert_epi8(sse_qe[0],\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[1].op],\n+\t\t\t\tDLB2_QE_CMD_BYTE + 8);\n+\t\tsse_qe[1] = _mm_insert_epi8(sse_qe[1],\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[2].op],\n+\t\t\t\tDLB2_QE_CMD_BYTE);\n+\t\tsse_qe[1] = _mm_insert_epi8(sse_qe[1],\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[3].op],\n+\t\t\t\tDLB2_QE_CMD_BYTE + 8);\n+\n+\t\t/* Store priority, scheduling type, and queue ID in the sched\n+\t\t * word array because these values are re-used when the\n+\t\t * destination is a directed queue.\n+\t\t */\n+\t\tsched_word[0] = EV_TO_DLB2_PRIO(ev[0].priority) << 10 |\n+\t\t\t\tsched_type[0] << 8 |\n+\t\t\t\tqueue_id[0];\n+\t\tsched_word[1] = EV_TO_DLB2_PRIO(ev[1].priority) << 10 |\n+\t\t\t\tsched_type[1] << 8 |\n+\t\t\t\tqueue_id[1];\n+\t\tsched_word[2] = EV_TO_DLB2_PRIO(ev[2].priority) << 10 |\n+\t\t\t\tsched_type[2] << 8 |\n+\t\t\t\tqueue_id[2];\n+\t\tsched_word[3] = EV_TO_DLB2_PRIO(ev[3].priority) << 10 |\n+\t\t\t\tsched_type[3] << 8 |\n+\t\t\t\tqueue_id[3];\n+\n+\t\t/* Store the event priority, scheduling type, and queue ID in\n+\t\t * the metadata:\n+\t\t * sse_qe[0][31:16] = sched_word[0]\n+\t\t * sse_qe[0][95:80] = sched_word[1]\n+\t\t * sse_qe[1][31:16] = sched_word[2]\n+\t\t * sse_qe[1][95:80] = sched_word[3]\n+\t\t */\n+#define DLB2_QE_QID_SCHED_WORD 1\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t\t sched_word[0],\n+\t\t\t\t\t DLB2_QE_QID_SCHED_WORD);\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t\t sched_word[1],\n+\t\t\t\t\t DLB2_QE_QID_SCHED_WORD + 4);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t\t sched_word[2],\n+\t\t\t\t\t DLB2_QE_QID_SCHED_WORD);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t\t sched_word[3],\n+\t\t\t\t\t DLB2_QE_QID_SCHED_WORD + 4);\n+\n+\t\t/* If the destination is a load-balanced queue, store the lock\n+\t\t * ID. If it is a directed queue, DLB places this field in\n+\t\t * bytes 10-11 of the received QE, so we format it accordingly:\n+\t\t * sse_qe[0][47:32] = dir queue ? sched_word[0] : flow_id[0]\n+\t\t * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]\n+\t\t * sse_qe[1][47:32] = dir queue ? sched_word[2] : flow_id[2]\n+\t\t * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]\n+\t\t */\n+#define DLB2_QE_LOCK_ID_WORD 2\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t(sched_type[0] == DLB2_SCHED_DIRECTED) ?\n+\t\t\t\t\tsched_word[0] : ev[0].flow_id,\n+\t\t\t\tDLB2_QE_LOCK_ID_WORD);\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t(sched_type[1] == DLB2_SCHED_DIRECTED) ?\n+\t\t\t\t\tsched_word[1] : ev[1].flow_id,\n+\t\t\t\tDLB2_QE_LOCK_ID_WORD + 4);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t(sched_type[2] == DLB2_SCHED_DIRECTED) ?\n+\t\t\t\t\tsched_word[2] : ev[2].flow_id,\n+\t\t\t\tDLB2_QE_LOCK_ID_WORD);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t(sched_type[3] == DLB2_SCHED_DIRECTED) ?\n+\t\t\t\t\tsched_word[3] : ev[3].flow_id,\n+\t\t\t\tDLB2_QE_LOCK_ID_WORD + 4);\n+\n+\t\t/* Store the event type and sub event type in the metadata:\n+\t\t * sse_qe[0][15:0] = flow_id[0]\n+\t\t * sse_qe[0][79:64] = flow_id[1]\n+\t\t * sse_qe[1][15:0] = flow_id[2]\n+\t\t * sse_qe[1][79:64] = flow_id[3]\n+\t\t */\n+#define DLB2_QE_EV_TYPE_WORD 0\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t\t ev[0].sub_event_type << 8 |\n+\t\t\t\t\t\tev[0].event_type,\n+\t\t\t\t\t DLB2_QE_EV_TYPE_WORD);\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t\t ev[1].sub_event_type << 8 |\n+\t\t\t\t\t\tev[1].event_type,\n+\t\t\t\t\t DLB2_QE_EV_TYPE_WORD + 4);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t\t ev[2].sub_event_type << 8 |\n+\t\t\t\t\t\tev[2].event_type,\n+\t\t\t\t\t DLB2_QE_EV_TYPE_WORD);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t\t ev[3].sub_event_type << 8 |\n+\t\t\t\t\t\tev[3].event_type,\n+\t\t\t\t\t DLB2_QE_EV_TYPE_WORD + 4);\n+\n+\t\t/* Store the metadata to memory (use the double-precision\n+\t\t * _mm_storeh_pd because there is no integer function for\n+\t\t * storing the upper 64b):\n+\t\t * qe[0] metadata = sse_qe[0][63:0]\n+\t\t * qe[1] metadata = sse_qe[0][127:64]\n+\t\t * qe[2] metadata = sse_qe[1][63:0]\n+\t\t * qe[3] metadata = sse_qe[1][127:64]\n+\t\t */\n+\t\t_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);\n+\t\t_mm_storeh_pd((double *)&qe[1].u.opaque_data,\n+\t\t\t (__m128d)sse_qe[0]);\n+\t\t_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);\n+\t\t_mm_storeh_pd((double *)&qe[3].u.opaque_data,\n+\t\t\t (__m128d)sse_qe[1]);\n+\n+\t\tqe[0].data = ev[0].u64;\n+\t\tqe[1].data = ev[1].u64;\n+\t\tqe[2].data = ev[2].u64;\n+\t\tqe[3].data = ev[3].u64;\n+\n+\t\tbreak;\n+\tcase 3:\n+\tcase 2:\n+\tcase 1:\n+\t\t/* At least one QE will be valid, so only zero out three */\n+\t\tqe[1].cmd_byte = 0;\n+\t\tqe[2].cmd_byte = 0;\n+\t\tqe[3].cmd_byte = 0;\n+\n+\t\tfor (i = 0; i < num; i++) {\n+\t\t\tqe[i].cmd_byte =\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[i].op];\n+\t\t\tqe[i].sched_type = sched_type[i];\n+\t\t\tqe[i].data = ev[i].u64;\n+\t\t\tqe[i].qid = queue_id[i];\n+\t\t\tqe[i].priority = EV_TO_DLB2_PRIO(ev[i].priority);\n+\t\t\tqe[i].lock_id = ev[i].flow_id;\n+\t\t\tif (sched_type[i] == DLB2_SCHED_DIRECTED) {\n+\t\t\t\tstruct dlb2_msg_info *info =\n+\t\t\t\t\t(struct dlb2_msg_info *)&qe[i].lock_id;\n+\n+\t\t\t\tinfo->qid = queue_id[i];\n+\t\t\t\tinfo->sched_type = DLB2_SCHED_DIRECTED;\n+\t\t\t\tinfo->priority = qe[i].priority;\n+\t\t\t}\n+\t\t\tqe[i].u.event_type.major = ev[i].event_type;\n+\t\t\tqe[i].u.event_type.sub = ev[i].sub_event_type;\n+\t\t}\n+\t\tbreak;\n+\t}\n+}\n+\n+static inline int\n+dlb2_event_enqueue_prep(struct dlb2_eventdev_port *ev_port,\n+\t\t\tstruct dlb2_port *qm_port,\n+\t\t\tconst struct rte_event ev[],\n+\t\t\tuint8_t *sched_type,\n+\t\t\tuint8_t *queue_id)\n+{\n+\tstruct dlb2_eventdev *dlb2 = ev_port->dlb2;\n+\tstruct dlb2_eventdev_queue *ev_queue;\n+\tuint16_t *cached_credits = NULL;\n+\tstruct dlb2_queue *qm_queue;\n+\n+\tev_queue = &dlb2->ev_queues[ev->queue_id];\n+\tqm_queue = &ev_queue->qm_queue;\n+\t*queue_id = qm_queue->id;\n+\n+\t/* Ignore sched_type and hardware credits on release events */\n+\tif (ev->op == RTE_EVENT_OP_RELEASE)\n+\t\tgoto op_check;\n+\n+\tif (!qm_queue->is_directed) {\n+\t\t/* Load balanced destination queue */\n+\n+\t\tif (dlb2_check_enqueue_hw_ldb_credits(qm_port)) {\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\t\tcached_credits = &qm_port->cached_ldb_credits;\n+\n+\t\tswitch (ev->sched_type) {\n+\t\tcase RTE_SCHED_TYPE_ORDERED:\n+\t\t\tDLB2_LOG_DBG(\"dlb2: put_qe: RTE_SCHED_TYPE_ORDERED\\n\");\n+\t\t\tif (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {\n+\t\t\t\tDLB2_LOG_ERR(\"dlb2: tried to send ordered event to unordered queue %d\\n\",\n+\t\t\t\t\t *queue_id);\n+\t\t\t\trte_errno = -EINVAL;\n+\t\t\t\treturn 1;\n+\t\t\t}\n+\t\t\t*sched_type = DLB2_SCHED_ORDERED;\n+\t\t\tbreak;\n+\t\tcase RTE_SCHED_TYPE_ATOMIC:\n+\t\t\tDLB2_LOG_DBG(\"dlb2: put_qe: RTE_SCHED_TYPE_ATOMIC\\n\");\n+\t\t\t*sched_type = DLB2_SCHED_ATOMIC;\n+\t\t\tbreak;\n+\t\tcase RTE_SCHED_TYPE_PARALLEL:\n+\t\t\tDLB2_LOG_DBG(\"dlb2: put_qe: RTE_SCHED_TYPE_PARALLEL\\n\");\n+\t\t\tif (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)\n+\t\t\t\t*sched_type = DLB2_SCHED_ORDERED;\n+\t\t\telse\n+\t\t\t\t*sched_type = DLB2_SCHED_UNORDERED;\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tDLB2_LOG_ERR(\"Unsupported LDB sched type in put_qe\\n\");\n+\t\t\tDLB2_INC_STAT(ev_port->stats.tx_invalid, 1);\n+\t\t\trte_errno = -EINVAL;\n+\t\t\treturn 1;\n+\t\t}\n+\t} else {\n+\t\t/* Directed destination queue */\n+\n+\t\tif (dlb2_check_enqueue_hw_dir_credits(qm_port)) {\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\t\tcached_credits = &qm_port->cached_dir_credits;\n+\n+\t\tDLB2_LOG_DBG(\"dlb2: put_qe: RTE_SCHED_TYPE_DIRECTED\\n\");\n+\n+\t\t*sched_type = DLB2_SCHED_DIRECTED;\n+\t}\n+\n+op_check:\n+\tswitch (ev->op) {\n+\tcase RTE_EVENT_OP_NEW:\n+\t\t/* Check that a sw credit is available */\n+\t\tif (dlb2_check_enqueue_sw_credits(dlb2, ev_port)) {\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\t\tev_port->inflight_credits--;\n+\t\t(*cached_credits)--;\n+\t\tbreak;\n+\tcase RTE_EVENT_OP_FORWARD:\n+\t\t/* Check for outstanding_releases underflow. If this occurs,\n+\t\t * the application is not using the EVENT_OPs correctly; for\n+\t\t * example, forwarding or releasing events that were not\n+\t\t * dequeued.\n+\t\t */\n+\t\tRTE_ASSERT(ev_port->outstanding_releases > 0);\n+\t\tev_port->outstanding_releases--;\n+\t\tqm_port->issued_releases++;\n+\t\t(*cached_credits)--;\n+\t\tbreak;\n+\tcase RTE_EVENT_OP_RELEASE:\n+\t\tev_port->inflight_credits++;\n+\t\t/* Check for outstanding_releases underflow. If this occurs,\n+\t\t * the application is not using the EVENT_OPs correctly; for\n+\t\t * example, forwarding or releasing events that were not\n+\t\t * dequeued.\n+\t\t */\n+\t\tRTE_ASSERT(ev_port->outstanding_releases > 0);\n+\t\tev_port->outstanding_releases--;\n+\t\tqm_port->issued_releases++;\n+\n+\t\t/* Replenish s/w credits if enough are cached */\n+\t\tdlb2_replenish_sw_credits(dlb2, ev_port);\n+\t\tbreak;\n+\t}\n+\n+\tDLB2_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);\n+\tDLB2_INC_STAT(ev_port->stats.traffic.tx_ok, 1);\n+\n+#ifndef RTE_LIBRTE_PMD_DLB2_QUELL_STATS\n+\tif (ev->op != RTE_EVENT_OP_RELEASE) {\n+\t\tDLB2_INC_STAT(ev_port->stats.queue[ev->queue_id].enq_ok, 1);\n+\t\tDLB2_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);\n+\t}\n+#endif\n+\n+\treturn 0;\n+}\n+\n+static inline uint16_t\n+dlb2_event_enqueue_burst(void *event_port,\n+\t\t\t const struct rte_event events[],\n+\t\t\t uint16_t num)\n+{\n+\tstruct dlb2_eventdev_port *ev_port = event_port;\n+\tstruct dlb2_port *qm_port = &ev_port->qm_port;\n+\tstruct process_local_port_data *port_data;\n+\tint i, cnt;\n+\n+\tRTE_ASSERT(ev_port->enq_configured);\n+\tRTE_ASSERT(events != NULL);\n+\n+\tcnt = 0;\n+\n+\tport_data = &dlb2_port[qm_port->id][PORT_TYPE(qm_port)];\n+\n+\tfor (i = 0; i < num; i += DLB2_NUM_QES_PER_CACHE_LINE) {\n+\t\tuint8_t sched_types[DLB2_NUM_QES_PER_CACHE_LINE];\n+\t\tuint8_t queue_ids[DLB2_NUM_QES_PER_CACHE_LINE];\n+\t\tint j = 0;\n+\n+\t\tfor (; j < DLB2_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {\n+\t\t\tconst struct rte_event *ev = &events[i + j];\n+\n+\t\t\tif (dlb2_event_enqueue_prep(ev_port, qm_port, ev,\n+\t\t\t\t\t\t &sched_types[j],\n+\t\t\t\t\t\t &queue_ids[j]))\n+\t\t\t\tbreak;\n+\t\t}\n+\n+\t\tif (j == 0)\n+\t\t\tbreak;\n+\n+\t\tdlb2_event_build_hcws(qm_port, &events[i], j,\n+\t\t\t\t sched_types, queue_ids);\n+\n+\t\tif (qm_port->token_pop_mode == DELAYED_POP && j < 4 &&\n+\t\t qm_port->issued_releases >= qm_port->token_pop_thresh - 1) {\n+\t\t\tdlb2_construct_token_pop_qe(qm_port, j);\n+\n+\t\t\t/* Reset the releases counter for the next QE batch */\n+\t\t\tqm_port->issued_releases -= qm_port->token_pop_thresh;\n+\t\t}\n+\n+\t\tdlb2_hw_do_enqueue(qm_port, i == 0, port_data);\n+\n+\t\tcnt += j;\n+\n+\t\tif (j < DLB2_NUM_QES_PER_CACHE_LINE)\n+\t\t\tbreak;\n+\t}\n+\n+\tif (qm_port->token_pop_mode == DELAYED_POP &&\n+\t qm_port->issued_releases >= qm_port->token_pop_thresh - 1) {\n+\t\tdlb2_consume_qe_immediate(qm_port, qm_port->owed_tokens);\n+\t\tqm_port->issued_releases -= qm_port->token_pop_thresh;\n+\t}\n+\n+\treturn cnt;\n+}\n+\n+static inline uint16_t\n+dlb2_event_enqueue(void *event_port,\n+\t\t const struct rte_event events[])\n+{\n+\treturn dlb2_event_enqueue_burst(event_port, events, 1);\n+}\n+\n+static uint16_t\n+dlb2_event_enqueue_new_burst(void *event_port,\n+\t\t\t const struct rte_event events[],\n+\t\t\t uint16_t num)\n+{\n+\treturn dlb2_event_enqueue_burst(event_port, events, num);\n+}\n+\n+static uint16_t\n+dlb2_event_enqueue_forward_burst(void *event_port,\n+\t\t\t\t const struct rte_event events[],\n+\t\t\t\t uint16_t num)\n+{\n+\treturn dlb2_event_enqueue_burst(event_port, events, num);\n+}\n+\n static void\n dlb2_entry_points_init(struct rte_eventdev *dev)\n {\n@@ -2188,7 +2774,13 @@ dlb2_entry_points_init(struct rte_eventdev *dev)\n \t\t.xstats_reset\t = dlb2_eventdev_xstats_reset,\n \t};\n \n+\t/* Expose PMD's eventdev interface */\n+\n \tdev->dev_ops = &dlb2_eventdev_entry_ops;\n+\tdev->enqueue = dlb2_event_enqueue;\n+\tdev->enqueue_burst = dlb2_event_enqueue_burst;\n+\tdev->enqueue_new_burst = dlb2_event_enqueue_new_burst;\n+\tdev->enqueue_forward_burst = dlb2_event_enqueue_forward_burst;\n }\n \n int\n", "prefixes": [ "15/22" ] }