Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/77499/?format=api
{ "id": 77499, "url": "http://patches.dpdk.org/api/patches/77499/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1599851920-16802-16-git-send-email-timothy.mcdaniel@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1599851920-16802-16-git-send-email-timothy.mcdaniel@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1599851920-16802-16-git-send-email-timothy.mcdaniel@intel.com", "date": "2020-09-11T19:18:33", "name": "[v4,15/22] event/dlb: add enqueue and its burst variants", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "228451c625663fbc0f948524d29cdbb8be16bd12", "submitter": { "id": 826, "url": "http://patches.dpdk.org/api/people/826/?format=api", "name": "Timothy McDaniel", "email": "timothy.mcdaniel@intel.com" }, "delegate": { "id": 310, "url": "http://patches.dpdk.org/api/users/310/?format=api", "username": "jerin", "first_name": "Jerin", "last_name": "Jacob", "email": "jerinj@marvell.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1599851920-16802-16-git-send-email-timothy.mcdaniel@intel.com/mbox/", "series": [ { "id": 12163, "url": "http://patches.dpdk.org/api/series/12163/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=12163", "date": "2020-09-11T19:18:18", "name": "Add DLB PMD", "version": 4, "mbox": "http://patches.dpdk.org/series/12163/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/77499/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/77499/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 2FED6A04C1;\n\tFri, 11 Sep 2020 21:24:44 +0200 (CEST)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 823461C216;\n\tFri, 11 Sep 2020 21:22:29 +0200 (CEST)", "from mga18.intel.com (mga18.intel.com [134.134.136.126])\n by dpdk.org (Postfix) with ESMTP id 286281C1BA\n for <dev@dpdk.org>; Fri, 11 Sep 2020 21:22:14 +0200 (CEST)", "from fmsmga005.fm.intel.com ([10.253.24.32])\n by orsmga106.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 11 Sep 2020 12:22:12 -0700", "from txasoft-yocto.an.intel.com ([10.123.72.192])\n by fmsmga005.fm.intel.com with ESMTP; 11 Sep 2020 12:22:12 -0700" ], "IronPort-SDR": [ "\n 6sTkRrqU/r8ML411jNkj9y3W/GZRIEL8JcxwFoiMvrZXEZprD6t1WlpcJYcdiTuf4+qcU+unUC\n yXXiZDbFE3LA==", "\n uu0y4kIIFIb51asuExgRqcf31CTx8utMBniMoR9pT8WOHqpBkyX6ydEh23yg3O+qluof9p1IPe\n lVMZC1N5y0pQ==" ], "X-IronPort-AV": [ "E=McAfee;i=\"6000,8403,9741\"; a=\"146570556\"", "E=Sophos;i=\"5.76,416,1592895600\"; d=\"scan'208\";a=\"146570556\"", "E=Sophos;i=\"5.76,416,1592895600\"; d=\"scan'208\";a=\"506375692\"" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "From": "Timothy McDaniel <timothy.mcdaniel@intel.com>", "To": "", "Cc": "dev@dpdk.org, erik.g.carrillo@intel.com, gage.eads@intel.com,\n harry.van.haaren@intel.com, jerinj@marvell.com", "Date": "Fri, 11 Sep 2020 14:18:33 -0500", "Message-Id": "<1599851920-16802-16-git-send-email-timothy.mcdaniel@intel.com>", "X-Mailer": "git-send-email 1.7.10", "In-Reply-To": "<1599851920-16802-1-git-send-email-timothy.mcdaniel@intel.com>", "References": "<1599851920-16802-1-git-send-email-timothy.mcdaniel@intel.com>", "Subject": "[dpdk-dev] [PATCH v4 15/22] event/dlb: add enqueue and its burst\n\tvariants", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Add support for enqueue and its variants.\n\nSigned-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>\n---\n drivers/event/dlb/dlb.c | 725 ++++++++++++++++++++++++++++++++++++++++++++++++\n 1 file changed, 725 insertions(+)", "diff": "diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c\nindex 5f2a7fa..8fbd25a 100644\n--- a/drivers/event/dlb/dlb.c\n+++ b/drivers/event/dlb/dlb.c\n@@ -71,6 +71,25 @@ static struct rte_event_dev_info evdev_dlb_default_info = {\n struct process_local_port_data\n dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];\n \n+static inline uint16_t\n+dlb_event_enqueue_delayed(void *event_port,\n+\t\t\t const struct rte_event events[]);\n+\n+static inline uint16_t\n+dlb_event_enqueue_burst_delayed(void *event_port,\n+\t\t\t\tconst struct rte_event events[],\n+\t\t\t\tuint16_t num);\n+\n+static inline uint16_t\n+dlb_event_enqueue_new_burst_delayed(void *event_port,\n+\t\t\t\t const struct rte_event events[],\n+\t\t\t\t uint16_t num);\n+\n+static inline uint16_t\n+dlb_event_enqueue_forward_burst_delayed(void *event_port,\n+\t\t\t\t\tconst struct rte_event events[],\n+\t\t\t\t\tuint16_t num);\n+\n uint32_t\n dlb_get_queue_depth(struct dlb_eventdev *dlb,\n \t\t struct dlb_eventdev_queue *queue)\n@@ -1038,6 +1057,19 @@ dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,\n \t\t\tqm_port->token_pop_mode = DEFERRED_POP;\n \t}\n \n+\t/* The default enqueue functions do not include delayed-pop support for\n+\t * performance reasons.\n+\t */\n+\tif (qm_port->token_pop_mode == DELAYED_POP) {\n+\t\tdlb->event_dev->enqueue = dlb_event_enqueue_delayed;\n+\t\tdlb->event_dev->enqueue_burst =\n+\t\t\tdlb_event_enqueue_burst_delayed;\n+\t\tdlb->event_dev->enqueue_new_burst =\n+\t\t\tdlb_event_enqueue_new_burst_delayed;\n+\t\tdlb->event_dev->enqueue_forward_burst =\n+\t\t\tdlb_event_enqueue_forward_burst_delayed;\n+\t}\n+\n \tqm_port->owed_tokens = 0;\n \tqm_port->issued_releases = 0;\n \n@@ -2166,6 +2198,695 @@ dlb_eventdev_start(struct rte_eventdev *dev)\n \treturn 0;\n }\n \n+static inline int\n+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,\n+\t\t\t struct dlb_eventdev_port *ev_port)\n+{\n+\tuint32_t sw_inflights = __atomic_load_n(&dlb->inflights,\n+\t\t\t\t\t\t__ATOMIC_SEQ_CST);\n+\tconst int num = 1;\n+\n+\tif (unlikely(ev_port->inflight_max < sw_inflights)) {\n+\t\tDLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);\n+\t\trte_errno = -ENOSPC;\n+\t\treturn 1;\n+\t}\n+\n+\tif (ev_port->inflight_credits < num) {\n+\t\t/* check if event enqueue brings ev_port over max threshold */\n+\t\tuint32_t credit_update_quanta = ev_port->credit_update_quanta;\n+\n+\t\tif (sw_inflights + credit_update_quanta >\n+\t\t dlb->new_event_limit) {\n+\t\t\tDLB_INC_STAT(\n+\t\t\t\tev_port->stats.traffic.tx_nospc_new_event_limit,\n+\t\t\t\t1);\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\n+\t\t__atomic_fetch_add(&dlb->inflights, credit_update_quanta,\n+\t\t\t\t __ATOMIC_SEQ_CST);\n+\t\tev_port->inflight_credits += (credit_update_quanta);\n+\n+\t\tif (ev_port->inflight_credits < num) {\n+\t\t\tDLB_INC_STAT(\n+\t\t\t ev_port->stats.traffic.tx_nospc_inflight_credits,\n+\t\t\t 1);\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static inline void\n+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,\n+\t\t\t struct dlb_eventdev_port *ev_port)\n+{\n+\tuint16_t quanta = ev_port->credit_update_quanta;\n+\n+\tif (ev_port->inflight_credits >= quanta * 2) {\n+\t\t/* Replenish credits, saving one quanta for enqueues */\n+\t\tuint16_t val = ev_port->inflight_credits - quanta;\n+\n+\t\t__atomic_fetch_sub(&dlb->inflights, val, __ATOMIC_SEQ_CST);\n+\t\tev_port->inflight_credits -= val;\n+\t}\n+}\n+\n+static __rte_always_inline uint16_t\n+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)\n+{\n+\tvolatile uint16_t *popcount;\n+\n+\tif (ldb)\n+\t\tpopcount = port_data->ldb_popcount;\n+\telse\n+\t\tpopcount = port_data->dir_popcount;\n+\n+\treturn *popcount;\n+}\n+\n+static inline int\n+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,\n+\t\t\t\t struct process_local_port_data *port_data)\n+{\n+\tif (unlikely(qm_port->cached_ldb_credits == 0)) {\n+\t\tuint16_t pc;\n+\n+\t\tpc = dlb_read_pc(port_data, true);\n+\n+\t\tqm_port->cached_ldb_credits = pc -\n+\t\t\tqm_port->ldb_pushcount_at_credit_expiry;\n+\t\tif (unlikely(qm_port->cached_ldb_credits == 0)) {\n+\t\t\tDLB_INC_STAT(\n+\t\t\tqm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,\n+\t\t\t1);\n+\n+\t\t\tDLB_LOG_DBG(\"ldb credits exhausted\\n\");\n+\t\t\treturn 1;\n+\t\t}\n+\t\tqm_port->ldb_pushcount_at_credit_expiry +=\n+\t\t\tqm_port->cached_ldb_credits;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static inline int\n+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,\n+\t\t\t\t struct process_local_port_data *port_data)\n+{\n+\tif (unlikely(qm_port->cached_dir_credits == 0)) {\n+\t\tuint16_t pc;\n+\n+\t\tpc = dlb_read_pc(port_data, false);\n+\n+\t\tqm_port->cached_dir_credits = pc -\n+\t\t\tqm_port->dir_pushcount_at_credit_expiry;\n+\n+\t\tif (unlikely(qm_port->cached_dir_credits == 0)) {\n+\t\t\tDLB_INC_STAT(\n+\t\t\tqm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,\n+\t\t\t1);\n+\n+\t\t\tDLB_LOG_DBG(\"dir credits exhausted\\n\");\n+\t\t\treturn 1;\n+\t\t}\n+\t\tqm_port->dir_pushcount_at_credit_expiry +=\n+\t\t\tqm_port->cached_dir_credits;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static inline int\n+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,\n+\t\t struct dlb_port *qm_port,\n+\t\t const struct rte_event ev[],\n+\t\t struct process_local_port_data *port_data,\n+\t\t uint8_t *sched_type,\n+\t\t uint8_t *queue_id)\n+{\n+\tstruct dlb_eventdev *dlb = ev_port->dlb;\n+\tstruct dlb_eventdev_queue *ev_queue;\n+\tuint16_t *cached_credits = NULL;\n+\tstruct dlb_queue *qm_queue;\n+\n+\tev_queue = &dlb->ev_queues[ev->queue_id];\n+\tqm_queue = &ev_queue->qm_queue;\n+\t*queue_id = qm_queue->id;\n+\n+\t/* Ignore sched_type and hardware credits on release events */\n+\tif (ev->op == RTE_EVENT_OP_RELEASE)\n+\t\tgoto op_check;\n+\n+\tif (!qm_queue->is_directed) {\n+\t\t/* Load balanced destination queue */\n+\n+\t\tif (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\t\tcached_credits = &qm_port->cached_ldb_credits;\n+\n+\t\tswitch (ev->sched_type) {\n+\t\tcase RTE_SCHED_TYPE_ORDERED:\n+\t\t\tDLB_LOG_DBG(\"dlb: put_qe: RTE_SCHED_TYPE_ORDERED\\n\");\n+\t\t\tif (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {\n+\t\t\t\tDLB_LOG_ERR(\"dlb: tried to send ordered event to unordered queue %d\\n\",\n+\t\t\t\t\t *queue_id);\n+\t\t\t\trte_errno = -EINVAL;\n+\t\t\t\treturn 1;\n+\t\t\t}\n+\t\t\t*sched_type = DLB_SCHED_ORDERED;\n+\t\t\tbreak;\n+\t\tcase RTE_SCHED_TYPE_ATOMIC:\n+\t\t\tDLB_LOG_DBG(\"dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\\n\");\n+\t\t\t*sched_type = DLB_SCHED_ATOMIC;\n+\t\t\tbreak;\n+\t\tcase RTE_SCHED_TYPE_PARALLEL:\n+\t\t\tDLB_LOG_DBG(\"dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\\n\");\n+\t\t\tif (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)\n+\t\t\t\t*sched_type = DLB_SCHED_ORDERED;\n+\t\t\telse\n+\t\t\t\t*sched_type = DLB_SCHED_UNORDERED;\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tDLB_LOG_ERR(\"Unsupported LDB sched type in put_qe\\n\");\n+\t\t\tDLB_INC_STAT(ev_port->stats.tx_invalid, 1);\n+\t\t\trte_errno = -EINVAL;\n+\t\t\treturn 1;\n+\t\t}\n+\t} else {\n+\t\t/* Directed destination queue */\n+\n+\t\tif (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\t\tcached_credits = &qm_port->cached_dir_credits;\n+\n+\t\tDLB_LOG_DBG(\"dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\\n\");\n+\n+\t\t*sched_type = DLB_SCHED_DIRECTED;\n+\t}\n+\n+op_check:\n+\tswitch (ev->op) {\n+\tcase RTE_EVENT_OP_NEW:\n+\t\t/* Check that a sw credit is available */\n+\t\tif (dlb_check_enqueue_sw_credits(dlb, ev_port)) {\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\t\tev_port->inflight_credits--;\n+\t\t(*cached_credits)--;\n+\t\tbreak;\n+\tcase RTE_EVENT_OP_FORWARD:\n+\t\t/* Check for outstanding_releases underflow. If this occurs,\n+\t\t * the application is not using the EVENT_OPs correctly; for\n+\t\t * example, forwarding or releasing events that were not\n+\t\t * dequeued.\n+\t\t */\n+\t\tRTE_ASSERT(ev_port->outstanding_releases > 0);\n+\t\tev_port->outstanding_releases--;\n+\t\tqm_port->issued_releases++;\n+\t\t(*cached_credits)--;\n+\t\tbreak;\n+\tcase RTE_EVENT_OP_RELEASE:\n+\t\tev_port->inflight_credits++;\n+\t\t/* Check for outstanding_releases underflow. If this occurs,\n+\t\t * the application is not using the EVENT_OPs correctly; for\n+\t\t * example, forwarding or releasing events that were not\n+\t\t * dequeued.\n+\t\t */\n+\t\tRTE_ASSERT(ev_port->outstanding_releases > 0);\n+\t\tev_port->outstanding_releases--;\n+\t\tqm_port->issued_releases++;\n+\t\t/* Replenish s/w credits if enough are cached */\n+\t\tdlb_replenish_sw_credits(dlb, ev_port);\n+\t\tbreak;\n+\t}\n+\n+\tDLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);\n+\tDLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);\n+\n+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS\n+\tif (ev->op != RTE_EVENT_OP_RELEASE) {\n+\t\tDLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);\n+\t\tDLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);\n+\t}\n+#endif\n+\n+\treturn 0;\n+}\n+\n+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {\n+\t{\n+\t\t/* Load-balanced cmd bytes */\n+\t\t[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,\n+\t\t[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,\n+\t\t[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,\n+\t},\n+\t{\n+\t\t/* Directed cmd bytes */\n+\t\t[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,\n+\t\t[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,\n+\t\t[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,\n+\t},\n+};\n+\n+static inline void\n+dlb_event_build_hcws(struct dlb_port *qm_port,\n+\t\t const struct rte_event ev[],\n+\t\t int num,\n+\t\t uint8_t *sched_type,\n+\t\t uint8_t *queue_id)\n+{\n+\tstruct dlb_enqueue_qe *qe;\n+\tuint16_t sched_word[4];\n+\t__m128i sse_qe[2];\n+\tint i;\n+\n+\tqe = qm_port->qe4;\n+\n+\tsse_qe[0] = _mm_setzero_si128();\n+\tsse_qe[1] = _mm_setzero_si128();\n+\n+\tswitch (num) {\n+\tcase 4:\n+\t\t/* Construct the metadata portion of two HCWs in one 128b SSE\n+\t\t * register. HCW metadata is constructed in the SSE registers\n+\t\t * like so:\n+\t\t * sse_qe[0][63:0]: qe[0]'s metadata\n+\t\t * sse_qe[0][127:64]: qe[1]'s metadata\n+\t\t * sse_qe[1][63:0]: qe[2]'s metadata\n+\t\t * sse_qe[1][127:64]: qe[3]'s metadata\n+\t\t */\n+\n+\t\t/* Convert the event operation into a command byte and store it\n+\t\t * in the metadata:\n+\t\t * sse_qe[0][63:56] = cmd_byte_map[is_directed][ev[0].op]\n+\t\t * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]\n+\t\t * sse_qe[1][63:56] = cmd_byte_map[is_directed][ev[2].op]\n+\t\t * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]\n+\t\t */\n+#define DLB_QE_CMD_BYTE 7\n+\t\tsse_qe[0] = _mm_insert_epi8(sse_qe[0],\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[0].op],\n+\t\t\t\tDLB_QE_CMD_BYTE);\n+\t\tsse_qe[0] = _mm_insert_epi8(sse_qe[0],\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[1].op],\n+\t\t\t\tDLB_QE_CMD_BYTE + 8);\n+\t\tsse_qe[1] = _mm_insert_epi8(sse_qe[1],\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[2].op],\n+\t\t\t\tDLB_QE_CMD_BYTE);\n+\t\tsse_qe[1] = _mm_insert_epi8(sse_qe[1],\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[3].op],\n+\t\t\t\tDLB_QE_CMD_BYTE + 8);\n+\n+\t\t/* Store priority, scheduling type, and queue ID in the sched\n+\t\t * word array because these values are re-used when the\n+\t\t * destination is a directed queue.\n+\t\t */\n+\t\tsched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |\n+\t\t\t\tsched_type[0] << 8 |\n+\t\t\t\tqueue_id[0];\n+\t\tsched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |\n+\t\t\t\tsched_type[1] << 8 |\n+\t\t\t\tqueue_id[1];\n+\t\tsched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |\n+\t\t\t\tsched_type[2] << 8 |\n+\t\t\t\tqueue_id[2];\n+\t\tsched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |\n+\t\t\t\tsched_type[3] << 8 |\n+\t\t\t\tqueue_id[3];\n+\n+\t\t/* Store the event priority, scheduling type, and queue ID in\n+\t\t * the metadata:\n+\t\t * sse_qe[0][31:16] = sched_word[0]\n+\t\t * sse_qe[0][95:80] = sched_word[1]\n+\t\t * sse_qe[1][31:16] = sched_word[2]\n+\t\t * sse_qe[1][95:80] = sched_word[3]\n+\t\t */\n+#define DLB_QE_QID_SCHED_WORD 1\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t\t sched_word[0],\n+\t\t\t\t\t DLB_QE_QID_SCHED_WORD);\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t\t sched_word[1],\n+\t\t\t\t\t DLB_QE_QID_SCHED_WORD + 4);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t\t sched_word[2],\n+\t\t\t\t\t DLB_QE_QID_SCHED_WORD);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t\t sched_word[3],\n+\t\t\t\t\t DLB_QE_QID_SCHED_WORD + 4);\n+\n+\t\t/* If the destination is a load-balanced queue, store the lock\n+\t\t * ID. If it is a directed queue, DLB places this field in\n+\t\t * bytes 10-11 of the received QE, so we format it accordingly:\n+\t\t * sse_qe[0][47:32] = dir queue ? sched_word[0] : flow_id[0]\n+\t\t * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]\n+\t\t * sse_qe[1][47:32] = dir queue ? sched_word[2] : flow_id[2]\n+\t\t * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]\n+\t\t */\n+#define DLB_QE_LOCK_ID_WORD 2\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t(sched_type[0] == DLB_SCHED_DIRECTED) ?\n+\t\t\t\t\tsched_word[0] : ev[0].flow_id,\n+\t\t\t\tDLB_QE_LOCK_ID_WORD);\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t(sched_type[1] == DLB_SCHED_DIRECTED) ?\n+\t\t\t\t\tsched_word[1] : ev[1].flow_id,\n+\t\t\t\tDLB_QE_LOCK_ID_WORD + 4);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t(sched_type[2] == DLB_SCHED_DIRECTED) ?\n+\t\t\t\t\tsched_word[2] : ev[2].flow_id,\n+\t\t\t\tDLB_QE_LOCK_ID_WORD);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t(sched_type[3] == DLB_SCHED_DIRECTED) ?\n+\t\t\t\t\tsched_word[3] : ev[3].flow_id,\n+\t\t\t\tDLB_QE_LOCK_ID_WORD + 4);\n+\n+\t\t/* Store the event type and sub event type in the metadata:\n+\t\t * sse_qe[0][15:0] = flow_id[0]\n+\t\t * sse_qe[0][79:64] = flow_id[1]\n+\t\t * sse_qe[1][15:0] = flow_id[2]\n+\t\t * sse_qe[1][79:64] = flow_id[3]\n+\t\t */\n+#define DLB_QE_EV_TYPE_WORD 0\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t\t ev[0].sub_event_type << 8 |\n+\t\t\t\t\t\tev[0].event_type,\n+\t\t\t\t\t DLB_QE_EV_TYPE_WORD);\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t\t ev[1].sub_event_type << 8 |\n+\t\t\t\t\t\tev[1].event_type,\n+\t\t\t\t\t DLB_QE_EV_TYPE_WORD + 4);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t\t ev[2].sub_event_type << 8 |\n+\t\t\t\t\t\tev[2].event_type,\n+\t\t\t\t\t DLB_QE_EV_TYPE_WORD);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t\t ev[3].sub_event_type << 8 |\n+\t\t\t\t\t\tev[3].event_type,\n+\t\t\t\t\t DLB_QE_EV_TYPE_WORD + 4);\n+\n+\t\t/* Store the metadata to memory (use the double-precision\n+\t\t * _mm_storeh_pd because there is no integer function for\n+\t\t * storing the upper 64b):\n+\t\t * qe[0] metadata = sse_qe[0][63:0]\n+\t\t * qe[1] metadata = sse_qe[0][127:64]\n+\t\t * qe[2] metadata = sse_qe[1][63:0]\n+\t\t * qe[3] metadata = sse_qe[1][127:64]\n+\t\t */\n+\t\t_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);\n+\t\t_mm_storeh_pd((double *)&qe[1].u.opaque_data,\n+\t\t\t (__m128d) sse_qe[0]);\n+\t\t_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);\n+\t\t_mm_storeh_pd((double *)&qe[3].u.opaque_data,\n+\t\t\t (__m128d) sse_qe[1]);\n+\n+\t\tqe[0].data = ev[0].u64;\n+\t\tqe[1].data = ev[1].u64;\n+\t\tqe[2].data = ev[2].u64;\n+\t\tqe[3].data = ev[3].u64;\n+\n+\t\tbreak;\n+\tcase 3:\n+\tcase 2:\n+\tcase 1:\n+\t\tfor (i = 0; i < num; i++) {\n+\t\t\tqe[i].cmd_byte =\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[i].op];\n+\t\t\tqe[i].sched_type = sched_type[i];\n+\t\t\tqe[i].data = ev[i].u64;\n+\t\t\tqe[i].qid = queue_id[i];\n+\t\t\tqe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);\n+\t\t\tqe[i].lock_id = ev[i].flow_id;\n+\t\t\tif (sched_type[i] == DLB_SCHED_DIRECTED) {\n+\t\t\t\tstruct dlb_msg_info *info =\n+\t\t\t\t\t(struct dlb_msg_info *)&qe[i].lock_id;\n+\n+\t\t\t\tinfo->qid = queue_id[i];\n+\t\t\t\tinfo->sched_type = DLB_SCHED_DIRECTED;\n+\t\t\t\tinfo->priority = qe[i].priority;\n+\t\t\t}\n+\t\t\tqe[i].u.event_type.major = ev[i].event_type;\n+\t\t\tqe[i].u.event_type.sub = ev[i].sub_event_type;\n+\t\t}\n+\t\tbreak;\n+\tcase 0:\n+\t\tbreak;\n+\t}\n+}\n+\n+static inline void\n+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)\n+{\n+\tstruct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;\n+\tint num = qm_port->owed_tokens;\n+\n+\tif (qm_port->use_rsvd_token_scheme) {\n+\t\t/* Check if there's a deficit of reserved tokens, and return\n+\t\t * early if there are no (unreserved) tokens to consume.\n+\t\t */\n+\t\tif (num <= qm_port->cq_rsvd_token_deficit) {\n+\t\t\tqm_port->cq_rsvd_token_deficit -= num;\n+\t\t\tqm_port->owed_tokens = 0;\n+\t\t\treturn;\n+\t\t}\n+\t\tnum -= qm_port->cq_rsvd_token_deficit;\n+\t\tqm_port->cq_rsvd_token_deficit = 0;\n+\t}\n+\n+\tqe[idx].cmd_byte = DLB_POP_CMD_BYTE;\n+\tqe[idx].tokens = num - 1;\n+\n+\tqm_port->owed_tokens = 0;\n+}\n+\n+static __rte_always_inline void\n+dlb_pp_write(struct dlb_enqueue_qe *qe4,\n+\t struct process_local_port_data *port_data)\n+{\n+\tdlb_movdir64b(port_data->pp_addr, qe4);\n+}\n+\n+static inline void\n+dlb_hw_do_enqueue(struct dlb_port *qm_port,\n+\t\t bool do_sfence,\n+\t\t struct process_local_port_data *port_data)\n+{\n+\tDLB_LOG_DBG(\"dlb: Flushing QE(s) to DLB\\n\");\n+\n+\t/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that\n+\t * application writes complete before enqueueing the release HCW.\n+\t */\n+\tif (do_sfence)\n+\t\trte_wmb();\n+\n+\n+\tdlb_pp_write(qm_port->qe4, port_data);\n+}\n+\n+static inline int\n+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)\n+{\n+\tstruct process_local_port_data *port_data;\n+\tstruct dlb_cq_pop_qe *qe;\n+\n+\tRTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);\n+\n+\tif (qm_port->use_rsvd_token_scheme) {\n+\t\t/* Check if there's a deficit of reserved tokens, and return\n+\t\t * early if there are no (unreserved) tokens to consume.\n+\t\t */\n+\t\tif (num <= qm_port->cq_rsvd_token_deficit) {\n+\t\t\tqm_port->cq_rsvd_token_deficit -= num;\n+\t\t\tqm_port->owed_tokens = 0;\n+\t\t\treturn 0;\n+\t\t}\n+\t\tnum -= qm_port->cq_rsvd_token_deficit;\n+\t\tqm_port->cq_rsvd_token_deficit = 0;\n+\t}\n+\n+\tqe = qm_port->consume_qe;\n+\n+\tqe->tokens = num - 1;\n+\tqe->int_arm = 0;\n+\n+\t/* No store fence needed since no pointer is being sent, and CQ token\n+\t * pops can be safely reordered with other HCWs.\n+\t */\n+\tport_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];\n+\n+\tdlb_movntdq_single(port_data->pp_addr, qe);\n+\n+\tDLB_LOG_DBG(\"dlb: consume immediate - %d QEs\\n\", num);\n+\n+\tqm_port->owed_tokens = 0;\n+\n+\treturn 0;\n+}\n+\n+static inline uint16_t\n+__dlb_event_enqueue_burst(void *event_port,\n+\t\t\t const struct rte_event events[],\n+\t\t\t uint16_t num,\n+\t\t\t bool use_delayed)\n+{\n+\tstruct dlb_eventdev_port *ev_port = event_port;\n+\tstruct dlb_port *qm_port = &ev_port->qm_port;\n+\tstruct process_local_port_data *port_data;\n+\tint i;\n+\n+\tRTE_ASSERT(ev_port->enq_configured);\n+\tRTE_ASSERT(events != NULL);\n+\n+\trte_errno = 0;\n+\ti = 0;\n+\n+\tport_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];\n+\n+\twhile (i < num) {\n+\t\tuint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];\n+\t\tuint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];\n+\t\tint pop_offs = 0;\n+\t\tint j = 0;\n+\n+\t\tmemset(qm_port->qe4,\n+\t\t 0,\n+\t\t DLB_NUM_QES_PER_CACHE_LINE *\n+\t\t sizeof(struct dlb_enqueue_qe));\n+\n+\t\tfor (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {\n+\t\t\tconst struct rte_event *ev = &events[i + j];\n+\t\t\tint16_t thresh = qm_port->token_pop_thresh;\n+\n+\t\t\tif (use_delayed &&\n+\t\t\t qm_port->token_pop_mode == DELAYED_POP &&\n+\t\t\t (ev->op == RTE_EVENT_OP_FORWARD ||\n+\t\t\t ev->op == RTE_EVENT_OP_RELEASE) &&\n+\t\t\t qm_port->issued_releases >= thresh - 1) {\n+\t\t\t\t/* Insert the token pop QE and break out. This\n+\t\t\t\t * may result in a partial HCW, but that is\n+\t\t\t\t * simpler than supporting arbitrary QE\n+\t\t\t\t * insertion.\n+\t\t\t\t */\n+\t\t\t\tdlb_construct_token_pop_qe(qm_port, j);\n+\n+\t\t\t\t/* Reset the releases for the next QE batch */\n+\t\t\t\tqm_port->issued_releases -= thresh;\n+\n+\t\t\t\t/* When using delayed token pop mode, the\n+\t\t\t\t * initial token threshold is the full CQ\n+\t\t\t\t * depth. After the first token pop, we need to\n+\t\t\t\t * reset it to the dequeue_depth.\n+\t\t\t\t */\n+\t\t\t\tqm_port->token_pop_thresh =\n+\t\t\t\t\tqm_port->dequeue_depth;\n+\n+\t\t\t\tpop_offs = 1;\n+\t\t\t\tj++;\n+\t\t\t\tbreak;\n+\t\t\t}\n+\n+\t\t\tif (dlb_event_enqueue_prep(ev_port, qm_port, ev,\n+\t\t\t\t\t\t port_data, &sched_types[j],\n+\t\t\t\t\t\t &queue_ids[j]))\n+\t\t\t\tbreak;\n+\t\t}\n+\n+\t\tif (j == 0)\n+\t\t\tbreak;\n+\n+\t\tdlb_event_build_hcws(qm_port, &events[i], j - pop_offs,\n+\t\t\t\t sched_types, queue_ids);\n+\n+\t\tdlb_hw_do_enqueue(qm_port, i == 0, port_data);\n+\n+\t\t/* Don't include the token pop QE in the enqueue count */\n+\t\ti += j - pop_offs;\n+\n+\t\t/* Don't interpret j < DLB_NUM_... as out-of-credits if\n+\t\t * pop_offs != 0\n+\t\t */\n+\t\tif (j < DLB_NUM_QES_PER_CACHE_LINE && pop_offs == 0)\n+\t\t\tbreak;\n+\t}\n+\n+\tRTE_ASSERT(!((i == 0 && rte_errno != -ENOSPC)));\n+\n+\treturn i;\n+}\n+\n+static inline uint16_t\n+dlb_event_enqueue_burst(void *event_port,\n+\t\t\tconst struct rte_event events[],\n+\t\t\tuint16_t num)\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, num, false);\n+}\n+\n+static inline uint16_t\n+dlb_event_enqueue_burst_delayed(void *event_port,\n+\t\t\t\tconst struct rte_event events[],\n+\t\t\t\tuint16_t num)\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, num, true);\n+}\n+\n+static inline uint16_t\n+dlb_event_enqueue(void *event_port,\n+\t\t const struct rte_event events[])\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, 1, false);\n+}\n+\n+static inline uint16_t\n+dlb_event_enqueue_delayed(void *event_port,\n+\t\t\t const struct rte_event events[])\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, 1, true);\n+}\n+\n+static uint16_t\n+dlb_event_enqueue_new_burst(void *event_port,\n+\t\t\t const struct rte_event events[],\n+\t\t\t uint16_t num)\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, num, false);\n+}\n+\n+static uint16_t\n+dlb_event_enqueue_new_burst_delayed(void *event_port,\n+\t\t\t\t const struct rte_event events[],\n+\t\t\t\t uint16_t num)\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, num, true);\n+}\n+\n+static uint16_t\n+dlb_event_enqueue_forward_burst(void *event_port,\n+\t\t\t\tconst struct rte_event events[],\n+\t\t\t\tuint16_t num)\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, num, false);\n+}\n+\n+static uint16_t\n+dlb_event_enqueue_forward_burst_delayed(void *event_port,\n+\t\t\t\t\tconst struct rte_event events[],\n+\t\t\t\t\tuint16_t num)\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, num, true);\n+}\n+\n void\n dlb_entry_points_init(struct rte_eventdev *dev)\n {\n@@ -2190,6 +2911,10 @@ dlb_entry_points_init(struct rte_eventdev *dev)\n \n \t/* Expose PMD's eventdev interface */\n \tdev->dev_ops = &dlb_eventdev_entry_ops;\n+\tdev->enqueue = dlb_event_enqueue;\n+\tdev->enqueue_burst = dlb_event_enqueue_burst;\n+\tdev->enqueue_new_burst = dlb_event_enqueue_new_burst;\n+\tdev->enqueue_forward_burst = dlb_event_enqueue_forward_burst;\n \n }\n \n", "prefixes": [ "v4", "15/22" ] }