Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/73361/?format=api
{ "id": 73361, "url": "https://patches.dpdk.org/api/patches/73361/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/20200707050709.205480-3-patrick.fu@intel.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20200707050709.205480-3-patrick.fu@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20200707050709.205480-3-patrick.fu@intel.com", "date": "2020-07-07T05:07:09", "name": "[v6,2/2] vhost: introduce async enqueue for split ring", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "25b917c712f86daf6cfe18a053fc374fae732b2e", "submitter": { "id": 1781, "url": "https://patches.dpdk.org/api/people/1781/?format=api", "name": "Patrick Fu", "email": "patrick.fu@intel.com" }, "delegate": { "id": 319, "url": "https://patches.dpdk.org/api/users/319/?format=api", "username": "fyigit", "first_name": "Ferruh", "last_name": "Yigit", "email": "ferruh.yigit@amd.com" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/20200707050709.205480-3-patrick.fu@intel.com/mbox/", "series": [ { "id": 10832, "url": "https://patches.dpdk.org/api/series/10832/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=10832", "date": "2020-07-07T05:07:07", "name": "introduce asynchronous data path for vhost", "version": 6, "mbox": "https://patches.dpdk.org/series/10832/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/73361/comments/", "check": "success", "checks": "https://patches.dpdk.org/api/patches/73361/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 46018A00BE;\n\tTue, 7 Jul 2020 07:07:26 +0200 (CEST)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id CB1651DA64;\n\tTue, 7 Jul 2020 07:07:10 +0200 (CEST)", "from mga18.intel.com (mga18.intel.com [134.134.136.126])\n by dpdk.org (Postfix) with ESMTP id B6E4F1DA60\n for <dev@dpdk.org>; Tue, 7 Jul 2020 07:07:08 +0200 (CEST)", "from orsmga004.jf.intel.com ([10.7.209.38])\n by orsmga106.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 06 Jul 2020 22:07:08 -0700", "from npg-dpdk-patrickfu-casc2.sh.intel.com ([10.67.119.92])\n by orsmga004.jf.intel.com with ESMTP; 06 Jul 2020 22:07:05 -0700" ], "IronPort-SDR": [ "\n /EPk7OAc6OwBr2e68Z0rZNXGLUcyg3cr4XnqjCK4Qmd7C990Kxq9arffR95W3JO8EZ2jaM/lEV\n 8x2CSWaFfO/Q==", "\n Vj2MycyO34lO5gmc2xYknyLhDaBQiNY72DYOFZAf+AuQJtJlqCCuEoDAAcprAKVi6/fVR7QXGA\n ToCRXYXQ/K7Q==" ], "X-IronPort-AV": [ "E=McAfee;i=\"6000,8403,9674\"; a=\"135005159\"", "E=Sophos;i=\"5.75,321,1589266800\"; d=\"scan'208\";a=\"135005159\"", "E=Sophos;i=\"5.75,321,1589266800\"; d=\"scan'208\";a=\"427346681\"" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "From": "patrick.fu@intel.com", "To": "dev@dpdk.org, maxime.coquelin@redhat.com, chenbo.xia@intel.com,\n zhihong.wang@intel.com", "Cc": "patrick.fu@intel.com, yinan.wang@intel.com, cheng1.jiang@intel.com,\n cunming.liang@intel.com", "Date": "Tue, 7 Jul 2020 13:07:09 +0800", "Message-Id": "<20200707050709.205480-3-patrick.fu@intel.com>", "X-Mailer": "git-send-email 2.18.4", "In-Reply-To": "<20200707050709.205480-1-patrick.fu@intel.com>", "References": "<1591869725-13331-1-git-send-email-patrick.fu@intel.com>\n <20200707050709.205480-1-patrick.fu@intel.com>", "Subject": "[dpdk-dev] [PATCH v6 2/2] vhost: introduce async enqueue for split\n\tring", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "From: Patrick Fu <patrick.fu@intel.com>\n\nThis patch implements async enqueue data path for split ring. 2 new\nasync data path APIs are defined, by which applications can submit\nand poll packets to/from async engines. The async engine is either\na physical DMA device or it could also be a software emulated backend.\nThe async enqueue data path leverages callback functions registered by\napplications to work with the async engine.\n\nSigned-off-by: Patrick Fu <patrick.fu@intel.com>\n---\n lib/librte_vhost/rte_vhost_async.h | 40 +++\n lib/librte_vhost/virtio_net.c | 551 ++++++++++++++++++++++++++++-\n 2 files changed, 589 insertions(+), 2 deletions(-)", "diff": "diff --git a/lib/librte_vhost/rte_vhost_async.h b/lib/librte_vhost/rte_vhost_async.h\nindex d5a59279a..c8ad8dbc7 100644\n--- a/lib/librte_vhost/rte_vhost_async.h\n+++ b/lib/librte_vhost/rte_vhost_async.h\n@@ -133,4 +133,44 @@ int rte_vhost_async_channel_register(int vid, uint16_t queue_id,\n __rte_experimental\n int rte_vhost_async_channel_unregister(int vid, uint16_t queue_id);\n \n+/**\n+ * This function submit enqueue data to async engine. This function has\n+ * no guranttee to the transfer completion upon return. Applications\n+ * should poll transfer status by rte_vhost_poll_enqueue_completed()\n+ *\n+ * @param vid\n+ * id of vhost device to enqueue data\n+ * @param queue_id\n+ * queue id to enqueue data\n+ * @param pkts\n+ * array of packets to be enqueued\n+ * @param count\n+ * packets num to be enqueued\n+ * @return\n+ * num of packets enqueued\n+ */\n+__rte_experimental\n+uint16_t rte_vhost_submit_enqueue_burst(int vid, uint16_t queue_id,\n+\t\tstruct rte_mbuf **pkts, uint16_t count);\n+\n+/**\n+ * This function check async completion status for a specific vhost\n+ * device queue. Packets which finish copying (enqueue) operation\n+ * will be returned in an array.\n+ *\n+ * @param vid\n+ * id of vhost device to enqueue data\n+ * @param queue_id\n+ * queue id to enqueue data\n+ * @param pkts\n+ * blank array to get return packet pointer\n+ * @param count\n+ * size of the packet array\n+ * @return\n+ * num of packets returned\n+ */\n+__rte_experimental\n+uint16_t rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id,\n+\t\tstruct rte_mbuf **pkts, uint16_t count);\n+\n #endif /* _RTE_VHOST_ASYNC_H_ */\ndiff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c\nindex 751c1f373..236498f71 100644\n--- a/lib/librte_vhost/virtio_net.c\n+++ b/lib/librte_vhost/virtio_net.c\n@@ -17,14 +17,15 @@\n #include <rte_arp.h>\n #include <rte_spinlock.h>\n #include <rte_malloc.h>\n+#include <rte_vhost_async.h>\n \n #include \"iotlb.h\"\n #include \"vhost.h\"\n \n-#define MAX_PKT_BURST 32\n-\n #define MAX_BATCH_LEN 256\n \n+#define VHOST_ASYNC_BATCH_THRESHOLD 32\n+\n static __rte_always_inline bool\n rxvq_is_mergeable(struct virtio_net *dev)\n {\n@@ -116,6 +117,31 @@ flush_shadow_used_ring_split(struct virtio_net *dev, struct vhost_virtqueue *vq)\n \t\tsizeof(vq->used->idx));\n }\n \n+static __rte_always_inline void\n+async_flush_shadow_used_ring_split(struct virtio_net *dev,\n+\tstruct vhost_virtqueue *vq)\n+{\n+\tuint16_t used_idx = vq->last_used_idx & (vq->size - 1);\n+\n+\tif (used_idx + vq->shadow_used_idx <= vq->size) {\n+\t\tdo_flush_shadow_used_ring_split(dev, vq, used_idx, 0,\n+\t\t\t\t\t vq->shadow_used_idx);\n+\t} else {\n+\t\tuint16_t size;\n+\n+\t\t/* update used ring interval [used_idx, vq->size] */\n+\t\tsize = vq->size - used_idx;\n+\t\tdo_flush_shadow_used_ring_split(dev, vq, used_idx, 0, size);\n+\n+\t\t/* update the left half used ring interval [0, left_size] */\n+\t\tdo_flush_shadow_used_ring_split(dev, vq, 0, size,\n+\t\t\t\t\t vq->shadow_used_idx - size);\n+\t}\n+\n+\tvq->last_used_idx += vq->shadow_used_idx;\n+\tvq->shadow_used_idx = 0;\n+}\n+\n static __rte_always_inline void\n update_shadow_used_ring_split(struct vhost_virtqueue *vq,\n \t\t\t uint16_t desc_idx, uint32_t len)\n@@ -905,6 +931,200 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \treturn error;\n }\n \n+static __rte_always_inline void\n+async_fill_vec(struct iovec *v, void *base, size_t len)\n+{\n+\tv->iov_base = base;\n+\tv->iov_len = len;\n+}\n+\n+static __rte_always_inline void\n+async_fill_iter(struct rte_vhost_iov_iter *it, size_t count,\n+\tstruct iovec *vec, unsigned long nr_seg)\n+{\n+\tit->offset = 0;\n+\tit->count = count;\n+\n+\tif (count) {\n+\t\tit->iov = vec;\n+\t\tit->nr_segs = nr_seg;\n+\t} else {\n+\t\tit->iov = 0;\n+\t\tit->nr_segs = 0;\n+\t}\n+}\n+\n+static __rte_always_inline void\n+async_fill_desc(struct rte_vhost_async_desc *desc,\n+\tstruct rte_vhost_iov_iter *src, struct rte_vhost_iov_iter *dst)\n+{\n+\tdesc->src = src;\n+\tdesc->dst = dst;\n+}\n+\n+static __rte_always_inline int\n+async_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,\n+\t\t\tstruct rte_mbuf *m, struct buf_vector *buf_vec,\n+\t\t\tuint16_t nr_vec, uint16_t num_buffers,\n+\t\t\tstruct iovec *src_iovec, struct iovec *dst_iovec,\n+\t\t\tstruct rte_vhost_iov_iter *src_it,\n+\t\t\tstruct rte_vhost_iov_iter *dst_it)\n+{\n+\tuint32_t vec_idx = 0;\n+\tuint32_t mbuf_offset, mbuf_avail;\n+\tuint32_t buf_offset, buf_avail;\n+\tuint64_t buf_addr, buf_iova, buf_len;\n+\tuint32_t cpy_len, cpy_threshold;\n+\tuint64_t hdr_addr;\n+\tstruct rte_mbuf *hdr_mbuf;\n+\tstruct batch_copy_elem *batch_copy = vq->batch_copy_elems;\n+\tstruct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL;\n+\tint error = 0;\n+\n+\tuint32_t tlen = 0;\n+\tint tvec_idx = 0;\n+\tvoid *hpa;\n+\n+\tif (unlikely(m == NULL)) {\n+\t\terror = -1;\n+\t\tgoto out;\n+\t}\n+\n+\tcpy_threshold = vq->async_threshold;\n+\n+\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\tbuf_len = buf_vec[vec_idx].buf_len;\n+\n+\tif (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {\n+\t\terror = -1;\n+\t\tgoto out;\n+\t}\n+\n+\thdr_mbuf = m;\n+\thdr_addr = buf_addr;\n+\tif (unlikely(buf_len < dev->vhost_hlen))\n+\t\thdr = &tmp_hdr;\n+\telse\n+\t\thdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;\n+\n+\tVHOST_LOG_DATA(DEBUG, \"(%d) RX: num merge buffers %d\\n\",\n+\t\tdev->vid, num_buffers);\n+\n+\tif (unlikely(buf_len < dev->vhost_hlen)) {\n+\t\tbuf_offset = dev->vhost_hlen - buf_len;\n+\t\tvec_idx++;\n+\t\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\t\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\t\tbuf_len = buf_vec[vec_idx].buf_len;\n+\t\tbuf_avail = buf_len - buf_offset;\n+\t} else {\n+\t\tbuf_offset = dev->vhost_hlen;\n+\t\tbuf_avail = buf_len - dev->vhost_hlen;\n+\t}\n+\n+\tmbuf_avail = rte_pktmbuf_data_len(m);\n+\tmbuf_offset = 0;\n+\n+\twhile (mbuf_avail != 0 || m->next != NULL) {\n+\t\t/* done with current buf, get the next one */\n+\t\tif (buf_avail == 0) {\n+\t\t\tvec_idx++;\n+\t\t\tif (unlikely(vec_idx >= nr_vec)) {\n+\t\t\t\terror = -1;\n+\t\t\t\tgoto out;\n+\t\t\t}\n+\n+\t\t\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\t\t\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\t\t\tbuf_len = buf_vec[vec_idx].buf_len;\n+\n+\t\t\tbuf_offset = 0;\n+\t\t\tbuf_avail = buf_len;\n+\t\t}\n+\n+\t\t/* done with current mbuf, get the next one */\n+\t\tif (mbuf_avail == 0) {\n+\t\t\tm = m->next;\n+\n+\t\t\tmbuf_offset = 0;\n+\t\t\tmbuf_avail = rte_pktmbuf_data_len(m);\n+\t\t}\n+\n+\t\tif (hdr_addr) {\n+\t\t\tvirtio_enqueue_offload(hdr_mbuf, &hdr->hdr);\n+\t\t\tif (rxvq_is_mergeable(dev))\n+\t\t\t\tASSIGN_UNLESS_EQUAL(hdr->num_buffers,\n+\t\t\t\t\t\tnum_buffers);\n+\n+\t\t\tif (unlikely(hdr == &tmp_hdr)) {\n+\t\t\t\tcopy_vnet_hdr_to_desc(dev, vq, buf_vec, hdr);\n+\t\t\t} else {\n+\t\t\t\tPRINT_PACKET(dev, (uintptr_t)hdr_addr,\n+\t\t\t\t\t\tdev->vhost_hlen, 0);\n+\t\t\t\tvhost_log_cache_write_iova(dev, vq,\n+\t\t\t\t\t\tbuf_vec[0].buf_iova,\n+\t\t\t\t\t\tdev->vhost_hlen);\n+\t\t\t}\n+\n+\t\t\thdr_addr = 0;\n+\t\t}\n+\n+\t\tcpy_len = RTE_MIN(buf_avail, mbuf_avail);\n+\n+\t\tif (unlikely(cpy_len >= cpy_threshold)) {\n+\t\t\thpa = (void *)(uintptr_t)gpa_to_hpa(dev,\n+\t\t\t\t\tbuf_iova + buf_offset, cpy_len);\n+\n+\t\t\tif (unlikely(!hpa)) {\n+\t\t\t\terror = -1;\n+\t\t\t\tgoto out;\n+\t\t\t}\n+\n+\t\t\tasync_fill_vec(src_iovec + tvec_idx,\n+\t\t\t\t(void *)(uintptr_t)rte_pktmbuf_iova_offset(m,\n+\t\t\t\t\t\tmbuf_offset), cpy_len);\n+\n+\t\t\tasync_fill_vec(dst_iovec + tvec_idx, hpa, cpy_len);\n+\n+\t\t\ttlen += cpy_len;\n+\t\t\ttvec_idx++;\n+\t\t} else {\n+\t\t\tif (unlikely(vq->batch_copy_nb_elems >= vq->size)) {\n+\t\t\t\trte_memcpy(\n+\t\t\t\t(void *)((uintptr_t)(buf_addr + buf_offset)),\n+\t\t\t\trte_pktmbuf_mtod_offset(m, void *, mbuf_offset),\n+\t\t\t\tcpy_len);\n+\n+\t\t\t\tPRINT_PACKET(dev,\n+\t\t\t\t\t(uintptr_t)(buf_addr + buf_offset),\n+\t\t\t\t\tcpy_len, 0);\n+\t\t\t} else {\n+\t\t\t\tbatch_copy[vq->batch_copy_nb_elems].dst =\n+\t\t\t\t(void *)((uintptr_t)(buf_addr + buf_offset));\n+\t\t\t\tbatch_copy[vq->batch_copy_nb_elems].src =\n+\t\t\t\trte_pktmbuf_mtod_offset(m, void *, mbuf_offset);\n+\t\t\t\tbatch_copy[vq->batch_copy_nb_elems].log_addr =\n+\t\t\t\t\tbuf_iova + buf_offset;\n+\t\t\t\tbatch_copy[vq->batch_copy_nb_elems].len =\n+\t\t\t\t\tcpy_len;\n+\t\t\t\tvq->batch_copy_nb_elems++;\n+\t\t\t}\n+\t\t}\n+\n+\t\tmbuf_avail -= cpy_len;\n+\t\tmbuf_offset += cpy_len;\n+\t\tbuf_avail -= cpy_len;\n+\t\tbuf_offset += cpy_len;\n+\t}\n+\n+out:\n+\tasync_fill_iter(src_it, tlen, src_iovec, tvec_idx);\n+\tasync_fill_iter(dst_it, tlen, dst_iovec, tvec_idx);\n+\n+\treturn error;\n+}\n+\n static __rte_always_inline int\n vhost_enqueue_single_packed(struct virtio_net *dev,\n \t\t\t struct vhost_virtqueue *vq,\n@@ -1236,6 +1456,333 @@ rte_vhost_enqueue_burst(int vid, uint16_t queue_id,\n \treturn virtio_dev_rx(dev, queue_id, pkts, count);\n }\n \n+static __rte_always_inline uint16_t\n+virtio_dev_rx_async_get_info_idx(uint16_t pkts_idx,\n+\tuint16_t vq_size, uint16_t n_inflight)\n+{\n+\treturn pkts_idx > n_inflight ? (pkts_idx - n_inflight) :\n+\t\t(vq_size - n_inflight + pkts_idx) & (vq_size - 1);\n+}\n+\n+static __rte_always_inline void\n+virtio_dev_rx_async_submit_split_err(struct virtio_net *dev,\n+\tstruct vhost_virtqueue *vq, uint16_t queue_id,\n+\tuint16_t last_idx, uint16_t shadow_idx)\n+{\n+\tuint16_t start_idx, pkts_idx, vq_size;\n+\tuint64_t *async_pending_info;\n+\n+\tpkts_idx = vq->async_pkts_idx;\n+\tasync_pending_info = vq->async_pending_info;\n+\tvq_size = vq->size;\n+\tstart_idx = virtio_dev_rx_async_get_info_idx(pkts_idx,\n+\t\tvq_size, vq->async_pkts_inflight_n);\n+\n+\twhile (likely((start_idx & (vq_size - 1)) != pkts_idx)) {\n+\t\tuint64_t n_seg =\n+\t\t\tasync_pending_info[(start_idx) & (vq_size - 1)] >>\n+\t\t\tASYNC_PENDING_INFO_N_SFT;\n+\n+\t\twhile (n_seg)\n+\t\t\tn_seg -= vq->async_ops.check_completed_copies(dev->vid,\n+\t\t\t\tqueue_id, 0, 1);\n+\t}\n+\n+\tvq->async_pkts_inflight_n = 0;\n+\tvq->batch_copy_nb_elems = 0;\n+\n+\tvq->shadow_used_idx = shadow_idx;\n+\tvq->last_avail_idx = last_idx;\n+}\n+\n+static __rte_noinline uint32_t\n+virtio_dev_rx_async_submit_split(struct virtio_net *dev,\n+\tstruct vhost_virtqueue *vq, uint16_t queue_id,\n+\tstruct rte_mbuf **pkts, uint32_t count)\n+{\n+\tuint32_t pkt_idx = 0, pkt_burst_idx = 0;\n+\tuint16_t num_buffers;\n+\tstruct buf_vector buf_vec[BUF_VECTOR_MAX];\n+\tuint16_t avail_head, last_idx, shadow_idx;\n+\n+\tstruct rte_vhost_iov_iter *it_pool = vq->it_pool;\n+\tstruct iovec *vec_pool = vq->vec_pool;\n+\tstruct rte_vhost_async_desc tdes[MAX_PKT_BURST];\n+\tstruct iovec *src_iovec = vec_pool;\n+\tstruct iovec *dst_iovec = vec_pool + (VHOST_MAX_ASYNC_VEC >> 1);\n+\tstruct rte_vhost_iov_iter *src_it = it_pool;\n+\tstruct rte_vhost_iov_iter *dst_it = it_pool + 1;\n+\tuint16_t n_free_slot, slot_idx;\n+\tint n_pkts = 0;\n+\n+\tavail_head = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE);\n+\tlast_idx = vq->last_avail_idx;\n+\tshadow_idx = vq->shadow_used_idx;\n+\n+\t/*\n+\t * The ordering between avail index and\n+\t * desc reads needs to be enforced.\n+\t */\n+\trte_smp_rmb();\n+\n+\trte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);\n+\n+\tfor (pkt_idx = 0; pkt_idx < count; pkt_idx++) {\n+\t\tuint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;\n+\t\tuint16_t nr_vec = 0;\n+\n+\t\tif (unlikely(reserve_avail_buf_split(dev, vq,\n+\t\t\t\t\t\tpkt_len, buf_vec, &num_buffers,\n+\t\t\t\t\t\tavail_head, &nr_vec) < 0)) {\n+\t\t\tVHOST_LOG_DATA(DEBUG,\n+\t\t\t\t\"(%d) failed to get enough desc from vring\\n\",\n+\t\t\t\tdev->vid);\n+\t\t\tvq->shadow_used_idx -= num_buffers;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tVHOST_LOG_DATA(DEBUG, \"(%d) current index %d | end index %d\\n\",\n+\t\t\tdev->vid, vq->last_avail_idx,\n+\t\t\tvq->last_avail_idx + num_buffers);\n+\n+\t\tif (async_mbuf_to_desc(dev, vq, pkts[pkt_idx],\n+\t\t\t\tbuf_vec, nr_vec, num_buffers,\n+\t\t\t\tsrc_iovec, dst_iovec, src_it, dst_it) < 0) {\n+\t\t\tvq->shadow_used_idx -= num_buffers;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tslot_idx = (vq->async_pkts_idx + pkt_idx) & (vq->size - 1);\n+\t\tif (src_it->count) {\n+\t\t\tasync_fill_desc(&tdes[pkt_burst_idx], src_it, dst_it);\n+\t\t\tpkt_burst_idx++;\n+\t\t\tvq->async_pending_info[slot_idx] =\n+\t\t\t\tnum_buffers | (src_it->nr_segs << 16);\n+\t\t\tsrc_iovec += src_it->nr_segs;\n+\t\t\tdst_iovec += dst_it->nr_segs;\n+\t\t\tsrc_it += 2;\n+\t\t\tdst_it += 2;\n+\t\t} else {\n+\t\t\tvq->async_pending_info[slot_idx] = num_buffers;\n+\t\t\tvq->async_pkts_inflight_n++;\n+\t\t}\n+\n+\t\tvq->last_avail_idx += num_buffers;\n+\n+\t\tif (pkt_burst_idx >= VHOST_ASYNC_BATCH_THRESHOLD ||\n+\t\t\t\t(pkt_idx == count - 1 && pkt_burst_idx)) {\n+\t\t\tn_pkts = vq->async_ops.transfer_data(dev->vid,\n+\t\t\t\t\tqueue_id, tdes, 0, pkt_burst_idx);\n+\t\t\tsrc_iovec = vec_pool;\n+\t\t\tdst_iovec = vec_pool + (VHOST_MAX_ASYNC_VEC >> 1);\n+\t\t\tsrc_it = it_pool;\n+\t\t\tdst_it = it_pool + 1;\n+\n+\t\t\tif (unlikely(n_pkts < (int)pkt_burst_idx)) {\n+\t\t\t\tvq->async_pkts_inflight_n +=\n+\t\t\t\t\tn_pkts > 0 ? n_pkts : 0;\n+\t\t\t\tvirtio_dev_rx_async_submit_split_err(dev,\n+\t\t\t\t\tvq, queue_id, last_idx, shadow_idx);\n+\t\t\t\treturn 0;\n+\t\t\t}\n+\n+\t\t\tpkt_burst_idx = 0;\n+\t\t\tvq->async_pkts_inflight_n += n_pkts;\n+\t\t}\n+\t}\n+\n+\tif (pkt_burst_idx) {\n+\t\tn_pkts = vq->async_ops.transfer_data(dev->vid,\n+\t\t\t\tqueue_id, tdes, 0, pkt_burst_idx);\n+\t\tif (unlikely(n_pkts < (int)pkt_burst_idx)) {\n+\t\t\tvq->async_pkts_inflight_n += n_pkts > 0 ? n_pkts : 0;\n+\t\t\tvirtio_dev_rx_async_submit_split_err(dev, vq, queue_id,\n+\t\t\t\tlast_idx, shadow_idx);\n+\t\t\treturn 0;\n+\t\t}\n+\n+\t\tvq->async_pkts_inflight_n += n_pkts;\n+\t}\n+\n+\tdo_data_copy_enqueue(dev, vq);\n+\n+\tn_free_slot = vq->size - vq->async_pkts_idx;\n+\tif (n_free_slot > pkt_idx) {\n+\t\trte_memcpy(&vq->async_pkts_pending[vq->async_pkts_idx],\n+\t\t\tpkts, pkt_idx * sizeof(uintptr_t));\n+\t\tvq->async_pkts_idx += pkt_idx;\n+\t} else {\n+\t\trte_memcpy(&vq->async_pkts_pending[vq->async_pkts_idx],\n+\t\t\tpkts, n_free_slot * sizeof(uintptr_t));\n+\t\trte_memcpy(&vq->async_pkts_pending[0],\n+\t\t\t&pkts[n_free_slot],\n+\t\t\t(pkt_idx - n_free_slot) * sizeof(uintptr_t));\n+\t\tvq->async_pkts_idx = pkt_idx - n_free_slot;\n+\t}\n+\n+\tif (likely(vq->shadow_used_idx))\n+\t\tasync_flush_shadow_used_ring_split(dev, vq);\n+\n+\treturn pkt_idx;\n+}\n+\n+uint16_t rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id,\n+\t\tstruct rte_mbuf **pkts, uint16_t count)\n+{\n+\tstruct virtio_net *dev = get_device(vid);\n+\tstruct vhost_virtqueue *vq;\n+\tuint16_t n_pkts_cpl, n_pkts_put = 0, n_descs = 0;\n+\tuint16_t start_idx, pkts_idx, vq_size;\n+\tuint64_t *async_pending_info;\n+\n+\tVHOST_LOG_DATA(DEBUG, \"(%d) %s\\n\", dev->vid, __func__);\n+\tif (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {\n+\t\tVHOST_LOG_DATA(ERR, \"(%d) %s: invalid virtqueue idx %d.\\n\",\n+\t\t\tdev->vid, __func__, queue_id);\n+\t\treturn 0;\n+\t}\n+\n+\tvq = dev->virtqueue[queue_id];\n+\n+\trte_spinlock_lock(&vq->access_lock);\n+\n+\tpkts_idx = vq->async_pkts_idx;\n+\tasync_pending_info = vq->async_pending_info;\n+\tvq_size = vq->size;\n+\tstart_idx = virtio_dev_rx_async_get_info_idx(pkts_idx,\n+\t\tvq_size, vq->async_pkts_inflight_n);\n+\n+\tn_pkts_cpl =\n+\t\tvq->async_ops.check_completed_copies(vid, queue_id, 0, count);\n+\n+\trte_smp_wmb();\n+\n+\twhile (likely(((start_idx + n_pkts_put) & (vq_size - 1)) != pkts_idx)) {\n+\t\tuint64_t info = async_pending_info[\n+\t\t\t(start_idx + n_pkts_put) & (vq_size - 1)];\n+\t\tuint64_t n_segs;\n+\t\tn_pkts_put++;\n+\t\tn_descs += info & ASYNC_PENDING_INFO_N_MSK;\n+\t\tn_segs = info >> ASYNC_PENDING_INFO_N_SFT;\n+\n+\t\tif (n_segs) {\n+\t\t\tif (!n_pkts_cpl || n_pkts_cpl < n_segs) {\n+\t\t\t\tn_pkts_put--;\n+\t\t\t\tn_descs -= info & ASYNC_PENDING_INFO_N_MSK;\n+\t\t\t\tif (n_pkts_cpl) {\n+\t\t\t\t\tasync_pending_info[\n+\t\t\t\t\t\t(start_idx + n_pkts_put) &\n+\t\t\t\t\t\t(vq_size - 1)] =\n+\t\t\t\t\t((n_segs - n_pkts_cpl) <<\n+\t\t\t\t\t ASYNC_PENDING_INFO_N_SFT) |\n+\t\t\t\t\t(info & ASYNC_PENDING_INFO_N_MSK);\n+\t\t\t\t\tn_pkts_cpl = 0;\n+\t\t\t\t}\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t\tn_pkts_cpl -= n_segs;\n+\t\t}\n+\t}\n+\n+\tif (n_pkts_put) {\n+\t\tvq->async_pkts_inflight_n -= n_pkts_put;\n+\t\t__atomic_add_fetch(&vq->used->idx, n_descs, __ATOMIC_RELEASE);\n+\n+\t\tvhost_vring_call_split(dev, vq);\n+\t}\n+\n+\tif (start_idx + n_pkts_put <= vq_size) {\n+\t\trte_memcpy(pkts, &vq->async_pkts_pending[start_idx],\n+\t\t\tn_pkts_put * sizeof(uintptr_t));\n+\t} else {\n+\t\trte_memcpy(pkts, &vq->async_pkts_pending[start_idx],\n+\t\t\t(vq_size - start_idx) * sizeof(uintptr_t));\n+\t\trte_memcpy(&pkts[vq_size - start_idx], vq->async_pkts_pending,\n+\t\t\t(n_pkts_put - vq_size + start_idx) * sizeof(uintptr_t));\n+\t}\n+\n+\trte_spinlock_unlock(&vq->access_lock);\n+\n+\treturn n_pkts_put;\n+}\n+\n+static __rte_always_inline uint32_t\n+virtio_dev_rx_async_submit(struct virtio_net *dev, uint16_t queue_id,\n+\tstruct rte_mbuf **pkts, uint32_t count)\n+{\n+\tstruct vhost_virtqueue *vq;\n+\tuint32_t nb_tx = 0;\n+\tbool drawback = false;\n+\n+\tVHOST_LOG_DATA(DEBUG, \"(%d) %s\\n\", dev->vid, __func__);\n+\tif (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {\n+\t\tVHOST_LOG_DATA(ERR, \"(%d) %s: invalid virtqueue idx %d.\\n\",\n+\t\t\tdev->vid, __func__, queue_id);\n+\t\treturn 0;\n+\t}\n+\n+\tvq = dev->virtqueue[queue_id];\n+\n+\trte_spinlock_lock(&vq->access_lock);\n+\n+\tif (unlikely(vq->enabled == 0))\n+\t\tgoto out_access_unlock;\n+\n+\tif (unlikely(!vq->async_registered)) {\n+\t\tdrawback = true;\n+\t\tgoto out_access_unlock;\n+\t}\n+\n+\tif (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))\n+\t\tvhost_user_iotlb_rd_lock(vq);\n+\n+\tif (unlikely(vq->access_ok == 0))\n+\t\tif (unlikely(vring_translate(dev, vq) < 0))\n+\t\t\tgoto out;\n+\n+\tcount = RTE_MIN((uint32_t)MAX_PKT_BURST, count);\n+\tif (count == 0)\n+\t\tgoto out;\n+\n+\t/* TODO: packed queue not implemented */\n+\tif (vq_is_packed(dev))\n+\t\tnb_tx = 0;\n+\telse\n+\t\tnb_tx = virtio_dev_rx_async_submit_split(dev,\n+\t\t\t\tvq, queue_id, pkts, count);\n+\n+out:\n+\tif (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))\n+\t\tvhost_user_iotlb_rd_unlock(vq);\n+\n+out_access_unlock:\n+\trte_spinlock_unlock(&vq->access_lock);\n+\n+\tif (drawback)\n+\t\treturn rte_vhost_enqueue_burst(dev->vid, queue_id, pkts, count);\n+\n+\treturn nb_tx;\n+}\n+\n+uint16_t\n+rte_vhost_submit_enqueue_burst(int vid, uint16_t queue_id,\n+\t\tstruct rte_mbuf **pkts, uint16_t count)\n+{\n+\tstruct virtio_net *dev = get_device(vid);\n+\n+\tif (!dev)\n+\t\treturn 0;\n+\n+\tif (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {\n+\t\tVHOST_LOG_DATA(ERR,\n+\t\t\t\"(%d) %s: built-in vhost net backend is disabled.\\n\",\n+\t\t\tdev->vid, __func__);\n+\t\treturn 0;\n+\t}\n+\n+\treturn virtio_dev_rx_async_submit(dev, queue_id, pkts, count);\n+}\n+\n static inline bool\n virtio_net_with_host_offload(struct virtio_net *dev)\n {\n", "prefixes": [ "v6", "2/2" ] }