Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/121043/?format=api
{ "id": 121043, "url": "https://patches.dpdk.org/api/patches/121043/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/20221220004415.29576-3-cheng1.jiang@intel.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20221220004415.29576-3-cheng1.jiang@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20221220004415.29576-3-cheng1.jiang@intel.com", "date": "2022-12-20T00:44:14", "name": "[2/3] vhost: add batch enqueue in async vhost packed ring", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "ccdf9270e161f975dd6734d711970b0bfab98950", "submitter": { "id": 1530, "url": "https://patches.dpdk.org/api/people/1530/?format=api", "name": "Jiang, Cheng1", "email": "Cheng1.jiang@intel.com" }, "delegate": { "id": 2642, "url": "https://patches.dpdk.org/api/users/2642/?format=api", "username": "mcoquelin", "first_name": "Maxime", "last_name": "Coquelin", "email": "maxime.coquelin@redhat.com" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/20221220004415.29576-3-cheng1.jiang@intel.com/mbox/", "series": [ { "id": 26184, "url": "https://patches.dpdk.org/api/series/26184/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=26184", "date": "2022-12-20T00:44:12", "name": "Async vhost packed ring optimization", "version": 1, "mbox": "https://patches.dpdk.org/series/26184/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/121043/comments/", "check": "success", "checks": "https://patches.dpdk.org/api/patches/121043/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 8A942A00C5;\n\tTue, 20 Dec 2022 02:30:59 +0100 (CET)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 7D65F42D12;\n\tTue, 20 Dec 2022 02:30:52 +0100 (CET)", "from mga12.intel.com (mga12.intel.com [192.55.52.136])\n by mails.dpdk.org (Postfix) with ESMTP id E136042D10\n for <dev@dpdk.org>; Tue, 20 Dec 2022 02:30:50 +0100 (CET)", "from fmsmga005.fm.intel.com ([10.253.24.32])\n by fmsmga106.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 19 Dec 2022 17:30:50 -0800", "from dpdk_jiangcheng.sh.intel.com ([10.67.118.237])\n by fmsmga005.fm.intel.com with ESMTP; 19 Dec 2022 17:30:47 -0800" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple;\n d=intel.com; i=@intel.com; q=dns/txt; s=Intel;\n t=1671499851; x=1703035851;\n h=from:to:cc:subject:date:message-id:in-reply-to:\n references:mime-version:content-transfer-encoding;\n bh=KPnXEwWXK/DN8kmHD4idn8b7A6/MJQ3blhwP4SNQJJI=;\n b=VDJS68FQ3v+3IHrk+3Qld+b1xq0g3NQ/ixIp867JMdt90T5ekUltf5Yr\n q15QWY3ol1Qc7TjoVIZ62XWY6vYX9w855okfz+AUs8m9K9932jBUFCWfe\n JCvl1VCLfSOnJERIn6mWIeGe6uV2mvFPd1EXskwC3y3huV2le+wYvjniy\n NAIlSbKmlcee8mrGv+JxLKZi8mJyAWEzTxk1jtzWGgngHo6CI74Nz6Dvo\n H95OEvoIIusKPTjFA9sR1yNvckB3fP9ETIKS4Ke6LlnVrgXwMcYrjQVs3\n GDETROA0VsdlvzALUIZKzaRPSA/fVxHGNkcIIOCU8X7UND0r0tT8X/B52 g==;", "X-IronPort-AV": [ "E=McAfee;i=\"6500,9779,10566\"; a=\"299176767\"", "E=Sophos;i=\"5.96,258,1665471600\"; d=\"scan'208\";a=\"299176767\"", "E=McAfee;i=\"6500,9779,10566\"; a=\"979585057\"", "E=Sophos;i=\"5.96,258,1665471600\"; d=\"scan'208\";a=\"979585057\"" ], "X-ExtLoop1": "1", "From": "Cheng Jiang <cheng1.jiang@intel.com>", "To": "maxime.coquelin@redhat.com,\n\tchenbo.xia@intel.com", "Cc": "dev@dpdk.org, jiayu.hu@intel.com, xuan.ding@intel.com,\n wenwux.ma@intel.com,\n yuanx.wang@intel.com, yvonnex.yang@intel.com, xingguang.he@intel.com,\n Cheng Jiang <cheng1.jiang@intel.com>", "Subject": "[PATCH 2/3] vhost: add batch enqueue in async vhost packed ring", "Date": "Tue, 20 Dec 2022 00:44:14 +0000", "Message-Id": "<20221220004415.29576-3-cheng1.jiang@intel.com>", "X-Mailer": "git-send-email 2.35.1", "In-Reply-To": "<20221220004415.29576-1-cheng1.jiang@intel.com>", "References": "<20221220004415.29576-1-cheng1.jiang@intel.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "Add batch enqueue function in asynchronous vhost packed ring to\nimprove the performance. Chained mbufs are not supported, it will\nbe handled in single enqueue function.\n\nSigned-off-by: Cheng Jiang <cheng1.jiang@intel.com>\n---\n lib/vhost/virtio_net.c | 157 +++++++++++++++++++++++++++++++++++++++++\n 1 file changed, 157 insertions(+)", "diff": "diff --git a/lib/vhost/virtio_net.c b/lib/vhost/virtio_net.c\nindex 7c3ec128a0..ac8c404327 100644\n--- a/lib/vhost/virtio_net.c\n+++ b/lib/vhost/virtio_net.c\n@@ -432,6 +432,24 @@ vhost_flush_enqueue_batch_packed(struct virtio_net *dev,\n \tvq_inc_last_used_packed(vq, PACKED_BATCH_SIZE);\n }\n \n+static __rte_always_inline void\n+vhost_async_shadow_enqueue_packed_batch(struct vhost_virtqueue *vq,\n+\t\t\t\t uint64_t *lens,\n+\t\t\t\t uint16_t *ids)\n+{\n+\tuint16_t i;\n+\tstruct vhost_async *async = vq->async;\n+\n+\tvhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {\n+\t\tasync->buffers_packed[async->buffer_idx_packed].id = ids[i];\n+\t\tasync->buffers_packed[async->buffer_idx_packed].len = lens[i];\n+\t\tasync->buffers_packed[async->buffer_idx_packed].count = 1;\n+\t\tasync->buffer_idx_packed++;\n+\t\tif (async->buffer_idx_packed >= vq->size)\n+\t\t\tasync->buffer_idx_packed -= vq->size;\n+\t}\n+}\n+\n static __rte_always_inline void\n vhost_shadow_dequeue_batch_packed_inorder(struct vhost_virtqueue *vq,\n \t\t\t\t\t uint16_t id)\n@@ -1451,6 +1469,58 @@ virtio_dev_rx_sync_batch_check(struct virtio_net *dev,\n \treturn 0;\n }\n \n+static __rte_always_inline int\n+virtio_dev_rx_async_batch_check(struct vhost_virtqueue *vq,\n+\t\t\t struct rte_mbuf **pkts,\n+\t\t\t uint64_t *desc_addrs,\n+\t\t\t uint64_t *lens,\n+\t\t\t int16_t dma_id,\n+\t\t\t uint16_t vchan_id)\n+{\n+\tbool wrap_counter = vq->avail_wrap_counter;\n+\tstruct vring_packed_desc *descs = vq->desc_packed;\n+\tuint16_t avail_idx = vq->last_avail_idx;\n+\tuint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf);\n+\tuint16_t i;\n+\n+\tif (unlikely(avail_idx & PACKED_BATCH_MASK))\n+\t\treturn -1;\n+\n+\tif (unlikely((avail_idx + PACKED_BATCH_SIZE) > vq->size))\n+\t\treturn -1;\n+\n+\tvhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {\n+\t\tif (unlikely(pkts[i]->next != NULL))\n+\t\t\treturn -1;\n+\t\tif (unlikely(!desc_is_avail(&descs[avail_idx + i],\n+\t\t\t\t\t wrap_counter)))\n+\t\t\treturn -1;\n+\t}\n+\n+\tvhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)\n+\t\tlens[i] = descs[avail_idx + i].len;\n+\n+\tvhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {\n+\t\tif (unlikely(pkts[i]->pkt_len > (lens[i] - buf_offset)))\n+\t\t\treturn -1;\n+\t}\n+\n+\tvhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)\n+\t\tdesc_addrs[i] = descs[avail_idx + i].addr;\n+\n+\tvhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {\n+\t\tif (unlikely(!desc_addrs[i]))\n+\t\t\treturn -1;\n+\t\tif (unlikely(lens[i] != descs[avail_idx + i].len))\n+\t\t\treturn -1;\n+\t}\n+\n+\tif (rte_dma_burst_capacity(dma_id, vchan_id) < PACKED_BATCH_SIZE)\n+\t\treturn -1;\n+\n+\treturn 0;\n+}\n+\n static __rte_always_inline void\n virtio_dev_rx_batch_packed_copy(struct virtio_net *dev,\n \t\t\t struct vhost_virtqueue *vq,\n@@ -1850,6 +1920,78 @@ virtio_dev_rx_async_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \treturn 0;\n }\n \n+static __rte_always_inline void\n+virtio_dev_rx_async_packed_batch_enqueue(struct virtio_net *dev,\n+\t\t\t struct vhost_virtqueue *vq,\n+\t\t\t struct rte_mbuf **pkts,\n+\t\t\t uint64_t *desc_addrs,\n+\t\t\t uint64_t *lens)\n+{\n+\tuint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf);\n+\tstruct virtio_net_hdr_mrg_rxbuf *hdrs[PACKED_BATCH_SIZE];\n+\tstruct vring_packed_desc *descs = vq->desc_packed;\n+\tstruct vhost_async *async = vq->async;\n+\tuint16_t avail_idx = vq->last_avail_idx;\n+\tuint32_t mbuf_offset = 0;\n+\tuint16_t ids[PACKED_BATCH_SIZE];\n+\tuint64_t mapped_len[PACKED_BATCH_SIZE];\n+\tvoid *host_iova[PACKED_BATCH_SIZE];\n+\tuintptr_t desc;\n+\tuint16_t i;\n+\n+\tvhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {\n+\t\trte_prefetch0((void *)(uintptr_t)desc_addrs[i]);\n+\t\tdesc = vhost_iova_to_vva(dev, vq, desc_addrs[i], &lens[i], VHOST_ACCESS_RW);\n+\t\thdrs[i] = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)desc;\n+\t\tlens[i] = pkts[i]->pkt_len +\n+\t\t\tsizeof(struct virtio_net_hdr_mrg_rxbuf);\n+\t}\n+\n+\tvhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)\n+\t\tvirtio_enqueue_offload(pkts[i], &hdrs[i]->hdr);\n+\n+\tvq_inc_last_avail_packed(vq, PACKED_BATCH_SIZE);\n+\n+\tvhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {\n+\t\thost_iova[i] = (void *)(uintptr_t)gpa_to_first_hpa(dev,\n+\t\t\tdesc_addrs[i] + buf_offset, lens[i], &mapped_len[i]);\n+\t}\n+\n+\tvhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {\n+\t\tasync_iter_initialize(dev, async);\n+\t\tasync_iter_add_iovec(dev, async,\n+\t\t\t\t(void *)(uintptr_t)rte_pktmbuf_iova_offset(pkts[i], mbuf_offset),\n+\t\t\t\thost_iova[i],\n+\t\t\t\tmapped_len[i]);\n+\t\tasync->iter_idx++;\n+\t}\n+\n+\tvhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)\n+\t\tvhost_log_cache_write_iova(dev, vq, descs[avail_idx + i].addr, lens[i]);\n+\n+\tvhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)\n+\t\tids[i] = descs[avail_idx + i].id;\n+\n+\tvhost_async_shadow_enqueue_packed_batch(vq, lens, ids);\n+}\n+\n+static __rte_always_inline int\n+virtio_dev_rx_async_packed_batch(struct virtio_net *dev,\n+\t\t\t struct vhost_virtqueue *vq,\n+\t\t\t struct rte_mbuf **pkts,\n+\t\t\t int16_t dma_id, uint16_t vchan_id)\n+{\n+\tuint64_t desc_addrs[PACKED_BATCH_SIZE];\n+\tuint64_t lens[PACKED_BATCH_SIZE];\n+\n+\tif (virtio_dev_rx_async_batch_check(vq, pkts, desc_addrs, lens, dma_id, vchan_id) == -1)\n+\t\treturn -1;\n+\n+\tvirtio_dev_rx_async_packed_batch_enqueue(dev, vq, pkts, desc_addrs, lens);\n+\n+\treturn 0;\n+}\n+\n static __rte_always_inline void\n dma_error_handler_packed(struct vhost_virtqueue *vq, uint16_t slot_idx,\n \t\t\tuint32_t nr_err, uint32_t *pkt_idx)\n@@ -1893,10 +2035,25 @@ virtio_dev_rx_async_submit_packed(struct virtio_net *dev, struct vhost_virtqueue\n \tstruct async_inflight_info *pkts_info = async->pkts_info;\n \tuint32_t pkt_err = 0;\n \tuint16_t slot_idx = 0;\n+\tuint16_t i;\n \n \tdo {\n \t\trte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);\n \n+\t\tif (count - pkt_idx >= PACKED_BATCH_SIZE) {\n+\t\t\tif (!virtio_dev_rx_async_packed_batch(dev, vq, &pkts[pkt_idx],\n+\t\t\t\t\tdma_id, vchan_id)) {\n+\t\t\t\tfor (i = 0; i < PACKED_BATCH_SIZE; i++) {\n+\t\t\t\t\tslot_idx = (async->pkts_idx + pkt_idx) % vq->size;\n+\t\t\t\t\tpkts_info[slot_idx].descs = 1;\n+\t\t\t\t\tpkts_info[slot_idx].nr_buffers = 1;\n+\t\t\t\t\tpkts_info[slot_idx].mbuf = pkts[pkt_idx];\n+\t\t\t\t\tpkt_idx++;\n+\t\t\t\t}\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t}\n+\n \t\tnum_buffers = 0;\n \t\tnum_descs = 0;\n \t\tif (unlikely(virtio_dev_rx_async_packed(dev, vq, pkts[pkt_idx],\n", "prefixes": [ "2/3" ] }