Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/59393/?format=api
{ "id": 59393, "url": "https://patches.dpdk.org/api/patches/59393/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/20190919163643.24130-14-yong.liu@intel.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20190919163643.24130-14-yong.liu@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20190919163643.24130-14-yong.liu@intel.com", "date": "2019-09-19T16:36:40", "name": "[v2,13/16] vhost: optimize dequeue function of packed ring", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "449c24a1a4f8a533f960244088dadaeea17273a8", "submitter": { "id": 17, "url": "https://patches.dpdk.org/api/people/17/?format=api", "name": "Marvin Liu", "email": "yong.liu@intel.com" }, "delegate": null, "mbox": "https://patches.dpdk.org/project/dpdk/patch/20190919163643.24130-14-yong.liu@intel.com/mbox/", "series": [ { "id": 6444, "url": "https://patches.dpdk.org/api/series/6444/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=6444", "date": "2019-09-19T16:36:27", "name": "vhost packed ring performance optimization", "version": 2, "mbox": "https://patches.dpdk.org/series/6444/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/59393/comments/", "check": "fail", "checks": "https://patches.dpdk.org/api/patches/59393/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 092A31EB62;\n\tThu, 19 Sep 2019 10:57:06 +0200 (CEST)", "from mga14.intel.com (mga14.intel.com [192.55.52.115])\n\tby dpdk.org (Postfix) with ESMTP id 87EFF1E8FE\n\tfor <dev@dpdk.org>; Thu, 19 Sep 2019 10:56:44 +0200 (CEST)", "from orsmga001.jf.intel.com ([10.7.209.18])\n\tby fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t19 Sep 2019 01:56:44 -0700", "from npg-dpdk-virtual-marvin-dev.sh.intel.com ([10.67.119.142])\n\tby orsmga001.jf.intel.com with ESMTP; 19 Sep 2019 01:56:42 -0700" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.64,523,1559545200\"; d=\"scan'208\";a=\"271146156\"", "From": "Marvin Liu <yong.liu@intel.com>", "To": "maxime.coquelin@redhat.com, tiwei.bie@intel.com, zhihong.wang@intel.com", "Cc": "dev@dpdk.org,\n\tMarvin Liu <yong.liu@intel.com>", "Date": "Fri, 20 Sep 2019 00:36:40 +0800", "Message-Id": "<20190919163643.24130-14-yong.liu@intel.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20190919163643.24130-1-yong.liu@intel.com>", "References": "<20190905161421.55981-2-yong.liu@intel.com>\n\t<20190919163643.24130-1-yong.liu@intel.com>", "Subject": "[dpdk-dev] [PATCH v2 13/16] vhost: optimize dequeue function of\n\tpacked ring", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Optimize vhost device Rx datapath by separate functions. No-chained\nand direct descriptors will be handled by burst and other will be\nhandled one by one as before.\n\nSigned-off-by: Marvin Liu <yong.liu@intel.com>", "diff": "diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c\nindex a8df74f87..066514e43 100644\n--- a/lib/librte_vhost/virtio_net.c\n+++ b/lib/librte_vhost/virtio_net.c\n@@ -182,17 +182,6 @@ flush_dequeue_shadow_packed(struct virtio_net *dev, struct vhost_virtqueue *vq)\n \tvhost_log_cache_sync(dev, vq);\n }\n \n-static __rte_always_inline void\n-update_shadow_packed(struct vhost_virtqueue *vq,\n-\t\t\t uint16_t desc_idx, uint32_t len, uint16_t count)\n-{\n-\tuint16_t i = vq->shadow_used_idx++;\n-\n-\tvq->shadow_used_packed[i].id = desc_idx;\n-\tvq->shadow_used_packed[i].len = len;\n-\tvq->shadow_used_packed[i].count = count;\n-}\n-\n static __rte_always_inline void\n flush_burst_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \tuint64_t *lens, uint16_t *ids, uint16_t flags)\n@@ -383,7 +372,7 @@ flush_enqueue_packed(struct virtio_net *dev,\n \t}\n }\n \n-static __rte_unused __rte_always_inline void\n+static __rte_always_inline void\n flush_dequeue_packed(struct virtio_net *dev, struct vhost_virtqueue *vq)\n {\n \tif (!vq->shadow_used_idx)\n@@ -1809,7 +1798,7 @@ vhost_dequeue_burst_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \treturn -1;\n }\n \n-static __rte_unused int\n+static __rte_always_inline int\n virtio_dev_tx_burst_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \tstruct rte_mempool *mbuf_pool, struct rte_mbuf **pkts)\n {\n@@ -1887,7 +1876,7 @@ vhost_dequeue_single_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \treturn 0;\n }\n \n-static __rte_unused int\n+static __rte_always_inline int\n virtio_dev_tx_single_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \tstruct rte_mempool *mbuf_pool, struct rte_mbuf **pkts)\n {\n@@ -1909,7 +1898,7 @@ virtio_dev_tx_single_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \treturn 0;\n }\n \n-static __rte_unused __rte_always_inline int\n+static __rte_always_inline int\n virtio_dev_tx_burst_packed_zmbuf(struct virtio_net *dev,\n \t\t\t\t\tstruct vhost_virtqueue *vq,\n \t\t\t\t\tstruct rte_mempool *mbuf_pool,\n@@ -1971,7 +1960,7 @@ virtio_dev_tx_burst_packed_zmbuf(struct virtio_net *dev,\n \treturn -1;\n }\n \n-static __rte_unused int\n+static __rte_always_inline int\n virtio_dev_tx_single_packed_zmbuf(struct virtio_net *dev,\n \tstruct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool,\n \tstruct rte_mbuf **pkts)\n@@ -2006,7 +1995,7 @@ virtio_dev_tx_single_packed_zmbuf(struct virtio_net *dev,\n \treturn 0;\n }\n \n-static __rte_unused void\n+static __rte_always_inline void\n free_zmbuf(struct vhost_virtqueue *vq)\n {\n \tstruct zcopy_mbuf *next = NULL;\n@@ -2048,120 +2037,97 @@ free_zmbuf(struct vhost_virtqueue *vq)\n }\n \n static __rte_noinline uint16_t\n-virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,\n-\tstruct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)\n+virtio_dev_tx_packed_zmbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n+\tstruct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint32_t count)\n {\n-\tuint16_t i;\n-\n-\tif (unlikely(dev->dequeue_zero_copy)) {\n-\t\tstruct zcopy_mbuf *zmbuf, *next;\n+\tuint32_t pkt_idx = 0;\n+\tuint32_t remained = count;\n+\tint ret;\n \n-\t\tfor (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);\n-\t\t zmbuf != NULL; zmbuf = next) {\n-\t\t\tnext = TAILQ_NEXT(zmbuf, next);\n+\tfree_zmbuf(vq);\n \n-\t\t\tif (mbuf_is_consumed(zmbuf->mbuf)) {\n-\t\t\t\tupdate_shadow_packed(vq,\n-\t\t\t\t\t\tzmbuf->desc_idx,\n-\t\t\t\t\t\t0,\n-\t\t\t\t\t\tzmbuf->desc_count);\n+\tdo {\n+\t\tif (remained >= PACKED_DESCS_BURST) {\n+\t\t\tret = virtio_dev_tx_burst_packed_zmbuf(dev, vq,\n+\t\t\t\t\t\t\t mbuf_pool,\n+\t\t\t\t\t\t\t &pkts[pkt_idx]);\n \n-\t\t\t\tTAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);\n-\t\t\t\trestore_mbuf(zmbuf->mbuf);\n-\t\t\t\trte_pktmbuf_free(zmbuf->mbuf);\n-\t\t\t\tput_zmbuf(zmbuf);\n-\t\t\t\tvq->nr_zmbuf -= 1;\n+\t\t\tif (!ret) {\n+\t\t\t\tpkt_idx += PACKED_DESCS_BURST;\n+\t\t\t\tremained -= PACKED_DESCS_BURST;\n+\t\t\t\tcontinue;\n \t\t\t}\n \t\t}\n \n-\t\tif (likely(vq->shadow_used_idx)) {\n-\t\t\tflush_dequeue_shadow_packed(dev, vq);\n-\t\t\tvhost_vring_call_packed(dev, vq);\n-\t\t}\n-\t}\n-\n-\tVHOST_LOG_DEBUG(VHOST_DATA, \"(%d) %s\\n\", dev->vid, __func__);\n-\n-\tcount = RTE_MIN(count, MAX_PKT_BURST);\n-\tVHOST_LOG_DEBUG(VHOST_DATA, \"(%d) about to dequeue %u buffers\\n\",\n-\t\t\tdev->vid, count);\n+\t\tif (virtio_dev_tx_single_packed_zmbuf(dev, vq, mbuf_pool,\n+\t\t\t\t\t\t &pkts[pkt_idx]))\n+\t\t\tbreak;\n \n-\tfor (i = 0; i < count; i++) {\n-\t\tstruct buf_vector buf_vec[BUF_VECTOR_MAX];\n-\t\tuint16_t buf_id;\n-\t\tuint32_t dummy_len;\n-\t\tuint16_t desc_count, nr_vec = 0;\n-\t\tint err;\n+\t\tpkt_idx++;\n+\t\tremained--;\n+\t} while (remained);\n \n-\t\tif (unlikely(fill_vec_buf_packed(dev, vq,\n-\t\t\t\t\t\tvq->last_avail_idx, &desc_count,\n-\t\t\t\t\t\tbuf_vec, &nr_vec,\n-\t\t\t\t\t\t&buf_id, &dummy_len,\n-\t\t\t\t\t\tVHOST_ACCESS_RO) < 0))\n-\t\t\tbreak;\n+\tif (pkt_idx)\n+\t\tvhost_vring_call_packed(dev, vq);\n \n-\t\tif (likely(dev->dequeue_zero_copy == 0))\n-\t\t\tupdate_shadow_packed(vq, buf_id, 0,\n-\t\t\t\t\tdesc_count);\n+\treturn pkt_idx;\n+}\n \n-\t\tpkts[i] = rte_pktmbuf_alloc(mbuf_pool);\n-\t\tif (unlikely(pkts[i] == NULL)) {\n-\t\t\tRTE_LOG(ERR, VHOST_DATA,\n-\t\t\t\t\"Failed to allocate memory for mbuf.\\n\");\n-\t\t\tbreak;\n-\t\t}\n+static __rte_noinline uint16_t\n+virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,\n+\tstruct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint32_t count)\n+{\n+\tuint32_t pkt_idx = 0;\n+\tuint32_t remained = count;\n+\tuint16_t fetch_idx;\n+\tint ret;\n+\tstruct vring_packed_desc *descs = vq->desc_packed;\n \n-\t\terr = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],\n-\t\t\t\tmbuf_pool);\n-\t\tif (unlikely(err)) {\n-\t\t\trte_pktmbuf_free(pkts[i]);\n-\t\t\tbreak;\n+\tdo {\n+\t\tif ((vq->last_avail_idx & 0x7) == 0) {\n+\t\t\tfetch_idx = vq->last_avail_idx + 8;\n+\t\t\trte_prefetch0((void *)(uintptr_t)&descs[fetch_idx]);\n \t\t}\n \n-\t\tif (unlikely(dev->dequeue_zero_copy)) {\n-\t\t\tstruct zcopy_mbuf *zmbuf;\n+\t\tif (remained >= PACKED_DESCS_BURST) {\n+\t\t\tret = virtio_dev_tx_burst_packed(dev, vq, mbuf_pool,\n+\t\t\t\t\t\t\t &pkts[pkt_idx]);\n \n-\t\t\tzmbuf = get_zmbuf(vq);\n-\t\t\tif (!zmbuf) {\n-\t\t\t\trte_pktmbuf_free(pkts[i]);\n-\t\t\t\tbreak;\n+\t\t\tif (!ret) {\n+\t\t\t\tflush_dequeue_packed(dev, vq);\n+\t\t\t\tpkt_idx += PACKED_DESCS_BURST;\n+\t\t\t\tremained -= PACKED_DESCS_BURST;\n+\t\t\t\tcontinue;\n \t\t\t}\n-\t\t\tzmbuf->mbuf = pkts[i];\n-\t\t\tzmbuf->desc_idx = buf_id;\n-\t\t\tzmbuf->desc_count = desc_count;\n+\t\t}\n \n-\t\t\t/*\n-\t\t\t * Pin lock the mbuf; we will check later to see\n-\t\t\t * whether the mbuf is freed (when we are the last\n-\t\t\t * user) or not. If that's the case, we then could\n-\t\t\t * update the used ring safely.\n-\t\t\t */\n-\t\t\trte_mbuf_refcnt_update(pkts[i], 1);\n+\t\t/*\n+\t\t * If remained descs can't bundled into one burst, just skip to\n+\t\t * next round.\n+\t\t */\n+\t\tif (((vq->last_avail_idx & PACKED_BURST_MASK) + remained) <\n+\t\t\tPACKED_DESCS_BURST)\n+\t\t\tbreak;\n \n-\t\t\tvq->nr_zmbuf += 1;\n-\t\t\tTAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);\n-\t\t}\n+\t\tif (virtio_dev_tx_single_packed(dev, vq, mbuf_pool,\n+\t\t\t\t\t\t&pkts[pkt_idx]))\n+\t\t\tbreak;\n \n-\t\tvq->last_avail_idx += desc_count;\n-\t\tif (vq->last_avail_idx >= vq->size) {\n-\t\t\tvq->last_avail_idx -= vq->size;\n-\t\t\tvq->avail_wrap_counter ^= 1;\n-\t\t}\n-\t}\n+\t\tpkt_idx++;\n+\t\tremained--;\n+\t\tflush_dequeue_packed(dev, vq);\n \n-\tif (likely(dev->dequeue_zero_copy == 0)) {\n-\t\tdo_data_copy_dequeue(vq);\n-\t\tif (unlikely(i < count))\n-\t\t\tvq->shadow_used_idx = i;\n-\t\tif (likely(vq->shadow_used_idx)) {\n-\t\t\tflush_dequeue_shadow_packed(dev, vq);\n-\t\t\tvhost_vring_call_packed(dev, vq);\n-\t\t}\n+\t} while (remained);\n+\n+\tif (pkt_idx) {\n+\t\tif (vq->shadow_used_idx)\n+\t\t\tdo_data_copy_dequeue(vq);\n \t}\n \n-\treturn i;\n+\treturn pkt_idx;\n }\n \n+\n uint16_t\n rte_vhost_dequeue_burst(int vid, uint16_t queue_id,\n \tstruct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)\n@@ -2235,9 +2201,14 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,\n \t\tcount -= 1;\n \t}\n \n-\tif (vq_is_packed(dev))\n-\t\tcount = virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count);\n-\telse\n+\tif (vq_is_packed(dev)) {\n+\t\tif (unlikely(dev->dequeue_zero_copy))\n+\t\t\tcount = virtio_dev_tx_packed_zmbuf(dev, vq, mbuf_pool,\n+\t\t\t\t\t\t\t pkts, count);\n+\t\telse\n+\t\t\tcount = virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts,\n+\t\t\t\t\t\t count);\n+\t} else\n \t\tcount = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count);\n \n out:\n", "prefixes": [ "v2", "13/16" ] }