Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/42458/?format=api
{ "id": 42458, "url": "http://patches.dpdk.org/api/patches/42458/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20180706070449.1946-3-maxime.coquelin@redhat.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20180706070449.1946-3-maxime.coquelin@redhat.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20180706070449.1946-3-maxime.coquelin@redhat.com", "date": "2018-07-06T07:04:46", "name": "[v4,2/5] vhost: use buffer vectors in dequeue path", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "275cfcf77a19ff68814cb578a5309a390984c87a", "submitter": { "id": 512, "url": "http://patches.dpdk.org/api/people/512/?format=api", "name": "Maxime Coquelin", "email": "maxime.coquelin@redhat.com" }, "delegate": { "id": 2642, "url": "http://patches.dpdk.org/api/users/2642/?format=api", "username": "mcoquelin", "first_name": "Maxime", "last_name": "Coquelin", "email": "maxime.coquelin@redhat.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20180706070449.1946-3-maxime.coquelin@redhat.com/mbox/", "series": [ { "id": 449, "url": "http://patches.dpdk.org/api/series/449/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=449", "date": "2018-07-06T07:04:44", "name": "vhost: generalize buffer vectors", "version": 4, "mbox": "http://patches.dpdk.org/series/449/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/42458/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/42458/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id E28491B5B9;\n\tFri, 6 Jul 2018 09:04:58 +0200 (CEST)", "from mx1.redhat.com (mx3-rdu2.redhat.com [66.187.233.73])\n\tby dpdk.org (Postfix) with ESMTP id 87DFB1B533\n\tfor <dev@dpdk.org>; Fri, 6 Jul 2018 09:04:56 +0200 (CEST)", "from smtp.corp.redhat.com\n\t(int-mx06.intmail.prod.int.rdu2.redhat.com [10.11.54.6])\n\t(using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits))\n\t(No client certificate requested)\n\tby mx1.redhat.com (Postfix) with ESMTPS id CB60BE9002;\n\tFri, 6 Jul 2018 07:04:55 +0000 (UTC)", "from localhost.localdomain (unknown [10.36.112.10])\n\tby smtp.corp.redhat.com (Postfix) with ESMTP id 9306D2156891;\n\tFri, 6 Jul 2018 07:04:54 +0000 (UTC)" ], "From": "Maxime Coquelin <maxime.coquelin@redhat.com>", "To": "tiwei.bie@intel.com,\n\tzhihong.wang@intel.com,\n\tdev@dpdk.org", "Cc": "Maxime Coquelin <maxime.coquelin@redhat.com>", "Date": "Fri, 6 Jul 2018 09:04:46 +0200", "Message-Id": "<20180706070449.1946-3-maxime.coquelin@redhat.com>", "In-Reply-To": "<20180706070449.1946-1-maxime.coquelin@redhat.com>", "References": "<20180706070449.1946-1-maxime.coquelin@redhat.com>", "X-Scanned-By": "MIMEDefang 2.78 on 10.11.54.6", "X-Greylist": [ "Sender IP whitelisted, not delayed by milter-greylist-4.5.16\n\t(mx1.redhat.com [10.11.55.1]); Fri, 06 Jul 2018 07:04:55 +0000 (UTC)", "inspected by milter-greylist-4.5.16 (mx1.redhat.com\n\t[10.11.55.1]); \n\tFri, 06 Jul 2018 07:04:55 +0000 (UTC) for IP:'10.11.54.6'\n\tDOMAIN:'int-mx06.intmail.prod.int.rdu2.redhat.com'\n\tHELO:'smtp.corp.redhat.com' FROM:'maxime.coquelin@redhat.com' RCPT:''" ], "Subject": "[dpdk-dev] [PATCH v4 2/5] vhost: use buffer vectors in dequeue path", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "To ease packed ring layout integration, this patch makes\nthe dequeue path to re-use buffer vectors implemented for\nenqueue path.\n\nDoing this, copy_desc_to_mbuf() is now ring layout type\nagnostic.\n\nSigned-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>\n---\n lib/librte_vhost/vhost.h | 1 +\n lib/librte_vhost/virtio_net.c | 451 ++++++++++++++++--------------------------\n 2 files changed, 167 insertions(+), 285 deletions(-)", "diff": "diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h\nindex 3437b996b..79e3117d2 100644\n--- a/lib/librte_vhost/vhost.h\n+++ b/lib/librte_vhost/vhost.h\n@@ -43,6 +43,7 @@\n * from vring to do scatter RX.\n */\n struct buf_vector {\n+\tuint64_t buf_iova;\n \tuint64_t buf_addr;\n \tuint32_t buf_len;\n \tuint32_t desc_idx;\ndiff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c\nindex 741267345..6339296c7 100644\n--- a/lib/librte_vhost/virtio_net.c\n+++ b/lib/librte_vhost/virtio_net.c\n@@ -225,12 +225,12 @@ static __rte_always_inline int\n fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\t\t uint32_t avail_idx, uint32_t *vec_idx,\n \t\t\t struct buf_vector *buf_vec, uint16_t *desc_chain_head,\n-\t\t\t uint16_t *desc_chain_len)\n+\t\t\t uint16_t *desc_chain_len, uint8_t perm)\n {\n \tuint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)];\n \tuint32_t vec_id = *vec_idx;\n \tuint32_t len = 0;\n-\tuint64_t dlen;\n+\tuint64_t dlen, desc_avail, desc_iova;\n \tstruct vring_desc *descs = vq->desc;\n \tstruct vring_desc *idesc = NULL;\n \n@@ -261,16 +261,43 @@ fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t}\n \n \twhile (1) {\n-\t\tif (unlikely(vec_id >= BUF_VECTOR_MAX || idx >= vq->size)) {\n+\t\tif (unlikely(idx >= vq->size)) {\n \t\t\tfree_ind_table(idesc);\n \t\t\treturn -1;\n \t\t}\n \n+\n \t\tlen += descs[idx].len;\n-\t\tbuf_vec[vec_id].buf_addr = descs[idx].addr;\n-\t\tbuf_vec[vec_id].buf_len = descs[idx].len;\n-\t\tbuf_vec[vec_id].desc_idx = idx;\n-\t\tvec_id++;\n+\t\tdesc_avail = descs[idx].len;\n+\t\tdesc_iova = descs[idx].addr;\n+\n+\t\twhile (desc_avail) {\n+\t\t\tuint64_t desc_addr;\n+\t\t\tuint64_t desc_chunck_len = desc_avail;\n+\n+\t\t\tif (unlikely(vec_id >= BUF_VECTOR_MAX)) {\n+\t\t\t\tfree_ind_table(idesc);\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\n+\t\t\tdesc_addr = vhost_iova_to_vva(dev, vq,\n+\t\t\t\t\tdesc_iova,\n+\t\t\t\t\t&desc_chunck_len,\n+\t\t\t\t\tperm);\n+\t\t\tif (unlikely(!desc_addr)) {\n+\t\t\t\tfree_ind_table(idesc);\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\n+\t\t\tbuf_vec[vec_id].buf_iova = desc_iova;\n+\t\t\tbuf_vec[vec_id].buf_addr = desc_addr;\n+\t\t\tbuf_vec[vec_id].buf_len = desc_chunck_len;\n+\t\t\tbuf_vec[vec_id].desc_idx = idx;\n+\n+\t\t\tdesc_avail -= desc_chunck_len;\n+\t\t\tdesc_iova += desc_chunck_len;\n+\t\t\tvec_id++;\n+\t\t}\n \n \t\tif ((descs[idx].flags & VRING_DESC_F_NEXT) == 0)\n \t\t\tbreak;\n@@ -293,7 +320,8 @@ fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n static inline int\n reserve_avail_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\t\t\tuint32_t size, struct buf_vector *buf_vec,\n-\t\t\t\tuint16_t *num_buffers, uint16_t avail_head)\n+\t\t\t\tuint16_t *num_buffers, uint16_t avail_head,\n+\t\t\t\tuint16_t *nr_vec)\n {\n \tuint16_t cur_idx;\n \tuint32_t vec_idx = 0;\n@@ -315,7 +343,8 @@ reserve_avail_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\t\treturn -1;\n \n \t\tif (unlikely(fill_vec_buf(dev, vq, cur_idx, &vec_idx, buf_vec,\n-\t\t\t\t\t\t&head_idx, &len) < 0))\n+\t\t\t\t\t\t&head_idx, &len,\n+\t\t\t\t\t\tVHOST_ACCESS_RW) < 0))\n \t\t\treturn -1;\n \t\tlen = RTE_MIN(len, size);\n \t\tupdate_shadow_used_ring(vq, head_idx, len);\n@@ -334,21 +363,22 @@ reserve_avail_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\t\treturn -1;\n \t}\n \n+\t*nr_vec = vec_idx;\n+\n \treturn 0;\n }\n \n static __rte_always_inline int\n copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\t\t struct rte_mbuf *m, struct buf_vector *buf_vec,\n-\t\t\t uint16_t num_buffers)\n+\t\t\t uint16_t nr_vec, uint16_t num_buffers)\n {\n \tuint32_t vec_idx = 0;\n-\tuint64_t desc_addr, desc_gaddr;\n \tuint32_t mbuf_offset, mbuf_avail;\n-\tuint32_t desc_offset, desc_avail;\n+\tuint32_t buf_offset, buf_avail;\n+\tuint64_t buf_addr, buf_iova, buf_len;\n \tuint32_t cpy_len;\n-\tuint64_t desc_chunck_len;\n-\tuint64_t hdr_addr, hdr_phys_addr;\n+\tuint64_t hdr_addr;\n \tstruct rte_mbuf *hdr_mbuf;\n \tstruct batch_copy_elem *batch_copy = vq->batch_copy_elems;\n \tstruct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL;\n@@ -359,82 +389,57 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\tgoto out;\n \t}\n \n-\tdesc_chunck_len = buf_vec[vec_idx].buf_len;\n-\tdesc_gaddr = buf_vec[vec_idx].buf_addr;\n-\tdesc_addr = vhost_iova_to_vva(dev, vq,\n-\t\t\t\t\tdesc_gaddr,\n-\t\t\t\t\t&desc_chunck_len,\n-\t\t\t\t\tVHOST_ACCESS_RW);\n-\tif (buf_vec[vec_idx].buf_len < dev->vhost_hlen || !desc_addr) {\n+\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\tbuf_len = buf_vec[vec_idx].buf_len;\n+\n+\tif (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {\n \t\terror = -1;\n \t\tgoto out;\n \t}\n \n \thdr_mbuf = m;\n-\thdr_addr = desc_addr;\n-\tif (unlikely(desc_chunck_len < dev->vhost_hlen))\n+\thdr_addr = buf_addr;\n+\tif (unlikely(buf_len < dev->vhost_hlen))\n \t\thdr = &tmp_hdr;\n \telse\n \t\thdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;\n-\thdr_phys_addr = desc_gaddr;\n \trte_prefetch0((void *)(uintptr_t)hdr_addr);\n \n \tVHOST_LOG_DEBUG(VHOST_DATA, \"(%d) RX: num merge buffers %d\\n\",\n \t\tdev->vid, num_buffers);\n \n-\tdesc_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen;\n-\tif (unlikely(desc_chunck_len < dev->vhost_hlen)) {\n-\t\tdesc_chunck_len = desc_avail;\n-\t\tdesc_gaddr += dev->vhost_hlen;\n-\t\tdesc_addr = vhost_iova_to_vva(dev, vq,\n-\t\t\t\tdesc_gaddr,\n-\t\t\t\t&desc_chunck_len,\n-\t\t\t\tVHOST_ACCESS_RW);\n-\t\tif (unlikely(!desc_addr)) {\n-\t\t\terror = -1;\n-\t\t\tgoto out;\n-\t\t}\n-\n-\t\tdesc_offset = 0;\n+\tif (unlikely(buf_len < dev->vhost_hlen)) {\n+\t\tbuf_offset = dev->vhost_hlen - buf_len;\n+\t\tvec_idx++;\n+\t\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\t\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\t\tbuf_len = buf_vec[vec_idx].buf_len;\n+\t\tbuf_avail = buf_len - buf_offset;\n \t} else {\n-\t\tdesc_offset = dev->vhost_hlen;\n-\t\tdesc_chunck_len -= dev->vhost_hlen;\n+\t\tbuf_offset = dev->vhost_hlen;\n+\t\tbuf_avail = buf_len - dev->vhost_hlen;\n \t}\n \n-\n \tmbuf_avail = rte_pktmbuf_data_len(m);\n \tmbuf_offset = 0;\n \twhile (mbuf_avail != 0 || m->next != NULL) {\n-\t\t/* done with current desc buf, get the next one */\n-\t\tif (desc_avail == 0) {\n+\t\t/* done with current buf, get the next one */\n+\t\tif (buf_avail == 0) {\n \t\t\tvec_idx++;\n-\t\t\tdesc_chunck_len = buf_vec[vec_idx].buf_len;\n-\t\t\tdesc_gaddr = buf_vec[vec_idx].buf_addr;\n-\t\t\tdesc_addr =\n-\t\t\t\tvhost_iova_to_vva(dev, vq,\n-\t\t\t\t\tdesc_gaddr,\n-\t\t\t\t\t&desc_chunck_len,\n-\t\t\t\t\tVHOST_ACCESS_RW);\n-\t\t\tif (unlikely(!desc_addr)) {\n+\t\t\tif (unlikely(vec_idx >= nr_vec)) {\n \t\t\t\terror = -1;\n \t\t\t\tgoto out;\n \t\t\t}\n \n+\t\t\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\t\t\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\t\t\tbuf_len = buf_vec[vec_idx].buf_len;\n+\n \t\t\t/* Prefetch buffer address. */\n-\t\t\trte_prefetch0((void *)(uintptr_t)desc_addr);\n-\t\t\tdesc_offset = 0;\n-\t\t\tdesc_avail = buf_vec[vec_idx].buf_len;\n-\t\t} else if (unlikely(desc_chunck_len == 0)) {\n-\t\t\tdesc_chunck_len = desc_avail;\n-\t\t\tdesc_gaddr += desc_offset;\n-\t\t\tdesc_addr = vhost_iova_to_vva(dev, vq,\n-\t\t\t\t\tdesc_gaddr,\n-\t\t\t\t\t&desc_chunck_len, VHOST_ACCESS_RW);\n-\t\t\tif (unlikely(!desc_addr)) {\n-\t\t\t\terror = -1;\n-\t\t\t\tgoto out;\n-\t\t\t}\n-\t\t\tdesc_offset = 0;\n+\t\t\trte_prefetch0((void *)(uintptr_t)buf_addr);\n+\t\t\tbuf_offset = 0;\n+\t\t\tbuf_avail = buf_len;\n \t\t}\n \n \t\t/* done with current mbuf, get the next one */\n@@ -455,18 +460,12 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\t\t\tuint64_t len;\n \t\t\t\tuint64_t remain = dev->vhost_hlen;\n \t\t\t\tuint64_t src = (uint64_t)(uintptr_t)hdr, dst;\n-\t\t\t\tuint64_t guest_addr = hdr_phys_addr;\n+\t\t\t\tuint64_t iova = buf_vec[0].buf_iova;\n+\t\t\t\tuint16_t hdr_vec_idx = 0;\n \n \t\t\t\twhile (remain) {\n \t\t\t\t\tlen = remain;\n-\t\t\t\t\tdst = vhost_iova_to_vva(dev, vq,\n-\t\t\t\t\t\t\tguest_addr, &len,\n-\t\t\t\t\t\t\tVHOST_ACCESS_RW);\n-\t\t\t\t\tif (unlikely(!dst || !len)) {\n-\t\t\t\t\t\terror = -1;\n-\t\t\t\t\t\tgoto out;\n-\t\t\t\t\t}\n-\n+\t\t\t\t\tdst = buf_vec[hdr_vec_idx].buf_addr;\n \t\t\t\t\trte_memcpy((void *)(uintptr_t)dst,\n \t\t\t\t\t\t\t(void *)(uintptr_t)src,\n \t\t\t\t\t\t\tlen);\n@@ -474,50 +473,50 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\t\t\t\tPRINT_PACKET(dev, (uintptr_t)dst,\n \t\t\t\t\t\t\t(uint32_t)len, 0);\n \t\t\t\t\tvhost_log_cache_write(dev, vq,\n-\t\t\t\t\t\t\tguest_addr, len);\n+\t\t\t\t\t\t\tiova, len);\n \n \t\t\t\t\tremain -= len;\n-\t\t\t\t\tguest_addr += len;\n+\t\t\t\t\tiova += len;\n \t\t\t\t\tsrc += len;\n+\t\t\t\t\thdr_vec_idx++;\n \t\t\t\t}\n \t\t\t} else {\n \t\t\t\tPRINT_PACKET(dev, (uintptr_t)hdr_addr,\n \t\t\t\t\t\tdev->vhost_hlen, 0);\n-\t\t\t\tvhost_log_cache_write(dev, vq, hdr_phys_addr,\n+\t\t\t\tvhost_log_cache_write(dev, vq,\n+\t\t\t\t\t\tbuf_vec[0].buf_iova,\n \t\t\t\t\t\tdev->vhost_hlen);\n \t\t\t}\n \n \t\t\thdr_addr = 0;\n \t\t}\n \n-\t\tcpy_len = RTE_MIN(desc_chunck_len, mbuf_avail);\n+\t\tcpy_len = RTE_MIN(buf_len, mbuf_avail);\n \n \t\tif (likely(cpy_len > MAX_BATCH_LEN ||\n \t\t\t\t\tvq->batch_copy_nb_elems >= vq->size)) {\n-\t\t\trte_memcpy((void *)((uintptr_t)(desc_addr +\n-\t\t\t\t\t\t\tdesc_offset)),\n+\t\t\trte_memcpy((void *)((uintptr_t)(buf_addr + buf_offset)),\n \t\t\t\trte_pktmbuf_mtod_offset(m, void *, mbuf_offset),\n \t\t\t\tcpy_len);\n-\t\t\tvhost_log_cache_write(dev, vq, desc_gaddr + desc_offset,\n+\t\t\tvhost_log_cache_write(dev, vq, buf_iova + buf_offset,\n \t\t\t\t\tcpy_len);\n-\t\t\tPRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),\n+\t\t\tPRINT_PACKET(dev, (uintptr_t)(buf_addr + buf_offset),\n \t\t\t\tcpy_len, 0);\n \t\t} else {\n \t\t\tbatch_copy[vq->batch_copy_nb_elems].dst =\n-\t\t\t\t(void *)((uintptr_t)(desc_addr + desc_offset));\n+\t\t\t\t(void *)((uintptr_t)(buf_addr + buf_offset));\n \t\t\tbatch_copy[vq->batch_copy_nb_elems].src =\n \t\t\t\trte_pktmbuf_mtod_offset(m, void *, mbuf_offset);\n \t\t\tbatch_copy[vq->batch_copy_nb_elems].log_addr =\n-\t\t\t\tdesc_gaddr + desc_offset;\n+\t\t\t\tbuf_iova + buf_offset;\n \t\t\tbatch_copy[vq->batch_copy_nb_elems].len = cpy_len;\n \t\t\tvq->batch_copy_nb_elems++;\n \t\t}\n \n \t\tmbuf_avail -= cpy_len;\n \t\tmbuf_offset += cpy_len;\n-\t\tdesc_avail -= cpy_len;\n-\t\tdesc_offset += cpy_len;\n-\t\tdesc_chunck_len -= cpy_len;\n+\t\tbuf_avail -= cpy_len;\n+\t\tbuf_offset += cpy_len;\n \t}\n \n out:\n@@ -568,10 +567,11 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,\n \tavail_head = *((volatile uint16_t *)&vq->avail->idx);\n \tfor (pkt_idx = 0; pkt_idx < count; pkt_idx++) {\n \t\tuint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;\n+\t\tuint16_t nr_vec = 0;\n \n \t\tif (unlikely(reserve_avail_buf(dev, vq,\n \t\t\t\t\t\tpkt_len, buf_vec, &num_buffers,\n-\t\t\t\t\t\tavail_head) < 0)) {\n+\t\t\t\t\t\tavail_head, &nr_vec) < 0)) {\n \t\t\tVHOST_LOG_DEBUG(VHOST_DATA,\n \t\t\t\t\"(%d) failed to get enough desc from vring\\n\",\n \t\t\t\tdev->vid);\n@@ -584,7 +584,8 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,\n \t\t\tvq->last_avail_idx + num_buffers);\n \n \t\tif (copy_mbuf_to_desc(dev, vq, pkts[pkt_idx],\n-\t\t\t\t\t\tbuf_vec, num_buffers) < 0) {\n+\t\t\t\t\t\tbuf_vec, nr_vec,\n+\t\t\t\t\t\tnum_buffers) < 0) {\n \t\t\tvq->shadow_used_idx -= num_buffers;\n \t\t\tbreak;\n \t\t}\n@@ -750,49 +751,40 @@ put_zmbuf(struct zcopy_mbuf *zmbuf)\n \n static __rte_always_inline int\n copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n-\t\t struct vring_desc *descs, uint16_t max_desc,\n-\t\t struct rte_mbuf *m, uint16_t desc_idx,\n-\t\t struct rte_mempool *mbuf_pool)\n+\t\t struct buf_vector *buf_vec, uint16_t nr_vec,\n+\t\t struct rte_mbuf *m, struct rte_mempool *mbuf_pool)\n {\n-\tstruct vring_desc *desc;\n-\tuint64_t desc_addr, desc_gaddr;\n-\tuint32_t desc_avail, desc_offset;\n+\tuint32_t buf_avail, buf_offset;\n+\tuint64_t buf_addr, buf_iova, buf_len;\n \tuint32_t mbuf_avail, mbuf_offset;\n \tuint32_t cpy_len;\n-\tuint64_t desc_chunck_len;\n \tstruct rte_mbuf *cur = m, *prev = m;\n \tstruct virtio_net_hdr tmp_hdr;\n \tstruct virtio_net_hdr *hdr = NULL;\n \t/* A counter to avoid desc dead loop chain */\n-\tuint32_t nr_desc = 1;\n+\tuint16_t vec_idx = 0;\n \tstruct batch_copy_elem *batch_copy = vq->batch_copy_elems;\n \tint error = 0;\n \n-\tdesc = &descs[desc_idx];\n-\tif (unlikely((desc->len < dev->vhost_hlen)) ||\n-\t\t\t(desc->flags & VRING_DESC_F_INDIRECT)) {\n-\t\terror = -1;\n-\t\tgoto out;\n-\t}\n+\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\tbuf_len = buf_vec[vec_idx].buf_len;\n \n-\tdesc_chunck_len = desc->len;\n-\tdesc_gaddr = desc->addr;\n-\tdesc_addr = vhost_iova_to_vva(dev,\n-\t\t\t\t\tvq, desc_gaddr,\n-\t\t\t\t\t&desc_chunck_len,\n-\t\t\t\t\tVHOST_ACCESS_RO);\n-\tif (unlikely(!desc_addr)) {\n+\tif (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {\n \t\terror = -1;\n \t\tgoto out;\n \t}\n \n+\tif (likely(nr_vec > 1))\n+\t\trte_prefetch0((void *)(uintptr_t)buf_vec[1].buf_addr);\n+\n \tif (virtio_net_with_host_offload(dev)) {\n-\t\tif (unlikely(desc_chunck_len < sizeof(struct virtio_net_hdr))) {\n-\t\t\tuint64_t len = desc_chunck_len;\n+\t\tif (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {\n+\t\t\tuint64_t len;\n \t\t\tuint64_t remain = sizeof(struct virtio_net_hdr);\n-\t\t\tuint64_t src = desc_addr;\n+\t\t\tuint64_t src;\n \t\t\tuint64_t dst = (uint64_t)(uintptr_t)&tmp_hdr;\n-\t\t\tuint64_t guest_addr = desc_gaddr;\n+\t\t\tuint16_t hdr_vec_idx = 0;\n \n \t\t\t/*\n \t\t\t * No luck, the virtio-net header doesn't fit\n@@ -800,25 +792,18 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\t\t */\n \t\t\twhile (remain) {\n \t\t\t\tlen = remain;\n-\t\t\t\tsrc = vhost_iova_to_vva(dev, vq,\n-\t\t\t\t\t\tguest_addr, &len,\n-\t\t\t\t\t\tVHOST_ACCESS_RO);\n-\t\t\t\tif (unlikely(!src || !len)) {\n-\t\t\t\t\terror = -1;\n-\t\t\t\t\tgoto out;\n-\t\t\t\t}\n-\n+\t\t\t\tsrc = buf_vec[hdr_vec_idx].buf_addr;\n \t\t\t\trte_memcpy((void *)(uintptr_t)dst,\n \t\t\t\t\t\t (void *)(uintptr_t)src, len);\n \n-\t\t\t\tguest_addr += len;\n \t\t\t\tremain -= len;\n \t\t\t\tdst += len;\n+\t\t\t\thdr_vec_idx++;\n \t\t\t}\n \n \t\t\thdr = &tmp_hdr;\n \t\t} else {\n-\t\t\thdr = (struct virtio_net_hdr *)((uintptr_t)desc_addr);\n+\t\t\thdr = (struct virtio_net_hdr *)((uintptr_t)buf_addr);\n \t\t\trte_prefetch0(hdr);\n \t\t}\n \t}\n@@ -828,61 +813,40 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t * for Tx: the first for storing the header, and others\n \t * for storing the data.\n \t */\n-\tif (likely((desc->len == dev->vhost_hlen) &&\n-\t\t (desc->flags & VRING_DESC_F_NEXT) != 0)) {\n-\t\tdesc = &descs[desc->next];\n-\t\tif (unlikely(desc->flags & VRING_DESC_F_INDIRECT)) {\n-\t\t\terror = -1;\n+\tif (unlikely(buf_len < dev->vhost_hlen)) {\n+\t\tbuf_offset = dev->vhost_hlen - buf_len;\n+\t\tvec_idx++;\n+\t\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\t\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\t\tbuf_len = buf_vec[vec_idx].buf_len;\n+\t\tbuf_avail = buf_len - buf_offset;\n+\t} else if (buf_len == dev->vhost_hlen) {\n+\t\tif (unlikely(++vec_idx >= nr_vec))\n \t\t\tgoto out;\n-\t\t}\n+\t\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\t\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\t\tbuf_len = buf_vec[vec_idx].buf_len;\n \n-\t\tdesc_chunck_len = desc->len;\n-\t\tdesc_gaddr = desc->addr;\n-\t\tdesc_addr = vhost_iova_to_vva(dev,\n-\t\t\t\t\t\t\tvq, desc_gaddr,\n-\t\t\t\t\t\t\t&desc_chunck_len,\n-\t\t\t\t\t\t\tVHOST_ACCESS_RO);\n-\t\tif (unlikely(!desc_addr)) {\n-\t\t\terror = -1;\n-\t\t\tgoto out;\n-\t\t}\n-\n-\t\tdesc_offset = 0;\n-\t\tdesc_avail = desc->len;\n-\t\tnr_desc += 1;\n+\t\tbuf_offset = 0;\n+\t\tbuf_avail = buf_len;\n \t} else {\n-\t\tdesc_avail = desc->len - dev->vhost_hlen;\n-\n-\t\tif (unlikely(desc_chunck_len < dev->vhost_hlen)) {\n-\t\t\tdesc_chunck_len = desc_avail;\n-\t\t\tdesc_gaddr += dev->vhost_hlen;\n-\t\t\tdesc_addr = vhost_iova_to_vva(dev,\n-\t\t\t\t\tvq, desc_gaddr,\n-\t\t\t\t\t&desc_chunck_len,\n-\t\t\t\t\tVHOST_ACCESS_RO);\n-\t\t\tif (unlikely(!desc_addr)) {\n-\t\t\t\terror = -1;\n-\t\t\t\tgoto out;\n-\t\t\t}\n-\n-\t\t\tdesc_offset = 0;\n-\t\t} else {\n-\t\t\tdesc_offset = dev->vhost_hlen;\n-\t\t\tdesc_chunck_len -= dev->vhost_hlen;\n-\t\t}\n+\t\tbuf_offset = dev->vhost_hlen;\n+\t\tbuf_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen;\n \t}\n \n-\trte_prefetch0((void *)(uintptr_t)(desc_addr + desc_offset));\n+\trte_prefetch0((void *)(uintptr_t)\n+\t\t\t(buf_addr + buf_offset));\n \n-\tPRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),\n-\t\t\t(uint32_t)desc_chunck_len, 0);\n+\tPRINT_PACKET(dev,\n+\t\t\t(uintptr_t)(buf_addr + buf_offset),\n+\t\t\t(uint32_t)buf_avail, 0);\n \n \tmbuf_offset = 0;\n \tmbuf_avail = m->buf_len - RTE_PKTMBUF_HEADROOM;\n \twhile (1) {\n \t\tuint64_t hpa;\n \n-\t\tcpy_len = RTE_MIN(desc_chunck_len, mbuf_avail);\n+\t\tcpy_len = RTE_MIN(buf_avail, mbuf_avail);\n \n \t\t/*\n \t\t * A desc buf might across two host physical pages that are\n@@ -890,11 +854,11 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\t * will be copied even though zero copy is enabled.\n \t\t */\n \t\tif (unlikely(dev->dequeue_zero_copy && (hpa = gpa_to_hpa(dev,\n-\t\t\t\t\tdesc_gaddr + desc_offset, cpy_len)))) {\n+\t\t\t\t\tbuf_iova + buf_offset, cpy_len)))) {\n \t\t\tcur->data_len = cpy_len;\n \t\t\tcur->data_off = 0;\n-\t\t\tcur->buf_addr = (void *)(uintptr_t)(desc_addr\n-\t\t\t\t+ desc_offset);\n+\t\t\tcur->buf_addr =\n+\t\t\t\t(void *)(uintptr_t)(buf_addr + buf_offset);\n \t\t\tcur->buf_iova = hpa;\n \n \t\t\t/*\n@@ -905,20 +869,19 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\t} else {\n \t\t\tif (likely(cpy_len > MAX_BATCH_LEN ||\n \t\t\t\t vq->batch_copy_nb_elems >= vq->size ||\n-\t\t\t\t (hdr && cur == m) ||\n-\t\t\t\t desc->len != desc_chunck_len)) {\n+\t\t\t\t (hdr && cur == m))) {\n \t\t\t\trte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,\n \t\t\t\t\t\t\t\t mbuf_offset),\n-\t\t\t\t\t (void *)((uintptr_t)(desc_addr +\n-\t\t\t\t\t\t\t\tdesc_offset)),\n+\t\t\t\t\t (void *)((uintptr_t)(buf_addr +\n+\t\t\t\t\t\t\t buf_offset)),\n \t\t\t\t\t cpy_len);\n \t\t\t} else {\n \t\t\t\tbatch_copy[vq->batch_copy_nb_elems].dst =\n \t\t\t\t\trte_pktmbuf_mtod_offset(cur, void *,\n \t\t\t\t\t\t\t\tmbuf_offset);\n \t\t\t\tbatch_copy[vq->batch_copy_nb_elems].src =\n-\t\t\t\t\t(void *)((uintptr_t)(desc_addr +\n-\t\t\t\t\t\t\t desc_offset));\n+\t\t\t\t\t(void *)((uintptr_t)(buf_addr +\n+\t\t\t\t\t\t\t\tbuf_offset));\n \t\t\t\tbatch_copy[vq->batch_copy_nb_elems].len =\n \t\t\t\t\tcpy_len;\n \t\t\t\tvq->batch_copy_nb_elems++;\n@@ -927,59 +890,25 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \n \t\tmbuf_avail -= cpy_len;\n \t\tmbuf_offset += cpy_len;\n-\t\tdesc_avail -= cpy_len;\n-\t\tdesc_chunck_len -= cpy_len;\n-\t\tdesc_offset += cpy_len;\n+\t\tbuf_avail -= cpy_len;\n+\t\tbuf_offset += cpy_len;\n \n-\t\t/* This desc reaches to its end, get the next one */\n-\t\tif (desc_avail == 0) {\n-\t\t\tif ((desc->flags & VRING_DESC_F_NEXT) == 0)\n+\t\t/* This buf reaches to its end, get the next one */\n+\t\tif (buf_avail == 0) {\n+\t\t\tif (++vec_idx >= nr_vec)\n \t\t\t\tbreak;\n \n-\t\t\tif (unlikely(desc->next >= max_desc ||\n-\t\t\t\t ++nr_desc > max_desc)) {\n-\t\t\t\terror = -1;\n-\t\t\t\tgoto out;\n-\t\t\t}\n-\t\t\tdesc = &descs[desc->next];\n-\t\t\tif (unlikely(desc->flags & VRING_DESC_F_INDIRECT)) {\n-\t\t\t\terror = -1;\n-\t\t\t\tgoto out;\n-\t\t\t}\n-\n-\t\t\tdesc_chunck_len = desc->len;\n-\t\t\tdesc_gaddr = desc->addr;\n-\t\t\tdesc_addr = vhost_iova_to_vva(dev,\n-\t\t\t\t\t\t\tvq, desc_gaddr,\n-\t\t\t\t\t\t\t&desc_chunck_len,\n-\t\t\t\t\t\t\tVHOST_ACCESS_RO);\n-\t\t\tif (unlikely(!desc_addr)) {\n-\t\t\t\terror = -1;\n-\t\t\t\tgoto out;\n-\t\t\t}\n+\t\t\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\t\t\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\t\t\tbuf_len = buf_vec[vec_idx].buf_len;\n \n-\t\t\trte_prefetch0((void *)(uintptr_t)desc_addr);\n+\t\t\trte_prefetch0((void *)(uintptr_t)buf_addr);\n \n-\t\t\tdesc_offset = 0;\n-\t\t\tdesc_avail = desc->len;\n-\n-\t\t\tPRINT_PACKET(dev, (uintptr_t)desc_addr,\n-\t\t\t\t\t(uint32_t)desc_chunck_len, 0);\n-\t\t} else if (unlikely(desc_chunck_len == 0)) {\n-\t\t\tdesc_chunck_len = desc_avail;\n-\t\t\tdesc_gaddr += desc_offset;\n-\t\t\tdesc_addr = vhost_iova_to_vva(dev, vq,\n-\t\t\t\t\tdesc_gaddr,\n-\t\t\t\t\t&desc_chunck_len,\n-\t\t\t\t\tVHOST_ACCESS_RO);\n-\t\t\tif (unlikely(!desc_addr)) {\n-\t\t\t\terror = -1;\n-\t\t\t\tgoto out;\n-\t\t\t}\n-\t\t\tdesc_offset = 0;\n+\t\t\tbuf_offset = 0;\n+\t\t\tbuf_avail = buf_len;\n \n-\t\t\tPRINT_PACKET(dev, (uintptr_t)desc_addr,\n-\t\t\t\t\t(uint32_t)desc_chunck_len, 0);\n+\t\t\tPRINT_PACKET(dev, (uintptr_t)buf_addr,\n+\t\t\t\t\t(uint32_t)buf_avail, 0);\n \t\t}\n \n \t\t/*\n@@ -1085,10 +1014,8 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,\n \tstruct virtio_net *dev;\n \tstruct rte_mbuf *rarp_mbuf = NULL;\n \tstruct vhost_virtqueue *vq;\n-\tuint32_t desc_indexes[MAX_PKT_BURST];\n \tuint32_t i = 0;\n \tuint16_t free_entries;\n-\tuint16_t avail_idx;\n \n \tdev = get_device(vid);\n \tif (!dev)\n@@ -1186,80 +1113,38 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,\n \n \tVHOST_LOG_DEBUG(VHOST_DATA, \"(%d) %s\\n\", dev->vid, __func__);\n \n-\t/* Prefetch available and used ring */\n-\tavail_idx = vq->last_avail_idx & (vq->size - 1);\n-\trte_prefetch0(&vq->avail->ring[avail_idx]);\n-\n \tcount = RTE_MIN(count, MAX_PKT_BURST);\n \tcount = RTE_MIN(count, free_entries);\n \tVHOST_LOG_DEBUG(VHOST_DATA, \"(%d) about to dequeue %u buffers\\n\",\n \t\t\tdev->vid, count);\n \n-\t/* Retrieve all of the head indexes first to avoid caching issues. */\n-\tfor (i = 0; i < count; i++) {\n-\t\tavail_idx = (vq->last_avail_idx + i) & (vq->size - 1);\n-\t\tdesc_indexes[i] = vq->avail->ring[avail_idx];\n-\n-\t\tif (likely(dev->dequeue_zero_copy == 0))\n-\t\t\tupdate_shadow_used_ring(vq, desc_indexes[i], 0);\n-\t}\n-\n-\t/* Prefetch descriptor index. */\n-\trte_prefetch0(&vq->desc[desc_indexes[0]]);\n \tfor (i = 0; i < count; i++) {\n-\t\tstruct vring_desc *desc, *idesc = NULL;\n-\t\tuint16_t sz, idx;\n-\t\tuint64_t dlen;\n+\t\tstruct buf_vector buf_vec[BUF_VECTOR_MAX];\n+\t\tuint16_t head_idx, dummy_len;\n+\t\tuint32_t nr_vec = 0;\n \t\tint err;\n \n-\t\tif (likely(i + 1 < count))\n-\t\t\trte_prefetch0(&vq->desc[desc_indexes[i + 1]]);\n-\n-\t\tif (vq->desc[desc_indexes[i]].flags & VRING_DESC_F_INDIRECT) {\n-\t\t\tdlen = vq->desc[desc_indexes[i]].len;\n-\t\t\tdesc = (struct vring_desc *)(uintptr_t)\n-\t\t\t\tvhost_iova_to_vva(dev, vq,\n-\t\t\t\t\t\tvq->desc[desc_indexes[i]].addr,\n-\t\t\t\t\t\t&dlen,\n-\t\t\t\t\t\tVHOST_ACCESS_RO);\n-\t\t\tif (unlikely(!desc))\n-\t\t\t\tbreak;\n-\n-\t\t\tif (unlikely(dlen < vq->desc[desc_indexes[i]].len)) {\n-\t\t\t\t/*\n-\t\t\t\t * The indirect desc table is not contiguous\n-\t\t\t\t * in process VA space, we have to copy it.\n-\t\t\t\t */\n-\t\t\t\tidesc = alloc_copy_ind_table(dev, vq,\n-\t\t\t\t\t\t&vq->desc[desc_indexes[i]]);\n-\t\t\t\tif (unlikely(!idesc))\n-\t\t\t\t\tbreak;\n-\n-\t\t\t\tdesc = idesc;\n-\t\t\t}\n+\t\tif (unlikely(fill_vec_buf(dev, vq,\n+\t\t\t\t\t\tvq->last_avail_idx + i,\n+\t\t\t\t\t\t&nr_vec, buf_vec,\n+\t\t\t\t\t\t&head_idx, &dummy_len,\n+\t\t\t\t\t\tVHOST_ACCESS_RO) < 0))\n+\t\t\tbreak;\n \n-\t\t\trte_prefetch0(desc);\n-\t\t\tsz = vq->desc[desc_indexes[i]].len / sizeof(*desc);\n-\t\t\tidx = 0;\n-\t\t} else {\n-\t\t\tdesc = vq->desc;\n-\t\t\tsz = vq->size;\n-\t\t\tidx = desc_indexes[i];\n-\t\t}\n+\t\tif (likely(dev->dequeue_zero_copy == 0))\n+\t\t\tupdate_shadow_used_ring(vq, head_idx, 0);\n \n \t\tpkts[i] = rte_pktmbuf_alloc(mbuf_pool);\n \t\tif (unlikely(pkts[i] == NULL)) {\n \t\t\tRTE_LOG(ERR, VHOST_DATA,\n \t\t\t\t\"Failed to allocate memory for mbuf.\\n\");\n-\t\t\tfree_ind_table(idesc);\n \t\t\tbreak;\n \t\t}\n \n-\t\terr = copy_desc_to_mbuf(dev, vq, desc, sz, pkts[i], idx,\n-\t\t\t\t\tmbuf_pool);\n+\t\terr = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i],\n+\t\t\t\tmbuf_pool);\n \t\tif (unlikely(err)) {\n \t\t\trte_pktmbuf_free(pkts[i]);\n-\t\t\tfree_ind_table(idesc);\n \t\t\tbreak;\n \t\t}\n \n@@ -1269,11 +1154,10 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,\n \t\t\tzmbuf = get_zmbuf(vq);\n \t\t\tif (!zmbuf) {\n \t\t\t\trte_pktmbuf_free(pkts[i]);\n-\t\t\t\tfree_ind_table(idesc);\n \t\t\t\tbreak;\n \t\t\t}\n \t\t\tzmbuf->mbuf = pkts[i];\n-\t\t\tzmbuf->desc_idx = desc_indexes[i];\n+\t\t\tzmbuf->desc_idx = head_idx;\n \n \t\t\t/*\n \t\t\t * Pin lock the mbuf; we will check later to see\n@@ -1286,9 +1170,6 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,\n \t\t\tvq->nr_zmbuf += 1;\n \t\t\tTAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next);\n \t\t}\n-\n-\t\tif (unlikely(!!idesc))\n-\t\t\tfree_ind_table(idesc);\n \t}\n \tvq->last_avail_idx += i;\n \n", "prefixes": [ "v4", "2/5" ] }