Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/77289/?format=api
{ "id": 77289, "url": "http://patches.dpdk.org/api/patches/77289/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20200911015316.1903181-2-patrick.fu@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20200911015316.1903181-2-patrick.fu@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20200911015316.1903181-2-patrick.fu@intel.com", "date": "2020-09-11T01:53:13", "name": "[v1,1/4] vhost: simplify async copy completion", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "8d59415cd71a8388a67e71c815803fe2eefb669c", "submitter": { "id": 1781, "url": "http://patches.dpdk.org/api/people/1781/?format=api", "name": "Patrick Fu", "email": "patrick.fu@intel.com" }, "delegate": { "id": 2642, "url": "http://patches.dpdk.org/api/users/2642/?format=api", "username": "mcoquelin", "first_name": "Maxime", "last_name": "Coquelin", "email": "maxime.coquelin@redhat.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20200911015316.1903181-2-patrick.fu@intel.com/mbox/", "series": [ { "id": 12117, "url": "http://patches.dpdk.org/api/series/12117/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=12117", "date": "2020-09-11T01:53:12", "name": "optimize async data path", "version": 1, "mbox": "http://patches.dpdk.org/series/12117/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/77289/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/77289/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id B7006A04B5;\n\tFri, 11 Sep 2020 04:00:56 +0200 (CEST)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 4FF751C135;\n\tFri, 11 Sep 2020 03:59:31 +0200 (CEST)", "from mga04.intel.com (mga04.intel.com [192.55.52.120])\n by dpdk.org (Postfix) with ESMTP id 924891C134\n for <dev@dpdk.org>; Fri, 11 Sep 2020 03:59:29 +0200 (CEST)", "from orsmga002.jf.intel.com ([10.7.209.21])\n by fmsmga104.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 10 Sep 2020 18:59:29 -0700", "from npg-dpdk-patrickfu-casc2.sh.intel.com ([10.67.119.92])\n by orsmga002.jf.intel.com with ESMTP; 10 Sep 2020 18:59:26 -0700" ], "IronPort-SDR": [ "\n lthzyn7dhPywrOcRDF2MQ59HTuW4HHXXCNXdMBZqttrUH7HdDkKGCLPzqbiWIEjm8EQtMfOcU6\n FGL4WoXQq+sg==", "\n bTJXN1dB73nDfBrocd7j5idHqPYgo9tnS5QjgNLAUXLecXD9HUXM4RG5eA4dk0YXixi3ykiJCq\n zhsSqbO3Mvgw==" ], "X-IronPort-AV": [ "E=McAfee;i=\"6000,8403,9740\"; a=\"156072275\"", "E=Sophos;i=\"5.76,413,1592895600\"; d=\"scan'208\";a=\"156072275\"", "E=Sophos;i=\"5.76,413,1592895600\"; d=\"scan'208\";a=\"318121856\"" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "From": "Patrick Fu <patrick.fu@intel.com>", "To": "dev@dpdk.org,\n\tmaxime.coquelin@redhat.com,\n\tchenbo.xia@intel.com", "Cc": "zhihong.wang@intel.com,\n\tcheng1.jiang@intel.com,\n\tpatrick.fu@intel.com", "Date": "Fri, 11 Sep 2020 09:53:13 +0800", "Message-Id": "<20200911015316.1903181-2-patrick.fu@intel.com>", "X-Mailer": "git-send-email 2.18.4", "In-Reply-To": "<20200911015316.1903181-1-patrick.fu@intel.com>", "References": "<20200911015316.1903181-1-patrick.fu@intel.com>", "Subject": "[dpdk-dev] [PATCH v1 1/4] vhost: simplify async copy completion", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Current async ops allows check_completed_copies() callback to return\narbitrary number of async iov segments finished from backend async\ndevices. This design creates complexity for vhost to handle breaking\ntransfer of a single packet (i.e. transfer completes in the middle\nof a async descriptor) and prevents application callbacks from\nleveraging hardware capability to offload the work. Thus, this patch\nenforces the check_completed_copies() callback to return the number\nof async memory descriptors, which is aligned with async transfer\ndata ops callbacks. vHost async data path are revised to work with\nnew ops define, which provides a clean and simplified processing.\n\nSigned-off-by: Patrick Fu <patrick.fu@intel.com>\n---\n lib/librte_vhost/rte_vhost_async.h | 15 ++-\n lib/librte_vhost/vhost.c | 20 ++--\n lib/librte_vhost/vhost.h | 8 +-\n lib/librte_vhost/vhost_user.c | 6 +-\n lib/librte_vhost/virtio_net.c | 149 ++++++++++++-----------------\n 5 files changed, 88 insertions(+), 110 deletions(-)", "diff": "diff --git a/lib/librte_vhost/rte_vhost_async.h b/lib/librte_vhost/rte_vhost_async.h\nindex cb6284539..c73bd7c99 100644\n--- a/lib/librte_vhost/rte_vhost_async.h\n+++ b/lib/librte_vhost/rte_vhost_async.h\n@@ -76,13 +76,26 @@ struct rte_vhost_async_channel_ops {\n \t * @param max_packets\n \t * max number of packets could be completed\n \t * @return\n-\t * number of iov segments completed\n+\t * number of async descs completed\n \t */\n \tuint32_t (*check_completed_copies)(int vid, uint16_t queue_id,\n \t\tstruct rte_vhost_async_status *opaque_data,\n \t\tuint16_t max_packets);\n };\n \n+/**\n+ * inflight async packet information\n+ */\n+struct async_inflight_info {\n+\tunion {\n+\t\tuint32_t info;\n+\t\tstruct {\n+\t\t\tuint16_t descs; /* num of descs inflight */\n+\t\t\tuint16_t segs; /* iov segs inflight */\n+\t\t};\n+\t};\n+};\n+\n /**\n * dma channel feature bit definition\n */\ndiff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c\nindex 8f20a0818..eca507836 100644\n--- a/lib/librte_vhost/vhost.c\n+++ b/lib/librte_vhost/vhost.c\n@@ -333,8 +333,8 @@ free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq)\n \t\trte_free(vq->shadow_used_split);\n \t\tif (vq->async_pkts_pending)\n \t\t\trte_free(vq->async_pkts_pending);\n-\t\tif (vq->async_pending_info)\n-\t\t\trte_free(vq->async_pending_info);\n+\t\tif (vq->async_pkts_info)\n+\t\t\trte_free(vq->async_pkts_info);\n \t}\n \trte_free(vq->batch_copy_elems);\n \trte_mempool_free(vq->iotlb_pool);\n@@ -1573,15 +1573,15 @@ int rte_vhost_async_channel_register(int vid, uint16_t queue_id,\n \tvq->async_pkts_pending = rte_malloc(NULL,\n \t\t\tvq->size * sizeof(uintptr_t),\n \t\t\tRTE_CACHE_LINE_SIZE);\n-\tvq->async_pending_info = rte_malloc(NULL,\n-\t\t\tvq->size * sizeof(uint64_t),\n+\tvq->async_pkts_info = rte_malloc(NULL,\n+\t\t\tvq->size * sizeof(struct async_inflight_info),\n \t\t\tRTE_CACHE_LINE_SIZE);\n-\tif (!vq->async_pkts_pending || !vq->async_pending_info) {\n+\tif (!vq->async_pkts_pending || !vq->async_pkts_info) {\n \t\tif (vq->async_pkts_pending)\n \t\t\trte_free(vq->async_pkts_pending);\n \n-\t\tif (vq->async_pending_info)\n-\t\t\trte_free(vq->async_pending_info);\n+\t\tif (vq->async_pkts_info)\n+\t\t\trte_free(vq->async_pkts_info);\n \n \t\tVHOST_LOG_CONFIG(ERR,\n \t\t\t\t\"async register failed: cannot allocate memory for vq data \"\n@@ -1635,9 +1635,9 @@ int rte_vhost_async_channel_unregister(int vid, uint16_t queue_id)\n \t\tvq->async_pkts_pending = NULL;\n \t}\n \n-\tif (vq->async_pending_info) {\n-\t\trte_free(vq->async_pending_info);\n-\t\tvq->async_pending_info = NULL;\n+\tif (vq->async_pkts_info) {\n+\t\trte_free(vq->async_pkts_info);\n+\t\tvq->async_pkts_info = NULL;\n \t}\n \n \tvq->async_ops.transfer_data = NULL;\ndiff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h\nindex 632f66d53..28aa77380 100644\n--- a/lib/librte_vhost/vhost.h\n+++ b/lib/librte_vhost/vhost.h\n@@ -46,8 +46,6 @@\n \n #define MAX_PKT_BURST 32\n \n-#define ASYNC_MAX_POLL_SEG 255\n-\n #define VHOST_MAX_ASYNC_IT (MAX_PKT_BURST * 2)\n #define VHOST_MAX_ASYNC_VEC (BUF_VECTOR_MAX * 2)\n \n@@ -225,12 +223,10 @@ struct vhost_virtqueue {\n \n \t/* async data transfer status */\n \tuintptr_t\t**async_pkts_pending;\n-\t#define\t\tASYNC_PENDING_INFO_N_MSK 0xFFFF\n-\t#define\t\tASYNC_PENDING_INFO_N_SFT 16\n-\tuint64_t\t*async_pending_info;\n+\tstruct async_inflight_info *async_pkts_info;\n \tuint16_t\tasync_pkts_idx;\n \tuint16_t\tasync_pkts_inflight_n;\n-\tuint16_t\tasync_last_seg_n;\n+\tuint16_t\tasync_last_pkts_n;\n \n \t/* vq async features */\n \tbool\t\tasync_inorder;\ndiff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c\nindex c3c924fae..f6cdbede7 100644\n--- a/lib/librte_vhost/vhost_user.c\n+++ b/lib/librte_vhost/vhost_user.c\n@@ -2006,10 +2006,10 @@ vhost_user_get_vring_base(struct virtio_net **pdev,\n \t\tvq->shadow_used_split = NULL;\n \t\tif (vq->async_pkts_pending)\n \t\t\trte_free(vq->async_pkts_pending);\n-\t\tif (vq->async_pending_info)\n-\t\t\trte_free(vq->async_pending_info);\n+\t\tif (vq->async_pkts_info)\n+\t\t\trte_free(vq->async_pkts_info);\n \t\tvq->async_pkts_pending = NULL;\n-\t\tvq->async_pending_info = NULL;\n+\t\tvq->async_pkts_info = NULL;\n \t}\n \n \trte_free(vq->batch_copy_elems);\ndiff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c\nindex bd9303c8a..70c3377fb 100644\n--- a/lib/librte_vhost/virtio_net.c\n+++ b/lib/librte_vhost/virtio_net.c\n@@ -1473,37 +1473,6 @@ virtio_dev_rx_async_get_info_idx(uint16_t pkts_idx,\n \t\t(vq_size - n_inflight + pkts_idx) & (vq_size - 1);\n }\n \n-static __rte_always_inline void\n-virtio_dev_rx_async_submit_split_err(struct virtio_net *dev,\n-\tstruct vhost_virtqueue *vq, uint16_t queue_id,\n-\tuint16_t last_idx, uint16_t shadow_idx)\n-{\n-\tuint16_t start_idx, pkts_idx, vq_size;\n-\tuint64_t *async_pending_info;\n-\n-\tpkts_idx = vq->async_pkts_idx;\n-\tasync_pending_info = vq->async_pending_info;\n-\tvq_size = vq->size;\n-\tstart_idx = virtio_dev_rx_async_get_info_idx(pkts_idx,\n-\t\tvq_size, vq->async_pkts_inflight_n);\n-\n-\twhile (likely((start_idx & (vq_size - 1)) != pkts_idx)) {\n-\t\tuint64_t n_seg =\n-\t\t\tasync_pending_info[(start_idx) & (vq_size - 1)] >>\n-\t\t\tASYNC_PENDING_INFO_N_SFT;\n-\n-\t\twhile (n_seg)\n-\t\t\tn_seg -= vq->async_ops.check_completed_copies(dev->vid,\n-\t\t\t\tqueue_id, 0, 1);\n-\t}\n-\n-\tvq->async_pkts_inflight_n = 0;\n-\tvq->batch_copy_nb_elems = 0;\n-\n-\tvq->shadow_used_idx = shadow_idx;\n-\tvq->last_avail_idx = last_idx;\n-}\n-\n static __rte_noinline uint32_t\n virtio_dev_rx_async_submit_split(struct virtio_net *dev,\n \tstruct vhost_virtqueue *vq, uint16_t queue_id,\n@@ -1512,7 +1481,7 @@ virtio_dev_rx_async_submit_split(struct virtio_net *dev,\n \tuint32_t pkt_idx = 0, pkt_burst_idx = 0;\n \tuint16_t num_buffers;\n \tstruct buf_vector buf_vec[BUF_VECTOR_MAX];\n-\tuint16_t avail_head, last_idx, shadow_idx;\n+\tuint16_t avail_head;\n \n \tstruct rte_vhost_iov_iter *it_pool = vq->it_pool;\n \tstruct iovec *vec_pool = vq->vec_pool;\n@@ -1522,11 +1491,11 @@ virtio_dev_rx_async_submit_split(struct virtio_net *dev,\n \tstruct rte_vhost_iov_iter *src_it = it_pool;\n \tstruct rte_vhost_iov_iter *dst_it = it_pool + 1;\n \tuint16_t n_free_slot, slot_idx;\n+\tuint16_t pkt_err = 0;\n+\tstruct async_inflight_info *pkts_info = vq->async_pkts_info;\n \tint n_pkts = 0;\n \n \tavail_head = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE);\n-\tlast_idx = vq->last_avail_idx;\n-\tshadow_idx = vq->shadow_used_idx;\n \n \t/*\n \t * The ordering between avail index and\n@@ -1565,14 +1534,14 @@ virtio_dev_rx_async_submit_split(struct virtio_net *dev,\n \t\tif (src_it->count) {\n \t\t\tasync_fill_desc(&tdes[pkt_burst_idx], src_it, dst_it);\n \t\t\tpkt_burst_idx++;\n-\t\t\tvq->async_pending_info[slot_idx] =\n-\t\t\t\tnum_buffers | (src_it->nr_segs << 16);\n+\t\t\tpkts_info[slot_idx].descs = num_buffers;\n+\t\t\tpkts_info[slot_idx].segs = src_it->nr_segs;\n \t\t\tsrc_iovec += src_it->nr_segs;\n \t\t\tdst_iovec += dst_it->nr_segs;\n \t\t\tsrc_it += 2;\n \t\t\tdst_it += 2;\n \t\t} else {\n-\t\t\tvq->async_pending_info[slot_idx] = num_buffers;\n+\t\t\tpkts_info[slot_idx].info = num_buffers;\n \t\t\tvq->async_pkts_inflight_n++;\n \t\t}\n \n@@ -1586,35 +1555,46 @@ virtio_dev_rx_async_submit_split(struct virtio_net *dev,\n \t\t\tdst_iovec = vec_pool + (VHOST_MAX_ASYNC_VEC >> 1);\n \t\t\tsrc_it = it_pool;\n \t\t\tdst_it = it_pool + 1;\n+\t\t\tvq->async_pkts_inflight_n += n_pkts;\n \n \t\t\tif (unlikely(n_pkts < (int)pkt_burst_idx)) {\n-\t\t\t\tvq->async_pkts_inflight_n +=\n-\t\t\t\t\tn_pkts > 0 ? n_pkts : 0;\n-\t\t\t\tvirtio_dev_rx_async_submit_split_err(dev,\n-\t\t\t\t\tvq, queue_id, last_idx, shadow_idx);\n-\t\t\t\treturn 0;\n+\t\t\t\t/*\n+\t\t\t\t * log error packets number here and do actual\n+\t\t\t\t * error processing when applications poll\n+\t\t\t\t * completion\n+\t\t\t\t */\n+\t\t\t\tpkt_err = pkt_burst_idx - n_pkts;\n+\t\t\t\tpkt_burst_idx = 0;\n+\t\t\t\tbreak;\n \t\t\t}\n \n \t\t\tpkt_burst_idx = 0;\n-\t\t\tvq->async_pkts_inflight_n += n_pkts;\n \t\t}\n \t}\n \n \tif (pkt_burst_idx) {\n \t\tn_pkts = vq->async_ops.transfer_data(dev->vid,\n \t\t\t\tqueue_id, tdes, 0, pkt_burst_idx);\n-\t\tif (unlikely(n_pkts < (int)pkt_burst_idx)) {\n-\t\t\tvq->async_pkts_inflight_n += n_pkts > 0 ? n_pkts : 0;\n-\t\t\tvirtio_dev_rx_async_submit_split_err(dev, vq, queue_id,\n-\t\t\t\tlast_idx, shadow_idx);\n-\t\t\treturn 0;\n-\t\t}\n-\n \t\tvq->async_pkts_inflight_n += n_pkts;\n+\n+\t\tif (unlikely(n_pkts < (int)pkt_burst_idx))\n+\t\t\tpkt_err = pkt_burst_idx - n_pkts;\n \t}\n \n \tdo_data_copy_enqueue(dev, vq);\n \n+\tif (unlikely(pkt_err)) {\n+\t\tdo {\n+\t\t\tif (pkts_info[slot_idx].segs)\n+\t\t\t\tpkt_err--;\n+\t\t\tvq->last_avail_idx -= pkts_info[slot_idx].descs;\n+\t\t\tvq->shadow_used_idx -= pkts_info[slot_idx].descs;\n+\t\t\tvq->async_pkts_inflight_n--;\n+\t\t\tslot_idx--;\n+\t\t\tpkt_idx--;\n+\t\t} while (pkt_err);\n+\t}\n+\n \tn_free_slot = vq->size - vq->async_pkts_idx;\n \tif (n_free_slot > pkt_idx) {\n \t\trte_memcpy(&vq->async_pkts_pending[vq->async_pkts_idx],\n@@ -1640,10 +1620,10 @@ uint16_t rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id,\n {\n \tstruct virtio_net *dev = get_device(vid);\n \tstruct vhost_virtqueue *vq;\n-\tuint16_t n_segs_cpl, n_pkts_put = 0, n_descs = 0;\n+\tuint16_t n_pkts_cpl = 0, n_pkts_put = 0, n_descs = 0;\n \tuint16_t start_idx, pkts_idx, vq_size;\n \tuint16_t n_inflight;\n-\tuint64_t *async_pending_info;\n+\tstruct async_inflight_info *pkts_info;\n \n \tif (!dev)\n \t\treturn 0;\n@@ -1657,51 +1637,38 @@ uint16_t rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id,\n \n \tvq = dev->virtqueue[queue_id];\n \n+\tif (unlikely(!vq->async_registered)) {\n+\t\tVHOST_LOG_DATA(ERR, \"(%d) %s: async not registered for queue id %d.\\n\",\n+\t\t\tdev->vid, __func__, queue_id);\n+\t\treturn 0;\n+\t}\n+\n \trte_spinlock_lock(&vq->access_lock);\n \n \tn_inflight = vq->async_pkts_inflight_n;\n \tpkts_idx = vq->async_pkts_idx;\n-\tasync_pending_info = vq->async_pending_info;\n+\tpkts_info = vq->async_pkts_info;\n \tvq_size = vq->size;\n \tstart_idx = virtio_dev_rx_async_get_info_idx(pkts_idx,\n \t\tvq_size, vq->async_pkts_inflight_n);\n \n-\tn_segs_cpl = vq->async_ops.check_completed_copies(vid, queue_id,\n-\t\t0, ASYNC_MAX_POLL_SEG - vq->async_last_seg_n) +\n-\t\tvq->async_last_seg_n;\n+\tif (count > vq->async_last_pkts_n)\n+\t\tn_pkts_cpl = vq->async_ops.check_completed_copies(vid,\n+\t\t\tqueue_id, 0, count - vq->async_last_pkts_n);\n+\tn_pkts_cpl += vq->async_last_pkts_n;\n \n \trte_smp_wmb();\n \n \twhile (likely((n_pkts_put < count) && n_inflight)) {\n-\t\tuint64_t info = async_pending_info[\n-\t\t\t(start_idx + n_pkts_put) & (vq_size - 1)];\n-\t\tuint64_t n_segs;\n+\t\tuint16_t info_idx = (start_idx + n_pkts_put) & (vq_size - 1);\n \t\tn_pkts_put++;\n \t\tn_inflight--;\n-\t\tn_descs += info & ASYNC_PENDING_INFO_N_MSK;\n-\t\tn_segs = info >> ASYNC_PENDING_INFO_N_SFT;\n-\n-\t\tif (n_segs) {\n-\t\t\tif (unlikely(n_segs_cpl < n_segs)) {\n-\t\t\t\tn_pkts_put--;\n-\t\t\t\tn_inflight++;\n-\t\t\t\tn_descs -= info & ASYNC_PENDING_INFO_N_MSK;\n-\t\t\t\tif (n_segs_cpl) {\n-\t\t\t\t\tasync_pending_info[\n-\t\t\t\t\t\t(start_idx + n_pkts_put) &\n-\t\t\t\t\t\t(vq_size - 1)] =\n-\t\t\t\t\t((n_segs - n_segs_cpl) <<\n-\t\t\t\t\t ASYNC_PENDING_INFO_N_SFT) |\n-\t\t\t\t\t(info & ASYNC_PENDING_INFO_N_MSK);\n-\t\t\t\t\tn_segs_cpl = 0;\n-\t\t\t\t}\n-\t\t\t\tbreak;\n-\t\t\t}\n-\t\t\tn_segs_cpl -= n_segs;\n-\t\t}\n+\t\tn_descs += pkts_info[info_idx].descs;\n+\t\tif (pkts_info[info_idx].segs)\n+\t\t\tn_pkts_cpl--;\n \t}\n \n-\tvq->async_last_seg_n = n_segs_cpl;\n+\tvq->async_last_pkts_n = n_pkts_cpl;\n \n \tif (n_pkts_put) {\n \t\tvq->async_pkts_inflight_n = n_inflight;\n@@ -1710,16 +1677,18 @@ uint16_t rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id,\n \t\t\t\t\tn_descs, __ATOMIC_RELEASE);\n \t\t\tvhost_vring_call_split(dev, vq);\n \t\t}\n-\t}\n \n-\tif (start_idx + n_pkts_put <= vq_size) {\n-\t\trte_memcpy(pkts, &vq->async_pkts_pending[start_idx],\n-\t\t\tn_pkts_put * sizeof(uintptr_t));\n-\t} else {\n-\t\trte_memcpy(pkts, &vq->async_pkts_pending[start_idx],\n-\t\t\t(vq_size - start_idx) * sizeof(uintptr_t));\n-\t\trte_memcpy(&pkts[vq_size - start_idx], vq->async_pkts_pending,\n-\t\t\t(n_pkts_put - vq_size + start_idx) * sizeof(uintptr_t));\n+\t\tif (start_idx + n_pkts_put <= vq_size) {\n+\t\t\trte_memcpy(pkts, &vq->async_pkts_pending[start_idx],\n+\t\t\t\tn_pkts_put * sizeof(uintptr_t));\n+\t\t} else {\n+\t\t\trte_memcpy(pkts, &vq->async_pkts_pending[start_idx],\n+\t\t\t\t(vq_size - start_idx) * sizeof(uintptr_t));\n+\t\t\trte_memcpy(&pkts[vq_size - start_idx],\n+\t\t\t\tvq->async_pkts_pending,\n+\t\t\t\t(n_pkts_put + start_idx - vq_size) *\n+\t\t\t\tsizeof(uintptr_t));\n+\t\t}\n \t}\n \n \trte_spinlock_unlock(&vq->access_lock);\n", "prefixes": [ "v1", "1/4" ] }