Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 15542,
    "url": "https://patches.dpdk.org/api/patches/15542/?format=api",
    "web_url": "https://patches.dpdk.org/project/dpdk/patch/1472528164-54296-3-git-send-email-zhihong.wang@intel.com/",
    "project": {
        "id": 1,
        "url": "https://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<1472528164-54296-3-git-send-email-zhihong.wang@intel.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/1472528164-54296-3-git-send-email-zhihong.wang@intel.com",
    "date": "2016-08-30T03:36:00",
    "name": "[dpdk-dev,v4,2/6] vhost: rewrite enqueue",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "dcfe38ecdfebc6f83295bc8d23f72a19022266d8",
    "submitter": {
        "id": 156,
        "url": "https://patches.dpdk.org/api/people/156/?format=api",
        "name": "Zhihong Wang",
        "email": "zhihong.wang@intel.com"
    },
    "delegate": {
        "id": 355,
        "url": "https://patches.dpdk.org/api/users/355/?format=api",
        "username": "yliu",
        "first_name": "Yuanhan",
        "last_name": "Liu",
        "email": "yuanhan.liu@linux.intel.com"
    },
    "mbox": "https://patches.dpdk.org/project/dpdk/patch/1472528164-54296-3-git-send-email-zhihong.wang@intel.com/mbox/",
    "series": [],
    "comments": "https://patches.dpdk.org/api/patches/15542/comments/",
    "check": "pending",
    "checks": "https://patches.dpdk.org/api/patches/15542/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@dpdk.org",
        "Delivered-To": "patchwork@dpdk.org",
        "Received": [
            "from [92.243.14.124] (localhost [IPv6:::1])\n\tby dpdk.org (Postfix) with ESMTP id 95AEF530A;\n\tTue, 30 Aug 2016 12:44:04 +0200 (CEST)",
            "from mga14.intel.com (mga14.intel.com [192.55.52.115])\n\tby dpdk.org (Postfix) with ESMTP id ACA96379B\n\tfor <dev@dpdk.org>; Tue, 30 Aug 2016 12:43:59 +0200 (CEST)",
            "from orsmga004.jf.intel.com ([10.7.209.38])\n\tby fmsmga103.fm.intel.com with ESMTP; 30 Aug 2016 03:43:59 -0700",
            "from unknown (HELO dpdk5.sh.intel.com) ([10.239.129.118])\n\tby orsmga004.jf.intel.com with ESMTP; 30 Aug 2016 03:43:57 -0700"
        ],
        "X-ExtLoop1": "1",
        "X-IronPort-AV": "E=Sophos;i=\"5.30,255,1470726000\"; d=\"scan'208\";a=\"2722976\"",
        "From": "Zhihong Wang <zhihong.wang@intel.com>",
        "To": "dev@dpdk.org",
        "Cc": "maxime.coquelin@redhat.com, yuanhan.liu@linux.intel.com,\n\tthomas.monjalon@6wind.com, Zhihong Wang <zhihong.wang@intel.com>",
        "Date": "Mon, 29 Aug 2016 23:36:00 -0400",
        "Message-Id": "<1472528164-54296-3-git-send-email-zhihong.wang@intel.com>",
        "X-Mailer": "git-send-email 2.7.4",
        "In-Reply-To": "<1472528164-54296-1-git-send-email-zhihong.wang@intel.com>",
        "References": "<1471319402-112998-1-git-send-email-zhihong.wang@intel.com>\n\t<1472528164-54296-1-git-send-email-zhihong.wang@intel.com>",
        "Subject": "[dpdk-dev] [PATCH v4 2/6] vhost: rewrite enqueue",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.15",
        "Precedence": "list",
        "List-Id": "patches and discussions about DPDK <dev.dpdk.org>",
        "List-Unsubscribe": "<http://dpdk.org/ml/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://dpdk.org/ml/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<http://dpdk.org/ml/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org",
        "Sender": "\"dev\" <dev-bounces@dpdk.org>"
    },
    "content": "This patch implements the vhost logic from scratch into a single function\ndesigned for high performance and better maintainability.\n\nThis is the baseline version of the new code, more optimization will be\nadded in the following patches in this patch set.\n\n---\nChanges in v4:\n\n 1. Refactor the code for clearer logic.\n\n 2. Add PRINT_PACKET for debugging.\n\n---\nChanges in v3:\n\n 1. Rewrite enqueue and delete the obsolete in the same patch.\n\nSigned-off-by: Zhihong Wang <zhihong.wang@intel.com>\n---\n lib/librte_vhost/vhost_rxtx.c | 525 ++++++++++++------------------------------\n 1 file changed, 145 insertions(+), 380 deletions(-)",
    "diff": "diff --git a/lib/librte_vhost/vhost_rxtx.c b/lib/librte_vhost/vhost_rxtx.c\nindex 5806f99..629e8ae 100644\n--- a/lib/librte_vhost/vhost_rxtx.c\n+++ b/lib/librte_vhost/vhost_rxtx.c\n@@ -91,7 +91,7 @@ is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t qp_nb)\n \treturn (is_tx ^ (idx & 1)) == 0 && idx < qp_nb * VIRTIO_QNUM;\n }\n \n-static void\n+static inline void __attribute__((always_inline))\n virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)\n {\n \tif (m_buf->ol_flags & PKT_TX_L4_MASK) {\n@@ -112,6 +112,10 @@ virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)\n \t\t\t\t\t\tcksum));\n \t\t\tbreak;\n \t\t}\n+\t} else {\n+\t\tnet_hdr->flags = 0;\n+\t\tnet_hdr->csum_start = 0;\n+\t\tnet_hdr->csum_offset = 0;\n \t}\n \n \tif (m_buf->ol_flags & PKT_TX_TCP_SEG) {\n@@ -122,437 +126,198 @@ virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)\n \t\tnet_hdr->gso_size = m_buf->tso_segsz;\n \t\tnet_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len\n \t\t\t\t\t+ m_buf->l4_len;\n+\t} else {\n+\t\tnet_hdr->gso_type = 0;\n+\t\tnet_hdr->hdr_len = 0;\n+\t\tnet_hdr->gso_size = 0;\n \t}\n }\n \n-static inline void\n-copy_virtio_net_hdr(struct virtio_net *dev, uint64_t desc_addr,\n-\t\t    struct virtio_net_hdr_mrg_rxbuf hdr)\n+static inline void __attribute__((always_inline))\n+update_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,\n+\t\tuint32_t desc_chain_head, uint32_t desc_chain_len)\n {\n-\tif (dev->vhost_hlen == sizeof(struct virtio_net_hdr_mrg_rxbuf))\n-\t\t*(struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)desc_addr = hdr;\n-\telse\n-\t\t*(struct virtio_net_hdr *)(uintptr_t)desc_addr = hdr.hdr;\n+\tuint32_t used_idx_round = vq->last_used_idx & (vq->size - 1);\n+\n+\tvq->used->ring[used_idx_round].id = desc_chain_head;\n+\tvq->used->ring[used_idx_round].len = desc_chain_len;\n+\tvhost_log_used_vring(dev, vq, offsetof(struct vring_used,\n+\t\t\t\tring[used_idx_round]),\n+\t\t\tsizeof(vq->used->ring[used_idx_round]));\n }\n \n-static inline int __attribute__((always_inline))\n-copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,\n-\t\t  struct rte_mbuf *m, uint16_t desc_idx)\n+static inline uint32_t __attribute__((always_inline))\n+enqueue_packet(struct virtio_net *dev, struct vhost_virtqueue *vq,\n+\t\tuint16_t avail_idx, struct rte_mbuf *mbuf,\n+\t\tuint32_t is_mrg_rxbuf)\n {\n-\tuint32_t desc_avail, desc_offset;\n-\tuint32_t mbuf_avail, mbuf_offset;\n-\tuint32_t cpy_len;\n+\tstruct virtio_net_hdr_mrg_rxbuf *virtio_hdr;\n \tstruct vring_desc *desc;\n \tuint64_t desc_addr;\n-\tstruct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0, 0, 0, 0, 0, 0}, 0};\n-\n-\tdesc = &vq->desc[desc_idx];\n+\tuint32_t desc_chain_head;\n+\tuint32_t desc_chain_len;\n+\tuint32_t desc_current;\n+\tuint32_t desc_offset;\n+\tuint32_t mbuf_len;\n+\tuint32_t mbuf_avail;\n+\tuint32_t copy_len;\n+\tuint32_t extra_buffers = 0;\n+\n+\t/* start with the first mbuf of the packet */\n+\tmbuf_len = rte_pktmbuf_data_len(mbuf);\n+\tmbuf_avail = mbuf_len;\n+\n+\t/* get the current desc */\n+\tdesc_current = vq->avail->ring[(vq->last_used_idx) & (vq->size - 1)];\n+\tdesc_chain_head = desc_current;\n+\tdesc = &vq->desc[desc_current];\n \tdesc_addr = gpa_to_vva(dev, desc->addr);\n-\t/*\n-\t * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid\n-\t * performance issue with some versions of gcc (4.8.4 and 5.3.0) which\n-\t * otherwise stores offset on the stack instead of in a register.\n-\t */\n-\tif (unlikely(desc->len < dev->vhost_hlen) || !desc_addr)\n-\t\treturn -1;\n+\tif (unlikely(!desc_addr))\n+\t\tgoto error;\n \n-\trte_prefetch0((void *)(uintptr_t)desc_addr);\n+\t/* handle virtio header */\n+\tvirtio_hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)desc_addr;\n+\tvirtio_enqueue_offload(mbuf, &(virtio_hdr->hdr));\n+\tif (is_mrg_rxbuf)\n+\t\tvirtio_hdr->num_buffers = extra_buffers + 1;\n \n-\tvirtio_enqueue_offload(m, &virtio_hdr.hdr);\n-\tcopy_virtio_net_hdr(dev, desc_addr, virtio_hdr);\n \tvhost_log_write(dev, desc->addr, dev->vhost_hlen);\n \tPRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0);\n-\n \tdesc_offset = dev->vhost_hlen;\n-\tdesc_avail  = desc->len - dev->vhost_hlen;\n-\n-\tmbuf_avail  = rte_pktmbuf_data_len(m);\n-\tmbuf_offset = 0;\n-\twhile (mbuf_avail != 0 || m->next != NULL) {\n-\t\t/* done with current mbuf, fetch next */\n-\t\tif (mbuf_avail == 0) {\n-\t\t\tm = m->next;\n-\n-\t\t\tmbuf_offset = 0;\n-\t\t\tmbuf_avail  = rte_pktmbuf_data_len(m);\n+\tdesc_chain_len = desc_offset;\n+\tdesc_addr += desc_offset;\n+\n+\t/* start copy from mbuf to desc */\n+\twhile (mbuf_avail || mbuf->next) {\n+\t\t/* get the next mbuf if the current done */\n+\t\tif (!mbuf_avail) {\n+\t\t\tmbuf = mbuf->next;\n+\t\t\tmbuf_len = rte_pktmbuf_data_len(mbuf);\n+\t\t\tmbuf_avail = mbuf_len;\n \t\t}\n \n-\t\t/* done with current desc buf, fetch next */\n-\t\tif (desc_avail == 0) {\n-\t\t\tif ((desc->flags & VRING_DESC_F_NEXT) == 0) {\n-\t\t\t\t/* Room in vring buffer is not enough */\n-\t\t\t\treturn -1;\n-\t\t\t}\n-\t\t\tif (unlikely(desc->next >= vq->size))\n-\t\t\t\treturn -1;\n+\t\t/* get the next desc if the current done */\n+\t\tif (desc->len <= desc_offset) {\n+\t\t\tif (desc->flags & VRING_DESC_F_NEXT) {\n+\t\t\t\t/* go on with the current desc chain */\n+\t\t\t\tdesc_offset = 0;\n+\t\t\t\tdesc_current = desc->next;\n+\t\t\t\tdesc = &vq->desc[desc_current];\n+\t\t\t\tdesc_addr = gpa_to_vva(dev, desc->addr);\n+\t\t\t\tif (unlikely(!desc_addr))\n+\t\t\t\t\tgoto error;\n+\t\t\t} else if (is_mrg_rxbuf) {\n+\t\t\t\t/* start with the next desc chain */\n+\t\t\t\tupdate_used_ring(dev, vq, desc_chain_head,\n+\t\t\t\t\t\tdesc_chain_len);\n+\t\t\t\tvq->last_used_idx++;\n+\t\t\t\textra_buffers++;\n+\t\t\t\tvirtio_hdr->num_buffers++;\n+\t\t\t\tif (avail_idx == vq->last_used_idx)\n+\t\t\t\t\tgoto error;\n+\n+\t\t\t\tdesc_current =\n+\t\t\t\t\tvq->avail->ring[(vq->last_used_idx) &\n+\t\t\t\t\t(vq->size - 1)];\n+\t\t\t\tdesc_chain_head = desc_current;\n+\t\t\t\tdesc = &vq->desc[desc_current];\n+\t\t\t\tdesc_addr = gpa_to_vva(dev, desc->addr);\n+\t\t\t\tif (unlikely(!desc_addr))\n+\t\t\t\t\tgoto error;\n \n-\t\t\tdesc = &vq->desc[desc->next];\n-\t\t\tdesc_addr = gpa_to_vva(dev, desc->addr);\n-\t\t\tif (unlikely(!desc_addr))\n-\t\t\t\treturn -1;\n-\n-\t\t\tdesc_offset = 0;\n-\t\t\tdesc_avail  = desc->len;\n+\t\t\t\tdesc_chain_len = 0;\n+\t\t\t\tdesc_offset = 0;\n+\t\t\t} else\n+\t\t\t\tgoto error;\n \t\t}\n \n-\t\tcpy_len = RTE_MIN(desc_avail, mbuf_avail);\n-\t\trte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)),\n-\t\t\trte_pktmbuf_mtod_offset(m, void *, mbuf_offset),\n-\t\t\tcpy_len);\n-\t\tvhost_log_write(dev, desc->addr + desc_offset, cpy_len);\n-\t\tPRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),\n-\t\t\t     cpy_len, 0);\n-\n-\t\tmbuf_avail  -= cpy_len;\n-\t\tmbuf_offset += cpy_len;\n-\t\tdesc_avail  -= cpy_len;\n-\t\tdesc_offset += cpy_len;\n+\t\t/* copy mbuf data */\n+\t\tcopy_len = RTE_MIN(desc->len - desc_offset, mbuf_avail);\n+\t\trte_memcpy((void *)(uintptr_t)desc_addr,\n+\t\t\t\trte_pktmbuf_mtod_offset(mbuf, void *,\n+\t\t\t\t\tmbuf_len - mbuf_avail),\n+\t\t\t\tcopy_len);\n+\t\tvhost_log_write(dev, desc->addr + desc_offset, copy_len);\n+\t\tPRINT_PACKET(dev, (uintptr_t)desc_addr, copy_len, 0);\n+\t\tmbuf_avail -= copy_len;\n+\t\tdesc_offset += copy_len;\n+\t\tdesc_addr += copy_len;\n+\t\tdesc_chain_len += copy_len;\n \t}\n \n-\treturn 0;\n-}\n+\tupdate_used_ring(dev, vq, desc_chain_head, desc_chain_len);\n+\tvq->last_used_idx++;\n \n-/**\n- * This function adds buffers to the virtio devices RX virtqueue. Buffers can\n- * be received from the physical port or from another virtio device. A packet\n- * count is returned to indicate the number of packets that are succesfully\n- * added to the RX queue. This function works when the mbuf is scattered, but\n- * it doesn't support the mergeable feature.\n- */\n-static inline uint32_t __attribute__((always_inline))\n-virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,\n-\t      struct rte_mbuf **pkts, uint32_t count)\n-{\n-\tstruct vhost_virtqueue *vq;\n-\tuint16_t avail_idx, free_entries, start_idx;\n-\tuint16_t desc_indexes[MAX_PKT_BURST];\n-\tuint16_t used_idx;\n-\tuint32_t i;\n-\n-\tLOG_DEBUG(VHOST_DATA, \"(%d) %s\\n\", dev->vid, __func__);\n-\tif (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->virt_qp_nb))) {\n-\t\tRTE_LOG(ERR, VHOST_DATA, \"(%d) %s: invalid virtqueue idx %d.\\n\",\n-\t\t\tdev->vid, __func__, queue_id);\n-\t\treturn 0;\n-\t}\n-\n-\tvq = dev->virtqueue[queue_id];\n-\tif (unlikely(vq->enabled == 0))\n-\t\treturn 0;\n-\n-\tavail_idx = *((volatile uint16_t *)&vq->avail->idx);\n-\tstart_idx = vq->last_used_idx;\n-\tfree_entries = avail_idx - start_idx;\n-\tcount = RTE_MIN(count, free_entries);\n-\tcount = RTE_MIN(count, (uint32_t)MAX_PKT_BURST);\n-\tif (count == 0)\n-\t\treturn 0;\n-\n-\tLOG_DEBUG(VHOST_DATA, \"(%d) start_idx %d | end_idx %d\\n\",\n-\t\tdev->vid, start_idx, start_idx + count);\n-\n-\t/* Retrieve all of the desc indexes first to avoid caching issues. */\n-\trte_prefetch0(&vq->avail->ring[start_idx & (vq->size - 1)]);\n-\tfor (i = 0; i < count; i++) {\n-\t\tused_idx = (start_idx + i) & (vq->size - 1);\n-\t\tdesc_indexes[i] = vq->avail->ring[used_idx];\n-\t\tvq->used->ring[used_idx].id = desc_indexes[i];\n-\t\tvq->used->ring[used_idx].len = pkts[i]->pkt_len +\n-\t\t\t\t\t       dev->vhost_hlen;\n-\t\tvhost_log_used_vring(dev, vq,\n-\t\t\toffsetof(struct vring_used, ring[used_idx]),\n-\t\t\tsizeof(vq->used->ring[used_idx]));\n-\t}\n-\n-\trte_prefetch0(&vq->desc[desc_indexes[0]]);\n-\tfor (i = 0; i < count; i++) {\n-\t\tuint16_t desc_idx = desc_indexes[i];\n-\t\tint err;\n+\treturn 0;\n \n-\t\terr = copy_mbuf_to_desc(dev, vq, pkts[i], desc_idx);\n-\t\tif (unlikely(err)) {\n-\t\t\tused_idx = (start_idx + i) & (vq->size - 1);\n-\t\t\tvq->used->ring[used_idx].len = dev->vhost_hlen;\n-\t\t\tvhost_log_used_vring(dev, vq,\n-\t\t\t\toffsetof(struct vring_used, ring[used_idx]),\n-\t\t\t\tsizeof(vq->used->ring[used_idx]));\n-\t\t}\n+error:\n+\t/* rollback on any error if last_used_idx update on-the-fly */\n+\tvq->last_used_idx -= extra_buffers;\n \n-\t\tif (i + 1 < count)\n-\t\t\trte_prefetch0(&vq->desc[desc_indexes[i+1]]);\n-\t}\n+\treturn 1;\n+}\n \n+static inline void __attribute__((always_inline))\n+notify_guest(struct virtio_net *dev, struct vhost_virtqueue *vq)\n+{\n \trte_smp_wmb();\n-\n-\t*(volatile uint16_t *)&vq->used->idx += count;\n-\tvq->last_used_idx += count;\n-\tvhost_log_used_vring(dev, vq,\n-\t\toffsetof(struct vring_used, idx),\n-\t\tsizeof(vq->used->idx));\n-\n-\t/* flush used->idx update before we read avail->flags. */\n+\tvq->used->idx = vq->last_used_idx;\n+\tvhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),\n+\t\t\tsizeof(vq->used->idx));\n \trte_mb();\n-\n-\t/* Kick the guest if necessary. */\n \tif (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)\n \t\t\t&& (vq->callfd >= 0))\n \t\teventfd_write(vq->callfd, (eventfd_t)1);\n-\treturn count;\n }\n \n-static inline int\n-fill_vec_buf(struct vhost_virtqueue *vq, uint32_t avail_idx,\n-\t     uint32_t *allocated, uint32_t *vec_idx,\n-\t     struct buf_vector *buf_vec)\n-{\n-\tuint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)];\n-\tuint32_t vec_id = *vec_idx;\n-\tuint32_t len    = *allocated;\n-\n-\twhile (1) {\n-\t\tif (unlikely(vec_id >= BUF_VECTOR_MAX || idx >= vq->size))\n-\t\t\treturn -1;\n-\n-\t\tlen += vq->desc[idx].len;\n-\t\tbuf_vec[vec_id].buf_addr = vq->desc[idx].addr;\n-\t\tbuf_vec[vec_id].buf_len  = vq->desc[idx].len;\n-\t\tbuf_vec[vec_id].desc_idx = idx;\n-\t\tvec_id++;\n-\n-\t\tif ((vq->desc[idx].flags & VRING_DESC_F_NEXT) == 0)\n-\t\t\tbreak;\n-\n-\t\tidx = vq->desc[idx].next;\n-\t}\n-\n-\t*allocated = len;\n-\t*vec_idx   = vec_id;\n-\n-\treturn 0;\n-}\n-\n-/*\n- * Returns -1 on fail, 0 on success\n- */\n-static inline int\n-reserve_avail_buf_mergeable(struct vhost_virtqueue *vq, uint32_t size,\n-\t\t\t    uint16_t *end, struct buf_vector *buf_vec)\n+uint16_t\n+rte_vhost_enqueue_burst(int vid, uint16_t queue_id,\n+\tstruct rte_mbuf **pkts, uint16_t count)\n {\n-\tuint16_t cur_idx;\n+\tstruct vhost_virtqueue *vq;\n+\tstruct virtio_net *dev;\n+\tuint32_t pkt_left = count;\n+\tuint32_t pkt_idx = 0;\n+\tuint32_t pkt_sent = 0;\n+\tuint32_t is_mrg_rxbuf = 0;\n \tuint16_t avail_idx;\n-\tuint32_t allocated = 0;\n-\tuint32_t vec_idx = 0;\n-\tuint16_t tries = 0;\n \n-\tcur_idx  = vq->last_used_idx;\n-\n-\twhile (1) {\n-\t\tavail_idx = *((volatile uint16_t *)&vq->avail->idx);\n-\t\tif (unlikely(cur_idx == avail_idx))\n-\t\t\treturn -1;\n-\n-\t\tif (unlikely(fill_vec_buf(vq, cur_idx, &allocated,\n-\t\t\t\t\t  &vec_idx, buf_vec) < 0))\n-\t\t\treturn -1;\n-\n-\t\tcur_idx++;\n-\t\ttries++;\n-\n-\t\tif (allocated >= size)\n-\t\t\tbreak;\n-\n-\t\t/*\n-\t\t * if we tried all available ring items, and still\n-\t\t * can't get enough buf, it means something abnormal\n-\t\t * happened.\n-\t\t */\n-\t\tif (unlikely(tries >= vq->size))\n-\t\t\treturn -1;\n-\t}\n-\n-\t*end = cur_idx;\n-\treturn 0;\n-}\n-\n-static inline uint32_t __attribute__((always_inline))\n-copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,\n-\t\t\t    uint16_t end_idx, struct rte_mbuf *m,\n-\t\t\t    struct buf_vector *buf_vec)\n-{\n-\tstruct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0, 0, 0, 0, 0, 0}, 0};\n-\tuint32_t vec_idx = 0;\n-\tuint16_t start_idx = vq->last_used_idx;\n-\tuint16_t cur_idx = start_idx;\n-\tuint64_t desc_addr;\n-\tuint32_t desc_chain_head;\n-\tuint32_t desc_chain_len;\n-\tuint32_t mbuf_offset, mbuf_avail;\n-\tuint32_t desc_offset, desc_avail;\n-\tuint32_t cpy_len;\n-\tuint16_t desc_idx, used_idx;\n-\n-\tif (unlikely(m == NULL))\n+\tif (unlikely(!pkt_left))\n \t\treturn 0;\n \n-\tLOG_DEBUG(VHOST_DATA, \"(%d) current index %d | end index %d\\n\",\n-\t\tdev->vid, cur_idx, end_idx);\n+\tpkt_left = RTE_MIN((uint32_t)MAX_PKT_BURST, pkt_left);\n \n-\tdesc_addr = gpa_to_vva(dev, buf_vec[vec_idx].buf_addr);\n-\tif (buf_vec[vec_idx].buf_len < dev->vhost_hlen || !desc_addr)\n+\tdev = get_device(vid);\n+\tif (unlikely(!dev))\n \t\treturn 0;\n \n-\trte_prefetch0((void *)(uintptr_t)desc_addr);\n-\n-\tvirtio_hdr.num_buffers = end_idx - start_idx;\n-\tLOG_DEBUG(VHOST_DATA, \"(%d) RX: num merge buffers %d\\n\",\n-\t\tdev->vid, virtio_hdr.num_buffers);\n-\n-\tvirtio_enqueue_offload(m, &virtio_hdr.hdr);\n-\tcopy_virtio_net_hdr(dev, desc_addr, virtio_hdr);\n-\tvhost_log_write(dev, buf_vec[vec_idx].buf_addr, dev->vhost_hlen);\n-\tPRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0);\n-\n-\tdesc_avail  = buf_vec[vec_idx].buf_len - dev->vhost_hlen;\n-\tdesc_offset = dev->vhost_hlen;\n-\tdesc_chain_head = buf_vec[vec_idx].desc_idx;\n-\tdesc_chain_len = desc_offset;\n-\n-\tmbuf_avail  = rte_pktmbuf_data_len(m);\n-\tmbuf_offset = 0;\n-\twhile (mbuf_avail != 0 || m->next != NULL) {\n-\t\t/* done with current desc buf, get the next one */\n-\t\tif (desc_avail == 0) {\n-\t\t\tdesc_idx = buf_vec[vec_idx].desc_idx;\n-\t\t\tvec_idx++;\n-\n-\t\t\tif (!(vq->desc[desc_idx].flags & VRING_DESC_F_NEXT)) {\n-\t\t\t\t/* Update used ring with desc information */\n-\t\t\t\tused_idx = cur_idx++ & (vq->size - 1);\n-\t\t\t\tvq->used->ring[used_idx].id = desc_chain_head;\n-\t\t\t\tvq->used->ring[used_idx].len = desc_chain_len;\n-\t\t\t\tvhost_log_used_vring(dev, vq,\n-\t\t\t\t\toffsetof(struct vring_used,\n-\t\t\t\t\t\t ring[used_idx]),\n-\t\t\t\t\tsizeof(vq->used->ring[used_idx]));\n-\t\t\t\tdesc_chain_head = buf_vec[vec_idx].desc_idx;\n-\t\t\t\tdesc_chain_len = 0;\n-\t\t\t}\n-\n-\t\t\tdesc_addr = gpa_to_vva(dev, buf_vec[vec_idx].buf_addr);\n-\t\t\tif (unlikely(!desc_addr))\n-\t\t\t\treturn 0;\n-\n-\t\t\t/* Prefetch buffer address. */\n-\t\t\trte_prefetch0((void *)(uintptr_t)desc_addr);\n-\t\t\tdesc_offset = 0;\n-\t\t\tdesc_avail  = buf_vec[vec_idx].buf_len;\n-\t\t}\n-\n-\t\t/* done with current mbuf, get the next one */\n-\t\tif (mbuf_avail == 0) {\n-\t\t\tm = m->next;\n-\n-\t\t\tmbuf_offset = 0;\n-\t\t\tmbuf_avail  = rte_pktmbuf_data_len(m);\n-\t\t}\n-\n-\t\tcpy_len = RTE_MIN(desc_avail, mbuf_avail);\n-\t\trte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)),\n-\t\t\trte_pktmbuf_mtod_offset(m, void *, mbuf_offset),\n-\t\t\tcpy_len);\n-\t\tvhost_log_write(dev, buf_vec[vec_idx].buf_addr + desc_offset,\n-\t\t\tcpy_len);\n-\t\tPRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),\n-\t\t\tcpy_len, 0);\n-\n-\t\tmbuf_avail  -= cpy_len;\n-\t\tmbuf_offset += cpy_len;\n-\t\tdesc_avail  -= cpy_len;\n-\t\tdesc_offset += cpy_len;\n-\t\tdesc_chain_len += cpy_len;\n-\t}\n-\n-\tused_idx = cur_idx & (vq->size - 1);\n-\tvq->used->ring[used_idx].id = desc_chain_head;\n-\tvq->used->ring[used_idx].len = desc_chain_len;\n-\tvhost_log_used_vring(dev, vq,\n-\t\toffsetof(struct vring_used, ring[used_idx]),\n-\t\tsizeof(vq->used->ring[used_idx]));\n-\n-\treturn end_idx - start_idx;\n-}\n-\n-static inline uint32_t __attribute__((always_inline))\n-virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,\n-\tstruct rte_mbuf **pkts, uint32_t count)\n-{\n-\tstruct vhost_virtqueue *vq;\n-\tuint32_t pkt_idx = 0, nr_used = 0;\n-\tuint16_t end;\n-\tstruct buf_vector buf_vec[BUF_VECTOR_MAX];\n-\n-\tLOG_DEBUG(VHOST_DATA, \"(%d) %s\\n\", dev->vid, __func__);\n-\tif (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->virt_qp_nb))) {\n-\t\tRTE_LOG(ERR, VHOST_DATA, \"(%d) %s: invalid virtqueue idx %d.\\n\",\n-\t\t\tdev->vid, __func__, queue_id);\n+\tif (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->virt_qp_nb)))\n \t\treturn 0;\n-\t}\n \n \tvq = dev->virtqueue[queue_id];\n-\tif (unlikely(vq->enabled == 0))\n+\tif (unlikely(!vq->enabled))\n \t\treturn 0;\n \n-\tcount = RTE_MIN((uint32_t)MAX_PKT_BURST, count);\n-\tif (count == 0)\n-\t\treturn 0;\n-\n-\tfor (pkt_idx = 0; pkt_idx < count; pkt_idx++) {\n-\t\tuint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;\n+\tif (dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF))\n+\t\tis_mrg_rxbuf = 1;\n \n-\t\tif (unlikely(reserve_avail_buf_mergeable(vq, pkt_len,\n-\t\t\t\t\t\t\t &end, buf_vec) < 0)) {\n-\t\t\tLOG_DEBUG(VHOST_DATA,\n-\t\t\t\t\"(%d) failed to get enough desc from vring\\n\",\n-\t\t\t\tdev->vid);\n+\t/* start enqueuing packets 1 by 1 */\n+\tavail_idx = *((volatile uint16_t *)&vq->avail->idx);\n+\twhile (pkt_left && avail_idx != vq->last_used_idx) {\n+\t\tif (enqueue_packet(dev, vq, avail_idx, pkts[pkt_idx],\n+\t\t\t\t\tis_mrg_rxbuf))\n \t\t\tbreak;\n-\t\t}\n-\n-\t\tnr_used = copy_mbuf_to_desc_mergeable(dev, vq, end,\n-\t\t\t\t\t\t      pkts[pkt_idx], buf_vec);\n-\t\trte_smp_wmb();\n-\n-\t\t*(volatile uint16_t *)&vq->used->idx += nr_used;\n-\t\tvhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),\n-\t\t\tsizeof(vq->used->idx));\n-\t\tvq->last_used_idx += nr_used;\n-\t}\n-\n-\tif (likely(pkt_idx)) {\n-\t\t/* flush used->idx update before we read avail->flags. */\n-\t\trte_mb();\n \n-\t\t/* Kick the guest if necessary. */\n-\t\tif (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)\n-\t\t\t\t&& (vq->callfd >= 0))\n-\t\t\teventfd_write(vq->callfd, (eventfd_t)1);\n+\t\tpkt_idx++;\n+\t\tpkt_sent++;\n+\t\tpkt_left--;\n \t}\n \n-\treturn pkt_idx;\n-}\n-\n-uint16_t\n-rte_vhost_enqueue_burst(int vid, uint16_t queue_id,\n-\tstruct rte_mbuf **pkts, uint16_t count)\n-{\n-\tstruct virtio_net *dev = get_device(vid);\n-\n-\tif (!dev)\n-\t\treturn 0;\n+\t/* update used idx and kick the guest if necessary */\n+\tif (pkt_sent)\n+\t\tnotify_guest(dev, vq);\n \n-\tif (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF))\n-\t\treturn virtio_dev_merge_rx(dev, queue_id, pkts, count);\n-\telse\n-\t\treturn virtio_dev_rx(dev, queue_id, pkts, count);\n+\treturn pkt_sent;\n }\n \n static void\n",
    "prefixes": [
        "dpdk-dev",
        "v4",
        "2/6"
    ]
}