Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 126336,
    "url": "http://patches.dpdk.org/api/patches/126336/?format=api",
    "web_url": "http://patches.dpdk.org/project/dpdk/patch/20230421010932.46847-3-cheng1.jiang@intel.com/",
    "project": {
        "id": 1,
        "url": "http://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<20230421010932.46847-3-cheng1.jiang@intel.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/20230421010932.46847-3-cheng1.jiang@intel.com",
    "date": "2023-04-21T01:09:32",
    "name": "[2/2] vhost: add egress API for port mirroring datapath",
    "commit_ref": null,
    "pull_url": null,
    "state": "rejected",
    "archived": true,
    "hash": "663a0b5c07f4ebb593decf6170580222518a2e2c",
    "submitter": {
        "id": 1530,
        "url": "http://patches.dpdk.org/api/people/1530/?format=api",
        "name": "Jiang, Cheng1",
        "email": "Cheng1.jiang@intel.com"
    },
    "delegate": {
        "id": 2642,
        "url": "http://patches.dpdk.org/api/users/2642/?format=api",
        "username": "mcoquelin",
        "first_name": "Maxime",
        "last_name": "Coquelin",
        "email": "maxime.coquelin@redhat.com"
    },
    "mbox": "http://patches.dpdk.org/project/dpdk/patch/20230421010932.46847-3-cheng1.jiang@intel.com/mbox/",
    "series": [
        {
            "id": 27806,
            "url": "http://patches.dpdk.org/api/series/27806/?format=api",
            "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=27806",
            "date": "2023-04-21T01:09:30",
            "name": "vhost: add port mirroring function in the vhost lib",
            "version": 1,
            "mbox": "http://patches.dpdk.org/series/27806/mbox/"
        }
    ],
    "comments": "http://patches.dpdk.org/api/patches/126336/comments/",
    "check": "fail",
    "checks": "http://patches.dpdk.org/api/patches/126336/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 14F16429A2;\n\tFri, 21 Apr 2023 03:44:42 +0200 (CEST)",
            "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 7EAD742D0B;\n\tFri, 21 Apr 2023 03:44:36 +0200 (CEST)",
            "from mga02.intel.com (mga02.intel.com [134.134.136.20])\n by mails.dpdk.org (Postfix) with ESMTP id 18D574113C\n for <dev@dpdk.org>; Fri, 21 Apr 2023 03:44:34 +0200 (CEST)",
            "from orsmga006.jf.intel.com ([10.7.209.51])\n by orsmga101.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 20 Apr 2023 18:44:34 -0700",
            "from dpdk_jiangcheng.sh.intel.com ([10.67.119.139])\n by orsmga006.jf.intel.com with ESMTP; 20 Apr 2023 18:44:31 -0700"
        ],
        "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple;\n d=intel.com; i=@intel.com; q=dns/txt; s=Intel;\n t=1682041475; x=1713577475;\n h=from:to:cc:subject:date:message-id:in-reply-to:\n references:mime-version:content-transfer-encoding;\n bh=mr5264wRwmBCcovGt2/K2W9BrzD1b1sNL+DQlA8fq3E=;\n b=MPq7v86vVUoFK4+1cYiSE5punI1Cqki8xQlQtzPTeLncbe+6HAw52g2o\n EUKxuKCwanl2A7jBMbQx16cUGcTbRxgZ2IV5UyWb4FKvrLiKDX5Ew7Pkj\n M8ZjrkTVNbzG2EPAIdaPOjkTR+XVZQbMI/ZdOaLuypyWzMa/XRt6Xe9AQ\n HlgbwBcx1h2F4gx5+f2wjs+dpV/oIICdM7X6rx3E+r1S9V7tcPLM3XqI5\n w5Eg69LZ83p7cVh5YJx8HITfYEWvIj05NSk6DsCgLwMOw2OHhQxmVfUt6\n tGhs64lPaHIL8DX9RKUyZ+l/0ZwUVeUIMUoQ5cLG+fc+v174C7Mk2CKb+ w==;",
        "X-IronPort-AV": [
            "E=McAfee;i=\"6600,9927,10686\"; a=\"334763704\"",
            "E=Sophos;i=\"5.99,214,1677571200\"; d=\"scan'208\";a=\"334763704\"",
            "E=McAfee;i=\"6600,9927,10686\"; a=\"669558579\"",
            "E=Sophos;i=\"5.99,214,1677571200\"; d=\"scan'208\";a=\"669558579\""
        ],
        "X-ExtLoop1": "1",
        "From": "Cheng Jiang <cheng1.jiang@intel.com>",
        "To": "maxime.coquelin@redhat.com,\n\tchenbo.xia@intel.com",
        "Cc": "dev@dpdk.org, jiayu.hu@intel.com, xuan.ding@intel.com,\n wenwux.ma@intel.com,\n yuanx.wang@intel.com, xingguang.he@intel.com,\n Cheng Jiang <cheng1.jiang@intel.com>",
        "Subject": "[PATCH 2/2] vhost: add egress API for port mirroring datapath",
        "Date": "Fri, 21 Apr 2023 01:09:32 +0000",
        "Message-Id": "<20230421010932.46847-3-cheng1.jiang@intel.com>",
        "X-Mailer": "git-send-email 2.35.1",
        "In-Reply-To": "<20230421010932.46847-1-cheng1.jiang@intel.com>",
        "References": "<20230421010932.46847-1-cheng1.jiang@intel.com>",
        "MIME-Version": "1.0",
        "Content-Transfer-Encoding": "8bit",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org"
    },
    "content": "This patch implements egress function on the Vhost lib. When packets are\nreceived from a front-end, it will also send the packets to its mirror\nfront-end.\n\nSigned-off-by: Cheng Jiang <cheng1.jiang@intel.com>\nSigned-off-by: Yuan Wang <yuanx.wang@intel.com>\nSigned-off-by: Wenwu Ma <wenwux.ma@intel.com>\n---\n lib/vhost/rte_vhost_async.h |  11 +\n lib/vhost/version.map       |   2 +\n lib/vhost/virtio_net.c      | 682 +++++++++++++++++++++++++++++++++---\n 3 files changed, 643 insertions(+), 52 deletions(-)",
    "diff": "diff --git a/lib/vhost/rte_vhost_async.h b/lib/vhost/rte_vhost_async.h\nindex 30aaf66b60..4df473f1ec 100644\n--- a/lib/vhost/rte_vhost_async.h\n+++ b/lib/vhost/rte_vhost_async.h\n@@ -286,6 +286,17 @@ __rte_experimental\n int\n rte_vhost_async_dma_unconfigure(int16_t dma_id, uint16_t vchan_id);\n \n+__rte_experimental\n+uint16_t rte_vhost_submit_ingress_mirroring_burst(int vid, uint16_t queue_id,\n+\t\tint mirror_vid, uint16_t mirror_queue_id,\n+\t\tstruct rte_mbuf **pkts, uint16_t count,\n+\t\tint16_t dma_id, uint16_t vchan_id);\n+\n+__rte_experimental\n+uint16_t rte_vhost_poll_ingress_completed(int vid, uint16_t queue_id, int mr_vid,\n+\t\tuint16_t mr_queue_id, struct rte_mbuf **pkts, uint16_t count,\n+\t\tint16_t dma_id, uint16_t vchan_id);\n+\n __rte_experimental\n uint16_t rte_vhost_async_try_egress_burst(int vid, uint16_t queue_id,\n \t\tint mr_vid, uint16_t mr_queue_id,\ndiff --git a/lib/vhost/version.map b/lib/vhost/version.map\nindex 95f75a6928..347ea6ac9c 100644\n--- a/lib/vhost/version.map\n+++ b/lib/vhost/version.map\n@@ -98,6 +98,8 @@ EXPERIMENTAL {\n \t# added in 22.11\n \trte_vhost_async_dma_unconfigure;\n \trte_vhost_vring_call_nonblock;\n+\trte_vhost_submit_ingress_mirroring_burst;\n+\trte_vhost_poll_ingress_completed;\n \trte_vhost_async_try_egress_burst;\n };\n \ndiff --git a/lib/vhost/virtio_net.c b/lib/vhost/virtio_net.c\nindex c7e99d403e..f4c96c3216 100644\n--- a/lib/vhost/virtio_net.c\n+++ b/lib/vhost/virtio_net.c\n@@ -4263,6 +4263,634 @@ rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,\n \treturn count;\n }\n \n+static __rte_always_inline int\n+async_mirror_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, uint64_t buf_iova,\n+\t\tstruct virtio_net *mr_dev, uint64_t mr_buf_iova,\n+\t\tstruct rte_mbuf *m, uint32_t mbuf_offset, uint32_t cpy_len, bool is_ingress)\n+{\n+\tstruct vhost_async *async = vq->async;\n+\tuint64_t mapped_len, mr_mapped_len;\n+\tuint32_t buf_offset = 0;\n+\tvoid *src, *dst, *mr_dst;\n+\tvoid *host_iova, *mr_host_iova;\n+\n+\twhile (cpy_len) {\n+\t\thost_iova = (void *)(uintptr_t)gpa_to_first_hpa(dev,\n+\t\t\t\tbuf_iova + buf_offset, cpy_len, &mapped_len);\n+\t\tif (unlikely(!host_iova)) {\n+\t\t\tVHOST_LOG_DATA(dev->ifname, ERR, \"%s: failed to get host iova.\\n\", __func__);\n+\t\t\treturn -1;\n+\t\t}\n+\t\tmr_host_iova = (void *)(uintptr_t)gpa_to_first_hpa(mr_dev,\n+\t\t\t\tmr_buf_iova + buf_offset, cpy_len, &mr_mapped_len);\n+\t\tif (unlikely(!mr_host_iova)) {\n+\t\t\tVHOST_LOG_DATA(mr_dev->ifname, ERR, \"%s: failed to get mirror host iova.\\n\", __func__);\n+\t\t\treturn -1;\n+\t\t}\n+\n+\t\tif (unlikely(mr_mapped_len != mapped_len)) {\n+\t\t\tVHOST_LOG_DATA(dev->ifname, ERR, \"original mapped len is not equal to mirror len\\n\");\n+\t\t\treturn -1;\n+\t\t}\n+\n+\t\tif (is_ingress) {\n+\t\t\tsrc = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);\n+\t\t\tdst = host_iova;\n+\t\t\tmr_dst = mr_host_iova;\n+\t\t} else {\n+\t\t\tsrc = host_iova;\n+\t\t\tdst = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);\n+\t\t\tmr_dst = mr_host_iova;\n+\t\t}\n+\n+\t\tif (unlikely(async_iter_add_iovec(dev, async, src, dst, (size_t)mapped_len)))\n+\t\t\treturn -1;\n+\t\tif (unlikely(async_iter_add_iovec(mr_dev, async, src, mr_dst, (size_t)mapped_len)))\n+\t\t\treturn -1;\n+\n+\t\tcpy_len -= (uint32_t)mapped_len;\n+\t\tmbuf_offset += (uint32_t)mapped_len;\n+\t\tbuf_offset += (uint32_t)mapped_len;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static __rte_always_inline uint16_t\n+vhost_poll_ingress_completed(struct virtio_net *dev, struct vhost_virtqueue *vq,\n+\t\t\tstruct virtio_net *mr_dev, struct vhost_virtqueue *mr_vq,\n+\t\t\tstruct rte_mbuf **pkts, uint16_t count,\tint16_t dma_id, uint16_t vchan_id)\n+{\n+\tstruct vhost_async *async = vq->async;\n+\tstruct vhost_async *mr_async = mr_vq->async;\n+\tstruct async_inflight_info *pkts_info = async->pkts_info;\n+\n+\tuint16_t nr_cpl_pkts = 0, n_descs = 0;\n+\tuint16_t mr_n_descs = 0;\n+\tuint16_t start_idx, from, i;\n+\n+\t/* Check completed copies for the given DMA vChannel */\n+\tvhost_async_dma_check_completed(dev, dma_id, vchan_id, VHOST_DMA_MAX_COPY_COMPLETE);\n+\n+\tstart_idx = async_get_first_inflight_pkt_idx(vq);\n+\t/*\n+\t * Calculate the number of copy completed packets.\n+\t * Note that there may be completed packets even if\n+\t * no copies are reported done by the given DMA vChannel,\n+\t * as it's possible that a virtqueue uses multiple DMA\n+\t * vChannels.\n+\t */\n+\tfrom = start_idx;\n+\twhile (vq->async->pkts_cmpl_flag[from] && count--) {\n+\t\tvq->async->pkts_cmpl_flag[from] = false;\n+\t\tfrom++;\n+\t\tif (from >= vq->size)\n+\t\t\tfrom -= vq->size;\n+\t\tnr_cpl_pkts++;\n+\t}\n+\n+\tif (nr_cpl_pkts == 0)\n+\t\treturn 0;\n+\n+\tfor (i = 0; i < nr_cpl_pkts; i++) {\n+\t\tfrom = (start_idx + i) % vq->size;\n+\t\t/* Only used with split ring */\n+\t\tn_descs += pkts_info[from].descs;\n+\t\tmr_n_descs += pkts_info[from].nr_buffers;\n+\t\tpkts[i] = pkts_info[from].mbuf;\n+\t}\n+\n+\tasync->pkts_inflight_n -= nr_cpl_pkts;\n+\n+\tif (likely(vq->enabled && vq->access_ok)) {\n+\t\twrite_back_completed_descs_split(vq, n_descs);\n+\t\t__atomic_add_fetch(&vq->used->idx, n_descs, __ATOMIC_RELEASE);\n+\t\tvhost_vring_call_split(dev, vq);\n+\t} else {\n+\t\tasync->last_desc_idx_split += n_descs;\n+\t}\n+\n+\tif (likely(mr_vq->enabled && mr_vq->access_ok)) {\n+\t\twrite_back_completed_descs_split(mr_vq, mr_n_descs);\n+\t\t__atomic_add_fetch(&mr_vq->used->idx, mr_n_descs, __ATOMIC_RELEASE);\n+\t\tvhost_vring_call_split(mr_dev, mr_vq);\n+\t} else {\n+\t\tmr_async->last_desc_idx_split += mr_n_descs;\n+\t}\n+\n+\treturn nr_cpl_pkts;\n+}\n+\n+uint16_t\n+rte_vhost_poll_ingress_completed(int vid, uint16_t queue_id, int mr_vid, uint16_t mr_queue_id,\n+\t\t\tstruct rte_mbuf **pkts, uint16_t count, int16_t dma_id, uint16_t vchan_id)\n+{\n+\tstruct virtio_net *dev, *mr_dev;\n+\tstruct vhost_virtqueue *vq, *mr_vq;\n+\tuint16_t n_pkts_cpl = 0;\n+\n+\tdev = get_device(vid);\n+\tif (unlikely(!dev))\n+\t\treturn 0;\n+\n+\tmr_dev = get_device(mr_vid);\n+\tif (unlikely(!mr_dev))\n+\t\treturn 0;\n+\n+\tVHOST_LOG_DATA(dev->ifname, DEBUG, \"%s\\n\", __func__);\n+\tif (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {\n+\t\tVHOST_LOG_DATA(dev->ifname, ERR,\n+\t\t\t\"%s: invalid virtqueue idx %d.\\n\",\n+\t\t\t__func__, queue_id);\n+\t\treturn 0;\n+\t}\n+\tif (unlikely(!is_valid_virt_queue_idx(mr_queue_id, 0, mr_dev->nr_vring))) {\n+\t\tVHOST_LOG_DATA(mr_dev->ifname, ERR,\n+\t\t\t\"%s: invalid virtqueue idx %d.\\n\",\n+\t\t\t__func__, mr_queue_id);\n+\t\treturn 0;\n+\t}\n+\n+\tif (unlikely(!dma_copy_track[dma_id].vchans ||\n+\t\t\t!dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr)) {\n+\t\tVHOST_LOG_DATA(dev->ifname, ERR,\n+\t\t\t\t\"%s: invalid channel %d:%u.\\n\",\n+\t\t\t\t__func__, dma_id, vchan_id);\n+\t\treturn 0;\n+\t}\n+\n+\tvq = dev->virtqueue[queue_id];\n+\tmr_vq = mr_dev->virtqueue[mr_queue_id];\n+\n+\tif (!rte_spinlock_trylock(&vq->access_lock)) {\n+\t\tVHOST_LOG_DATA(dev->ifname, DEBUG,\n+\t\t\t\t\"%s: virtqueue %u is busy.\\n\",\n+\t\t\t\t__func__, queue_id);\n+\t\treturn 0;\n+\t}\n+\n+\tif (!rte_spinlock_trylock(&mr_vq->access_lock)) {\n+\t\tVHOST_LOG_DATA(mr_dev->ifname, DEBUG,\n+\t\t\t\t\"%s: mirror virtqueue %u is busy.\\n\",\n+\t\t\t\t__func__, queue_id);\n+\t\tgoto out_org_access_unlock;\n+\t}\n+\n+\tif (unlikely(!vq->async)) {\n+\t\tVHOST_LOG_DATA(dev->ifname, ERR,\n+\t\t\t\t\"%s: async not registered for virtqueue %d.\\n\",\n+\t\t\t\t__func__, queue_id);\n+\t\tgoto out;\n+\t}\n+\n+\tif (unlikely(!mr_vq->async)) {\n+\t\tVHOST_LOG_DATA(mr_dev->ifname, ERR,\n+\t\t\t\t\"%s: async not registered for mirror virtqueue %d.\\n\",\n+\t\t\t\t__func__, queue_id);\n+\t\tgoto out;\n+\t}\n+\n+\tn_pkts_cpl = vhost_poll_ingress_completed(dev, vq, mr_dev, mr_vq,\n+\t\t\t\t\tpkts, count, dma_id, vchan_id);\n+\n+\tvhost_queue_stats_update(dev, vq, pkts, n_pkts_cpl);\n+\tvhost_queue_stats_update(mr_dev, mr_vq, pkts, n_pkts_cpl);\n+\tvq->stats.inflight_completed += n_pkts_cpl;\n+\n+out:\n+\trte_spinlock_unlock(&mr_vq->access_lock);\n+out_org_access_unlock:\n+\trte_spinlock_unlock(&vq->access_lock);\n+\n+\treturn n_pkts_cpl;\n+}\n+\n+static __rte_always_inline int\n+ingress_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,\n+\t\tstruct virtio_net *mr_dev, struct vhost_virtqueue *mr_vq,\n+\t\tstruct buf_vector *buf_vec, uint16_t nr_vec, uint16_t num_buffers,\n+\t\tstruct buf_vector *mr_buf_vec, uint16_t mr_nr_vec, uint16_t mr_num_buffers,\n+\t\tstruct rte_mbuf *m)\n+{\n+\tuint32_t vec_idx = 0;\n+\tuint32_t mr_vec_idx = 0;\n+\tuint32_t mbuf_offset, mbuf_avail;\n+\tuint32_t buf_offset, buf_avail, mr_buf_offset, mr_buf_avail;\n+\tuint64_t buf_addr, mr_buf_addr, buf_iova, mr_buf_iova;\n+\tuint32_t cpy_len, buf_len, mr_buf_len;\n+\tuint64_t hdr_addr, mr_hdr_addr;\n+\tstruct rte_mbuf *hdr_mbuf;\n+\tstruct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL;\n+\tstruct virtio_net_hdr_mrg_rxbuf mr_tmp_hdr, *mr_hdr = NULL;\n+\tstruct vhost_async *async = vq->async;\n+\n+\tif (unlikely(m == NULL))\n+\t\treturn -1;\n+\n+\t/* handle original port header */\n+\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\tbuf_len = buf_vec[vec_idx].buf_len;\n+\n+\tif (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1))\n+\t\treturn -1;\n+\n+\thdr_mbuf = m;\n+\thdr_addr = buf_addr;\n+\tif (unlikely(buf_len < dev->vhost_hlen)) {\n+\t\tmemset(&tmp_hdr, 0, sizeof(struct virtio_net_hdr_mrg_rxbuf));\n+\t\thdr = &tmp_hdr;\n+\t} else {\n+\t\thdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;\n+\t}\n+\n+\tVHOST_LOG_DATA(dev->ifname, DEBUG, \"RX: num merge buffers %d\\n\", num_buffers);\n+\n+\tif (unlikely(buf_len < dev->vhost_hlen)) {\n+\t\tbuf_offset = dev->vhost_hlen - buf_len;\n+\t\tvec_idx++;\n+\t\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\t\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\t\tbuf_len = buf_vec[vec_idx].buf_len;\n+\t\tbuf_avail = buf_len - buf_offset;\n+\t} else {\n+\t\tbuf_offset = dev->vhost_hlen;\n+\t\tbuf_avail = buf_len - dev->vhost_hlen;\n+\t}\n+\n+\t/* handle mirror port header */\n+\tmr_buf_addr = mr_buf_vec[mr_vec_idx].buf_addr;\n+\tmr_buf_iova = mr_buf_vec[mr_vec_idx].buf_iova;\n+\tmr_buf_len = mr_buf_vec[mr_vec_idx].buf_len;\n+\n+\tif (unlikely(mr_buf_len < mr_dev->vhost_hlen && mr_nr_vec <= 1))\n+\t\treturn -1;\n+\n+\tmr_hdr_addr = mr_buf_addr;\n+\tif (unlikely(mr_buf_len < mr_dev->vhost_hlen)) {\n+\t\tmemset(&mr_tmp_hdr, 0, sizeof(struct virtio_net_hdr_mrg_rxbuf));\n+\t\tmr_hdr = &mr_tmp_hdr;\n+\t} else {\n+\t\tmr_hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)mr_hdr_addr;\n+\t}\n+\n+\tif (unlikely(mr_buf_len < mr_dev->vhost_hlen)) {\n+\t\tmr_buf_offset = mr_dev->vhost_hlen - mr_buf_len;\n+\t\tmr_vec_idx++;\n+\t\tmr_buf_addr = mr_buf_vec[mr_vec_idx].buf_addr;\n+\t\tmr_buf_iova = mr_buf_vec[mr_vec_idx].buf_iova;\n+\t\tmr_buf_len = mr_buf_vec[mr_vec_idx].buf_len;\n+\t\tmr_buf_avail = mr_buf_len - mr_buf_offset;\n+\t} else {\n+\t\tmr_buf_offset = mr_dev->vhost_hlen;\n+\t\tmr_buf_avail = mr_buf_len - mr_dev->vhost_hlen;\n+\t}\n+\n+\t/* copy mbuf to desc */\n+\tmbuf_avail = rte_pktmbuf_data_len(m);\n+\tmbuf_offset = 0;\n+\n+\tif (async_iter_initialize(dev, async))\n+\t\treturn -1;\n+\n+\twhile (mbuf_avail != 0 || m->next != NULL) {\n+\t\t/* done with current buf, get the next one */\n+\t\tif (buf_avail == 0) {\n+\t\t\tvec_idx++;\n+\t\t\tif (unlikely(vec_idx >= nr_vec))\n+\t\t\t\tgoto error;\n+\n+\t\t\tbuf_addr = buf_vec[vec_idx].buf_addr;\n+\t\t\tbuf_iova = buf_vec[vec_idx].buf_iova;\n+\t\t\tbuf_len = buf_vec[vec_idx].buf_len;\n+\n+\t\t\tbuf_offset = 0;\n+\t\t\tbuf_avail = buf_len;\n+\t\t}\n+\n+\t\tif (mr_buf_avail == 0) {\n+\t\t\tmr_vec_idx++;\n+\t\t\tif (unlikely(mr_vec_idx >= mr_nr_vec))\n+\t\t\t\tgoto error;\n+\n+\t\t\tmr_buf_addr = mr_buf_vec[mr_vec_idx].buf_addr;\n+\t\t\tmr_buf_iova = mr_buf_vec[mr_vec_idx].buf_iova;\n+\t\t\tmr_buf_len = mr_buf_vec[mr_vec_idx].buf_len;\n+\n+\t\t\tmr_buf_offset = 0;\n+\t\t\tmr_buf_avail = mr_buf_len;\n+\t\t}\n+\n+\t\t/* done with current mbuf, get the next one */\n+\t\tif (mbuf_avail == 0) {\n+\t\t\tm = m->next;\n+\n+\t\t\tmbuf_offset = 0;\n+\t\t\tmbuf_avail = rte_pktmbuf_data_len(m);\n+\t\t}\n+\n+\t\tif (hdr_addr) {\n+\t\t\tvirtio_enqueue_offload(hdr_mbuf, &hdr->hdr);\n+\n+\t\t\tif (rxvq_is_mergeable(dev))\n+\t\t\t\tASSIGN_UNLESS_EQUAL(hdr->num_buffers,\n+\t\t\t\t\t\t\tnum_buffers);\n+\n+\t\t\tif (unlikely(hdr == &tmp_hdr)) {\n+\t\t\t\tcopy_vnet_hdr_to_desc(dev, vq, buf_vec, hdr);\n+\t\t\t} else {\n+\t\t\t\tPRINT_PACKET(dev, (uintptr_t)hdr_addr,\n+\t\t\t\t\t\tdev->vhost_hlen, 0);\n+\t\t\t\tvhost_log_cache_write_iova(dev, vq,\n+\t\t\t\t\t\t\t\tbuf_vec[0].buf_iova,\n+\t\t\t\t\t\t\t\tdev->vhost_hlen);\n+\t\t\t}\n+\n+\t\t\thdr_addr = 0;\n+\t\t}\n+\n+\t\tif (mr_hdr_addr) {\n+\t\t\trte_memcpy(&mr_hdr->hdr, &hdr->hdr, sizeof(struct virtio_net_hdr));\n+\t\t\tif (rxvq_is_mergeable(mr_dev))\n+\t\t\t\tASSIGN_UNLESS_EQUAL(mr_hdr->num_buffers,\n+\t\t\t\t\t\t\tmr_num_buffers);\n+\n+\t\t\tif (unlikely(mr_hdr == &mr_tmp_hdr)) {\n+\t\t\t\tcopy_vnet_hdr_to_desc(mr_dev, mr_vq, mr_buf_vec, mr_hdr);\n+\t\t\t} else {\n+\t\t\t\tPRINT_PACKET(mr_dev, (uintptr_t)mr_hdr_addr,\n+\t\t\t\t\t\tmr_dev->vhost_hlen, 0);\n+\t\t\t\tvhost_log_cache_write_iova(mr_dev, mr_vq,\n+\t\t\t\t\t\t\t\tmr_buf_vec[0].buf_iova,\n+\t\t\t\t\t\t\t\tmr_dev->vhost_hlen);\n+\t\t\t}\n+\n+\t\t\tmr_hdr_addr = 0;\n+\t\t}\n+\n+\t\tcpy_len = RTE_MIN(buf_avail, mbuf_avail);\n+\t\tcpy_len = RTE_MIN(mr_buf_avail, cpy_len);\n+\n+\t\tif (async_mirror_fill_seg(dev, vq, buf_iova + buf_offset,\n+\t\t\t\t\tmr_dev,\tmr_buf_iova + mr_buf_offset,\n+\t\t\t\t\tm, mbuf_offset, cpy_len, true) < 0)\n+\t\t\tgoto error;\n+\n+\t\tmbuf_avail -= cpy_len;\n+\t\tmbuf_offset += cpy_len;\n+\t\tbuf_avail -= cpy_len;\n+\t\tbuf_offset += cpy_len;\n+\t\tmr_buf_avail -= cpy_len;\n+\t\tmr_buf_offset += cpy_len;\n+\t}\n+\n+\tasync_iter_finalize(async);\n+\n+\treturn 0;\n+error:\n+\tasync_iter_cancel(async);\n+\n+\treturn -1;\n+}\n+\n+static __rte_noinline uint32_t\n+virtio_dev_ingress_async_submit_split(struct virtio_net *dev, struct vhost_virtqueue *vq,\n+\t\t\tstruct virtio_net *mr_dev, struct vhost_virtqueue *mr_vq,\n+\t\t\tstruct rte_mbuf **pkts, uint32_t count, int16_t dma_id, uint16_t vchan_id)\n+{\n+\tstruct buf_vector buf_vec[BUF_VECTOR_MAX];\n+\tstruct buf_vector mr_buf_vec[BUF_VECTOR_MAX];\n+\tuint32_t pkt_idx = 0;\n+\tuint16_t num_buffers;\n+\tuint16_t mr_num_buffers;\n+\tuint16_t avail_head;\n+\tuint16_t mr_avail_head;\n+\n+\tstruct vhost_async *async = vq->async;\n+\tstruct vhost_async *mr_async = mr_vq->async;\n+\tstruct async_inflight_info *pkts_info = async->pkts_info;\n+\tuint32_t pkt_err = 0;\n+\tint32_t n_xfer;\n+\tuint16_t slot_idx = 0;\n+\tuint16_t nr_vec, mr_nr_vec;\n+\n+\t/* The ordering between avail index and desc reads need to be enforced. */\n+\tavail_head = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE);\n+\tmr_avail_head = __atomic_load_n(&mr_vq->avail->idx, __ATOMIC_ACQUIRE);\n+\n+\trte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);\n+\trte_prefetch0(&mr_vq->avail->ring[mr_vq->last_avail_idx & (mr_vq->size - 1)]);\n+\n+\tasync_iter_reset(async);\n+\n+\tfor (pkt_idx = 0; pkt_idx < count; pkt_idx++) {\n+\t\tuint16_t vhost_hlen = dev->vhost_hlen > mr_dev->vhost_hlen ?\n+\t\t\t\t\tdev->vhost_hlen : mr_dev->vhost_hlen;\n+\t\tuint32_t pkt_len = pkts[pkt_idx]->pkt_len + vhost_hlen;\n+\n+\n+\t\tif (unlikely(reserve_avail_buf_split(dev, vq, pkt_len, buf_vec,\n+\t\t\t\t\t&num_buffers, avail_head, &nr_vec) < 0)) {\n+\t\t\tVHOST_LOG_DATA(dev->ifname, DEBUG,\n+\t\t\t\t\t\"failed to get enough desc from vring\\n\");\n+\t\t\tvq->shadow_used_idx -= num_buffers;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tif (unlikely(reserve_avail_buf_split(mr_dev, mr_vq, pkt_len, mr_buf_vec,\n+\t\t\t\t\t&mr_num_buffers, mr_avail_head, &mr_nr_vec) < 0)) {\n+\t\t\tVHOST_LOG_DATA(mr_dev->ifname, DEBUG,\n+\t\t\t\t\t\"failed to get enough desc from mirror vring\\n\");\n+\t\t\tvq->shadow_used_idx -= num_buffers;\n+\t\t\tmr_vq->shadow_used_idx -= mr_num_buffers;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tVHOST_LOG_DATA(dev->ifname, DEBUG,\n+\t\t\t\t\"current index %d | end index %d\\n\",\n+\t\t\t\tvq->last_avail_idx, vq->last_avail_idx + num_buffers);\n+\n+\t\tif (ingress_mbuf_to_desc(dev, vq, mr_dev, mr_vq,\n+\t\t\t\t\tbuf_vec, nr_vec, num_buffers,\n+\t\t\t\t\tmr_buf_vec, mr_nr_vec, mr_num_buffers,\n+\t\t\t\t\tpkts[pkt_idx]) < 0) {\n+\t\t\tvq->shadow_used_idx -= num_buffers;\n+\t\t\tmr_vq->shadow_used_idx -= mr_num_buffers;\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tslot_idx = (async->pkts_idx + pkt_idx) & (vq->size - 1);\n+\t\tpkts_info[slot_idx].descs = num_buffers;\n+\t\tpkts_info[slot_idx].nr_buffers = mr_num_buffers;\n+\t\tpkts_info[slot_idx].mbuf = pkts[pkt_idx];\n+\n+\t\tvq->last_avail_idx += num_buffers;\n+\t\tmr_vq->last_avail_idx += mr_num_buffers;\n+\n+\t}\n+\n+\tif (unlikely(pkt_idx == 0))\n+\t\treturn 0;\n+\n+\tn_xfer = vhost_async_dma_transfer(dev, vq, dma_id, vchan_id, async->pkts_idx,\n+\t\t\tasync->iov_iter, pkt_idx);\n+\n+\tpkt_err = pkt_idx - n_xfer;\n+\tif (unlikely(pkt_err)) {\n+\t\tuint16_t num_descs = 0;\n+\t\tuint16_t mr_num_descs = 0;\n+\n+\t\tVHOST_LOG_DATA(dev->ifname, DEBUG,\n+\t\t\t\"%s: failed to transfer %u packets for queue %u.\\n\",\n+\t\t\t__func__, pkt_err, vq->index);\n+\n+\t\t/* update number of completed packets */\n+\t\tpkt_idx = n_xfer;\n+\n+\t\t/* calculate the sum of descriptors to revert */\n+\t\twhile (pkt_err-- > 0) {\n+\t\t\tnum_descs += pkts_info[slot_idx & (vq->size - 1)].descs;\n+\t\t\tmr_num_descs += pkts_info[slot_idx & (vq->size - 1)].nr_buffers;\n+\t\t\tslot_idx--;\n+\t\t}\n+\n+\t\t/* recover shadow used ring and available ring */\n+\t\tvq->shadow_used_idx -= num_descs;\n+\t\tvq->last_avail_idx -= num_descs;\n+\t\tmr_vq->shadow_used_idx -= mr_num_descs;\n+\t\tmr_vq->last_avail_idx -= mr_num_descs;\n+\t}\n+\n+\t/* keep used descriptors */\n+\tif (likely(vq->shadow_used_idx)) {\n+\t\tuint16_t to = async->desc_idx_split & (vq->size - 1);\n+\n+\t\tstore_dma_desc_info_split(vq->shadow_used_split,\n+\t\t\t\tasync->descs_split, vq->size, 0, to,\n+\t\t\t\tvq->shadow_used_idx);\n+\n+\t\tasync->desc_idx_split += vq->shadow_used_idx;\n+\t\tasync->pkts_idx += pkt_idx;\n+\t\tif (async->pkts_idx >= vq->size)\n+\t\t\tasync->pkts_idx -= vq->size;\n+\n+\t\tasync->pkts_inflight_n += pkt_idx;\n+\t\tvq->shadow_used_idx = 0;\n+\t}\n+\n+\tif (likely(mr_vq->shadow_used_idx)) {\n+\t\tuint16_t to = mr_async->desc_idx_split & (mr_vq->size - 1);\n+\n+\t\tstore_dma_desc_info_split(mr_vq->shadow_used_split, mr_async->descs_split,\n+\t\t\t\t\tmr_vq->size, 0, to, mr_vq->shadow_used_idx);\n+\n+\t\tmr_async->desc_idx_split += mr_vq->shadow_used_idx;\n+\t\tmr_vq->shadow_used_idx = 0;\n+\t}\n+\n+\treturn pkt_idx;\n+}\n+\n+static __rte_always_inline uint32_t\n+virtio_dev_ingress_async_submit(struct virtio_net *dev, struct vhost_virtqueue *vq,\n+\t\t\tstruct virtio_net *mr_dev, struct vhost_virtqueue *mr_vq,\n+\t\t\tstruct rte_mbuf **pkts, uint32_t count,\n+\t\t\tint16_t dma_id, uint16_t vchan_id)\n+{\n+\tuint32_t nb_tx = 0;\n+\n+\tVHOST_LOG_DATA(dev->ifname, DEBUG, \"%s\\n\", __func__);\n+\n+\tif (unlikely(!dma_copy_track[dma_id].vchans ||\n+\t\t\t\t!dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr)) {\n+\t\tVHOST_LOG_DATA(dev->ifname, ERR,\n+\t\t\t\"%s: invalid channel %d:%u.\\n\",\n+\t\t\t __func__, dma_id, vchan_id);\n+\t\treturn 0;\n+\t}\n+\n+\trte_spinlock_lock(&vq->access_lock);\n+\trte_spinlock_lock(&mr_vq->access_lock);\n+\n+\tif (unlikely(!vq->enabled || !vq->async ||\n+\t\t!mr_vq->enabled || !mr_vq->async))\n+\t\tgoto out_access_unlock;\n+\n+\tif (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))\n+\t\tvhost_user_iotlb_rd_lock(vq);\n+\tif (mr_dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))\n+\t\tvhost_user_iotlb_rd_lock(mr_vq);\n+\n+\tif (unlikely(!vq->access_ok))\n+\t\tif (unlikely(vring_translate(dev, vq) < 0))\n+\t\t\tgoto out;\n+\tif (unlikely(!mr_vq->access_ok))\n+\t\tif (unlikely(vring_translate(mr_dev, mr_vq) < 0))\n+\t\t\tgoto out;\n+\n+\tcount = RTE_MIN((uint32_t)MAX_PKT_BURST, count);\n+\tif (count == 0)\n+\t\tgoto out;\n+\n+\tnb_tx = virtio_dev_ingress_async_submit_split(dev, vq,\n+\t\t\t\t\t\tmr_dev, mr_vq,\n+\t\t\t\t\t\tpkts, count, dma_id, vchan_id);\n+\n+out:\n+\tif (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))\n+\t\tvhost_user_iotlb_rd_unlock(vq);\n+\tif (mr_dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))\n+\t\tvhost_user_iotlb_rd_unlock(mr_vq);\n+\n+out_access_unlock:\n+\trte_spinlock_unlock(&mr_vq->access_lock);\n+\trte_spinlock_unlock(&vq->access_lock);\n+\n+\treturn nb_tx;\n+}\n+\n+uint16_t\n+rte_vhost_submit_ingress_mirroring_burst(int vid, uint16_t queue_id,\n+\t\t\tint mr_vid, uint16_t mr_queue_id,\n+\t\t\tstruct rte_mbuf **pkts, uint16_t count,\n+\t\t\tint16_t dma_id, uint16_t vchan_id)\n+{\n+\tstruct virtio_net *dev = get_device(vid);\n+\tstruct virtio_net *mr_dev = get_device(mr_vid);\n+\n+\tif (!dev || !mr_dev)\n+\t\treturn 0;\n+\n+\tif (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {\n+\t\tVHOST_LOG_DATA(dev->ifname, ERR,\n+\t\t\t\"%s: built-in vhost net backend is disabled.\\n\",\n+\t\t\t__func__);\n+\t\treturn 0;\n+\t}\n+\tif (unlikely(!(mr_dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {\n+\t\tVHOST_LOG_DATA(mr_dev->ifname, ERR,\n+\t\t\t\"%s: built-in vhost net backend is disabled.\\n\",\n+\t\t\t__func__);\n+\t\treturn 0;\n+\t}\n+\n+\tif (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {\n+\t\tVHOST_LOG_DATA(dev->ifname, ERR,\n+\t\t\t\"%s: invalid virtqueue idx %d.\\n\",\n+\t\t\t__func__, queue_id);\n+\t\treturn 0;\n+\t}\n+\tif (unlikely(!is_valid_virt_queue_idx(mr_queue_id, 0, mr_dev->nr_vring))) {\n+\t\tVHOST_LOG_DATA(mr_dev->ifname, ERR,\n+\t\t\t\"%s: invalid virtqueue idx %d.\\n\",\n+\t\t\t__func__, mr_queue_id);\n+\t\treturn 0;\n+\t}\n+\n+\treturn virtio_dev_ingress_async_submit(dev, dev->virtqueue[queue_id],\n+\t\t\t\tmr_dev, mr_dev->virtqueue[mr_queue_id],\n+\t\t\t\tpkts, count, dma_id, vchan_id);\n+}\n \n static __rte_always_inline uint16_t\n async_poll_egress_completed_split(struct virtio_net *dev, struct vhost_virtqueue *vq,\n@@ -4313,56 +4941,6 @@ async_poll_egress_completed_split(struct virtio_net *dev, struct vhost_virtqueue\n \treturn nr_cpl_pkts;\n }\n \n-static __rte_always_inline int\n-egress_async_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, uint64_t buf_iova,\n-\t\tstruct virtio_net *mr_dev, uint64_t mr_buf_iova,\n-\t\tstruct rte_mbuf *m, uint32_t mbuf_offset, uint32_t cpy_len)\n-{\n-\tstruct vhost_async *async = vq->async;\n-\tuint64_t mapped_len, mr_mapped_len;\n-\tuint32_t buf_offset = 0;\n-\tvoid *src, *dst, *mr_dst;\n-\tvoid *host_iova, *mr_host_iova;\n-\n-\twhile (cpy_len) {\n-\t\thost_iova = (void *)(uintptr_t)gpa_to_first_hpa(dev,\n-\t\t\t\t\tbuf_iova + buf_offset, cpy_len,\t&mapped_len);\n-\t\tif (unlikely(!host_iova)) {\n-\t\t\tVHOST_LOG_DATA(dev->ifname, ERR, \"%s: failed to get host iova.\\n\", __func__);\n-\t\t\treturn -1;\n-\t\t}\n-\n-\t\tmr_host_iova = (void *)(uintptr_t)gpa_to_first_hpa(mr_dev,\n-\t\t\t\t\tmr_buf_iova + buf_offset, cpy_len, &mr_mapped_len);\n-\t\tif (unlikely(!mr_host_iova)) {\n-\t\t\tVHOST_LOG_DATA(mr_dev->ifname, ERR, \"%s: failed to get mirror hpa.\\n\", __func__);\n-\t\t\treturn -1;\n-\t\t}\n-\n-\t\tif (unlikely(mr_mapped_len != mapped_len)) {\n-\t\t\tVHOST_LOG_DATA(dev->ifname, ERR, \"original mapped len is not equal to mirror len\\n\");\n-\t\t\treturn -1;\n-\t\t}\n-\n-\t\tsrc = host_iova;\n-\t\tdst = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);\n-\n-\t\tmr_dst = mr_host_iova;\n-\n-\t\tif (unlikely(async_iter_add_iovec(dev, async, src, dst, (size_t)mapped_len)))\n-\t\t\treturn -1;\n-\n-\t\tif (unlikely(async_iter_add_iovec(dev, async, src, mr_dst, (size_t)mapped_len)))\n-\t\t\treturn -1;\n-\n-\t\tcpy_len -= (uint32_t)mapped_len;\n-\t\tmbuf_offset += (uint32_t)mapped_len;\n-\t\tbuf_offset += (uint32_t)mapped_len;\n-\t}\n-\n-\treturn 0;\n-}\n-\n static __rte_always_inline int\n egress_async_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\t  struct buf_vector *buf_vec, uint16_t nr_vec,\n@@ -4477,9 +5055,9 @@ egress_async_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,\n \t\tcpy_len = RTE_MIN(buf_avail, mbuf_avail);\n \t\tcpy_len = RTE_MIN(cpy_len, mr_buf_avail);\n \n-\t\tif (egress_async_fill_seg(dev, vq, buf_iova + buf_offset,\n+\t\tif (async_mirror_fill_seg(dev, vq, buf_iova + buf_offset,\n \t\t\t\tmr_dev, mr_buf_iova + mr_buf_offset,\n-\t\t\t\tm, mbuf_offset, cpy_len) < 0)\n+\t\t\t\tm, mbuf_offset, cpy_len, false) < 0)\n \t\t\tgoto error;\n \n \t\tmbuf_avail -= cpy_len;\n",
    "prefixes": [
        "2/2"
    ]
}