Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/66647/?format=api
{ "id": 66647, "url": "https://patches.dpdk.org/api/patches/66647/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/20200313174230.74661-6-yong.liu@intel.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20200313174230.74661-6-yong.liu@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20200313174230.74661-6-yong.liu@intel.com", "date": "2020-03-13T17:42:28", "name": "[v1,5/7] net/virtio: add vectorized packed ring Tx function", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "59b1a448f84e8bfddf8b0b81e856b419bccc2b30", "submitter": { "id": 17, "url": "https://patches.dpdk.org/api/people/17/?format=api", "name": "Marvin Liu", "email": "yong.liu@intel.com" }, "delegate": { "id": 2642, "url": "https://patches.dpdk.org/api/users/2642/?format=api", "username": "mcoquelin", "first_name": "Maxime", "last_name": "Coquelin", "email": "maxime.coquelin@redhat.com" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/20200313174230.74661-6-yong.liu@intel.com/mbox/", "series": [ { "id": 8904, "url": "https://patches.dpdk.org/api/series/8904/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=8904", "date": "2020-03-13T17:42:23", "name": "vectorize virtio packed ring datapath", "version": 1, "mbox": "https://patches.dpdk.org/series/8904/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/66647/comments/", "check": "fail", "checks": "https://patches.dpdk.org/api/patches/66647/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id DA44FA0567;\n\tFri, 13 Mar 2020 11:08:11 +0100 (CET)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 35A0C1C0B3;\n\tFri, 13 Mar 2020 11:07:35 +0100 (CET)", "from mga02.intel.com (mga02.intel.com [134.134.136.20])\n by dpdk.org (Postfix) with ESMTP id 257E71C07E\n for <dev@dpdk.org>; Fri, 13 Mar 2020 11:07:33 +0100 (CET)", "from orsmga006.jf.intel.com ([10.7.209.51])\n by orsmga101.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n 13 Mar 2020 03:07:32 -0700", "from npg-dpdk-virtual-marvin-dev.sh.intel.com ([10.67.119.58])\n by orsmga006.jf.intel.com with ESMTP; 13 Mar 2020 03:07:31 -0700" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.70,548,1574150400\"; d=\"scan'208\";a=\"246653919\"", "From": "Marvin Liu <yong.liu@intel.com>", "To": "maxime.coquelin@redhat.com, xiaolong.ye@intel.com, zhihong.wang@intel.com", "Cc": "dev@dpdk.org,\n\tMarvin Liu <yong.liu@intel.com>", "Date": "Sat, 14 Mar 2020 01:42:28 +0800", "Message-Id": "<20200313174230.74661-6-yong.liu@intel.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20200313174230.74661-1-yong.liu@intel.com>", "References": "<20200313174230.74661-1-yong.liu@intel.com>", "Subject": "[dpdk-dev] [PATCH v1 5/7] net/virtio: add vectorized packed ring Tx\n\tfunction", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Optimize packed ring Tx datapath alike Rx datapath. Split Rx datapath\ninto batch and single Tx functions.\n\nSigned-off-by: Marvin Liu <yong.liu@intel.com>", "diff": "diff --git a/drivers/net/virtio/virtio_ethdev.h b/drivers/net/virtio/virtio_ethdev.h\nindex 10e39670e..c9aaef0af 100644\n--- a/drivers/net/virtio/virtio_ethdev.h\n+++ b/drivers/net/virtio/virtio_ethdev.h\n@@ -107,6 +107,9 @@ uint16_t virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,\n uint16_t virtio_recv_pkts_packed_vec(void *rx_queue, struct rte_mbuf **rx_pkts,\n \t\tuint16_t nb_pkts);\n \n+uint16_t virtio_xmit_pkts_packed_vec(void *tx_queue, struct rte_mbuf **tx_pkts,\n+\t\tuint16_t nb_pkts);\n+\n int eth_virtio_dev_init(struct rte_eth_dev *eth_dev);\n \n void virtio_interrupt_handler(void *param);\ndiff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c\nindex b8b4d3c25..125df3a13 100644\n--- a/drivers/net/virtio/virtio_rxtx.c\n+++ b/drivers/net/virtio/virtio_rxtx.c\n@@ -2174,3 +2174,11 @@ virtio_recv_pkts_packed_vec(void __rte_unused *rx_queue,\n {\n \treturn 0;\n }\n+\n+__rte_weak uint16_t\n+virtio_xmit_pkts_packed_vec(void __rte_unused *tx_queue,\n+\t\t\t struct rte_mbuf __rte_unused **tx_pkts,\n+\t\t\t uint16_t __rte_unused nb_pkts)\n+{\n+\treturn 0;\n+}\ndiff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.c b/drivers/net/virtio/virtio_rxtx_packed_avx.c\nindex d8cda9d71..0872f2083 100644\n--- a/drivers/net/virtio/virtio_rxtx_packed_avx.c\n+++ b/drivers/net/virtio/virtio_rxtx_packed_avx.c\n@@ -15,6 +15,11 @@\n #include \"virtio_pci.h\"\n #include \"virtqueue.h\"\n \n+#define REF_CNT_OFFSET 16\n+#define SEG_NUM_OFFSET 32\n+#define BATCH_REARM_DATA (1ULL << SEG_NUM_OFFSET | \\\n+\t\t\t 1ULL << REF_CNT_OFFSET | \\\n+\t\t\t RTE_PKTMBUF_HEADROOM)\n #define PACKED_FLAGS_MASK (1ULL << 55 | 1ULL << 63)\n \n #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \\\n@@ -41,6 +46,48 @@\n \tfor (iter = val; iter < num; iter++)\n #endif\n \n+static void\n+virtio_xmit_cleanup_packed_vec(struct virtqueue *vq)\n+{\n+\tstruct vring_packed_desc *desc = vq->vq_packed.ring.desc;\n+\tstruct vq_desc_extra *dxp;\n+\tuint16_t used_idx, id, curr_id, free_cnt = 0;\n+\tuint16_t size = vq->vq_nentries;\n+\tstruct rte_mbuf *mbufs[size];\n+\tuint16_t nb_mbuf = 0, i;\n+\n+\tused_idx = vq->vq_used_cons_idx;\n+\n+\tif (desc_is_used(&desc[used_idx], vq))\n+\t\tid = desc[used_idx].id;\n+\telse\n+\t\treturn;\n+\n+\tdo {\n+\t\tcurr_id = used_idx;\n+\t\tdxp = &vq->vq_descx[used_idx];\n+\t\tused_idx += dxp->ndescs;\n+\t\tfree_cnt += dxp->ndescs;\n+\n+\t\tif (dxp->cookie != NULL) {\n+\t\t\tmbufs[nb_mbuf] = dxp->cookie;\n+\t\t\tdxp->cookie = NULL;\n+\t\t\tnb_mbuf++;\n+\t\t}\n+\n+\t\tif (used_idx >= size) {\n+\t\t\tused_idx -= size;\n+\t\t\tvq->vq_packed.used_wrap_counter ^= 1;\n+\t\t}\n+\t} while (curr_id != id);\n+\n+\tfor (i = 0; i < nb_mbuf; i++)\n+\t\trte_pktmbuf_free(mbufs[i]);\n+\n+\tvq->vq_used_cons_idx = used_idx;\n+\tvq->vq_free_cnt += free_cnt;\n+}\n+\n static inline void\n virtio_update_batch_stats(struct virtnet_stats *stats,\n \t\t\t uint16_t pkt_len1,\n@@ -54,6 +101,185 @@ virtio_update_batch_stats(struct virtnet_stats *stats,\n \tstats->bytes += pkt_len4;\n }\n \n+static inline int\n+virtqueue_enqueue_batch_packed_vec(struct virtnet_tx *txvq,\n+\t\t\t\t struct rte_mbuf **tx_pkts)\n+{\n+\tstruct virtqueue *vq = txvq->vq;\n+\tuint16_t head_size = vq->hw->vtnet_hdr_size;\n+\tstruct vq_desc_extra *dxps[PACKED_BATCH_SIZE];\n+\tuint16_t idx = vq->vq_avail_idx;\n+\tuint64_t descs[PACKED_BATCH_SIZE];\n+\tstruct virtio_net_hdr *hdrs[PACKED_BATCH_SIZE];\n+\tuint16_t i;\n+\n+\tif (vq->vq_avail_idx & PACKED_BATCH_MASK)\n+\t\treturn -1;\n+\n+\t/* Load four mbufs rearm data */\n+\t__m256i mbufs = _mm256_set_epi64x(\n+\t\t\t*tx_pkts[3]->rearm_data,\n+\t\t\t*tx_pkts[2]->rearm_data,\n+\t\t\t*tx_pkts[1]->rearm_data,\n+\t\t\t*tx_pkts[0]->rearm_data);\n+\n+\t/* hdr_room=128, refcnt=1 and nb_segs=1 */\n+\t__m256i mbuf_ref = _mm256_set_epi64x(\n+\t\t\tBATCH_REARM_DATA, BATCH_REARM_DATA,\n+\t\t\tBATCH_REARM_DATA, BATCH_REARM_DATA);\n+\n+\t/* Check hdr_room,refcnt and nb_segs */\n+\tuint16_t cmp = _mm256_cmpneq_epu16_mask(mbufs, mbuf_ref);\n+\tif (cmp & 0x7777)\n+\t\treturn -1;\n+\n+\tvirtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {\n+\t\tdxps[i] = &vq->vq_descx[idx + i];\n+\t\tdxps[i]->ndescs = 1;\n+\t\tdxps[i]->cookie = tx_pkts[i];\n+\t}\n+\n+\tvirtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {\n+\t\trte_pktmbuf_prepend(tx_pkts[i], head_size);\n+\t\ttx_pkts[i]->pkt_len -= head_size;\n+\t}\n+\n+\tvirtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)\n+\t\tdescs[i] = (uint64_t)tx_pkts[i]->data_len |\n+\t\t(uint64_t)(idx + i) << 32 |\n+\t\t(uint64_t)vq->vq_packed.cached_flags << 48;\n+\n+\t__m512i new_descs = _mm512_set_epi64(\n+\t\t\tdescs[3], VIRTIO_MBUF_DATA_DMA_ADDR(tx_pkts[3], vq),\n+\t\t\tdescs[2], VIRTIO_MBUF_DATA_DMA_ADDR(tx_pkts[2], vq),\n+\t\t\tdescs[1], VIRTIO_MBUF_DATA_DMA_ADDR(tx_pkts[1], vq),\n+\t\t\tdescs[0], VIRTIO_MBUF_DATA_DMA_ADDR(tx_pkts[0], vq));\n+\n+\tvirtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)\n+\t\thdrs[i] = rte_pktmbuf_mtod_offset(tx_pkts[i],\n+\t\t\t\tstruct virtio_net_hdr *, -head_size);\n+\n+\tif (!vq->hw->has_tx_offload) {\n+\t\tvirtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)\n+\t\t\tvirtqueue_clear_net_hdr(hdrs[i]);\n+\t} else {\n+\t\tvirtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)\n+\t\t\tvirtqueue_xmit_offload(hdrs[i], tx_pkts[i], true);\n+\t}\n+\n+\t/* Enqueue Packet buffers */\n+\trte_smp_wmb();\n+\t_mm512_storeu_si512((void *)&vq->vq_packed.ring.desc[idx], new_descs);\n+\n+\tvirtio_update_batch_stats(&txvq->stats, tx_pkts[0]->pkt_len,\n+\t\t\ttx_pkts[1]->pkt_len, tx_pkts[2]->pkt_len,\n+\t\t\ttx_pkts[3]->pkt_len);\n+\n+\tvq->vq_avail_idx += PACKED_BATCH_SIZE;\n+\tvq->vq_free_cnt -= PACKED_BATCH_SIZE;\n+\n+\tif (vq->vq_avail_idx >= vq->vq_nentries) {\n+\t\tvq->vq_avail_idx -= vq->vq_nentries;\n+\t\tvq->vq_packed.cached_flags ^=\n+\t\t\tVRING_PACKED_DESC_F_AVAIL_USED;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static inline int\n+virtqueue_enqueue_single_packed_vec(struct virtnet_tx *txvq,\n+\t\t\t\t struct rte_mbuf *txm)\n+{\n+\tstruct virtqueue *vq = txvq->vq;\n+\tstruct virtio_hw *hw = vq->hw;\n+\tuint16_t hdr_size = hw->vtnet_hdr_size;\n+\tuint16_t slots, can_push;\n+\tint16_t need;\n+\n+\t/* How many main ring entries are needed to this Tx?\n+\t * any_layout => number of segments\n+\t * default => number of segments + 1\n+\t */\n+\tcan_push = rte_mbuf_refcnt_read(txm) == 1 &&\n+\t\t RTE_MBUF_DIRECT(txm) &&\n+\t\t txm->nb_segs == 1 &&\n+\t\t rte_pktmbuf_headroom(txm) >= hdr_size;\n+\n+\tslots = txm->nb_segs + !can_push;\n+\tneed = slots - vq->vq_free_cnt;\n+\n+\t/* Positive value indicates it need free vring descriptors */\n+\tif (unlikely(need > 0)) {\n+\t\tvirtio_xmit_cleanup_packed_vec(vq);\n+\t\tneed = slots - vq->vq_free_cnt;\n+\t\tif (unlikely(need > 0)) {\n+\t\t\tPMD_TX_LOG(ERR,\n+\t\t\t\t \"No free tx descriptors to transmit\");\n+\t\t\treturn -1;\n+\t\t}\n+\t}\n+\n+\t/* Enqueue Packet buffers */\n+\tvirtqueue_enqueue_xmit_packed(txvq, txm, slots, can_push, 1);\n+\n+\ttxvq->stats.bytes += txm->pkt_len;\n+\treturn 0;\n+}\n+\n+uint16_t\n+virtio_xmit_pkts_packed_vec(void *tx_queue, struct rte_mbuf **tx_pkts,\n+\t\t\tuint16_t nb_pkts)\n+{\n+\tstruct virtnet_tx *txvq = tx_queue;\n+\tstruct virtqueue *vq = txvq->vq;\n+\tstruct virtio_hw *hw = vq->hw;\n+\tuint16_t nb_tx = 0;\n+\tuint16_t remained;\n+\n+\tif (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))\n+\t\treturn nb_tx;\n+\n+\tif (unlikely(nb_pkts < 1))\n+\t\treturn nb_pkts;\n+\n+\tPMD_TX_LOG(DEBUG, \"%d packets to xmit\", nb_pkts);\n+\n+\tif (vq->vq_free_cnt <= vq->vq_nentries - vq->vq_free_thresh)\n+\t\tvirtio_xmit_cleanup_packed_vec(vq);\n+\n+\tremained = RTE_MIN(nb_pkts, vq->vq_free_cnt);\n+\n+\twhile (remained) {\n+\t\tif (remained >= PACKED_BATCH_SIZE) {\n+\t\t\tif (!virtqueue_enqueue_batch_packed_vec(txvq,\n+\t\t\t\t\t\t&tx_pkts[nb_tx])) {\n+\t\t\t\tnb_tx += PACKED_BATCH_SIZE;\n+\t\t\t\tremained -= PACKED_BATCH_SIZE;\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t}\n+\t\tif (!virtqueue_enqueue_single_packed_vec(txvq,\n+\t\t\t\t\ttx_pkts[nb_tx])) {\n+\t\t\tnb_tx++;\n+\t\t\tremained--;\n+\t\t\tcontinue;\n+\t\t}\n+\t\tbreak;\n+\t};\n+\n+\ttxvq->stats.packets += nb_tx;\n+\n+\tif (likely(nb_tx)) {\n+\t\tif (unlikely(virtqueue_kick_prepare_packed(vq))) {\n+\t\t\tvirtqueue_notify(vq);\n+\t\t\tPMD_TX_LOG(DEBUG, \"Notified backend after xmit\");\n+\t\t}\n+\t}\n+\n+\treturn nb_tx;\n+}\n+\n /* Optionally fill offload information in structure */\n static inline int\n virtio_vec_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr)\n", "prefixes": [ "v1", "5/7" ] }