Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/30282/?format=api
{ "id": 30282, "url": "http://patches.dpdk.org/api/patches/30282/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/5900a2c9893c879debccf5c14cbceda952332858.1507810956.git.adrien.mazarguil@6wind.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<5900a2c9893c879debccf5c14cbceda952332858.1507810956.git.adrien.mazarguil@6wind.com>", "list_archive_url": "https://inbox.dpdk.org/dev/5900a2c9893c879debccf5c14cbceda952332858.1507810956.git.adrien.mazarguil@6wind.com", "date": "2017-10-12T12:29:57", "name": "[dpdk-dev,v6,2/5] net/mlx4: add Rx bypassing Verbs", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "0fa850446f615e088bf5a7dc7434065421443c3d", "submitter": { "id": 165, "url": "http://patches.dpdk.org/api/people/165/?format=api", "name": "Adrien Mazarguil", "email": "adrien.mazarguil@6wind.com" }, "delegate": { "id": 319, "url": "http://patches.dpdk.org/api/users/319/?format=api", "username": "fyigit", "first_name": "Ferruh", "last_name": "Yigit", "email": "ferruh.yigit@amd.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/5900a2c9893c879debccf5c14cbceda952332858.1507810956.git.adrien.mazarguil@6wind.com/mbox/", "series": [], "comments": "http://patches.dpdk.org/api/patches/30282/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/30282/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 82D601B2CE;\n\tThu, 12 Oct 2017 14:30:37 +0200 (CEST)", "from mail-wm0-f42.google.com (mail-wm0-f42.google.com\n\t[74.125.82.42]) by dpdk.org (Postfix) with ESMTP id 684651B2CB\n\tfor <dev@dpdk.org>; Thu, 12 Oct 2017 14:30:32 +0200 (CEST)", "by mail-wm0-f42.google.com with SMTP id i124so12644774wmf.3\n\tfor <dev@dpdk.org>; Thu, 12 Oct 2017 05:30:32 -0700 (PDT)", "from 6wind.com (host.78.145.23.62.rev.coltfrance.com.\n\t[62.23.145.78]) by smtp.gmail.com with ESMTPSA id\n\te8sm108985wmc.46.2017.10.12.05.30.30\n\t(version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);\n\tThu, 12 Oct 2017 05:30:30 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=6wind-com.20150623.gappssmtp.com; s=20150623;\n\th=from:to:cc:subject:date:message-id:in-reply-to:references;\n\tbh=rPyXIXeSk4I6hQJ1CKrHGGy873h8mx91GQ4ExiGpELU=;\n\tb=jpQuQMXENrILpDyaAQx3jQ+ja7UfadsqsTTKMEUphPR6/gxc2hfkjGaHX5dP9a7N7V\n\tp+ZvADPUTl8te6wSGJTK3i2Ki/bqCGroaLGzSxo3rkfgzE6Ixcl+TT/TcrGQqDvgOXQ3\n\thu9pO8d+rWRQWUovBPjjuQ/AJwJPyTATOkRnSPN97R5qITNc/uPaKHZOWAOXIF9m1rDm\n\tZldpZN7VdiIkZDAWH22pmxS+v9BDWu7d02Q2WYqkbq0SUpiKohkbn3VljnBUdHOFFbhk\n\tYe0+2Hms096pyFGbVf9iPfCltx76pMbhhj4SDks44KjY/WbFRn/YbNKIreWMaBe6TlC9\n\toSOA==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to\n\t:references;\n\tbh=rPyXIXeSk4I6hQJ1CKrHGGy873h8mx91GQ4ExiGpELU=;\n\tb=ggjcBlWmCuCQD4iJh/GSVBNUKq5yd16VRaDnssPLVJrYdxQhGv2qTKX09YU5x5z88w\n\tRtOznUaia2BG6dp8odSSmU9F0FaBY1o8hHBKaWdo4VhyOlEFDfPT4O8lzpXYhZsQnSzu\n\tmibfb8uApnN1NdNyyO4V7B/0WODWjypSewewyCUfuqBpPcvm4QuQlTe/VaqIi7/Y5hsL\n\tmY+geqKA8QVGruFQA31avp92E2qQ/SzTlS9OPFACukx7nFchkU/VDVsMbnZkzoSA8Acq\n\t+8IyWlEfDoF+mv8NanaoOUSmMbgchKQYiECLtoaSnSK3GuTloaAeQkaKcb9RVJe+FgC9\n\tR/eA==", "X-Gm-Message-State": "AMCzsaVMoIpww0bJglWz6d0X/fMEwB/hIqIM5rx2GVfL4xq3Pj1D+Rns\n\tRiYc+vSQwU5flTnY1e2nBk9Dwg==", "X-Google-Smtp-Source": "AOwi7QBOfhs5sKcYOTjpGw5ka8Mm58NakTEVYhZP3WskZpMoh05DBhg0F/osj+HnTKnFh0/14I8hJg==", "X-Received": "by 10.28.74.89 with SMTP id x86mr1690949wma.57.1507811431589;\n\tThu, 12 Oct 2017 05:30:31 -0700 (PDT)", "From": "Adrien Mazarguil <adrien.mazarguil@6wind.com>", "To": "Ferruh Yigit <ferruh.yigit@intel.com>", "Cc": "dev@dpdk.org, Matan Azrad <matan@mellanox.com>,\n\tOphir Munk <ophirmu@mellanox.com>, Moti Haimovsky <motih@mellanox.com>,\n\tVasily Philipov <vasilyf@mellanox.com>", "Date": "Thu, 12 Oct 2017 14:29:57 +0200", "Message-Id": "<5900a2c9893c879debccf5c14cbceda952332858.1507810956.git.adrien.mazarguil@6wind.com>", "X-Mailer": "git-send-email 2.1.4", "In-Reply-To": "<cover.1507810956.git.adrien.mazarguil@6wind.com>", "References": "<cover.1507746059.git.adrien.mazarguil@6wind.com>\n\t<cover.1507810956.git.adrien.mazarguil@6wind.com>", "Subject": "[dpdk-dev] [PATCH v6 2/5] net/mlx4: add Rx bypassing Verbs", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<http://dpdk.org/ml/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://dpdk.org/ml/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<http://dpdk.org/ml/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "From: Moti Haimovsky <motih@mellanox.com>\n\nThis patch adds support for accessing the hardware directly when\nhandling Rx packets eliminating the need to use Verbs in the Rx data\npath.\n\nRx scatter support: calculate the number of scatters on the fly\naccording to the maximum expected packet size.\n\nSigned-off-by: Vasily Philipov <vasilyf@mellanox.com>\nSigned-off-by: Moti Haimovsky <motih@mellanox.com>\nSigned-off-by: Ophir Munk <ophirmu@mellanox.com>\nAcked-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>\n---\n doc/guides/nics/features/mlx4.ini | 1 +\n drivers/net/mlx4/mlx4_rxq.c | 151 +++++++++++++++-------\n drivers/net/mlx4/mlx4_rxtx.c | 226 +++++++++++++++++++--------------\n drivers/net/mlx4/mlx4_rxtx.h | 19 ++-\n 4 files changed, 242 insertions(+), 155 deletions(-)", "diff": "diff --git a/doc/guides/nics/features/mlx4.ini b/doc/guides/nics/features/mlx4.ini\nindex 9750ebf..19ae688 100644\n--- a/doc/guides/nics/features/mlx4.ini\n+++ b/doc/guides/nics/features/mlx4.ini\n@@ -12,6 +12,7 @@ Rx interrupt = Y\n Queue start/stop = Y\n MTU update = Y\n Jumbo frame = Y\n+Scattered Rx = Y\n Promiscuous mode = Y\n Allmulticast mode = Y\n Unicast MAC filter = Y\ndiff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c\nindex 483fe9b..39c83bc 100644\n--- a/drivers/net/mlx4/mlx4_rxq.c\n+++ b/drivers/net/mlx4/mlx4_rxq.c\n@@ -51,6 +51,7 @@\n #pragma GCC diagnostic error \"-Wpedantic\"\n #endif\n \n+#include <rte_byteorder.h>\n #include <rte_common.h>\n #include <rte_errno.h>\n #include <rte_ethdev.h>\n@@ -312,45 +313,46 @@ void mlx4_rss_detach(struct mlx4_rss *rss)\n static int\n mlx4_rxq_alloc_elts(struct rxq *rxq)\n {\n-\tstruct rxq_elt (*elts)[rxq->elts_n] = rxq->elts;\n+\tconst uint32_t elts_n = 1 << rxq->elts_n;\n+\tconst uint32_t sges_n = 1 << rxq->sges_n;\n+\tstruct rte_mbuf *(*elts)[elts_n] = rxq->elts;\n \tunsigned int i;\n \n-\t/* For each WR (packet). */\n+\tassert(rte_is_power_of_2(elts_n));\n \tfor (i = 0; i != RTE_DIM(*elts); ++i) {\n-\t\tstruct rxq_elt *elt = &(*elts)[i];\n-\t\tstruct ibv_recv_wr *wr = &elt->wr;\n-\t\tstruct ibv_sge *sge = &(*elts)[i].sge;\n+\t\tvolatile struct mlx4_wqe_data_seg *scat = &(*rxq->wqes)[i];\n \t\tstruct rte_mbuf *buf = rte_pktmbuf_alloc(rxq->mp);\n \n \t\tif (buf == NULL) {\n \t\t\twhile (i--) {\n-\t\t\t\trte_pktmbuf_free_seg((*elts)[i].buf);\n-\t\t\t\t(*elts)[i].buf = NULL;\n+\t\t\t\trte_pktmbuf_free_seg((*elts)[i]);\n+\t\t\t\t(*elts)[i] = NULL;\n \t\t\t}\n \t\t\trte_errno = ENOMEM;\n \t\t\treturn -rte_errno;\n \t\t}\n-\t\telt->buf = buf;\n-\t\twr->next = &(*elts)[(i + 1)].wr;\n-\t\twr->sg_list = sge;\n-\t\twr->num_sge = 1;\n \t\t/* Headroom is reserved by rte_pktmbuf_alloc(). */\n \t\tassert(buf->data_off == RTE_PKTMBUF_HEADROOM);\n \t\t/* Buffer is supposed to be empty. */\n \t\tassert(rte_pktmbuf_data_len(buf) == 0);\n \t\tassert(rte_pktmbuf_pkt_len(buf) == 0);\n-\t\t/* sge->addr must be able to store a pointer. */\n-\t\tassert(sizeof(sge->addr) >= sizeof(uintptr_t));\n-\t\t/* SGE keeps its headroom. */\n-\t\tsge->addr = (uintptr_t)\n-\t\t\t((uint8_t *)buf->buf_addr + RTE_PKTMBUF_HEADROOM);\n-\t\tsge->length = (buf->buf_len - RTE_PKTMBUF_HEADROOM);\n-\t\tsge->lkey = rxq->mr->lkey;\n-\t\t/* Redundant check for tailroom. */\n-\t\tassert(sge->length == rte_pktmbuf_tailroom(buf));\n+\t\t/* Only the first segment keeps headroom. */\n+\t\tif (i % sges_n)\n+\t\t\tbuf->data_off = 0;\n+\t\tbuf->port = rxq->port_id;\n+\t\tbuf->data_len = rte_pktmbuf_tailroom(buf);\n+\t\tbuf->pkt_len = rte_pktmbuf_tailroom(buf);\n+\t\tbuf->nb_segs = 1;\n+\t\t*scat = (struct mlx4_wqe_data_seg){\n+\t\t\t.addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,\n+\t\t\t\t\t\t\t\t uintptr_t)),\n+\t\t\t.byte_count = rte_cpu_to_be_32(buf->data_len),\n+\t\t\t.lkey = rte_cpu_to_be_32(rxq->mr->lkey),\n+\t\t};\n+\t\t(*elts)[i] = buf;\n \t}\n-\t/* The last WR pointer must be NULL. */\n-\t(*elts)[(i - 1)].wr.next = NULL;\n+\tDEBUG(\"%p: allocated and configured %u segments (max %u packets)\",\n+\t (void *)rxq, elts_n, elts_n / sges_n);\n \treturn 0;\n }\n \n@@ -364,14 +366,14 @@ static void\n mlx4_rxq_free_elts(struct rxq *rxq)\n {\n \tunsigned int i;\n-\tstruct rxq_elt (*elts)[rxq->elts_n] = rxq->elts;\n+\tstruct rte_mbuf *(*elts)[1 << rxq->elts_n] = rxq->elts;\n \n-\tDEBUG(\"%p: freeing WRs\", (void *)rxq);\n+\tDEBUG(\"%p: freeing Rx queue elements\", (void *)rxq);\n \tfor (i = 0; (i != RTE_DIM(*elts)); ++i) {\n-\t\tif (!(*elts)[i].buf)\n+\t\tif (!(*elts)[i])\n \t\t\tcontinue;\n-\t\trte_pktmbuf_free_seg((*elts)[i].buf);\n-\t\t(*elts)[i].buf = NULL;\n+\t\trte_pktmbuf_free_seg((*elts)[i]);\n+\t\t(*elts)[i] = NULL;\n \t}\n }\n \n@@ -400,8 +402,11 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,\n \t\t struct rte_mempool *mp)\n {\n \tstruct priv *priv = dev->data->dev_private;\n+\tstruct mlx4dv_obj mlxdv;\n+\tstruct mlx4dv_rwq dv_rwq;\n+\tstruct mlx4dv_cq dv_cq;\n \tuint32_t mb_len = rte_pktmbuf_data_room_size(mp);\n-\tstruct rxq_elt (*elts)[desc];\n+\tstruct rte_mbuf *(*elts)[rte_align32pow2(desc)];\n \tstruct rte_flow_error error;\n \tstruct rxq *rxq;\n \tstruct mlx4_malloc_vec vec[] = {\n@@ -439,6 +444,12 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,\n \t\tERROR(\"%p: invalid number of Rx descriptors\", (void *)dev);\n \t\treturn -rte_errno;\n \t}\n+\tif (desc != RTE_DIM(*elts)) {\n+\t\tdesc = RTE_DIM(*elts);\n+\t\tWARN(\"%p: increased number of descriptors in Rx queue %u\"\n+\t\t \" to the next power of two (%u)\",\n+\t\t (void *)dev, idx, desc);\n+\t}\n \t/* Allocate and initialize Rx queue. */\n \tmlx4_zmallocv_socket(\"RXQ\", vec, RTE_DIM(vec), socket);\n \tif (!rxq) {\n@@ -450,8 +461,8 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,\n \t\t.priv = priv,\n \t\t.mp = mp,\n \t\t.port_id = dev->data->port_id,\n-\t\t.elts_n = desc,\n-\t\t.elts_head = 0,\n+\t\t.sges_n = 0,\n+\t\t.elts_n = rte_log2_u32(desc),\n \t\t.elts = elts,\n \t\t.stats.idx = idx,\n \t\t.socket = socket,\n@@ -462,9 +473,29 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,\n \t (mb_len - RTE_PKTMBUF_HEADROOM)) {\n \t\t;\n \t} else if (dev->data->dev_conf.rxmode.enable_scatter) {\n-\t\tWARN(\"%p: scattered mode has been requested but is\"\n-\t\t \" not supported, this may lead to packet loss\",\n-\t\t (void *)dev);\n+\t\tuint32_t size =\n+\t\t\tRTE_PKTMBUF_HEADROOM +\n+\t\t\tdev->data->dev_conf.rxmode.max_rx_pkt_len;\n+\t\tuint32_t sges_n;\n+\n+\t\t/*\n+\t\t * Determine the number of SGEs needed for a full packet\n+\t\t * and round it to the next power of two.\n+\t\t */\n+\t\tsges_n = rte_log2_u32((size / mb_len) + !!(size % mb_len));\n+\t\trxq->sges_n = sges_n;\n+\t\t/* Make sure sges_n did not overflow. */\n+\t\tsize = mb_len * (1 << rxq->sges_n);\n+\t\tsize -= RTE_PKTMBUF_HEADROOM;\n+\t\tif (size < dev->data->dev_conf.rxmode.max_rx_pkt_len) {\n+\t\t\trte_errno = EOVERFLOW;\n+\t\t\tERROR(\"%p: too many SGEs (%u) needed to handle\"\n+\t\t\t \" requested maximum packet size %u\",\n+\t\t\t (void *)dev,\n+\t\t\t 1 << sges_n,\n+\t\t\t dev->data->dev_conf.rxmode.max_rx_pkt_len);\n+\t\t\tgoto error;\n+\t\t}\n \t} else {\n \t\tWARN(\"%p: the requested maximum Rx packet size (%u) is\"\n \t\t \" larger than a single mbuf (%u) and scattered\"\n@@ -473,6 +504,17 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,\n \t\t dev->data->dev_conf.rxmode.max_rx_pkt_len,\n \t\t mb_len - RTE_PKTMBUF_HEADROOM);\n \t}\n+\tDEBUG(\"%p: maximum number of segments per packet: %u\",\n+\t (void *)dev, 1 << rxq->sges_n);\n+\tif (desc % (1 << rxq->sges_n)) {\n+\t\trte_errno = EINVAL;\n+\t\tERROR(\"%p: number of Rx queue descriptors (%u) is not a\"\n+\t\t \" multiple of maximum segments per packet (%u)\",\n+\t\t (void *)dev,\n+\t\t desc,\n+\t\t 1 << rxq->sges_n);\n+\t\tgoto error;\n+\t}\n \t/* Use the entire Rx mempool as the memory region. */\n \trxq->mr = mlx4_mp2mr(priv->pd, mp);\n \tif (!rxq->mr) {\n@@ -497,7 +539,8 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,\n \t\t\tgoto error;\n \t\t}\n \t}\n-\trxq->cq = ibv_create_cq(priv->ctx, desc, NULL, rxq->channel, 0);\n+\trxq->cq = ibv_create_cq(priv->ctx, desc >> rxq->sges_n, NULL,\n+\t\t\t\trxq->channel, 0);\n \tif (!rxq->cq) {\n \t\trte_errno = ENOMEM;\n \t\tERROR(\"%p: CQ creation failure: %s\",\n@@ -508,8 +551,8 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,\n \t\t(priv->ctx,\n \t\t &(struct ibv_wq_init_attr){\n \t\t\t.wq_type = IBV_WQT_RQ,\n-\t\t\t.max_wr = RTE_MIN(priv->device_attr.max_qp_wr, desc),\n-\t\t\t.max_sge = 1,\n+\t\t\t.max_wr = desc >> rxq->sges_n,\n+\t\t\t.max_sge = 1 << rxq->sges_n,\n \t\t\t.pd = priv->pd,\n \t\t\t.cq = rxq->cq,\n \t\t });\n@@ -531,27 +574,43 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,\n \t\t (void *)dev, strerror(rte_errno));\n \t\tgoto error;\n \t}\n-\tret = mlx4_rxq_alloc_elts(rxq);\n+\t/* Retrieve device queue information. */\n+\tmlxdv.cq.in = rxq->cq;\n+\tmlxdv.cq.out = &dv_cq;\n+\tmlxdv.rwq.in = rxq->wq;\n+\tmlxdv.rwq.out = &dv_rwq;\n+\tret = mlx4dv_init_obj(&mlxdv, MLX4DV_OBJ_RWQ | MLX4DV_OBJ_CQ);\n \tif (ret) {\n-\t\tERROR(\"%p: RXQ allocation failed: %s\",\n-\t\t (void *)dev, strerror(rte_errno));\n+\t\trte_errno = EINVAL;\n+\t\tERROR(\"%p: failed to obtain device information\", (void *)dev);\n \t\tgoto error;\n \t}\n-\tret = ibv_post_wq_recv(rxq->wq, &(*rxq->elts)[0].wr,\n-\t\t\t &(struct ibv_recv_wr *){ NULL });\n+\trxq->wqes =\n+\t\t(volatile struct mlx4_wqe_data_seg (*)[])\n+\t\t((uintptr_t)dv_rwq.buf.buf + dv_rwq.rq.offset);\n+\trxq->rq_db = dv_rwq.rdb;\n+\trxq->rq_ci = 0;\n+\trxq->mcq.buf = dv_cq.buf.buf;\n+\trxq->mcq.cqe_cnt = dv_cq.cqe_cnt;\n+\trxq->mcq.set_ci_db = dv_cq.set_ci_db;\n+\trxq->mcq.cqe_64 = (dv_cq.cqe_size & 64) ? 1 : 0;\n+\tret = mlx4_rxq_alloc_elts(rxq);\n \tif (ret) {\n-\t\trte_errno = ret;\n-\t\tERROR(\"%p: ibv_post_recv() failed: %s\",\n-\t\t (void *)dev,\n-\t\t strerror(rte_errno));\n+\t\tERROR(\"%p: RXQ allocation failed: %s\",\n+\t\t (void *)dev, strerror(rte_errno));\n \t\tgoto error;\n \t}\n \tDEBUG(\"%p: adding Rx queue %p to list\", (void *)dev, (void *)rxq);\n \tdev->data->rx_queues[idx] = rxq;\n \t/* Enable associated flows. */\n \tret = mlx4_flow_sync(priv, &error);\n-\tif (!ret)\n+\tif (!ret) {\n+\t\t/* Update doorbell counter. */\n+\t\trxq->rq_ci = desc >> rxq->sges_n;\n+\t\trte_wmb();\n+\t\t*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);\n \t\treturn 0;\n+\t}\n \tERROR(\"cannot re-attach flow rules to queue %u\"\n \t \" (code %d, \\\"%s\\\"), flow error type %d, cause %p, message: %s\",\n \t idx, -ret, strerror(-ret), error.type, error.cause,\ndiff --git a/drivers/net/mlx4/mlx4_rxtx.c b/drivers/net/mlx4/mlx4_rxtx.c\nindex 38b87a0..cc0baaa 100644\n--- a/drivers/net/mlx4/mlx4_rxtx.c\n+++ b/drivers/net/mlx4/mlx4_rxtx.c\n@@ -538,9 +538,44 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)\n }\n \n /**\n- * DPDK callback for Rx.\n+ * Poll one CQE from CQ.\n *\n- * The following function doesn't manage scattered packets.\n+ * @param rxq\n+ * Pointer to the receive queue structure.\n+ * @param[out] out\n+ * Just polled CQE.\n+ *\n+ * @return\n+ * Number of bytes of the CQE, 0 in case there is no completion.\n+ */\n+static unsigned int\n+mlx4_cq_poll_one(struct rxq *rxq, struct mlx4_cqe **out)\n+{\n+\tint ret = 0;\n+\tstruct mlx4_cqe *cqe = NULL;\n+\tstruct mlx4_cq *cq = &rxq->mcq;\n+\n+\tcqe = (struct mlx4_cqe *)mlx4_get_cqe(cq, cq->cons_index);\n+\tif (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^\n+\t !!(cq->cons_index & cq->cqe_cnt))\n+\t\tgoto out;\n+\t/*\n+\t * Make sure we read CQ entry contents after we've checked the\n+\t * ownership bit.\n+\t */\n+\trte_rmb();\n+\tassert(!(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK));\n+\tassert((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) !=\n+\t MLX4_CQE_OPCODE_ERROR);\n+\tret = rte_be_to_cpu_32(cqe->byte_cnt);\n+\t++cq->cons_index;\n+out:\n+\t*out = cqe;\n+\treturn ret;\n+}\n+\n+/**\n+ * DPDK callback for Rx with scattered packets support.\n *\n * @param dpdk_rxq\n * Generic pointer to Rx queue structure.\n@@ -555,112 +590,107 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)\n uint16_t\n mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)\n {\n-\tstruct rxq *rxq = (struct rxq *)dpdk_rxq;\n-\tstruct rxq_elt (*elts)[rxq->elts_n] = rxq->elts;\n-\tconst unsigned int elts_n = rxq->elts_n;\n-\tunsigned int elts_head = rxq->elts_head;\n-\tstruct ibv_wc wcs[pkts_n];\n-\tstruct ibv_recv_wr *wr_head = NULL;\n-\tstruct ibv_recv_wr **wr_next = &wr_head;\n-\tstruct ibv_recv_wr *wr_bad = NULL;\n-\tunsigned int i;\n-\tunsigned int pkts_ret = 0;\n-\tint ret;\n+\tstruct rxq *rxq = dpdk_rxq;\n+\tconst uint32_t wr_cnt = (1 << rxq->elts_n) - 1;\n+\tconst uint16_t sges_n = rxq->sges_n;\n+\tstruct rte_mbuf *pkt = NULL;\n+\tstruct rte_mbuf *seg = NULL;\n+\tunsigned int i = 0;\n+\tuint32_t rq_ci = rxq->rq_ci << sges_n;\n+\tint len = 0;\n \n-\tret = ibv_poll_cq(rxq->cq, pkts_n, wcs);\n-\tif (unlikely(ret == 0))\n-\t\treturn 0;\n-\tif (unlikely(ret < 0)) {\n-\t\tDEBUG(\"rxq=%p, ibv_poll_cq() failed (wc_n=%d)\",\n-\t\t (void *)rxq, ret);\n-\t\treturn 0;\n-\t}\n-\tassert(ret <= (int)pkts_n);\n-\t/* For each work completion. */\n-\tfor (i = 0; i != (unsigned int)ret; ++i) {\n-\t\tstruct ibv_wc *wc = &wcs[i];\n-\t\tstruct rxq_elt *elt = &(*elts)[elts_head];\n-\t\tstruct ibv_recv_wr *wr = &elt->wr;\n-\t\tuint32_t len = wc->byte_len;\n-\t\tstruct rte_mbuf *seg = elt->buf;\n-\t\tstruct rte_mbuf *rep;\n+\twhile (pkts_n) {\n+\t\tstruct mlx4_cqe *cqe;\n+\t\tuint32_t idx = rq_ci & wr_cnt;\n+\t\tstruct rte_mbuf *rep = (*rxq->elts)[idx];\n+\t\tvolatile struct mlx4_wqe_data_seg *scat = &(*rxq->wqes)[idx];\n \n-\t\t/* Sanity checks. */\n-\t\tassert(wr->sg_list == &elt->sge);\n-\t\tassert(wr->num_sge == 1);\n-\t\tassert(elts_head < rxq->elts_n);\n-\t\tassert(rxq->elts_head < rxq->elts_n);\n-\t\t/*\n-\t\t * Fetch initial bytes of packet descriptor into a\n-\t\t * cacheline while allocating rep.\n-\t\t */\n-\t\trte_mbuf_prefetch_part1(seg);\n-\t\trte_mbuf_prefetch_part2(seg);\n-\t\t/* Link completed WRs together for repost. */\n-\t\t*wr_next = wr;\n-\t\twr_next = &wr->next;\n-\t\tif (unlikely(wc->status != IBV_WC_SUCCESS)) {\n-\t\t\t/* Whatever, just repost the offending WR. */\n-\t\t\tDEBUG(\"rxq=%p: bad work completion status (%d): %s\",\n-\t\t\t (void *)rxq, wc->status,\n-\t\t\t ibv_wc_status_str(wc->status));\n-\t\t\t/* Increment dropped packets counter. */\n-\t\t\t++rxq->stats.idropped;\n-\t\t\tgoto repost;\n-\t\t}\n+\t\t/* Update the 'next' pointer of the previous segment. */\n+\t\tif (pkt)\n+\t\t\tseg->next = rep;\n+\t\tseg = rep;\n+\t\trte_prefetch0(seg);\n+\t\trte_prefetch0(scat);\n \t\trep = rte_mbuf_raw_alloc(rxq->mp);\n \t\tif (unlikely(rep == NULL)) {\n-\t\t\t/*\n-\t\t\t * Unable to allocate a replacement mbuf,\n-\t\t\t * repost WR.\n-\t\t\t */\n-\t\t\tDEBUG(\"rxq=%p: can't allocate a new mbuf\",\n-\t\t\t (void *)rxq);\n-\t\t\t/* Increase out of memory counters. */\n \t\t\t++rxq->stats.rx_nombuf;\n-\t\t\t++rxq->priv->dev->data->rx_mbuf_alloc_failed;\n-\t\t\tgoto repost;\n+\t\t\tif (!pkt) {\n+\t\t\t\t/*\n+\t\t\t\t * No buffers before we even started,\n+\t\t\t\t * bail out silently.\n+\t\t\t\t */\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t\twhile (pkt != seg) {\n+\t\t\t\tassert(pkt != (*rxq->elts)[idx]);\n+\t\t\t\trep = pkt->next;\n+\t\t\t\tpkt->next = NULL;\n+\t\t\t\tpkt->nb_segs = 1;\n+\t\t\t\trte_mbuf_raw_free(pkt);\n+\t\t\t\tpkt = rep;\n+\t\t\t}\n+\t\t\tbreak;\n+\t\t}\n+\t\tif (!pkt) {\n+\t\t\t/* Looking for the new packet. */\n+\t\t\tlen = mlx4_cq_poll_one(rxq, &cqe);\n+\t\t\tif (!len) {\n+\t\t\t\trte_mbuf_raw_free(rep);\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t\tif (unlikely(len < 0)) {\n+\t\t\t\t/* Rx error, packet is likely too large. */\n+\t\t\t\trte_mbuf_raw_free(rep);\n+\t\t\t\t++rxq->stats.idropped;\n+\t\t\t\tgoto skip;\n+\t\t\t}\n+\t\t\tpkt = seg;\n+\t\t\tpkt->packet_type = 0;\n+\t\t\tpkt->ol_flags = 0;\n+\t\t\tpkt->pkt_len = len;\n+\t\t}\n+\t\trep->nb_segs = 1;\n+\t\trep->port = rxq->port_id;\n+\t\trep->data_len = seg->data_len;\n+\t\trep->data_off = seg->data_off;\n+\t\t(*rxq->elts)[idx] = rep;\n+\t\t/*\n+\t\t * Fill NIC descriptor with the new buffer. The lkey and size\n+\t\t * of the buffers are already known, only the buffer address\n+\t\t * changes.\n+\t\t */\n+\t\tscat->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));\n+\t\tif (len > seg->data_len) {\n+\t\t\tlen -= seg->data_len;\n+\t\t\t++pkt->nb_segs;\n+\t\t\t++rq_ci;\n+\t\t\tcontinue;\n \t\t}\n-\t\t/* Reconfigure sge to use rep instead of seg. */\n-\t\telt->sge.addr = (uintptr_t)rep->buf_addr + RTE_PKTMBUF_HEADROOM;\n-\t\tassert(elt->sge.lkey == rxq->mr->lkey);\n-\t\telt->buf = rep;\n-\t\t/* Update seg information. */\n-\t\tseg->data_off = RTE_PKTMBUF_HEADROOM;\n-\t\tseg->nb_segs = 1;\n-\t\tseg->port = rxq->port_id;\n-\t\tseg->next = NULL;\n-\t\tseg->pkt_len = len;\n+\t\t/* The last segment. */\n \t\tseg->data_len = len;\n-\t\tseg->packet_type = 0;\n-\t\tseg->ol_flags = 0;\n+\t\t/* Increment bytes counter. */\n+\t\trxq->stats.ibytes += pkt->pkt_len;\n \t\t/* Return packet. */\n-\t\t*(pkts++) = seg;\n-\t\t++pkts_ret;\n-\t\t/* Increase bytes counter. */\n-\t\trxq->stats.ibytes += len;\n-repost:\n-\t\tif (++elts_head >= elts_n)\n-\t\t\telts_head = 0;\n-\t\tcontinue;\n+\t\t*(pkts++) = pkt;\n+\t\tpkt = NULL;\n+\t\t--pkts_n;\n+\t\t++i;\n+skip:\n+\t\t/* Align consumer index to the next stride. */\n+\t\trq_ci >>= sges_n;\n+\t\t++rq_ci;\n+\t\trq_ci <<= sges_n;\n \t}\n-\tif (unlikely(i == 0))\n+\tif (unlikely(i == 0 && (rq_ci >> sges_n) == rxq->rq_ci))\n \t\treturn 0;\n-\t/* Repost WRs. */\n-\t*wr_next = NULL;\n-\tassert(wr_head);\n-\tret = ibv_post_wq_recv(rxq->wq, wr_head, &wr_bad);\n-\tif (unlikely(ret)) {\n-\t\t/* Inability to repost WRs is fatal. */\n-\t\tDEBUG(\"%p: recv_burst(): failed (ret=%d)\",\n-\t\t (void *)rxq->priv,\n-\t\t ret);\n-\t\tabort();\n-\t}\n-\trxq->elts_head = elts_head;\n-\t/* Increase packets counter. */\n-\trxq->stats.ipackets += pkts_ret;\n-\treturn pkts_ret;\n+\t/* Update the consumer index. */\n+\trxq->rq_ci = rq_ci >> sges_n;\n+\trte_wmb();\n+\t*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);\n+\t*rxq->mcq.set_ci_db = rte_cpu_to_be_32(rxq->mcq.cons_index & 0xffffff);\n+\t/* Increment packets counter. */\n+\trxq->stats.ipackets += i;\n+\treturn i;\n }\n \n /**\ndiff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h\nindex ff27126..fa5738f 100644\n--- a/drivers/net/mlx4/mlx4_rxtx.h\n+++ b/drivers/net/mlx4/mlx4_rxtx.h\n@@ -63,13 +63,6 @@ struct mlx4_rxq_stats {\n \tuint64_t rx_nombuf; /**< Total of Rx mbuf allocation failures. */\n };\n \n-/** Rx element. */\n-struct rxq_elt {\n-\tstruct ibv_recv_wr wr; /**< Work request. */\n-\tstruct ibv_sge sge; /**< Scatter/gather element. */\n-\tstruct rte_mbuf *buf; /**< Buffer. */\n-};\n-\n /** Rx queue descriptor. */\n struct rxq {\n \tstruct priv *priv; /**< Back pointer to private data. */\n@@ -78,10 +71,14 @@ struct rxq {\n \tstruct ibv_cq *cq; /**< Completion queue. */\n \tstruct ibv_wq *wq; /**< Work queue. */\n \tstruct ibv_comp_channel *channel; /**< Rx completion channel. */\n-\tunsigned int port_id; /**< Port ID for incoming packets. */\n-\tunsigned int elts_n; /**< (*elts)[] length. */\n-\tunsigned int elts_head; /**< Current index in (*elts)[]. */\n-\tstruct rxq_elt (*elts)[]; /**< Rx elements. */\n+\tuint16_t rq_ci; /**< Saved RQ consumer index. */\n+\tuint16_t port_id; /**< Port ID for incoming packets. */\n+\tuint16_t sges_n; /**< Number of segments per packet (log2 value). */\n+\tuint16_t elts_n; /**< Mbuf queue size (log2 value). */\n+\tstruct rte_mbuf *(*elts)[]; /**< Rx elements. */\n+\tvolatile struct mlx4_wqe_data_seg (*wqes)[]; /**< HW queue entries. */\n+\tvolatile uint32_t *rq_db; /**< RQ doorbell record. */\n+\tstruct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */\n \tstruct mlx4_rxq_stats stats; /**< Rx queue counters. */\n \tunsigned int socket; /**< CPU socket ID for allocations. */\n \tuint8_t data[]; /**< Remaining queue resources. */\n", "prefixes": [ "dpdk-dev", "v6", "2/5" ] }