Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 74047,
    "url": "http://patches.dpdk.org/api/patches/74047/?format=api",
    "web_url": "http://patches.dpdk.org/project/dpdk/patch/1594794114-16313-6-git-send-email-viacheslavo@mellanox.com/",
    "project": {
        "id": 1,
        "url": "http://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<1594794114-16313-6-git-send-email-viacheslavo@mellanox.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/1594794114-16313-6-git-send-email-viacheslavo@mellanox.com",
    "date": "2020-07-15T06:21:42",
    "name": "[v2,05/17] net/mlx5: create clock queue for packet pacing",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "58b9890781671f5f7111a963522735e681ff10c6",
    "submitter": {
        "id": 1102,
        "url": "http://patches.dpdk.org/api/people/1102/?format=api",
        "name": "Slava Ovsiienko",
        "email": "viacheslavo@mellanox.com"
    },
    "delegate": {
        "id": 3268,
        "url": "http://patches.dpdk.org/api/users/3268/?format=api",
        "username": "rasland",
        "first_name": "Raslan",
        "last_name": "Darawsheh",
        "email": "rasland@nvidia.com"
    },
    "mbox": "http://patches.dpdk.org/project/dpdk/patch/1594794114-16313-6-git-send-email-viacheslavo@mellanox.com/mbox/",
    "series": [
        {
            "id": 11032,
            "url": "http://patches.dpdk.org/api/series/11032/?format=api",
            "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=11032",
            "date": "2020-07-15T06:21:37",
            "name": "net/mlx5: introduce accurate packet Tx scheduling",
            "version": 2,
            "mbox": "http://patches.dpdk.org/series/11032/mbox/"
        }
    ],
    "comments": "http://patches.dpdk.org/api/patches/74047/comments/",
    "check": "warning",
    "checks": "http://patches.dpdk.org/api/patches/74047/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 1B22EA0540;\n\tWed, 15 Jul 2020 08:22:58 +0200 (CEST)",
            "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 87B901C12C;\n\tWed, 15 Jul 2020 08:22:15 +0200 (CEST)",
            "from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129])\n by dpdk.org (Postfix) with ESMTP id 7A5B01C126\n for <dev@dpdk.org>; Wed, 15 Jul 2020 08:22:12 +0200 (CEST)",
            "from Internal Mail-Server by MTLPINE1 (envelope-from\n viacheslavo@mellanox.com) with SMTP; 15 Jul 2020 09:22:09 +0300",
            "from pegasus12.mtr.labs.mlnx (pegasus12.mtr.labs.mlnx\n [10.210.17.40])\n by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 06F6M9q9007021;\n Wed, 15 Jul 2020 09:22:09 +0300",
            "from pegasus12.mtr.labs.mlnx (localhost [127.0.0.1])\n by pegasus12.mtr.labs.mlnx (8.14.7/8.14.7) with ESMTP id 06F6M928016436;\n Wed, 15 Jul 2020 06:22:09 GMT",
            "(from viacheslavo@localhost)\n by pegasus12.mtr.labs.mlnx (8.14.7/8.14.7/Submit) id 06F6M9uT016435;\n Wed, 15 Jul 2020 06:22:09 GMT"
        ],
        "X-Authentication-Warning": "pegasus12.mtr.labs.mlnx: viacheslavo set sender to\n viacheslavo@mellanox.com using -f",
        "From": "Viacheslav Ovsiienko <viacheslavo@mellanox.com>",
        "To": "dev@dpdk.org",
        "Cc": "matan@mellanox.com, rasland@mellanox.com, olivier.matz@6wind.com,\n thomas@monjalon.net, ferruh.yigit@intel.com",
        "Date": "Wed, 15 Jul 2020 06:21:42 +0000",
        "Message-Id": "<1594794114-16313-6-git-send-email-viacheslavo@mellanox.com>",
        "X-Mailer": "git-send-email 1.8.3.1",
        "In-Reply-To": "<1594794114-16313-1-git-send-email-viacheslavo@mellanox.com>",
        "References": "<1591771085-24959-1-git-send-email-viacheslavo@mellanox.com>\n <1594794114-16313-1-git-send-email-viacheslavo@mellanox.com>",
        "Subject": "[dpdk-dev] [PATCH v2 05/17] net/mlx5: create clock queue for packet\n\tpacing",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.15",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org",
        "Sender": "\"dev\" <dev-bounces@dpdk.org>"
    },
    "content": "This patch creates the special completion queue providing\nreference completions to schedule packet send from\nother transmitting queues.\n\nSigned-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>\nAcked-by: Matan Azrad <matan@mellanox.com>\n---\n drivers/net/mlx5/Makefile        |   1 +\n drivers/net/mlx5/linux/mlx5_os.c |   3 +\n drivers/net/mlx5/meson.build     |   1 +\n drivers/net/mlx5/mlx5.c          |   2 +\n drivers/net/mlx5/mlx5.h          |  47 ++++\n drivers/net/mlx5/mlx5_defs.h     |   7 +\n drivers/net/mlx5/mlx5_trigger.c  |  16 +-\n drivers/net/mlx5/mlx5_txpp.c     | 449 +++++++++++++++++++++++++++++++++++++++\n 8 files changed, 521 insertions(+), 5 deletions(-)\n create mode 100644 drivers/net/mlx5/mlx5_txpp.c",
    "diff": "diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile\nindex a458402..9eaac6b 100644\n--- a/drivers/net/mlx5/Makefile\n+++ b/drivers/net/mlx5/Makefile\n@@ -11,6 +11,7 @@ LIB = librte_pmd_mlx5.a\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxq.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_txq.c\n+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_txpp.c\n SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx.c\n ifneq ($(filter y,$(CONFIG_RTE_ARCH_X86_64) \\\n \t\t\t$(CONFIG_RTE_ARCH_PPC_64) \\\ndiff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c\nindex 63e9350..ea36309 100644\n--- a/drivers/net/mlx5/linux/mlx5_os.c\n+++ b/drivers/net/mlx5/linux/mlx5_os.c\n@@ -1869,6 +1869,9 @@\n {\n \tint dbmap_env;\n \tint err = 0;\n+\n+\tsh->numa_node = spawn->pci_dev->device.numa_node;\n+\tpthread_mutex_init(&sh->txpp.mutex, NULL);\n \t/*\n \t * Configure environment variable \"MLX5_BF_SHUT_UP\"\n \t * before the device creation. The rdma_core library\ndiff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build\nindex e95ce02..c06b153 100644\n--- a/drivers/net/mlx5/meson.build\n+++ b/drivers/net/mlx5/meson.build\n@@ -26,6 +26,7 @@ sources = files(\n \t'mlx5_stats.c',\n \t'mlx5_trigger.c',\n \t'mlx5_txq.c',\n+\t'mlx5_txpp.c',\n \t'mlx5_vlan.c',\n \t'mlx5_utils.c',\n )\ndiff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c\nindex 71e59ac..10196ac 100644\n--- a/drivers/net/mlx5/mlx5.c\n+++ b/drivers/net/mlx5/mlx5.c\n@@ -767,6 +767,7 @@ struct mlx5_dev_ctx_shared *\n \tpthread_mutex_unlock(&mlx5_dev_ctx_list_mutex);\n \treturn sh;\n error:\n+\tpthread_mutex_destroy(&sh->txpp.mutex);\n \tpthread_mutex_unlock(&mlx5_dev_ctx_list_mutex);\n \tMLX5_ASSERT(sh);\n \tif (sh->cnt_id_tbl) {\n@@ -856,6 +857,7 @@ struct mlx5_dev_ctx_shared *\n \t\tclaim_zero(mlx5_glue->close_device(sh->ctx));\n \tif (sh->flow_id_pool)\n \t\tmlx5_flow_id_pool_release(sh->flow_id_pool);\n+\tpthread_mutex_destroy(&sh->txpp.mutex);\n \trte_free(sh);\n exit:\n \tpthread_mutex_unlock(&mlx5_dev_ctx_list_mutex);\ndiff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h\nindex c760aff..0b73b2a 100644\n--- a/drivers/net/mlx5/mlx5.h\n+++ b/drivers/net/mlx5/mlx5.h\n@@ -531,6 +531,44 @@ struct mlx5_flow_id_pool {\n \tuint32_t max_id; /**< Maximum id can be allocated from the pool. */\n };\n \n+/* Tx pacing queue structure - for Clock and Rearm queues. */\n+struct mlx5_txpp_wq {\n+\t/* Completion Queue related data.*/\n+\tstruct mlx5_devx_obj *cq;\n+\tstruct mlx5dv_devx_umem *cq_umem;\n+\tunion {\n+\t\tvolatile void *cq_buf;\n+\t\tvolatile struct mlx5_cqe *cqes;\n+\t};\n+\tvolatile uint32_t *cq_dbrec;\n+\tuint32_t cq_ci:24;\n+\tuint32_t arm_sn:2;\n+\t/* Send Queue related data.*/\n+\tstruct mlx5_devx_obj *sq;\n+\tstruct mlx5dv_devx_umem *sq_umem;\n+\tunion {\n+\t\tvolatile void *sq_buf;\n+\t\tvolatile struct mlx5_wqe *wqes;\n+\t};\n+\tuint16_t sq_size; /* Number of WQEs in the queue. */\n+\tuint16_t sq_ci; /* Next WQE to execute. */\n+\tvolatile uint32_t *sq_dbrec;\n+};\n+\n+/* Tx packet pacing structure. */\n+struct mlx5_dev_txpp {\n+\tpthread_mutex_t mutex; /* Pacing create/destroy mutex. */\n+\tuint32_t refcnt; /* Pacing reference counter. */\n+\tuint32_t freq; /* Timestamp frequency, Hz. */\n+\tuint32_t tick; /* Completion tick duration in nanoseconds. */\n+\tuint32_t test; /* Packet pacing test mode. */\n+\tint32_t skew; /* Scheduling skew. */\n+\tuint32_t eqn; /* Event Queue number. */\n+\tstruct rte_intr_handle intr_handle; /* Periodic interrupt. */\n+\tstruct mlx5dv_devx_event_channel *echan; /* Event Channel. */\n+\tstruct mlx5_txpp_wq clock_queue; /* Clock Queue. */\n+};\n+\n /*\n  * Shared Infiniband device context for Master/Representors\n  * which belong to same IB device with multiple IB ports.\n@@ -547,9 +585,12 @@ struct mlx5_dev_ctx_shared {\n \tchar ibdev_name[DEV_SYSFS_NAME_MAX]; /* SYSFS dev name. */\n \tchar ibdev_path[DEV_SYSFS_PATH_MAX]; /* SYSFS dev path for secondary */\n \tstruct mlx5_dev_attr device_attr; /* Device properties. */\n+\tint numa_node; /* Numa node of backing physical device. */\n \tLIST_ENTRY(mlx5_dev_ctx_shared) mem_event_cb;\n \t/**< Called by memory event callback. */\n \tstruct mlx5_mr_share_cache share_cache;\n+\t/* Packet pacing related structure. */\n+\tstruct mlx5_dev_txpp txpp;\n \t/* Shared DV/DR flow data section. */\n \tpthread_mutex_t dv_mutex; /* DV context mutex. */\n \tuint32_t dv_meta_mask; /* flow META metadata supported mask. */\n@@ -622,6 +663,7 @@ struct mlx5_priv {\n \tunsigned int representor:1; /* Device is a port representor. */\n \tunsigned int master:1; /* Device is a E-Switch master. */\n \tunsigned int dr_shared:1; /* DV/DR data is shared. */\n+\tunsigned int txpp_en:1; /* Tx packet pacing enabled. */\n \tunsigned int counter_fallback:1; /* Use counter fallback management. */\n \tunsigned int mtr_en:1; /* Whether support meter. */\n \tunsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */\n@@ -944,4 +986,9 @@ int mlx5_os_read_dev_stat(struct mlx5_priv *priv,\n void mlx5_os_stats_init(struct rte_eth_dev *dev);\n void mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb,\n \t\t\t   mlx5_dereg_mr_t *dereg_mr_cb);\n+/* mlx5_txpp.c */\n+\n+int mlx5_txpp_start(struct rte_eth_dev *dev);\n+void mlx5_txpp_stop(struct rte_eth_dev *dev);\n+\n #endif /* RTE_PMD_MLX5_H_ */\ndiff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h\nindex 260f584..07a2b59 100644\n--- a/drivers/net/mlx5/mlx5_defs.h\n+++ b/drivers/net/mlx5/mlx5_defs.h\n@@ -171,6 +171,13 @@\n #define MLX5_TXDB_NCACHED 1\n #define MLX5_TXDB_HEURISTIC 2\n \n+/* Tx accurate scheduling on timestamps parameters. */\n+#define MLX5_TXPP_CLKQ_SIZE 1\n+/* The minimal size test packet to put into one WQE, padded by HW. */\n+#define MLX5_TXPP_TEST_PKT_SIZE (sizeof(struct rte_ether_hdr) +\t\\\n+\t\t\t\t sizeof(struct rte_ipv4_hdr))\n+\n+\n /* Size of the simple hash table for metadata register table. */\n #define MLX5_FLOW_MREG_HTABLE_SZ 4096\n #define MLX5_FLOW_MREG_HNAME \"MARK_COPY_TABLE\"\ndiff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c\nindex ef74609..ca25ad9 100644\n--- a/drivers/net/mlx5/mlx5_trigger.c\n+++ b/drivers/net/mlx5/mlx5_trigger.c\n@@ -288,25 +288,29 @@\n \t\t\treturn -rte_errno;\n \t\t}\n \t}\n+\tret = mlx5_txpp_start(dev);\n+\tif (ret) {\n+\t\tDRV_LOG(ERR, \"port %u Tx packet pacing init failed: %s\",\n+\t\t\tdev->data->port_id, strerror(rte_errno));\n+\t\tgoto error;\n+\t}\n \tret = mlx5_txq_start(dev);\n \tif (ret) {\n \t\tDRV_LOG(ERR, \"port %u Tx queue allocation failed: %s\",\n \t\t\tdev->data->port_id, strerror(rte_errno));\n-\t\treturn -rte_errno;\n+\t\tgoto error;\n \t}\n \tret = mlx5_rxq_start(dev);\n \tif (ret) {\n \t\tDRV_LOG(ERR, \"port %u Rx queue allocation failed: %s\",\n \t\t\tdev->data->port_id, strerror(rte_errno));\n-\t\tmlx5_txq_stop(dev);\n-\t\treturn -rte_errno;\n+\t\tgoto error;\n \t}\n \tret = mlx5_hairpin_bind(dev);\n \tif (ret) {\n \t\tDRV_LOG(ERR, \"port %u hairpin binding failed: %s\",\n \t\t\tdev->data->port_id, strerror(rte_errno));\n-\t\tmlx5_txq_stop(dev);\n-\t\treturn -rte_errno;\n+\t\tgoto error;\n \t}\n \t/* Set started flag here for the following steps like control flow. */\n \tdev->data->dev_started = 1;\n@@ -362,6 +366,7 @@\n \tmlx5_traffic_disable(dev);\n \tmlx5_txq_stop(dev);\n \tmlx5_rxq_stop(dev);\n+\tmlx5_txpp_stop(dev); /* Stop last. */\n \trte_errno = ret; /* Restore rte_errno. */\n \treturn -rte_errno;\n }\n@@ -398,6 +403,7 @@\n \tpriv->sh->port[priv->dev_port - 1].devx_ih_port_id = RTE_MAX_ETHPORTS;\n \tmlx5_txq_stop(dev);\n \tmlx5_rxq_stop(dev);\n+\tmlx5_txpp_stop(dev);\n }\n \n /**\ndiff --git a/drivers/net/mlx5/mlx5_txpp.c b/drivers/net/mlx5/mlx5_txpp.c\nnew file mode 100644\nindex 0000000..382bd20\n--- /dev/null\n+++ b/drivers/net/mlx5/mlx5_txpp.c\n@@ -0,0 +1,449 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright 2020 Mellanox Technologies, Ltd\n+ */\n+#include <rte_ether.h>\n+#include <rte_ethdev_driver.h>\n+#include <rte_interrupts.h>\n+#include <rte_alarm.h>\n+#include <rte_malloc.h>\n+\n+#include \"mlx5.h\"\n+#include \"mlx5_rxtx.h\"\n+\n+/* Destroy Event Queue Notification Channel. */\n+static void\n+mlx5_txpp_destroy_eqn(struct mlx5_dev_ctx_shared *sh)\n+{\n+\tif (sh->txpp.echan) {\n+\t\tmlx5_glue->devx_destroy_event_channel(sh->txpp.echan);\n+\t\tsh->txpp.echan = NULL;\n+\t}\n+\tsh->txpp.eqn = 0;\n+}\n+\n+/* Create Event Queue Notification Channel. */\n+static int\n+mlx5_txpp_create_eqn(struct mlx5_dev_ctx_shared *sh)\n+{\n+\tuint32_t lcore;\n+\n+\tMLX5_ASSERT(!sh->txpp.echan);\n+\tlcore = (uint32_t)rte_lcore_to_cpu_id(-1);\n+\tif (mlx5_glue->devx_query_eqn(sh->ctx, lcore, &sh->txpp.eqn)) {\n+\t\trte_errno = errno;\n+\t\tDRV_LOG(ERR, \"Failed to query EQ number %d.\", rte_errno);\n+\t\tsh->txpp.eqn = 0;\n+\t\treturn -rte_errno;\n+\t}\n+\tsh->txpp.echan = mlx5_glue->devx_create_event_channel(sh->ctx,\n+\t\t\tMLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA);\n+\tif (!sh->txpp.echan) {\n+\t\tsh->txpp.eqn = 0;\n+\t\trte_errno = errno;\n+\t\tDRV_LOG(ERR, \"Failed to create event channel %d.\",\n+\t\t\trte_errno);\n+\t\treturn -rte_errno;\n+\t}\n+\treturn 0;\n+}\n+\n+static void\n+mlx5_txpp_destroy_clock_queue(struct mlx5_dev_ctx_shared *sh)\n+{\n+\tstruct mlx5_txpp_wq *wq = &sh->txpp.clock_queue;\n+\n+\tif (wq->sq)\n+\t\tclaim_zero(mlx5_devx_cmd_destroy(wq->sq));\n+\tif (wq->sq_umem)\n+\t\tclaim_zero(mlx5_glue->devx_umem_dereg(wq->sq_umem));\n+\tif (wq->sq_buf)\n+\t\trte_free((void *)(uintptr_t)wq->sq_buf);\n+\tif (wq->cq)\n+\t\tclaim_zero(mlx5_devx_cmd_destroy(wq->cq));\n+\tif (wq->cq_umem)\n+\t\tclaim_zero(mlx5_glue->devx_umem_dereg(wq->cq_umem));\n+\tif (wq->cq_buf)\n+\t\trte_free((void *)(uintptr_t)wq->cq_buf);\n+\tmemset(wq, 0, sizeof(*wq));\n+}\n+\n+static void\n+mlx5_txpp_fill_wqe_clock_queue(struct mlx5_dev_ctx_shared *sh)\n+{\n+\tstruct mlx5_txpp_wq *wq = &sh->txpp.clock_queue;\n+\tstruct mlx5_wqe *wqe = (struct mlx5_wqe *)(uintptr_t)wq->wqes;\n+\tstruct mlx5_wqe_cseg *cs = &wqe->cseg;\n+\tuint32_t wqe_size, opcode, i;\n+\tuint8_t *dst;\n+\n+\t/* For test purposes fill the WQ with SEND inline packet. */\n+\tif (sh->txpp.test) {\n+\t\twqe_size = RTE_ALIGN(MLX5_TXPP_TEST_PKT_SIZE +\n+\t\t\t\t     MLX5_WQE_CSEG_SIZE +\n+\t\t\t\t     2 * MLX5_WQE_ESEG_SIZE -\n+\t\t\t\t     MLX5_ESEG_MIN_INLINE_SIZE,\n+\t\t\t\t     MLX5_WSEG_SIZE);\n+\t\topcode = MLX5_OPCODE_SEND;\n+\t} else {\n+\t\twqe_size = MLX5_WSEG_SIZE;\n+\t\topcode = MLX5_OPCODE_NOP;\n+\t}\n+\tcs->opcode = rte_cpu_to_be_32(opcode | 0); /* Index is ignored. */\n+\tcs->sq_ds = rte_cpu_to_be_32((wq->sq->id << 8) |\n+\t\t\t\t     (wqe_size / MLX5_WSEG_SIZE));\n+\tcs->flags = RTE_BE32(MLX5_COMP_ALWAYS << MLX5_COMP_MODE_OFFSET);\n+\tcs->misc = RTE_BE32(0);\n+\twqe_size = RTE_ALIGN(wqe_size, MLX5_WQE_SIZE);\n+\tif (sh->txpp.test) {\n+\t\tstruct mlx5_wqe_eseg *es = &wqe->eseg;\n+\t\tstruct rte_ether_hdr *eth_hdr;\n+\t\tstruct rte_ipv4_hdr *ip_hdr;\n+\t\tstruct rte_udp_hdr *udp_hdr;\n+\n+\t\t/* Build the inline test packet pattern. */\n+\t\tMLX5_ASSERT(wqe_size <= MLX5_WQE_SIZE_MAX);\n+\t\tMLX5_ASSERT(MLX5_TXPP_TEST_PKT_SIZE >=\n+\t\t\t\t(sizeof(struct rte_ether_hdr) +\n+\t\t\t\t sizeof(struct rte_ipv4_hdr)));\n+\t\tes->flags = 0;\n+\t\tes->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM;\n+\t\tes->swp_offs = 0;\n+\t\tes->metadata = 0;\n+\t\tes->swp_flags = 0;\n+\t\tes->mss = 0;\n+\t\tes->inline_hdr_sz = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE);\n+\t\t/* Build test packet L2 header (Ethernet). */\n+\t\tdst = (uint8_t *)&es->inline_data;\n+\t\teth_hdr = (struct rte_ether_hdr *)dst;\n+\t\trte_eth_random_addr(&eth_hdr->d_addr.addr_bytes[0]);\n+\t\trte_eth_random_addr(&eth_hdr->s_addr.addr_bytes[0]);\n+\t\teth_hdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);\n+\t\t/* Build test packet L3 header (IP v4). */\n+\t\tdst += sizeof(struct rte_ether_hdr);\n+\t\tip_hdr = (struct rte_ipv4_hdr *)dst;\n+\t\tip_hdr->version_ihl = RTE_IPV4_VHL_DEF;\n+\t\tip_hdr->type_of_service = 0;\n+\t\tip_hdr->fragment_offset = 0;\n+\t\tip_hdr->time_to_live = 64;\n+\t\tip_hdr->next_proto_id = IPPROTO_UDP;\n+\t\tip_hdr->packet_id = 0;\n+\t\tip_hdr->total_length = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE -\n+\t\t\t\t\t\tsizeof(struct rte_ether_hdr));\n+\t\t/* use RFC5735 / RFC2544 reserved network test addresses */\n+\t\tip_hdr->src_addr = RTE_BE32((198U << 24) | (18 << 16) |\n+\t\t\t\t\t    (0 << 8) | 1);\n+\t\tip_hdr->dst_addr = RTE_BE32((198U << 24) | (18 << 16) |\n+\t\t\t\t\t    (0 << 8) | 2);\n+\t\tif (MLX5_TXPP_TEST_PKT_SIZE <\n+\t\t\t\t\t(sizeof(struct rte_ether_hdr) +\n+\t\t\t\t\t sizeof(struct rte_ipv4_hdr) +\n+\t\t\t\t\t sizeof(struct rte_udp_hdr)))\n+\t\t\tgoto wcopy;\n+\t\t/* Build test packet L4 header (UDP). */\n+\t\tdst += sizeof(struct rte_ipv4_hdr);\n+\t\tudp_hdr = (struct rte_udp_hdr *)dst;\n+\t\tudp_hdr->src_port = RTE_BE16(9); /* RFC863 Discard. */\n+\t\tudp_hdr->dst_port = RTE_BE16(9);\n+\t\tudp_hdr->dgram_len = RTE_BE16(MLX5_TXPP_TEST_PKT_SIZE -\n+\t\t\t\t\t      sizeof(struct rte_ether_hdr) -\n+\t\t\t\t\t      sizeof(struct rte_ipv4_hdr));\n+\t\tudp_hdr->dgram_cksum = 0;\n+\t\t/* Fill the test packet data. */\n+\t\tdst += sizeof(struct rte_udp_hdr);\n+\t\tfor (i = sizeof(struct rte_ether_hdr) +\n+\t\t\tsizeof(struct rte_ipv4_hdr) +\n+\t\t\tsizeof(struct rte_udp_hdr);\n+\t\t\t\ti < MLX5_TXPP_TEST_PKT_SIZE; i++)\n+\t\t\t*dst++ = (uint8_t)(i & 0xFF);\n+\t}\n+wcopy:\n+\t/* Duplicate the pattern to the next WQEs. */\n+\tdst = (uint8_t *)(uintptr_t)wq->sq_buf;\n+\tfor (i = 1; i < MLX5_TXPP_CLKQ_SIZE; i++) {\n+\t\tdst += wqe_size;\n+\t\trte_memcpy(dst, (void *)(uintptr_t)wq->sq_buf, wqe_size);\n+\t}\n+}\n+\n+/* Creates the Clock Queue for packet pacing, returns zero on success. */\n+static int\n+mlx5_txpp_create_clock_queue(struct mlx5_dev_ctx_shared *sh)\n+{\n+\tstruct mlx5_devx_create_sq_attr sq_attr = { 0 };\n+\tstruct mlx5_devx_modify_sq_attr msq_attr = { 0 };\n+\tstruct mlx5_devx_cq_attr cq_attr = { 0 };\n+\tstruct mlx5_txpp_wq *wq = &sh->txpp.clock_queue;\n+\tsize_t page_size = sysconf(_SC_PAGESIZE);\n+\tuint32_t umem_size, umem_dbrec;\n+\tint ret;\n+\n+\t/* Allocate memory buffer for CQEs and doorbell record. */\n+\tumem_size = sizeof(struct mlx5_cqe) * MLX5_TXPP_CLKQ_SIZE;\n+\tumem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);\n+\tumem_size += MLX5_DBR_SIZE;\n+\twq->cq_buf = rte_zmalloc_socket(__func__, umem_size,\n+\t\t\t\t\tpage_size, sh->numa_node);\n+\tif (!wq->cq_buf) {\n+\t\tDRV_LOG(ERR, \"Failed to allocate memory for Clock Queue.\");\n+\t\treturn -ENOMEM;\n+\t}\n+\t/* Register allocated buffer in user space with DevX. */\n+\twq->cq_umem = mlx5_glue->devx_umem_reg(sh->ctx,\n+\t\t\t\t\t       (void *)(uintptr_t)wq->cq_buf,\n+\t\t\t\t\t       umem_size,\n+\t\t\t\t\t       IBV_ACCESS_LOCAL_WRITE);\n+\tif (!wq->cq_umem) {\n+\t\trte_errno = errno;\n+\t\tDRV_LOG(ERR, \"Failed to register umem for Clock Queue.\");\n+\t\tgoto error;\n+\t}\n+\t/* Create completion queue object for Clock Queue. */\n+\tcq_attr.cqe_size = (sizeof(struct mlx5_cqe) == 128) ?\n+\t\t\t    MLX5_CQE_SIZE_128B : MLX5_CQE_SIZE_64B;\n+\tcq_attr.use_first_only = 1;\n+\tcq_attr.overrun_ignore = 1;\n+\tcq_attr.uar_page_id = sh->tx_uar->page_id;\n+\tcq_attr.eqn = sh->txpp.eqn;\n+\tcq_attr.q_umem_valid = 1;\n+\tcq_attr.q_umem_offset = 0;\n+\tcq_attr.q_umem_id = wq->cq_umem->umem_id;\n+\tcq_attr.db_umem_valid = 1;\n+\tcq_attr.db_umem_offset = umem_dbrec;\n+\tcq_attr.db_umem_id = wq->cq_umem->umem_id;\n+\tcq_attr.log_cq_size = rte_log2_u32(MLX5_TXPP_CLKQ_SIZE);\n+\tcq_attr.log_page_size = rte_log2_u32(page_size);\n+\twq->cq = mlx5_devx_cmd_create_cq(sh->ctx, &cq_attr);\n+\tif (!wq->cq) {\n+\t\trte_errno = errno;\n+\t\tDRV_LOG(ERR, \"Failed to create CQ for Clock Queue.\");\n+\t\tgoto error;\n+\t}\n+\twq->cq_dbrec = RTE_PTR_ADD(wq->cq_buf, umem_dbrec);\n+\twq->cq_ci = 0;\n+\t/* Allocate memory buffer for Send Queue WQEs. */\n+\tif (sh->txpp.test) {\n+\t\twq->sq_size = RTE_ALIGN(MLX5_TXPP_TEST_PKT_SIZE +\n+\t\t\t\t\tMLX5_WQE_CSEG_SIZE +\n+\t\t\t\t\t2 * MLX5_WQE_ESEG_SIZE -\n+\t\t\t\t\tMLX5_ESEG_MIN_INLINE_SIZE,\n+\t\t\t\t\tMLX5_WQE_SIZE) / MLX5_WQE_SIZE;\n+\t\twq->sq_size *= MLX5_TXPP_CLKQ_SIZE;\n+\t} else {\n+\t\twq->sq_size = MLX5_TXPP_CLKQ_SIZE;\n+\t}\n+\t/* There should not be WQE leftovers in the cyclic queue. */\n+\tMLX5_ASSERT(wq->sq_size == (1 << log2above(wq->sq_size)));\n+\tumem_size =  MLX5_WQE_SIZE * wq->sq_size;\n+\tumem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE);\n+\tumem_size += MLX5_DBR_SIZE;\n+\twq->sq_buf = rte_zmalloc_socket(__func__, umem_size,\n+\t\t\t\t\tpage_size, sh->numa_node);\n+\tif (!wq->sq_buf) {\n+\t\tDRV_LOG(ERR, \"Failed to allocate memory for Clock Queue.\");\n+\t\trte_errno = ENOMEM;\n+\t\tgoto error;\n+\t}\n+\t/* Register allocated buffer in user space with DevX. */\n+\twq->sq_umem = mlx5_glue->devx_umem_reg(sh->ctx,\n+\t\t\t\t\t       (void *)(uintptr_t)wq->sq_buf,\n+\t\t\t\t\t       umem_size,\n+\t\t\t\t\t       IBV_ACCESS_LOCAL_WRITE);\n+\tif (!wq->sq_umem) {\n+\t\trte_errno = errno;\n+\t\tDRV_LOG(ERR, \"Failed to register umem for Clock Queue.\");\n+\t\tgoto error;\n+\t}\n+\t/* Create send queue object for Clock Queue. */\n+\tif (sh->txpp.test) {\n+\t\tsq_attr.tis_lst_sz = 1;\n+\t\tsq_attr.tis_num = sh->tis->id;\n+\t\tsq_attr.non_wire = 0;\n+\t\tsq_attr.static_sq_wq = 1;\n+\t} else {\n+\t\tsq_attr.non_wire = 1;\n+\t\tsq_attr.static_sq_wq = 1;\n+\t}\n+\tsq_attr.state = MLX5_SQC_STATE_RST;\n+\tsq_attr.cqn = wq->cq->id;\n+\tsq_attr.wq_attr.cd_slave = 1;\n+\tsq_attr.wq_attr.uar_page = sh->tx_uar->page_id;\n+\tsq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC;\n+\tsq_attr.wq_attr.pd = sh->pdn;\n+\tsq_attr.wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE);\n+\tsq_attr.wq_attr.log_wq_sz = rte_log2_u32(wq->sq_size);\n+\tsq_attr.wq_attr.dbr_umem_valid = 1;\n+\tsq_attr.wq_attr.dbr_addr = umem_dbrec;\n+\tsq_attr.wq_attr.dbr_umem_id = wq->sq_umem->umem_id;\n+\tsq_attr.wq_attr.wq_umem_valid = 1;\n+\tsq_attr.wq_attr.wq_umem_id = wq->sq_umem->umem_id;\n+\t/* umem_offset must be zero for static_sq_wq queue. */\n+\tsq_attr.wq_attr.wq_umem_offset = 0;\n+\twq->sq = mlx5_devx_cmd_create_sq(sh->ctx, &sq_attr);\n+\tif (!wq->sq) {\n+\t\trte_errno = errno;\n+\t\tDRV_LOG(ERR, \"Failed to create SQ for Clock Queue.\");\n+\t\tgoto error;\n+\t}\n+\twq->sq_dbrec = RTE_PTR_ADD(wq->sq_buf, umem_dbrec +\n+\t\t\t\t   MLX5_SND_DBR * sizeof(uint32_t));\n+\t/* Build the WQEs in the Send Queue before goto Ready state. */\n+\tmlx5_txpp_fill_wqe_clock_queue(sh);\n+\t/* Change queue state to ready. */\n+\tmsq_attr.sq_state = MLX5_SQC_STATE_RST;\n+\tmsq_attr.state = MLX5_SQC_STATE_RDY;\n+\twq->sq_ci = 0;\n+\tret = mlx5_devx_cmd_modify_sq(wq->sq, &msq_attr);\n+\tif (ret) {\n+\t\tDRV_LOG(ERR, \"Failed to set SQ ready state Clock Queue.\");\n+\t\tgoto error;\n+\t}\n+\treturn 0;\n+error:\n+\tret = -rte_errno;\n+\tmlx5_txpp_destroy_clock_queue(sh);\n+\trte_errno = -ret;\n+\treturn ret;\n+}\n+\n+/*\n+ * The routine initializes the packet pacing infrastructure:\n+ * - allocates PP context\n+ * - Clock CQ/SQ\n+ * - Rearm CQ/SQ\n+ * - attaches rearm interrupt handler\n+ *\n+ * Returns 0 on success, negative otherwise\n+ */\n+static int\n+mlx5_txpp_create(struct mlx5_dev_ctx_shared *sh, struct mlx5_priv *priv)\n+{\n+\tint tx_pp = priv->config.tx_pp;\n+\tint ret;\n+\n+\t/* Store the requested pacing parameters. */\n+\tsh->txpp.tick = tx_pp >= 0 ? tx_pp : -tx_pp;\n+\tsh->txpp.test = !!(tx_pp < 0);\n+\tsh->txpp.skew = priv->config.tx_skew;\n+\tsh->txpp.freq = priv->config.hca_attr.dev_freq_khz;\n+\tret = mlx5_txpp_create_eqn(sh);\n+\tif (ret)\n+\t\tgoto exit;\n+\tret = mlx5_txpp_create_clock_queue(sh);\n+\tif (ret)\n+\t\tgoto exit;\n+exit:\n+\tif (ret) {\n+\t\tmlx5_txpp_destroy_clock_queue(sh);\n+\t\tmlx5_txpp_destroy_eqn(sh);\n+\t\tsh->txpp.tick = 0;\n+\t\tsh->txpp.test = 0;\n+\t\tsh->txpp.skew = 0;\n+\t}\n+\treturn ret;\n+}\n+\n+/*\n+ * The routine destroys the packet pacing infrastructure:\n+ * - detaches rearm interrupt handler\n+ * - Rearm CQ/SQ\n+ * - Clock CQ/SQ\n+ * - PP context\n+ */\n+static void\n+mlx5_txpp_destroy(struct mlx5_dev_ctx_shared *sh)\n+{\n+\tmlx5_txpp_destroy_clock_queue(sh);\n+\tmlx5_txpp_destroy_eqn(sh);\n+\tsh->txpp.tick = 0;\n+\tsh->txpp.test = 0;\n+\tsh->txpp.skew = 0;\n+}\n+\n+/**\n+ * Creates and starts packet pacing infrastructure on specified device.\n+ *\n+ * @param dev\n+ *   Pointer to Ethernet device structure.\n+ *\n+ * @return\n+ *   0 on success, a negative errno value otherwise and rte_errno is set.\n+ */\n+int\n+mlx5_txpp_start(struct rte_eth_dev *dev)\n+{\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\tstruct mlx5_dev_ctx_shared *sh = priv->sh;\n+\tint err = 0;\n+\tint ret;\n+\n+\tif (!priv->config.tx_pp) {\n+\t\t/* Packet pacing is not requested for the device. */\n+\t\tMLX5_ASSERT(priv->txpp_en == 0);\n+\t\treturn 0;\n+\t}\n+\tif (priv->txpp_en) {\n+\t\t/* Packet pacing is already enabled for the device. */\n+\t\tMLX5_ASSERT(sh->txpp.refcnt);\n+\t\treturn 0;\n+\t}\n+\tif (priv->config.tx_pp > 0) {\n+\t\tret = rte_mbuf_dynflag_lookup\n+\t\t\t\t(RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL);\n+\t\tif (ret < 0)\n+\t\t\treturn 0;\n+\t}\n+\tret = pthread_mutex_lock(&sh->txpp.mutex);\n+\tMLX5_ASSERT(!ret);\n+\tRTE_SET_USED(ret);\n+\tif (sh->txpp.refcnt) {\n+\t\tpriv->txpp_en = 1;\n+\t\t++sh->txpp.refcnt;\n+\t} else {\n+\t\terr = mlx5_txpp_create(sh, priv);\n+\t\tif (!err) {\n+\t\t\tMLX5_ASSERT(sh->txpp.tick);\n+\t\t\tpriv->txpp_en = 1;\n+\t\t\tsh->txpp.refcnt = 1;\n+\t\t} else {\n+\t\t\trte_errno = -err;\n+\t\t}\n+\t}\n+\tret = pthread_mutex_unlock(&sh->txpp.mutex);\n+\tMLX5_ASSERT(!ret);\n+\tRTE_SET_USED(ret);\n+\treturn err;\n+}\n+\n+/**\n+ * Stops and destroys packet pacing infrastructure on specified device.\n+ *\n+ * @param dev\n+ *   Pointer to Ethernet device structure.\n+ *\n+ * @return\n+ *   0 on success, a negative errno value otherwise and rte_errno is set.\n+ */\n+void\n+mlx5_txpp_stop(struct rte_eth_dev *dev)\n+{\n+\tstruct mlx5_priv *priv = dev->data->dev_private;\n+\tstruct mlx5_dev_ctx_shared *sh = priv->sh;\n+\tint ret;\n+\n+\tif (!priv->txpp_en) {\n+\t\t/* Packet pacing is already disabled for the device. */\n+\t\treturn;\n+\t}\n+\tpriv->txpp_en = 0;\n+\tret = pthread_mutex_lock(&sh->txpp.mutex);\n+\tMLX5_ASSERT(!ret);\n+\tRTE_SET_USED(ret);\n+\tMLX5_ASSERT(sh->txpp.refcnt);\n+\tif (!sh->txpp.refcnt || --sh->txpp.refcnt)\n+\t\treturn;\n+\t/* No references any more, do actual destroy. */\n+\tmlx5_txpp_destroy(sh);\n+\tret = pthread_mutex_unlock(&sh->txpp.mutex);\n+\tMLX5_ASSERT(!ret);\n+\tRTE_SET_USED(ret);\n+}\n",
    "prefixes": [
        "v2",
        "05/17"
    ]
}