@@ -552,6 +552,27 @@ Run-time configuration
Also, if minimal data inlining is requested by non-zero ``txq_inline_min``
option or reported by the NIC, the eMPW feature is disengaged.
+- ``tx_db_nc`` parameter [int]
+
+ The rdma core library can map doorbell register in two ways, depending on the
+ environment variable "MLX5_SHUT_UP_BF":
+
+ - As regular cached memory, if the variable is either missing or set to zero.
+ - As non-cached memory, if the variable is present and set to not "0" value.
+
+ The type of mapping may slightly affect the Tx performance, the optimal choice
+ is strongly relied on the host architecture and should be deduced practically.
+
+ If ``tx_db_nc`` is either omitted or set to zero, the doorbell is forced to be
+ mapped to regular memory, the PMD will perform the extra write memory barrier
+ after writing to doorbell, it might increase the needed CPU clocks per packet
+ to send, but latency might be improved.
+
+ If ``tx_db_nc`` is set to not zero, the doorbell is forced to be mapped to
+ non cached memory, the PMD will not perform the extra write memory barrier
+ after writing to doorbell, on some architectures it might improve the
+ performance.
+
- ``tx_vec_en`` parameter [int]
A nonzero value enables Tx vector on ConnectX-5, ConnectX-6, ConnectX-6 DX
@@ -198,6 +198,11 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
func mlx5dv_dr_action_create_dest_devx_tir \
$(AUTOCONF_OUTPUT)
$Q sh -- '$<' '$@' \
+ HAVE_MLX5DV_MMAP_GET_NC_PAGES_CMD \
+ infiniband/mlx5dv.h \
+ enum MLX5_MMAP_GET_NC_PAGES_CMD \
+ $(AUTOCONF_OUTPUT)
+ $Q sh -- '$<' '$@' \
HAVE_ETHTOOL_LINK_MODE_25G \
/usr/include/linux/ethtool.h \
enum ETHTOOL_LINK_MODE_25000baseCR_Full_BIT \
@@ -129,6 +129,8 @@ if build
'mlx5dv_devx_obj_query_async' ],
[ 'HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR', 'infiniband/mlx5dv.h',
'mlx5dv_dr_action_create_dest_devx_tir' ],
+ [ 'HAVE_MLX5DV_MMAP_GET_NC_PAGES_CMD', 'infiniband/mlx5dv.h',
+ 'MLX5_MMAP_GET_NC_PAGES_CMD' ],
[ 'HAVE_MLX5DV_DR', 'infiniband/mlx5dv.h',
'MLX5DV_DR_DOMAIN_TYPE_NIC_RX' ],
[ 'HAVE_MLX5DV_DR_ESWITCH', 'infiniband/mlx5dv.h',
@@ -47,6 +47,9 @@
#include "mlx5_mr.h"
#include "mlx5_flow.h"
+/* Environment variable to control the doorbell register mapping. */
+#define MLX5_SHUT_UP_BF "MLX5_SHUT_UP_BF"
+
/* Device parameter to enable RX completion queue compression. */
#define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en"
@@ -96,6 +99,12 @@
#define MLX5_TXQ_MPW_EN "txq_mpw_en"
/*
+ * Device parameter to force doorbell register mapping
+ * to non-cahed region eliminating the extra write memory barrier.
+ */
+#define MLX5_TX_DB_NC "tx_db_nc"
+
+/*
* Device parameter to include 2 dsegs in the title WQEBB.
* Deprecated, ignored.
*/
@@ -418,6 +427,37 @@ struct mlx5_flow_id_pool *
}
#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+static int
+mlx5_config_doorbell_mapping_env(const struct mlx5_dev_config *config)
+{
+ char *env;
+ int value;
+
+ assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
+ /* The mapping type was not specified, follow the default. */
+ if (config->dbnc == MLX5_ARG_UNSET)
+ return MLX5_ARG_UNSET;
+ /* Get environment variable to store. */
+ env = getenv(MLX5_SHUT_UP_BF);
+ value = env ? !!strcmp(env, "0") : MLX5_ARG_UNSET;
+ setenv(MLX5_SHUT_UP_BF, config->dbnc ? "1" : "0", 1);
+ return value;
+}
+
+static void
+mlx5_restore_doorbell_mapping_env(const struct mlx5_dev_config *config,
+ int value)
+{
+ assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
+ if (config->dbnc == MLX5_ARG_UNSET)
+ return;
+ /* Restore the original environment variable state. */
+ if (value == MLX5_ARG_UNSET)
+ unsetenv(MLX5_SHUT_UP_BF);
+ else
+ setenv(MLX5_SHUT_UP_BF, value ? "1" : "0", 1);
+}
+
/**
* Allocate shared IB device context. If there is multiport device the
* master and representors will share this context, if there is single
@@ -431,22 +471,26 @@ struct mlx5_flow_id_pool *
*
* @param[in] spawn
* Pointer to the IB device attributes (name, port, etc).
+ * @param[in] config
+ * Pointer to device configuration structure.
*
* @return
* Pointer to mlx5_ibv_shared object on success,
* otherwise NULL and rte_errno is set.
*/
static struct mlx5_ibv_shared *
-mlx5_alloc_shared_ibctx(const struct mlx5_dev_spawn_data *spawn)
+mlx5_alloc_shared_ibctx(const struct mlx5_dev_spawn_data *spawn,
+ const struct mlx5_dev_config *config)
{
struct mlx5_ibv_shared *sh;
+ int dbmap_env;
int err = 0;
uint32_t i;
#ifdef HAVE_IBV_FLOW_DV_SUPPORT
struct mlx5_devx_tis_attr tis_attr = { 0 };
#endif
-assert(spawn);
+ assert(spawn);
/* Secondary process should not create the shared context. */
assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
pthread_mutex_lock(&mlx5_ibv_list_mutex);
@@ -469,16 +513,31 @@ struct mlx5_flow_id_pool *
rte_errno = ENOMEM;
goto exit;
}
+ /*
+ * Configure environment variable "MLX5_BF_SHUT_UP"
+ * before the device creation. The rdma_core library
+ * checks the variable at device creation and
+ * stores the result internally.
+ */
+ dbmap_env = mlx5_config_doorbell_mapping_env(config);
/* Try to open IB device with DV first, then usual Verbs. */
errno = 0;
sh->ctx = mlx5_glue->dv_open_device(spawn->ibv_dev);
if (sh->ctx) {
sh->devx = 1;
DRV_LOG(DEBUG, "DevX is supported");
+ /* The device is created, no need for environment. */
+ mlx5_restore_doorbell_mapping_env(config, dbmap_env);
} else {
+ /* The environment variable is still configured. */
sh->ctx = mlx5_glue->open_device(spawn->ibv_dev);
+ err = errno ? errno : ENODEV;
+ /*
+ * The environment variable is not needed anymore,
+ * all device creation attempts are completed.
+ */
+ mlx5_restore_doorbell_mapping_env(config, dbmap_env);
if (!sh->ctx) {
- err = errno ? errno : ENODEV;
goto error;
}
DRV_LOG(DEBUG, "DevX is NOT supported");
@@ -1292,6 +1351,8 @@ struct mlx5_flow_id_pool *
DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key);
} else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) {
config->mps = !!tmp;
+ } else if (strcmp(MLX5_TX_DB_NC, key) == 0) {
+ config->dbnc = !!tmp;
} else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) {
DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key);
} else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) {
@@ -1355,6 +1416,7 @@ struct mlx5_flow_id_pool *
MLX5_TXQ_MPW_EN,
MLX5_TXQ_MPW_HDR_DSEG_EN,
MLX5_TXQ_MAX_INLINE_LEN,
+ MLX5_TX_DB_NC,
MLX5_TX_VEC_EN,
MLX5_RX_VEC_EN,
MLX5_L3_VXLAN_EN,
@@ -1859,7 +1921,20 @@ struct mlx5_flow_id_pool *
eth_dev->tx_pkt_burst = mlx5_select_tx_function(eth_dev);
return eth_dev;
}
- sh = mlx5_alloc_shared_ibctx(spawn);
+ /*
+ * Some parameters ("tx_db_nc" in particularly) are needed in
+ * advance to create dv/verbs device context. We proceed the
+ * devargs here to get ones, and later proceed devargs again
+ * to override some hardware settings.
+ */
+ err = mlx5_args(&config, dpdk_dev->devargs);
+ if (err) {
+ err = rte_errno;
+ DRV_LOG(ERR, "failed to process device arguments: %s",
+ strerror(rte_errno));
+ goto error;
+ }
+ sh = mlx5_alloc_shared_ibctx(spawn, &config);
if (!sh)
return NULL;
config.devx = sh->devx;
@@ -2097,13 +2172,8 @@ struct mlx5_flow_id_pool *
}
own_domain_id = 1;
}
- err = mlx5_args(&config, dpdk_dev->devargs);
- if (err) {
- err = rte_errno;
- DRV_LOG(ERR, "failed to process device arguments: %s",
- strerror(rte_errno));
- goto error;
- }
+ /* Override some values set by hardware configuration. */
+ mlx5_args(&config, dpdk_dev->devargs);
err = mlx5_dev_check_sibling_config(priv, &config);
if (err)
goto error;
@@ -2868,6 +2938,7 @@ struct mlx5_flow_id_pool *
dev_config = (struct mlx5_dev_config){
.hw_padding = 0,
.mps = MLX5_ARG_UNSET,
+ .dbnc = MLX5_ARG_UNSET,
.rx_vec_en = 1,
.txq_inline_max = MLX5_ARG_UNSET,
.txq_inline_min = MLX5_ARG_UNSET,
@@ -253,6 +253,7 @@ struct mlx5_dev_config {
/* Rx queue count threshold to enable MPRQ. */
} mprq; /* Configurations for Multi-Packet RQ. */
int mps; /* Multi-packet send supported mode. */
+ int dbnc; /* Skip doorbell register write barrier. */
unsigned int flow_prio; /* Number of flow priorities. */
unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
unsigned int ind_table_max_size; /* Maximum indirection table size. */
@@ -123,6 +123,14 @@
#define MLX5_UAR_PAGE_NUM_MAX 64
#define MLX5_UAR_PAGE_NUM_MASK ((MLX5_UAR_PAGE_NUM_MAX) - 1)
+/* Fields of memory mapping type in offset parameter of mmap() */
+#define MLX5_UAR_MMAP_CMD_SHIFT 8
+#define MLX5_UAR_MMAP_CMD_MASK 0xff
+
+#ifndef HAVE_MLX5DV_MMAP_GET_NC_PAGES_CMD
+#define MLX5_MMAP_GET_NC_PAGES_CMD 3
+#endif
+
/* Log 2 of the default number of strides per WQE for Multi-Packet RQ. */
#define MLX5_MPRQ_STRIDE_NUM_N 6U
@@ -4749,8 +4749,23 @@ enum mlx5_txcmp_code {
* to improve latencies. The pure software related data treatment
* can be completed after doorbell. Tx CQEs for this SQ are
* processed in this thread only by the polling.
+ *
+ * The rdma core library can map doorbell register in two ways,
+ * depending on the environment variable "MLX5_SHUT_UP_BF":
+ *
+ * - as regular cached memory, the variable is either missing or
+ * set to zero. This type of mapping may cause the significant
+ * doorbell register writing latency and requires explicit
+ * memory write barrier to mitigate this issue and prevent
+ * write combining.
+ *
+ * - as non-cached memory, the variable is present and set to
+ * not "0" value. This type of mapping may cause performance
+ * impact under heavy loading conditions but the explicit write
+ * memory barrier is not required and it may improve core
+ * performance.
*/
- mlx5_tx_dbrec_cond_wmb(txq, loc.wqe_last, 0);
+ mlx5_tx_dbrec_cond_wmb(txq, loc.wqe_last, !txq->db_nc);
/* Not all of the mbufs may be stored into elts yet. */
part = MLX5_TXOFF_CONFIG(INLINE) ? 0 : loc.pkts_sent - loc.pkts_copy;
if (!MLX5_TXOFF_CONFIG(INLINE) && part) {
@@ -287,6 +287,7 @@ struct mlx5_txq_data {
/* When set TX offload for tunneled packets are supported. */
uint16_t swp_en:1; /* Whether SW parser is enabled. */
uint16_t vlan_en:1; /* VLAN insertion in WQE is supported. */
+ uint16_t db_nc:1; /* Doorbell mapped to non-cached region. */
uint16_t inlen_send; /* Ordinary send data inline size. */
uint16_t inlen_empw; /* eMPW max packet size to inline. */
uint16_t inlen_mode; /* Minimal data length to inline. */
@@ -18,6 +18,7 @@
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
#ifdef PEDANTIC
#pragma GCC diagnostic error "-Wpedantic"
#endif
@@ -302,6 +303,28 @@
}
/**
+ * Configure the doorbell register non-cached attribute.
+ *
+ * @param txq_ctrl
+ * Pointer to Tx queue control structure.
+ * @param page_size
+ * Systme page size
+ */
+static void
+txq_uar_ncattr_init(struct mlx5_txq_ctrl *txq_ctrl, size_t page_size)
+{
+ unsigned int cmd;
+
+ txq_ctrl->txq.db_nc = 0;
+ /* Check the doorbell register mapping type. */
+ cmd = txq_ctrl->uar_mmap_offset / page_size;
+ cmd >>= MLX5_UAR_MMAP_CMD_SHIFT;
+ cmd &= MLX5_UAR_MMAP_CMD_MASK;
+ if (cmd == MLX5_MMAP_GET_NC_PAGES_CMD)
+ txq_ctrl->txq.db_nc = 1;
+}
+
+/**
* Initialize Tx UAR registers for primary process.
*
* @param txq_ctrl
@@ -312,9 +335,9 @@
{
struct mlx5_priv *priv = txq_ctrl->priv;
struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(priv));
+ const size_t page_size = sysconf(_SC_PAGESIZE);
#ifndef RTE_ARCH_64
unsigned int lock_idx;
- const size_t page_size = sysconf(_SC_PAGESIZE);
#endif
if (txq_ctrl->type != MLX5_TXQ_TYPE_STANDARD)
@@ -322,6 +345,7 @@
assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
assert(ppriv);
ppriv->uar_table[txq_ctrl->txq.idx] = txq_ctrl->bf_reg;
+ txq_uar_ncattr_init(txq_ctrl, page_size);
#ifndef RTE_ARCH_64
/* Assign an UAR lock according to UAR page number */
lock_idx = (txq_ctrl->uar_mmap_offset / page_size) &
@@ -375,6 +399,7 @@
}
addr = RTE_PTR_ADD(addr, offset);
ppriv->uar_table[txq->idx] = addr;
+ txq_uar_ncattr_init(txq_ctrl, page_size);
return 0;
}