new file mode 100644
@@ -0,0 +1,430 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#include <unistd.h>
+#include <string.h>
+#include <stdio.h>
+#ifdef RTE_IBVERBS_LINK_DLOPEN
+#include <dlfcn.h>
+#endif
+#include <dirent.h>
+#include <net/if.h>
+
+#include <rte_errno.h>
+#include <rte_string_fns.h>
+
+#include "mlx5_common.h"
+#include "mlx5_common_log.h"
+#include "mlx5_common_os.h"
+#include "mlx5_glue.h"
+
+#ifdef MLX5_GLUE
+const struct mlx5_glue *mlx5_glue;
+#endif
+
+int
+mlx5_get_pci_addr(const char *dev_path, struct rte_pci_addr *pci_addr)
+{
+ FILE *file;
+ char line[32];
+ int rc = -ENOENT;
+ MKSTR(path, "%s/device/uevent", dev_path);
+
+ file = fopen(path, "rb");
+ if (file == NULL) {
+ rte_errno = errno;
+ return -rte_errno;
+ }
+ while (fgets(line, sizeof(line), file) == line) {
+ size_t len = strlen(line);
+
+ /* Truncate long lines. */
+ if (len == (sizeof(line) - 1)) {
+ while (line[(len - 1)] != '\n') {
+ int ret = fgetc(file);
+
+ if (ret == EOF)
+ goto exit;
+ line[(len - 1)] = ret;
+ }
+ /* No match for long lines. */
+ continue;
+ }
+ /* Extract information. */
+ if (sscanf(line,
+ "PCI_SLOT_NAME="
+ "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n",
+ &pci_addr->domain,
+ &pci_addr->bus,
+ &pci_addr->devid,
+ &pci_addr->function) == 4) {
+ rc = 0;
+ break;
+ }
+ }
+exit:
+ fclose(file);
+ if (rc)
+ rte_errno = -rc;
+ return rc;
+}
+
+/**
+ * Extract port name, as a number, from sysfs or netlink information.
+ *
+ * @param[in] port_name_in
+ * String representing the port name.
+ * @param[out] port_info_out
+ * Port information, including port name as a number and port name
+ * type if recognized
+ *
+ * @return
+ * port_name field set according to recognized name format.
+ */
+void
+mlx5_translate_port_name(const char *port_name_in,
+ struct mlx5_switch_info *port_info_out)
+{
+ char ctrl = 0, pf_c1, pf_c2, vf_c1, vf_c2, eol;
+ char *end;
+ int sc_items;
+
+ sc_items = sscanf(port_name_in, "%c%d",
+ &ctrl, &port_info_out->ctrl_num);
+ if (sc_items == 2 && ctrl == 'c') {
+ port_name_in++; /* 'c' */
+ port_name_in += snprintf(NULL, 0, "%d",
+ port_info_out->ctrl_num);
+ }
+ /* Check for port-name as a string of the form pf0vf0 or pf0sf0 */
+ sc_items = sscanf(port_name_in, "%c%c%d%c%c%d%c",
+ &pf_c1, &pf_c2, &port_info_out->pf_num,
+ &vf_c1, &vf_c2, &port_info_out->port_name, &eol);
+ if (sc_items == 6 && pf_c1 == 'p' && pf_c2 == 'f') {
+ if (vf_c1 == 'v' && vf_c2 == 'f') {
+ /* Kernel ver >= 5.0 or OFED ver >= 4.6 */
+ port_info_out->name_type =
+ MLX5_PHYS_PORT_NAME_TYPE_PFVF;
+ return;
+ }
+ if (vf_c1 == 's' && vf_c2 == 'f') {
+ /* Kernel ver >= 5.11 or OFED ver >= 5.1 */
+ port_info_out->name_type =
+ MLX5_PHYS_PORT_NAME_TYPE_PFSF;
+ return;
+ }
+ }
+ /*
+ * Check for port-name as a string of the form p0
+ * (support kernel ver >= 5.0, or OFED ver >= 4.6).
+ */
+ sc_items = sscanf(port_name_in, "%c%d%c",
+ &pf_c1, &port_info_out->port_name, &eol);
+ if (sc_items == 2 && pf_c1 == 'p') {
+ port_info_out->name_type = MLX5_PHYS_PORT_NAME_TYPE_UPLINK;
+ return;
+ }
+ /*
+ * Check for port-name as a string of the form pf0
+ * (support kernel ver >= 5.7 for HPF representor on BF).
+ */
+ sc_items = sscanf(port_name_in, "%c%c%d%c",
+ &pf_c1, &pf_c2, &port_info_out->pf_num, &eol);
+ if (sc_items == 3 && pf_c1 == 'p' && pf_c2 == 'f') {
+ port_info_out->port_name = -1;
+ port_info_out->name_type = MLX5_PHYS_PORT_NAME_TYPE_PFHPF;
+ return;
+ }
+ /* Check for port-name as a number (support kernel ver < 5.0 */
+ errno = 0;
+ port_info_out->port_name = strtol(port_name_in, &end, 0);
+ if (!errno &&
+ (size_t)(end - port_name_in) == strlen(port_name_in)) {
+ port_info_out->name_type = MLX5_PHYS_PORT_NAME_TYPE_LEGACY;
+ return;
+ }
+ port_info_out->name_type = MLX5_PHYS_PORT_NAME_TYPE_UNKNOWN;
+}
+
+int
+mlx5_get_ifname_sysfs(const char *ibdev_path, char *ifname)
+{
+ DIR *dir;
+ struct dirent *dent;
+ unsigned int dev_type = 0;
+ unsigned int dev_port_prev = ~0u;
+ char match[IF_NAMESIZE] = "";
+
+ MLX5_ASSERT(ibdev_path);
+ {
+ MKSTR(path, "%s/device/net", ibdev_path);
+
+ dir = opendir(path);
+ if (dir == NULL) {
+ rte_errno = errno;
+ return -rte_errno;
+ }
+ }
+ while ((dent = readdir(dir)) != NULL) {
+ char *name = dent->d_name;
+ FILE *file;
+ unsigned int dev_port;
+ int r;
+
+ if ((name[0] == '.') &&
+ ((name[1] == '\0') ||
+ ((name[1] == '.') && (name[2] == '\0'))))
+ continue;
+
+ MKSTR(path, "%s/device/net/%s/%s",
+ ibdev_path, name,
+ (dev_type ? "dev_id" : "dev_port"));
+
+ file = fopen(path, "rb");
+ if (file == NULL) {
+ if (errno != ENOENT)
+ continue;
+ /*
+ * Switch to dev_id when dev_port does not exist as
+ * is the case with Linux kernel versions < 3.15.
+ */
+try_dev_id:
+ match[0] = '\0';
+ if (dev_type)
+ break;
+ dev_type = 1;
+ dev_port_prev = ~0u;
+ rewinddir(dir);
+ continue;
+ }
+ r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port);
+ fclose(file);
+ if (r != 1)
+ continue;
+ /*
+ * Switch to dev_id when dev_port returns the same value for
+ * all ports. May happen when using a MOFED release older than
+ * 3.0 with a Linux kernel >= 3.15.
+ */
+ if (dev_port == dev_port_prev)
+ goto try_dev_id;
+ dev_port_prev = dev_port;
+ if (dev_port == 0)
+ strlcpy(match, name, IF_NAMESIZE);
+ }
+ closedir(dir);
+ if (match[0] == '\0') {
+ rte_errno = ENOENT;
+ return -rte_errno;
+ }
+ strncpy(ifname, match, IF_NAMESIZE);
+ return 0;
+}
+
+#ifdef MLX5_GLUE
+
+/**
+ * Suffix RTE_EAL_PMD_PATH with "-glue".
+ *
+ * This function performs a sanity check on RTE_EAL_PMD_PATH before
+ * suffixing its last component.
+ *
+ * @param buf[out]
+ * Output buffer, should be large enough otherwise NULL is returned.
+ * @param size
+ * Size of @p out.
+ *
+ * @return
+ * Pointer to @p buf or @p NULL in case suffix cannot be appended.
+ */
+static char *
+mlx5_glue_path(char *buf, size_t size)
+{
+ static const char *const bad[] = { "/", ".", "..", NULL };
+ const char *path = RTE_EAL_PMD_PATH;
+ size_t len = strlen(path);
+ size_t off;
+ int i;
+
+ while (len && path[len - 1] == '/')
+ --len;
+ for (off = len; off && path[off - 1] != '/'; --off)
+ ;
+ for (i = 0; bad[i]; ++i)
+ if (!strncmp(path + off, bad[i], (int)(len - off)))
+ goto error;
+ i = snprintf(buf, size, "%.*s-glue", (int)len, path);
+ if (i == -1 || (size_t)i >= size)
+ goto error;
+ return buf;
+error:
+ RTE_LOG(ERR, PMD, "unable to append \"-glue\" to last component of"
+ " RTE_EAL_PMD_PATH (\"" RTE_EAL_PMD_PATH "\"), please"
+ " re-configure DPDK");
+ return NULL;
+}
+
+static int
+mlx5_glue_dlopen(void)
+{
+ char glue_path[sizeof(RTE_EAL_PMD_PATH) - 1 + sizeof("-glue")];
+ void *handle = NULL;
+
+ char const *path[] = {
+ /*
+ * A basic security check is necessary before trusting
+ * MLX5_GLUE_PATH, which may override RTE_EAL_PMD_PATH.
+ */
+ (geteuid() == getuid() && getegid() == getgid() ?
+ getenv("MLX5_GLUE_PATH") : NULL),
+ /*
+ * When RTE_EAL_PMD_PATH is set, use its glue-suffixed
+ * variant, otherwise let dlopen() look up libraries on its
+ * own.
+ */
+ (*RTE_EAL_PMD_PATH ?
+ mlx5_glue_path(glue_path, sizeof(glue_path)) : ""),
+ };
+ unsigned int i = 0;
+ void **sym;
+ const char *dlmsg;
+
+ while (!handle && i != RTE_DIM(path)) {
+ const char *end;
+ size_t len;
+ int ret;
+
+ if (!path[i]) {
+ ++i;
+ continue;
+ }
+ end = strpbrk(path[i], ":;");
+ if (!end)
+ end = path[i] + strlen(path[i]);
+ len = end - path[i];
+ ret = 0;
+ do {
+ char name[ret + 1];
+
+ ret = snprintf(name, sizeof(name), "%.*s%s" MLX5_GLUE,
+ (int)len, path[i],
+ (!len || *(end - 1) == '/') ? "" : "/");
+ if (ret == -1)
+ break;
+ if (sizeof(name) != (size_t)ret + 1)
+ continue;
+ DRV_LOG(DEBUG, "Looking for rdma-core glue as "
+ "\"%s\"", name);
+ handle = dlopen(name, RTLD_LAZY);
+ break;
+ } while (1);
+ path[i] = end + 1;
+ if (!*end)
+ ++i;
+ }
+ if (!handle) {
+ rte_errno = EINVAL;
+ dlmsg = dlerror();
+ if (dlmsg)
+ DRV_LOG(WARNING, "Cannot load glue library: %s", dlmsg);
+ goto glue_error;
+ }
+ sym = dlsym(handle, "mlx5_glue");
+ if (!sym || !*sym) {
+ rte_errno = EINVAL;
+ dlmsg = dlerror();
+ if (dlmsg)
+ DRV_LOG(ERR, "Cannot resolve glue symbol: %s", dlmsg);
+ goto glue_error;
+ }
+ mlx5_glue = *sym;
+ return 0;
+
+glue_error:
+ if (handle)
+ dlclose(handle);
+ return -1;
+}
+
+#endif
+
+/**
+ * Initialization routine for run-time dependency on rdma-core.
+ */
+void
+mlx5_glue_constructor(void)
+{
+ /*
+ * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
+ * huge pages. Calling ibv_fork_init() during init allows
+ * applications to use fork() safely for purposes other than
+ * using this PMD, which is not supported in forked processes.
+ */
+ setenv("RDMAV_HUGEPAGES_SAFE", "1", 1);
+ /* Match the size of Rx completion entry to the size of a cacheline. */
+ if (RTE_CACHE_LINE_SIZE == 128)
+ setenv("MLX5_CQE_SIZE", "128", 0);
+ /*
+ * MLX5_DEVICE_FATAL_CLEANUP tells ibv_destroy functions to
+ * cleanup all the Verbs resources even when the device was removed.
+ */
+ setenv("MLX5_DEVICE_FATAL_CLEANUP", "1", 1);
+
+#ifdef MLX5_GLUE
+ if (mlx5_glue_dlopen() != 0)
+ goto glue_error;
+#endif
+
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+ /* Glue structure must not contain any NULL pointers. */
+ {
+ unsigned int i;
+
+ for (i = 0; i != sizeof(*mlx5_glue) / sizeof(void *); ++i)
+ MLX5_ASSERT(((const void *const *)mlx5_glue)[i]);
+ }
+#endif
+ if (strcmp(mlx5_glue->version, MLX5_GLUE_VERSION)) {
+ rte_errno = EINVAL;
+ DRV_LOG(ERR, "rdma-core glue \"%s\" mismatch: \"%s\" is "
+ "required", mlx5_glue->version, MLX5_GLUE_VERSION);
+ goto glue_error;
+ }
+ mlx5_glue->fork_init();
+ return;
+
+glue_error:
+ DRV_LOG(WARNING, "Cannot initialize MLX5 common due to missing"
+ " run-time dependency on rdma-core libraries (libibverbs,"
+ " libmlx5)");
+ mlx5_glue = NULL;
+}
+
+struct ibv_device *
+mlx5_os_get_ibv_device(const struct rte_pci_addr *addr)
+{
+ int n;
+ struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
+ struct ibv_device *ibv_match = NULL;
+
+ if (ibv_list == NULL) {
+ rte_errno = ENOSYS;
+ return NULL;
+ }
+ while (n-- > 0) {
+ struct rte_pci_addr paddr;
+
+ DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
+ if (mlx5_get_pci_addr(ibv_list[n]->ibdev_path, &paddr) != 0)
+ continue;
+ if (rte_pci_addr_cmp(addr, &paddr) != 0)
+ continue;
+ ibv_match = ibv_list[n];
+ break;
+ }
+ if (ibv_match == NULL)
+ rte_errno = ENOENT;
+ mlx5_glue->free_device_list(ibv_list);
+ return ibv_match;
+}
new file mode 100644
@@ -0,0 +1,299 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_COMMON_OS_H_
+#define RTE_PMD_MLX5_COMMON_OS_H_
+
+#include <stdio.h>
+#include <malloc.h>
+
+#include <rte_pci.h>
+#include <rte_debug.h>
+#include <rte_atomic.h>
+#include <rte_log.h>
+#include <rte_kvargs.h>
+#include <rte_devargs.h>
+
+#include "mlx5_autoconf.h"
+#include "mlx5_glue.h"
+#include "mlx5_malloc.h"
+
+/**
+ * Get device name. Given an ibv_device pointer - return a
+ * pointer to the corresponding device name.
+ *
+ * @param[in] dev
+ * Pointer to ibv device.
+ *
+ * @return
+ * Pointer to device name if dev is valid, NULL otherwise.
+ */
+static inline const char *
+mlx5_os_get_dev_device_name(void *dev)
+{
+ if (!dev)
+ return NULL;
+ return ((struct ibv_device *)dev)->name;
+}
+
+/**
+ * Get ibv device name. Given an ibv_context pointer - return a
+ * pointer to the corresponding device name.
+ *
+ * @param[in] ctx
+ * Pointer to ibv context.
+ *
+ * @return
+ * Pointer to device name if ctx is valid, NULL otherwise.
+ */
+static inline const char *
+mlx5_os_get_ctx_device_name(void *ctx)
+{
+ if (!ctx)
+ return NULL;
+ return ((struct ibv_context *)ctx)->device->name;
+}
+
+/**
+ * Get ibv device path name. Given an ibv_context pointer - return a
+ * pointer to the corresponding device path name.
+ *
+ * @param[in] ctx
+ * Pointer to ibv context.
+ *
+ * @return
+ * Pointer to device path name if ctx is valid, NULL otherwise.
+ */
+
+static inline const char *
+mlx5_os_get_ctx_device_path(void *ctx)
+{
+ if (!ctx)
+ return NULL;
+
+ return ((struct ibv_context *)ctx)->device->ibdev_path;
+}
+
+/**
+ * Get umem id. Given a pointer to umem object of type
+ * 'struct mlx5dv_devx_umem *' - return its id.
+ *
+ * @param[in] umem
+ * Pointer to umem object.
+ *
+ * @return
+ * The umem id if umem is valid, 0 otherwise.
+ */
+static inline uint32_t
+mlx5_os_get_umem_id(void *umem)
+{
+ if (!umem)
+ return 0;
+ return ((struct mlx5dv_devx_umem *)umem)->umem_id;
+}
+
+/**
+ * Get fd. Given a pointer to DevX channel object of type
+ * 'struct mlx5dv_devx_event_channel*' - return its fd.
+ *
+ * @param[in] channel
+ * Pointer to channel object.
+ *
+ * @return
+ * The fd if channel is valid, 0 otherwise.
+ */
+static inline int
+mlx5_os_get_devx_channel_fd(void *channel)
+{
+ if (!channel)
+ return 0;
+ return ((struct mlx5dv_devx_event_channel *)channel)->fd;
+}
+
+/**
+ * Get mmap offset. Given a pointer to an DevX UAR object of type
+ * 'struct mlx5dv_devx_uar *' - return its mmap offset.
+ *
+ * @param[in] uar
+ * Pointer to UAR object.
+ *
+ * @return
+ * The mmap offset if uar is valid, 0 otherwise.
+ */
+static inline off_t
+mlx5_os_get_devx_uar_mmap_offset(void *uar)
+{
+#ifdef HAVE_MLX5DV_DEVX_UAR_OFFSET
+ if (!uar)
+ return 0;
+ return ((struct mlx5dv_devx_uar *)uar)->mmap_off;
+#else
+ RTE_SET_USED(uar);
+ return 0;
+#endif
+}
+
+/**
+ * Get base addr pointer. Given a pointer to an UAR object of type
+ * 'struct mlx5dv_devx_uar *' - return its base address.
+ *
+ * @param[in] uar
+ * Pointer to an UAR object.
+ *
+ * @return
+ * The base address if UAR is valid, 0 otherwise.
+ */
+static inline void *
+mlx5_os_get_devx_uar_base_addr(void *uar)
+{
+#ifdef HAVE_MLX5DV_DEVX_UAR_OFFSET
+ if (!uar)
+ return NULL;
+ return ((struct mlx5dv_devx_uar *)uar)->base_addr;
+#else
+ RTE_SET_USED(uar);
+ return NULL;
+#endif
+}
+
+/**
+ * Get reg addr pointer. Given a pointer to an UAR object of type
+ * 'struct mlx5dv_devx_uar *' - return its reg address.
+ *
+ * @param[in] uar
+ * Pointer to an UAR object.
+ *
+ * @return
+ * The reg address if UAR is valid, 0 otherwise.
+ */
+static inline void *
+mlx5_os_get_devx_uar_reg_addr(void *uar)
+{
+#ifdef HAVE_MLX5DV_DEVX_UAR_OFFSET
+ if (!uar)
+ return NULL;
+ return ((struct mlx5dv_devx_uar *)uar)->reg_addr;
+#else
+ RTE_SET_USED(uar);
+ return NULL;
+#endif
+}
+
+/**
+ * Get page id. Given a pointer to an UAR object of type
+ * 'struct mlx5dv_devx_uar *' - return its page id.
+ *
+ * @param[in] uar
+ * Pointer to an UAR object.
+ *
+ * @return
+ * The page id if UAR is valid, 0 otherwise.
+ */
+static inline uint32_t
+mlx5_os_get_devx_uar_page_id(void *uar)
+{
+#ifdef HAVE_MLX5DV_DEVX_UAR_OFFSET
+ if (!uar)
+ return 0;
+ return ((struct mlx5dv_devx_uar *)uar)->page_id;
+#else
+ RTE_SET_USED(uar);
+ return 0;
+#endif
+}
+
+__rte_internal
+static inline void *
+mlx5_os_alloc_pd(void *ctx)
+{
+ return mlx5_glue->alloc_pd(ctx);
+}
+
+__rte_internal
+static inline int
+mlx5_os_dealloc_pd(void *pd)
+{
+ return mlx5_glue->dealloc_pd(pd);
+}
+
+__rte_internal
+static inline void *
+mlx5_os_umem_reg(void *ctx, void *addr, size_t size, uint32_t access)
+{
+ return mlx5_glue->devx_umem_reg(ctx, addr, size, access);
+}
+
+__rte_internal
+static inline int
+mlx5_os_umem_dereg(void *pumem)
+{
+ return mlx5_glue->devx_umem_dereg(pumem);
+}
+
+static inline void *
+mlx5_os_devx_create_event_channel(void *ctx, int flags)
+{
+ return mlx5_glue->devx_create_event_channel(ctx, flags);
+}
+
+static inline void
+mlx5_os_devx_destroy_event_channel(void *eventc)
+{
+ mlx5_glue->devx_destroy_event_channel(eventc);
+}
+
+static inline int
+mlx5_os_devx_subscribe_devx_event(void *eventc,
+ void *obj,
+ uint16_t events_sz, uint16_t events_num[],
+ uint64_t cookie)
+{
+ return mlx5_glue->devx_subscribe_devx_event(eventc, obj, events_sz,
+ events_num, cookie);
+}
+
+/**
+ * Memory allocation optionally with alignment.
+ *
+ * @param[in] align
+ * Alignment size (may be zero)
+ * @param[in] size
+ * Size in bytes to allocate
+ *
+ * @return
+ * Valid pointer to allocated memory, NULL in case of failure
+ */
+static inline void *
+mlx5_os_malloc(size_t align, size_t size)
+{
+ void *buf;
+
+ if (posix_memalign(&buf, align, size))
+ return NULL;
+ return buf;
+}
+
+/**
+ * This API de-allocates a memory that originally could have been
+ * allocated aligned or non-aligned. In Linux it is a wrapper
+ * around free().
+ *
+ * @param[in] addr
+ * Pointer to address to free
+ *
+ */
+static inline void
+mlx5_os_free(void *addr)
+{
+ free(addr);
+}
+
+struct ibv_device *
+mlx5_os_get_ibv_device(const struct rte_pci_addr *addr);
+
+__rte_internal
+struct ibv_device *
+mlx5_os_get_ibv_dev(const struct rte_device *dev);
+
+#endif /* RTE_PMD_MLX5_COMMON_OS_H_ */
new file mode 100644
@@ -0,0 +1,93 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#include <stddef.h>
+#include <errno.h>
+#include <string.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <inttypes.h>
+
+#include <rte_errno.h>
+#include <rte_bus_pci.h>
+#include <rte_bus_auxiliary.h>
+
+#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
+#include "mlx5_common_private.h"
+#include "mlx5_autoconf.h"
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_mr.h>
+
+struct ibv_device *
+mlx5_os_get_ibv_dev(const struct rte_device *dev)
+{
+ struct ibv_device *ibv;
+
+ if (mlx5_dev_is_pci(dev))
+ ibv = mlx5_os_get_ibv_device(&RTE_DEV_TO_PCI_CONST(dev)->addr);
+ else
+ ibv = mlx5_get_aux_ibv_device(RTE_DEV_TO_AUXILIARY_CONST(dev));
+ if (ibv == NULL) {
+ rte_errno = ENODEV;
+ DRV_LOG(ERR, "Verbs device not found: %s", dev->name);
+ }
+ return ibv;
+}
+
+/**
+ * Register mr. Given protection domain pointer, pointer to addr and length
+ * register the memory region.
+ *
+ * @param[in] pd
+ * Pointer to protection domain context.
+ * @param[in] addr
+ * Pointer to memory start address.
+ * @param[in] length
+ * Length of the memory to register.
+ * @param[out] pmd_mr
+ * pmd_mr struct set with lkey, address, length and pointer to mr object
+ *
+ * @return
+ * 0 on successful registration, -1 otherwise
+ */
+int
+mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
+ struct mlx5_pmd_mr *pmd_mr)
+{
+ struct ibv_mr *ibv_mr;
+
+ ibv_mr = mlx5_glue->reg_mr(pd, addr, length,
+ IBV_ACCESS_LOCAL_WRITE |
+ (haswell_broadwell_cpu ? 0 :
+ IBV_ACCESS_RELAXED_ORDERING));
+ if (!ibv_mr)
+ return -1;
+
+ *pmd_mr = (struct mlx5_pmd_mr){
+ .lkey = ibv_mr->lkey,
+ .addr = ibv_mr->addr,
+ .len = ibv_mr->length,
+ .obj = (void *)ibv_mr,
+ };
+ return 0;
+}
+
+/**
+ * Deregister mr. Given the mlx5 pmd MR - deregister the MR
+ *
+ * @param[in] pmd_mr
+ * pmd_mr struct set with lkey, address, length and pointer to mr object
+ *
+ */
+void
+mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
+{
+ if (pmd_mr && pmd_mr->obj != NULL) {
+ claim_zero(mlx5_glue->dereg_mr(pmd_mr->obj));
+ memset(pmd_mr, 0, sizeof(*pmd_mr));
+ }
+}
new file mode 100644
@@ -0,0 +1,1505 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#include <errno.h>
+#include <stdalign.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+/*
+ * Not needed by this file; included to work around the lack of off_t
+ * definition for mlx5dv.h with unpatched rdma-core versions.
+ */
+#include <sys/types.h>
+
+#include "mlx5_glue.h"
+
+static int
+mlx5_glue_fork_init(void)
+{
+ return ibv_fork_init();
+}
+
+static struct ibv_pd *
+mlx5_glue_alloc_pd(struct ibv_context *context)
+{
+ return ibv_alloc_pd(context);
+}
+
+static int
+mlx5_glue_dealloc_pd(struct ibv_pd *pd)
+{
+ return ibv_dealloc_pd(pd);
+}
+
+static struct ibv_device **
+mlx5_glue_get_device_list(int *num_devices)
+{
+ return ibv_get_device_list(num_devices);
+}
+
+static void
+mlx5_glue_free_device_list(struct ibv_device **list)
+{
+ ibv_free_device_list(list);
+}
+
+static struct ibv_context *
+mlx5_glue_open_device(struct ibv_device *device)
+{
+ return ibv_open_device(device);
+}
+
+static int
+mlx5_glue_close_device(struct ibv_context *context)
+{
+ return ibv_close_device(context);
+}
+
+static int
+mlx5_glue_query_device(struct ibv_context *context,
+ struct ibv_device_attr *device_attr)
+{
+ return ibv_query_device(context, device_attr);
+}
+
+static int
+mlx5_glue_query_device_ex(struct ibv_context *context,
+ const struct ibv_query_device_ex_input *input,
+ struct ibv_device_attr_ex *attr)
+{
+ return ibv_query_device_ex(context, input, attr);
+}
+
+static int
+mlx5_glue_query_rt_values_ex(struct ibv_context *context,
+ struct ibv_values_ex *values)
+{
+ return ibv_query_rt_values_ex(context, values);
+}
+
+static int
+mlx5_glue_query_port(struct ibv_context *context, uint8_t port_num,
+ struct ibv_port_attr *port_attr)
+{
+ return ibv_query_port(context, port_num, port_attr);
+}
+
+static struct ibv_comp_channel *
+mlx5_glue_create_comp_channel(struct ibv_context *context)
+{
+ return ibv_create_comp_channel(context);
+}
+
+static int
+mlx5_glue_destroy_comp_channel(struct ibv_comp_channel *channel)
+{
+ return ibv_destroy_comp_channel(channel);
+}
+
+static struct ibv_cq *
+mlx5_glue_create_cq(struct ibv_context *context, int cqe, void *cq_context,
+ struct ibv_comp_channel *channel, int comp_vector)
+{
+ return ibv_create_cq(context, cqe, cq_context, channel, comp_vector);
+}
+
+static int
+mlx5_glue_destroy_cq(struct ibv_cq *cq)
+{
+ return ibv_destroy_cq(cq);
+}
+
+static int
+mlx5_glue_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq,
+ void **cq_context)
+{
+ return ibv_get_cq_event(channel, cq, cq_context);
+}
+
+static void
+mlx5_glue_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
+{
+ ibv_ack_cq_events(cq, nevents);
+}
+
+static struct ibv_rwq_ind_table *
+mlx5_glue_create_rwq_ind_table(struct ibv_context *context,
+ struct ibv_rwq_ind_table_init_attr *init_attr)
+{
+ return ibv_create_rwq_ind_table(context, init_attr);
+}
+
+static int
+mlx5_glue_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table)
+{
+ return ibv_destroy_rwq_ind_table(rwq_ind_table);
+}
+
+static struct ibv_wq *
+mlx5_glue_create_wq(struct ibv_context *context,
+ struct ibv_wq_init_attr *wq_init_attr)
+{
+ return ibv_create_wq(context, wq_init_attr);
+}
+
+static int
+mlx5_glue_destroy_wq(struct ibv_wq *wq)
+{
+ return ibv_destroy_wq(wq);
+}
+static int
+mlx5_glue_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr)
+{
+ return ibv_modify_wq(wq, wq_attr);
+}
+
+static struct ibv_flow *
+mlx5_glue_create_flow(struct ibv_qp *qp, struct ibv_flow_attr *flow)
+{
+ return ibv_create_flow(qp, flow);
+}
+
+static int
+mlx5_glue_destroy_flow(struct ibv_flow *flow_id)
+{
+ return ibv_destroy_flow(flow_id);
+}
+
+static int
+mlx5_glue_destroy_flow_action(void *action)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_action_destroy(action);
+#else
+ struct mlx5dv_flow_action_attr *attr = action;
+ int res = 0;
+ switch (attr->type) {
+ case MLX5DV_FLOW_ACTION_TAG:
+ break;
+ default:
+ res = ibv_destroy_flow_action(attr->action);
+ break;
+ }
+ free(action);
+ return res;
+#endif
+#else
+ (void)action;
+ return -ENOTSUP;
+#endif
+}
+
+static struct ibv_qp *
+mlx5_glue_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
+{
+ return ibv_create_qp(pd, qp_init_attr);
+}
+
+static struct ibv_qp *
+mlx5_glue_create_qp_ex(struct ibv_context *context,
+ struct ibv_qp_init_attr_ex *qp_init_attr_ex)
+{
+ return ibv_create_qp_ex(context, qp_init_attr_ex);
+}
+
+static int
+mlx5_glue_destroy_qp(struct ibv_qp *qp)
+{
+ return ibv_destroy_qp(qp);
+}
+
+static int
+mlx5_glue_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
+{
+ return ibv_modify_qp(qp, attr, attr_mask);
+}
+
+static struct ibv_mr *
+mlx5_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access)
+{
+ return ibv_reg_mr(pd, addr, length, access);
+}
+
+static struct ibv_mr *
+mlx5_glue_alloc_null_mr(struct ibv_pd *pd)
+{
+#ifdef HAVE_IBV_DEVX_OBJ
+ return ibv_alloc_null_mr(pd);
+#else
+ (void)pd;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static int
+mlx5_glue_dereg_mr(struct ibv_mr *mr)
+{
+ return ibv_dereg_mr(mr);
+}
+
+static struct ibv_counter_set *
+mlx5_glue_create_counter_set(struct ibv_context *context,
+ struct ibv_counter_set_init_attr *init_attr)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+ (void)context;
+ (void)init_attr;
+ return NULL;
+#else
+ return ibv_create_counter_set(context, init_attr);
+#endif
+}
+
+static int
+mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+ (void)cs;
+ return -ENOTSUP;
+#else
+ return ibv_destroy_counter_set(cs);
+#endif
+}
+
+static int
+mlx5_glue_describe_counter_set(struct ibv_context *context,
+ uint16_t counter_set_id,
+ struct ibv_counter_set_description *cs_desc)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+ (void)context;
+ (void)counter_set_id;
+ (void)cs_desc;
+ return -ENOTSUP;
+#else
+ return ibv_describe_counter_set(context, counter_set_id, cs_desc);
+#endif
+}
+
+static int
+mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
+ struct ibv_counter_set_data *cs_data)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+ (void)query_attr;
+ (void)cs_data;
+ return -ENOTSUP;
+#else
+ return ibv_query_counter_set(query_attr, cs_data);
+#endif
+}
+
+static struct ibv_counters *
+mlx5_glue_create_counters(struct ibv_context *context,
+ struct ibv_counters_init_attr *init_attr)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+ (void)context;
+ (void)init_attr;
+ errno = ENOTSUP;
+ return NULL;
+#else
+ return ibv_create_counters(context, init_attr);
+#endif
+}
+
+static int
+mlx5_glue_destroy_counters(struct ibv_counters *counters)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+ (void)counters;
+ return -ENOTSUP;
+#else
+ return ibv_destroy_counters(counters);
+#endif
+}
+
+static int
+mlx5_glue_attach_counters(struct ibv_counters *counters,
+ struct ibv_counter_attach_attr *attr,
+ struct ibv_flow *flow)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+ (void)counters;
+ (void)attr;
+ (void)flow;
+ return -ENOTSUP;
+#else
+ return ibv_attach_counters_point_flow(counters, attr, flow);
+#endif
+}
+
+static int
+mlx5_glue_query_counters(struct ibv_counters *counters,
+ uint64_t *counters_value,
+ uint32_t ncounters,
+ uint32_t flags)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+ (void)counters;
+ (void)counters_value;
+ (void)ncounters;
+ (void)flags;
+ return -ENOTSUP;
+#else
+ return ibv_read_counters(counters, counters_value, ncounters, flags);
+#endif
+}
+
+static void
+mlx5_glue_ack_async_event(struct ibv_async_event *event)
+{
+ ibv_ack_async_event(event);
+}
+
+static int
+mlx5_glue_get_async_event(struct ibv_context *context,
+ struct ibv_async_event *event)
+{
+ return ibv_get_async_event(context, event);
+}
+
+static const char *
+mlx5_glue_port_state_str(enum ibv_port_state port_state)
+{
+ return ibv_port_state_str(port_state);
+}
+
+static struct ibv_cq *
+mlx5_glue_cq_ex_to_cq(struct ibv_cq_ex *cq)
+{
+ return ibv_cq_ex_to_cq(cq);
+}
+
+static void *
+mlx5_glue_dr_create_flow_action_dest_flow_tbl(void *tbl)
+{
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_action_create_dest_table(tbl);
+#else
+ (void)tbl;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dr_create_flow_action_dest_port(void *domain, uint32_t port)
+{
+#ifdef HAVE_MLX5DV_DR_CREATE_DEST_IB_PORT
+ return mlx5dv_dr_action_create_dest_ib_port(domain, port);
+#else
+#ifdef HAVE_MLX5DV_DR_ESWITCH
+ return mlx5dv_dr_action_create_dest_vport(domain, port);
+#else
+ (void)domain;
+ (void)port;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+#endif
+}
+
+static void *
+mlx5_glue_dr_create_flow_action_drop(void)
+{
+#ifdef HAVE_MLX5DV_DR_ESWITCH
+ return mlx5dv_dr_action_create_drop();
+#else
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dr_create_flow_action_push_vlan(struct mlx5dv_dr_domain *domain,
+ rte_be32_t vlan_tag)
+{
+#ifdef HAVE_MLX5DV_DR_VLAN
+ return mlx5dv_dr_action_create_push_vlan(domain, vlan_tag);
+#else
+ (void)domain;
+ (void)vlan_tag;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dr_create_flow_action_pop_vlan(void)
+{
+#ifdef HAVE_MLX5DV_DR_VLAN
+ return mlx5dv_dr_action_create_pop_vlan();
+#else
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dr_create_flow_tbl(void *domain, uint32_t level)
+{
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_table_create(domain, level);
+#else
+ (void)domain;
+ (void)level;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static int
+mlx5_glue_dr_destroy_flow_tbl(void *tbl)
+{
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_table_destroy(tbl);
+#else
+ (void)tbl;
+ errno = ENOTSUP;
+ return errno;
+#endif
+}
+
+static void *
+mlx5_glue_dr_create_domain(struct ibv_context *ctx,
+ enum mlx5dv_dr_domain_type domain)
+{
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_domain_create(ctx, domain);
+#else
+ (void)ctx;
+ (void)domain;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static int
+mlx5_glue_dr_destroy_domain(void *domain)
+{
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_domain_destroy(domain);
+#else
+ (void)domain;
+ errno = ENOTSUP;
+ return errno;
+#endif
+}
+
+static int
+mlx5_glue_dr_sync_domain(void *domain, uint32_t flags)
+{
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_domain_sync(domain, flags);
+#else
+ (void)domain;
+ (void)flags;
+ errno = ENOTSUP;
+ return errno;
+#endif
+}
+
+static struct ibv_cq_ex *
+mlx5_glue_dv_create_cq(struct ibv_context *context,
+ struct ibv_cq_init_attr_ex *cq_attr,
+ struct mlx5dv_cq_init_attr *mlx5_cq_attr)
+{
+ return mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr);
+}
+
+static struct ibv_wq *
+mlx5_glue_dv_create_wq(struct ibv_context *context,
+ struct ibv_wq_init_attr *wq_attr,
+ struct mlx5dv_wq_init_attr *mlx5_wq_attr)
+{
+#ifndef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
+ (void)context;
+ (void)wq_attr;
+ (void)mlx5_wq_attr;
+ errno = ENOTSUP;
+ return NULL;
+#else
+ return mlx5dv_create_wq(context, wq_attr, mlx5_wq_attr);
+#endif
+}
+
+static int
+mlx5_glue_dv_query_device(struct ibv_context *ctx,
+ struct mlx5dv_context *attrs_out)
+{
+ return mlx5dv_query_device(ctx, attrs_out);
+}
+
+static int
+mlx5_glue_dv_set_context_attr(struct ibv_context *ibv_ctx,
+ enum mlx5dv_set_ctx_attr_type type, void *attr)
+{
+ return mlx5dv_set_context_attr(ibv_ctx, type, attr);
+}
+
+static int
+mlx5_glue_dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type)
+{
+ return mlx5dv_init_obj(obj, obj_type);
+}
+
+static struct ibv_qp *
+mlx5_glue_dv_create_qp(struct ibv_context *context,
+ struct ibv_qp_init_attr_ex *qp_init_attr_ex,
+ struct mlx5dv_qp_init_attr *dv_qp_init_attr)
+{
+#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+ return mlx5dv_create_qp(context, qp_init_attr_ex, dv_qp_init_attr);
+#else
+ (void)context;
+ (void)qp_init_attr_ex;
+ (void)dv_qp_init_attr;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
+ struct mlx5dv_flow_matcher_attr *matcher_attr,
+ void *tbl)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+#ifdef HAVE_MLX5DV_DR
+ (void)context;
+ return mlx5dv_dr_matcher_create(tbl, matcher_attr->priority,
+ matcher_attr->match_criteria_enable,
+ matcher_attr->match_mask);
+#else
+ (void)tbl;
+ return mlx5dv_create_flow_matcher(context, matcher_attr);
+#endif
+#else
+ (void)context;
+ (void)matcher_attr;
+ (void)tbl;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dv_create_flow(void *matcher,
+ void *match_value,
+ size_t num_actions,
+ void *actions[])
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_rule_create(matcher, match_value, num_actions,
+ (struct mlx5dv_dr_action **)actions);
+#else
+ size_t i;
+ struct mlx5dv_flow_action_attr actions_attr[8];
+
+ if (num_actions > 8)
+ return NULL;
+ for (i = 0; i < num_actions; i++)
+ actions_attr[i] =
+ *((struct mlx5dv_flow_action_attr *)(actions[i]));
+ return mlx5dv_create_flow(matcher, match_value,
+ num_actions, actions_attr);
+#endif
+#else
+ (void)matcher;
+ (void)match_value;
+ (void)num_actions;
+ (void)actions;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dv_create_flow_action_counter(void *counter_obj, uint32_t offset)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_action_create_flow_counter(counter_obj, offset);
+#else
+ struct mlx5dv_flow_action_attr *action;
+
+ (void)offset;
+ action = malloc(sizeof(*action));
+ if (!action)
+ return NULL;
+ action->type = MLX5DV_FLOW_ACTION_COUNTERS_DEVX;
+ action->obj = counter_obj;
+ return action;
+#endif
+#else
+ (void)counter_obj;
+ (void)offset;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dv_create_flow_action_dest_ibv_qp(void *qp)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_action_create_dest_ibv_qp(qp);
+#else
+ struct mlx5dv_flow_action_attr *action;
+
+ action = malloc(sizeof(*action));
+ if (!action)
+ return NULL;
+ action->type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
+ action->obj = qp;
+ return action;
+#endif
+#else
+ (void)qp;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dv_create_flow_action_dest_devx_tir(void *tir)
+{
+#ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR
+ return mlx5dv_dr_action_create_dest_devx_tir(tir);
+#else
+ (void)tir;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dv_create_flow_action_modify_header
+ (struct ibv_context *ctx,
+ enum mlx5dv_flow_table_type ft_type,
+ void *domain, uint64_t flags,
+ size_t actions_sz,
+ uint64_t actions[])
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+#ifdef HAVE_MLX5DV_DR
+ (void)ctx;
+ (void)ft_type;
+ return mlx5dv_dr_action_create_modify_header(domain, flags, actions_sz,
+ (__be64 *)actions);
+#else
+ struct mlx5dv_flow_action_attr *action;
+
+ (void)domain;
+ (void)flags;
+ action = malloc(sizeof(*action));
+ if (!action)
+ return NULL;
+ action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
+ action->action = mlx5dv_create_flow_action_modify_header
+ (ctx, actions_sz, actions, ft_type);
+ return action;
+#endif
+#else
+ (void)ctx;
+ (void)ft_type;
+ (void)domain;
+ (void)flags;
+ (void)actions_sz;
+ (void)actions;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dv_create_flow_action_packet_reformat
+ (struct ibv_context *ctx,
+ enum mlx5dv_flow_action_packet_reformat_type reformat_type,
+ enum mlx5dv_flow_table_type ft_type,
+ struct mlx5dv_dr_domain *domain,
+ uint32_t flags, size_t data_sz, void *data)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+#ifdef HAVE_MLX5DV_DR
+ (void)ctx;
+ (void)ft_type;
+ return mlx5dv_dr_action_create_packet_reformat(domain, flags,
+ reformat_type, data_sz,
+ data);
+#else
+ (void)domain;
+ (void)flags;
+ struct mlx5dv_flow_action_attr *action;
+
+ action = malloc(sizeof(*action));
+ if (!action)
+ return NULL;
+ action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
+ action->action = mlx5dv_create_flow_action_packet_reformat
+ (ctx, data_sz, data, reformat_type, ft_type);
+ return action;
+#endif
+#else
+ (void)ctx;
+ (void)reformat_type;
+ (void)ft_type;
+ (void)domain;
+ (void)flags;
+ (void)data_sz;
+ (void)data;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dv_create_flow_action_tag(uint32_t tag)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_action_create_tag(tag);
+#else /* HAVE_MLX5DV_DR */
+ struct mlx5dv_flow_action_attr *action;
+
+ action = malloc(sizeof(*action));
+ if (!action)
+ return NULL;
+ action->type = MLX5DV_FLOW_ACTION_TAG;
+ action->tag_value = tag;
+ return action;
+#endif /* HAVE_MLX5DV_DR */
+#else /* HAVE_IBV_FLOW_DV_SUPPORT */
+ (void)tag;
+ errno = ENOTSUP;
+ return NULL;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static void *
+mlx5_glue_dv_create_flow_action_meter(struct mlx5dv_dr_flow_meter_attr *attr)
+{
+#if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER)
+ return mlx5dv_dr_action_create_flow_meter(attr);
+#else
+ (void)attr;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static int
+mlx5_glue_dv_modify_flow_action_meter(void *action,
+ struct mlx5dv_dr_flow_meter_attr *attr,
+ uint64_t modify_bits)
+{
+#if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER)
+ return mlx5dv_dr_action_modify_flow_meter(action, attr, modify_bits);
+#else
+ (void)action;
+ (void)attr;
+ (void)modify_bits;
+ errno = ENOTSUP;
+ return errno;
+#endif
+}
+
+static void *
+mlx5_glue_dv_create_flow_action_aso(struct mlx5dv_dr_domain *domain,
+ void *aso_obj,
+ uint32_t offset,
+ uint32_t flags,
+ uint8_t return_reg_c)
+{
+#if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_ASO)
+ return mlx5dv_dr_action_create_aso(domain, aso_obj, offset,
+ flags, return_reg_c);
+#else
+ (void)domain;
+ (void)aso_obj;
+ (void)offset;
+ (void)flags;
+ (void)return_reg_c;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dr_create_flow_action_default_miss(void)
+{
+#if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_DEFAULT_MISS)
+ return mlx5dv_dr_action_create_default_miss();
+#else
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static int
+mlx5_glue_dv_destroy_flow(void *flow_id)
+{
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_rule_destroy(flow_id);
+#else
+ return ibv_destroy_flow(flow_id);
+#endif
+}
+
+static int
+mlx5_glue_dv_destroy_flow_matcher(void *matcher)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+#ifdef HAVE_MLX5DV_DR
+ return mlx5dv_dr_matcher_destroy(matcher);
+#else
+ return mlx5dv_destroy_flow_matcher(matcher);
+#endif
+#else
+ (void)matcher;
+ errno = ENOTSUP;
+ return errno;
+#endif
+}
+
+static struct ibv_context *
+mlx5_glue_dv_open_device(struct ibv_device *device)
+{
+#ifdef HAVE_IBV_DEVX_OBJ
+ return mlx5dv_open_device(device,
+ &(struct mlx5dv_context_attr){
+ .flags = MLX5DV_CONTEXT_FLAGS_DEVX,
+ });
+#else
+ (void)device;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static struct mlx5dv_devx_obj *
+mlx5_glue_devx_obj_create(struct ibv_context *ctx,
+ const void *in, size_t inlen,
+ void *out, size_t outlen)
+{
+#ifdef HAVE_IBV_DEVX_OBJ
+ return mlx5dv_devx_obj_create(ctx, in, inlen, out, outlen);
+#else
+ (void)ctx;
+ (void)in;
+ (void)inlen;
+ (void)out;
+ (void)outlen;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static int
+mlx5_glue_devx_obj_destroy(struct mlx5dv_devx_obj *obj)
+{
+#ifdef HAVE_IBV_DEVX_OBJ
+ return mlx5dv_devx_obj_destroy(obj);
+#else
+ (void)obj;
+ return -ENOTSUP;
+#endif
+}
+
+static int
+mlx5_glue_devx_obj_query(struct mlx5dv_devx_obj *obj,
+ const void *in, size_t inlen,
+ void *out, size_t outlen)
+{
+#ifdef HAVE_IBV_DEVX_OBJ
+ return mlx5dv_devx_obj_query(obj, in, inlen, out, outlen);
+#else
+ (void)obj;
+ (void)in;
+ (void)inlen;
+ (void)out;
+ (void)outlen;
+ return -ENOTSUP;
+#endif
+}
+
+static int
+mlx5_glue_devx_obj_modify(struct mlx5dv_devx_obj *obj,
+ const void *in, size_t inlen,
+ void *out, size_t outlen)
+{
+#ifdef HAVE_IBV_DEVX_OBJ
+ return mlx5dv_devx_obj_modify(obj, in, inlen, out, outlen);
+#else
+ (void)obj;
+ (void)in;
+ (void)inlen;
+ (void)out;
+ (void)outlen;
+ return -ENOTSUP;
+#endif
+}
+
+static int
+mlx5_glue_devx_general_cmd(struct ibv_context *ctx,
+ const void *in, size_t inlen,
+ void *out, size_t outlen)
+{
+#ifdef HAVE_IBV_DEVX_OBJ
+ return mlx5dv_devx_general_cmd(ctx, in, inlen, out, outlen);
+#else
+ (void)ctx;
+ (void)in;
+ (void)inlen;
+ (void)out;
+ (void)outlen;
+ return -ENOTSUP;
+#endif
+}
+
+static struct mlx5dv_devx_cmd_comp *
+mlx5_glue_devx_create_cmd_comp(struct ibv_context *ctx)
+{
+#ifdef HAVE_IBV_DEVX_ASYNC
+ return mlx5dv_devx_create_cmd_comp(ctx);
+#else
+ (void)ctx;
+ errno = -ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void
+mlx5_glue_devx_destroy_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp)
+{
+#ifdef HAVE_IBV_DEVX_ASYNC
+ mlx5dv_devx_destroy_cmd_comp(cmd_comp);
+#else
+ (void)cmd_comp;
+ errno = -ENOTSUP;
+#endif
+}
+
+static int
+mlx5_glue_devx_obj_query_async(struct mlx5dv_devx_obj *obj, const void *in,
+ size_t inlen, size_t outlen, uint64_t wr_id,
+ struct mlx5dv_devx_cmd_comp *cmd_comp)
+{
+#ifdef HAVE_IBV_DEVX_ASYNC
+ return mlx5dv_devx_obj_query_async(obj, in, inlen, outlen, wr_id,
+ cmd_comp);
+#else
+ (void)obj;
+ (void)in;
+ (void)inlen;
+ (void)outlen;
+ (void)wr_id;
+ (void)cmd_comp;
+ return -ENOTSUP;
+#endif
+}
+
+static int
+mlx5_glue_devx_get_async_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp,
+ struct mlx5dv_devx_async_cmd_hdr *cmd_resp,
+ size_t cmd_resp_len)
+{
+#ifdef HAVE_IBV_DEVX_ASYNC
+ return mlx5dv_devx_get_async_cmd_comp(cmd_comp, cmd_resp,
+ cmd_resp_len);
+#else
+ (void)cmd_comp;
+ (void)cmd_resp;
+ (void)cmd_resp_len;
+ return -ENOTSUP;
+#endif
+}
+
+static struct mlx5dv_devx_umem *
+mlx5_glue_devx_umem_reg(struct ibv_context *context, void *addr, size_t size,
+ uint32_t access)
+{
+#ifdef HAVE_IBV_DEVX_OBJ
+ return mlx5dv_devx_umem_reg(context, addr, size, access);
+#else
+ (void)context;
+ (void)addr;
+ (void)size;
+ (void)access;
+ errno = -ENOTSUP;
+ return NULL;
+#endif
+}
+
+static int
+mlx5_glue_devx_umem_dereg(struct mlx5dv_devx_umem *dv_devx_umem)
+{
+#ifdef HAVE_IBV_DEVX_OBJ
+ return mlx5dv_devx_umem_dereg(dv_devx_umem);
+#else
+ (void)dv_devx_umem;
+ return -ENOTSUP;
+#endif
+}
+
+static int
+mlx5_glue_devx_qp_query(struct ibv_qp *qp,
+ const void *in, size_t inlen,
+ void *out, size_t outlen)
+{
+#ifdef HAVE_IBV_DEVX_QP
+ return mlx5dv_devx_qp_query(qp, in, inlen, out, outlen);
+#else
+ (void)qp;
+ (void)in;
+ (void)inlen;
+ (void)out;
+ (void)outlen;
+ errno = ENOTSUP;
+ return errno;
+#endif
+}
+
+static int
+mlx5_glue_devx_wq_query(struct ibv_wq *wq, const void *in, size_t inlen,
+ void *out, size_t outlen)
+{
+#ifdef HAVE_IBV_DEVX_QP
+ return mlx5dv_devx_wq_query(wq, in, inlen, out, outlen);
+#else
+ (void)wq;
+ (void)in;
+ (void)inlen;
+ (void)out;
+ (void)outlen;
+ errno = ENOTSUP;
+ return errno;
+#endif
+}
+
+static int
+mlx5_glue_devx_port_query(struct ibv_context *ctx,
+ uint32_t port_num,
+ struct mlx5_port_info *info)
+{
+ int err = 0;
+
+ info->query_flags = 0;
+#ifdef HAVE_MLX5DV_DR_DEVX_PORT_V35
+ /* The DevX port query API is implemented (rdma-core v35 and above). */
+ struct mlx5_ib_uapi_query_port devx_port;
+
+ memset(&devx_port, 0, sizeof(devx_port));
+ err = mlx5dv_query_port(ctx, port_num, &devx_port);
+ if (err)
+ return err;
+ if (devx_port.flags & MLX5DV_QUERY_PORT_VPORT_REG_C0) {
+ info->vport_meta_tag = devx_port.reg_c0.value;
+ info->vport_meta_mask = devx_port.reg_c0.mask;
+ info->query_flags |= MLX5_PORT_QUERY_REG_C0;
+ }
+ if (devx_port.flags & MLX5DV_QUERY_PORT_VPORT) {
+ info->vport_id = devx_port.vport;
+ info->query_flags |= MLX5_PORT_QUERY_VPORT;
+ }
+#else
+#ifdef HAVE_MLX5DV_DR_DEVX_PORT
+ /* The legacy DevX port query API is implemented (prior v35). */
+ struct mlx5dv_devx_port devx_port = {
+ .comp_mask = MLX5DV_DEVX_PORT_VPORT |
+ MLX5DV_DEVX_PORT_MATCH_REG_C_0
+ };
+
+ err = mlx5dv_query_devx_port(ctx, port_num, &devx_port);
+ if (err)
+ return err;
+ if (devx_port.comp_mask & MLX5DV_DEVX_PORT_MATCH_REG_C_0) {
+ info->vport_meta_tag = devx_port.reg_c_0.value;
+ info->vport_meta_mask = devx_port.reg_c_0.mask;
+ info->query_flags |= MLX5_PORT_QUERY_REG_C0;
+ }
+ if (devx_port.comp_mask & MLX5DV_DEVX_PORT_VPORT) {
+ info->vport_id = devx_port.vport_num;
+ info->query_flags |= MLX5_PORT_QUERY_VPORT;
+ }
+#else
+ RTE_SET_USED(ctx);
+ RTE_SET_USED(port_num);
+#endif /* HAVE_MLX5DV_DR_DEVX_PORT */
+#endif /* HAVE_MLX5DV_DR_DEVX_PORT_V35 */
+ return err;
+}
+
+static int
+mlx5_glue_dr_dump_single_rule(FILE *file, void *rule)
+{
+#ifdef HAVE_MLX5_DR_FLOW_DUMP_RULE
+ return mlx5dv_dump_dr_rule(file, rule);
+#else
+ RTE_SET_USED(file);
+ RTE_SET_USED(rule);
+ return -ENOTSUP;
+#endif
+}
+
+static int
+mlx5_glue_dr_dump_domain(FILE *file, void *domain)
+{
+#ifdef HAVE_MLX5_DR_FLOW_DUMP
+ return mlx5dv_dump_dr_domain(file, domain);
+#else
+ RTE_SET_USED(file);
+ RTE_SET_USED(domain);
+ return -ENOTSUP;
+#endif
+}
+
+static void *
+mlx5_glue_dr_create_flow_action_sampler
+ (struct mlx5dv_dr_flow_sampler_attr *attr)
+{
+#ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_SAMPLE
+ return mlx5dv_dr_action_create_flow_sampler(attr);
+#else
+ (void)attr;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void *
+mlx5_glue_dr_action_create_dest_array
+ (void *domain,
+ size_t num_dest,
+ struct mlx5dv_dr_action_dest_attr *dests[])
+{
+#ifdef HAVE_MLX5_DR_CREATE_ACTION_DEST_ARRAY
+ return mlx5dv_dr_action_create_dest_array
+ (domain,
+ num_dest,
+ dests);
+#else
+ (void)domain;
+ (void)num_dest;
+ (void)dests;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static int
+mlx5_glue_devx_query_eqn(struct ibv_context *ctx, uint32_t cpus,
+ uint32_t *eqn)
+{
+#ifdef HAVE_IBV_DEVX_OBJ
+ return mlx5dv_devx_query_eqn(ctx, cpus, eqn);
+#else
+ (void)ctx;
+ (void)cpus;
+ (void)eqn;
+ return -ENOTSUP;
+#endif
+}
+
+static struct mlx5dv_devx_event_channel *
+mlx5_glue_devx_create_event_channel(struct ibv_context *ctx, int flags)
+{
+#ifdef HAVE_IBV_DEVX_EVENT
+ return mlx5dv_devx_create_event_channel(ctx, flags);
+#else
+ (void)ctx;
+ (void)flags;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void
+mlx5_glue_devx_destroy_event_channel(struct mlx5dv_devx_event_channel *eventc)
+{
+#ifdef HAVE_IBV_DEVX_EVENT
+ mlx5dv_devx_destroy_event_channel(eventc);
+#else
+ (void)eventc;
+#endif
+}
+
+static int
+mlx5_glue_devx_subscribe_devx_event(struct mlx5dv_devx_event_channel *eventc,
+ struct mlx5dv_devx_obj *obj,
+ uint16_t events_sz, uint16_t events_num[],
+ uint64_t cookie)
+{
+#ifdef HAVE_IBV_DEVX_EVENT
+ return mlx5dv_devx_subscribe_devx_event(eventc, obj, events_sz,
+ events_num, cookie);
+#else
+ (void)eventc;
+ (void)obj;
+ (void)events_sz;
+ (void)events_num;
+ (void)cookie;
+ return -ENOTSUP;
+#endif
+}
+
+static int
+mlx5_glue_devx_subscribe_devx_event_fd(struct mlx5dv_devx_event_channel *eventc,
+ int fd, struct mlx5dv_devx_obj *obj,
+ uint16_t event_num)
+{
+#ifdef HAVE_IBV_DEVX_EVENT
+ return mlx5dv_devx_subscribe_devx_event_fd(eventc, fd, obj, event_num);
+#else
+ (void)eventc;
+ (void)fd;
+ (void)obj;
+ (void)event_num;
+ return -ENOTSUP;
+#endif
+}
+
+static ssize_t
+mlx5_glue_devx_get_event(struct mlx5dv_devx_event_channel *eventc,
+ struct mlx5dv_devx_async_event_hdr *event_data,
+ size_t event_resp_len)
+{
+#ifdef HAVE_IBV_DEVX_EVENT
+ return mlx5dv_devx_get_event(eventc, event_data, event_resp_len);
+#else
+ (void)eventc;
+ (void)event_data;
+ (void)event_resp_len;
+ errno = ENOTSUP;
+ return -1;
+#endif
+}
+
+static struct mlx5dv_devx_uar *
+mlx5_glue_devx_alloc_uar(struct ibv_context *context, uint32_t flags)
+{
+#ifdef HAVE_IBV_DEVX_OBJ
+ return mlx5dv_devx_alloc_uar(context, flags);
+#else
+ (void)context;
+ (void)flags;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void
+mlx5_glue_devx_free_uar(struct mlx5dv_devx_uar *devx_uar)
+{
+#ifdef HAVE_IBV_DEVX_OBJ
+ mlx5dv_devx_free_uar(devx_uar);
+#else
+ (void)devx_uar;
+#endif
+}
+
+static struct mlx5dv_var *
+mlx5_glue_dv_alloc_var(struct ibv_context *context, uint32_t flags)
+{
+#ifdef HAVE_IBV_VAR
+ return mlx5dv_alloc_var(context, flags);
+#else
+ (void)context;
+ (void)flags;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void
+mlx5_glue_dv_free_var(struct mlx5dv_var *var)
+{
+#ifdef HAVE_IBV_VAR
+ mlx5dv_free_var(var);
+#else
+ (void)var;
+ errno = ENOTSUP;
+#endif
+}
+
+static void
+mlx5_glue_dr_reclaim_domain_memory(void *domain, uint32_t enable)
+{
+#ifdef HAVE_MLX5DV_DR_MEM_RECLAIM
+ mlx5dv_dr_domain_set_reclaim_device_memory(domain, enable);
+#else
+ (void)(enable);
+ (void)(domain);
+#endif
+}
+
+static struct mlx5dv_pp *
+mlx5_glue_dv_alloc_pp(struct ibv_context *context,
+ size_t pp_context_sz,
+ const void *pp_context,
+ uint32_t flags)
+{
+#ifdef HAVE_MLX5DV_PP_ALLOC
+ return mlx5dv_pp_alloc(context, pp_context_sz, pp_context, flags);
+#else
+ RTE_SET_USED(context);
+ RTE_SET_USED(pp_context_sz);
+ RTE_SET_USED(pp_context);
+ RTE_SET_USED(flags);
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
+static void
+mlx5_glue_dr_allow_duplicate_rules(void *domain, uint32_t allow)
+{
+#ifdef HAVE_MLX5_DR_ALLOW_DUPLICATE
+ mlx5dv_dr_domain_allow_duplicate_rules(domain, allow);
+#else
+ (void)(allow);
+ (void)(domain);
+#endif
+}
+
+static void
+mlx5_glue_dv_free_pp(struct mlx5dv_pp *pp)
+{
+#ifdef HAVE_MLX5DV_PP_ALLOC
+ mlx5dv_pp_free(pp);
+#else
+ RTE_SET_USED(pp);
+#endif
+}
+
+__rte_cache_aligned
+const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue) {
+ .version = MLX5_GLUE_VERSION,
+ .fork_init = mlx5_glue_fork_init,
+ .alloc_pd = mlx5_glue_alloc_pd,
+ .dealloc_pd = mlx5_glue_dealloc_pd,
+ .get_device_list = mlx5_glue_get_device_list,
+ .free_device_list = mlx5_glue_free_device_list,
+ .open_device = mlx5_glue_open_device,
+ .close_device = mlx5_glue_close_device,
+ .query_device = mlx5_glue_query_device,
+ .query_device_ex = mlx5_glue_query_device_ex,
+ .query_rt_values_ex = mlx5_glue_query_rt_values_ex,
+ .query_port = mlx5_glue_query_port,
+ .create_comp_channel = mlx5_glue_create_comp_channel,
+ .destroy_comp_channel = mlx5_glue_destroy_comp_channel,
+ .create_cq = mlx5_glue_create_cq,
+ .destroy_cq = mlx5_glue_destroy_cq,
+ .get_cq_event = mlx5_glue_get_cq_event,
+ .ack_cq_events = mlx5_glue_ack_cq_events,
+ .create_rwq_ind_table = mlx5_glue_create_rwq_ind_table,
+ .destroy_rwq_ind_table = mlx5_glue_destroy_rwq_ind_table,
+ .create_wq = mlx5_glue_create_wq,
+ .destroy_wq = mlx5_glue_destroy_wq,
+ .modify_wq = mlx5_glue_modify_wq,
+ .create_flow = mlx5_glue_create_flow,
+ .destroy_flow = mlx5_glue_destroy_flow,
+ .destroy_flow_action = mlx5_glue_destroy_flow_action,
+ .create_qp = mlx5_glue_create_qp,
+ .create_qp_ex = mlx5_glue_create_qp_ex,
+ .destroy_qp = mlx5_glue_destroy_qp,
+ .modify_qp = mlx5_glue_modify_qp,
+ .reg_mr = mlx5_glue_reg_mr,
+ .alloc_null_mr = mlx5_glue_alloc_null_mr,
+ .dereg_mr = mlx5_glue_dereg_mr,
+ .create_counter_set = mlx5_glue_create_counter_set,
+ .destroy_counter_set = mlx5_glue_destroy_counter_set,
+ .describe_counter_set = mlx5_glue_describe_counter_set,
+ .query_counter_set = mlx5_glue_query_counter_set,
+ .create_counters = mlx5_glue_create_counters,
+ .destroy_counters = mlx5_glue_destroy_counters,
+ .attach_counters = mlx5_glue_attach_counters,
+ .query_counters = mlx5_glue_query_counters,
+ .ack_async_event = mlx5_glue_ack_async_event,
+ .get_async_event = mlx5_glue_get_async_event,
+ .port_state_str = mlx5_glue_port_state_str,
+ .cq_ex_to_cq = mlx5_glue_cq_ex_to_cq,
+ .dr_create_flow_action_dest_flow_tbl =
+ mlx5_glue_dr_create_flow_action_dest_flow_tbl,
+ .dr_create_flow_action_dest_port =
+ mlx5_glue_dr_create_flow_action_dest_port,
+ .dr_create_flow_action_drop =
+ mlx5_glue_dr_create_flow_action_drop,
+ .dr_create_flow_action_push_vlan =
+ mlx5_glue_dr_create_flow_action_push_vlan,
+ .dr_create_flow_action_pop_vlan =
+ mlx5_glue_dr_create_flow_action_pop_vlan,
+ .dr_create_flow_tbl = mlx5_glue_dr_create_flow_tbl,
+ .dr_destroy_flow_tbl = mlx5_glue_dr_destroy_flow_tbl,
+ .dr_create_domain = mlx5_glue_dr_create_domain,
+ .dr_destroy_domain = mlx5_glue_dr_destroy_domain,
+ .dr_sync_domain = mlx5_glue_dr_sync_domain,
+ .dv_create_cq = mlx5_glue_dv_create_cq,
+ .dv_create_wq = mlx5_glue_dv_create_wq,
+ .dv_query_device = mlx5_glue_dv_query_device,
+ .dv_set_context_attr = mlx5_glue_dv_set_context_attr,
+ .dv_init_obj = mlx5_glue_dv_init_obj,
+ .dv_create_qp = mlx5_glue_dv_create_qp,
+ .dv_create_flow_matcher = mlx5_glue_dv_create_flow_matcher,
+ .dv_create_flow = mlx5_glue_dv_create_flow,
+ .dv_create_flow_action_counter =
+ mlx5_glue_dv_create_flow_action_counter,
+ .dv_create_flow_action_dest_ibv_qp =
+ mlx5_glue_dv_create_flow_action_dest_ibv_qp,
+ .dv_create_flow_action_dest_devx_tir =
+ mlx5_glue_dv_create_flow_action_dest_devx_tir,
+ .dv_create_flow_action_modify_header =
+ mlx5_glue_dv_create_flow_action_modify_header,
+ .dv_create_flow_action_packet_reformat =
+ mlx5_glue_dv_create_flow_action_packet_reformat,
+ .dv_create_flow_action_tag = mlx5_glue_dv_create_flow_action_tag,
+ .dv_create_flow_action_meter = mlx5_glue_dv_create_flow_action_meter,
+ .dv_modify_flow_action_meter = mlx5_glue_dv_modify_flow_action_meter,
+ .dv_create_flow_action_aso = mlx5_glue_dv_create_flow_action_aso,
+ .dr_create_flow_action_default_miss =
+ mlx5_glue_dr_create_flow_action_default_miss,
+ .dv_destroy_flow = mlx5_glue_dv_destroy_flow,
+ .dv_destroy_flow_matcher = mlx5_glue_dv_destroy_flow_matcher,
+ .dv_open_device = mlx5_glue_dv_open_device,
+ .devx_obj_create = mlx5_glue_devx_obj_create,
+ .devx_obj_destroy = mlx5_glue_devx_obj_destroy,
+ .devx_obj_query = mlx5_glue_devx_obj_query,
+ .devx_obj_modify = mlx5_glue_devx_obj_modify,
+ .devx_general_cmd = mlx5_glue_devx_general_cmd,
+ .devx_create_cmd_comp = mlx5_glue_devx_create_cmd_comp,
+ .devx_destroy_cmd_comp = mlx5_glue_devx_destroy_cmd_comp,
+ .devx_obj_query_async = mlx5_glue_devx_obj_query_async,
+ .devx_get_async_cmd_comp = mlx5_glue_devx_get_async_cmd_comp,
+ .devx_umem_reg = mlx5_glue_devx_umem_reg,
+ .devx_umem_dereg = mlx5_glue_devx_umem_dereg,
+ .devx_qp_query = mlx5_glue_devx_qp_query,
+ .devx_wq_query = mlx5_glue_devx_wq_query,
+ .devx_port_query = mlx5_glue_devx_port_query,
+ .dr_dump_domain = mlx5_glue_dr_dump_domain,
+ .dr_dump_rule = mlx5_glue_dr_dump_single_rule,
+ .dr_reclaim_domain_memory = mlx5_glue_dr_reclaim_domain_memory,
+ .dr_create_flow_action_sampler =
+ mlx5_glue_dr_create_flow_action_sampler,
+ .dr_create_flow_action_dest_array =
+ mlx5_glue_dr_action_create_dest_array,
+ .dr_allow_duplicate_rules = mlx5_glue_dr_allow_duplicate_rules,
+ .devx_query_eqn = mlx5_glue_devx_query_eqn,
+ .devx_create_event_channel = mlx5_glue_devx_create_event_channel,
+ .devx_destroy_event_channel = mlx5_glue_devx_destroy_event_channel,
+ .devx_subscribe_devx_event = mlx5_glue_devx_subscribe_devx_event,
+ .devx_subscribe_devx_event_fd = mlx5_glue_devx_subscribe_devx_event_fd,
+ .devx_get_event = mlx5_glue_devx_get_event,
+ .devx_alloc_uar = mlx5_glue_devx_alloc_uar,
+ .devx_free_uar = mlx5_glue_devx_free_uar,
+ .dv_alloc_var = mlx5_glue_dv_alloc_var,
+ .dv_free_var = mlx5_glue_dv_free_var,
+ .dv_alloc_pp = mlx5_glue_dv_alloc_pp,
+ .dv_free_pp = mlx5_glue_dv_free_pp,
+};
new file mode 100644
@@ -0,0 +1,373 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#ifndef MLX5_GLUE_H_
+#define MLX5_GLUE_H_
+
+#include <stddef.h>
+#include <stdint.h>
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/mlx5dv.h>
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_byteorder.h>
+
+#include "mlx5_autoconf.h"
+
+#ifndef MLX5_GLUE_VERSION
+#define MLX5_GLUE_VERSION ""
+#endif
+
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+struct ibv_counter_set;
+struct ibv_counter_set_data;
+struct ibv_counter_set_description;
+struct ibv_counter_set_init_attr;
+struct ibv_query_counter_set_attr;
+#endif
+
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+struct ibv_counters;
+struct ibv_counters_init_attr;
+struct ibv_counter_attach_attr;
+#endif
+
+#ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+struct mlx5dv_qp_init_attr;
+#endif
+
+#ifndef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
+struct mlx5dv_wq_init_attr;
+#endif
+
+#ifndef HAVE_IBV_FLOW_DV_SUPPORT
+struct mlx5dv_flow_matcher;
+struct mlx5dv_flow_matcher_attr;
+struct mlx5dv_flow_action_attr;
+struct mlx5dv_flow_match_parameters;
+struct mlx5dv_dr_flow_meter_attr;
+struct ibv_flow_action;
+enum mlx5dv_flow_action_packet_reformat_type { packet_reformat_type = 0, };
+enum mlx5dv_flow_table_type { flow_table_type = 0, };
+#endif
+
+#ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
+#define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
+#endif
+
+#ifndef HAVE_IBV_DEVX_OBJ
+struct mlx5dv_devx_obj;
+struct mlx5dv_devx_umem { uint32_t umem_id; };
+struct mlx5dv_devx_uar { void *reg_addr; void *base_addr; uint32_t page_id; };
+#endif
+
+#ifndef HAVE_IBV_DEVX_ASYNC
+struct mlx5dv_devx_cmd_comp;
+struct mlx5dv_devx_async_cmd_hdr;
+#endif
+
+#ifndef HAVE_MLX5DV_DR
+enum mlx5dv_dr_domain_type { unused, };
+struct mlx5dv_dr_domain;
+struct mlx5dv_dr_action;
+#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
+#endif
+
+#ifndef HAVE_MLX5DV_DR_DEVX_PORT
+struct mlx5dv_devx_port;
+#endif
+
+#ifndef HAVE_MLX5DV_DR_DEVX_PORT_V35
+struct mlx5dv_port;
+#endif
+
+#define MLX5_PORT_QUERY_VPORT (1u << 0)
+#define MLX5_PORT_QUERY_REG_C0 (1u << 1)
+
+struct mlx5_port_info {
+ uint16_t query_flags;
+ uint16_t vport_id; /* Associated VF vport index (if any). */
+ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */
+ uint32_t vport_meta_mask; /* Used for vport index field match mask. */
+};
+
+#ifndef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
+struct mlx5dv_dr_flow_meter_attr;
+#endif
+
+#ifndef HAVE_MLX5_DR_CREATE_ACTION_FLOW_SAMPLE
+struct mlx5dv_dr_flow_sampler_attr {
+ uint32_t sample_ratio;
+ void *default_next_table;
+ size_t num_sample_actions;
+ struct mlx5dv_dr_action **sample_actions;
+ uint64_t action;
+};
+#endif
+
+#ifndef HAVE_MLX5_DR_CREATE_ACTION_DEST_ARRAY
+enum mlx5dv_dr_action_dest_type {
+ MLX5DV_DR_ACTION_DEST,
+ MLX5DV_DR_ACTION_DEST_REFORMAT,
+};
+struct mlx5dv_dr_action_dest_reformat {
+ struct mlx5dv_dr_action *reformat;
+ struct mlx5dv_dr_action *dest;
+};
+struct mlx5dv_dr_action_dest_attr {
+ enum mlx5dv_dr_action_dest_type type;
+ union {
+ struct mlx5dv_dr_action *dest;
+ struct mlx5dv_dr_action_dest_reformat *dest_reformat;
+ };
+};
+#endif
+
+#ifndef HAVE_IBV_DEVX_EVENT
+struct mlx5dv_devx_event_channel { int fd; };
+struct mlx5dv_devx_async_event_hdr;
+#define MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA 1
+#endif
+
+#ifndef HAVE_IBV_VAR
+struct mlx5dv_var { uint32_t page_id; uint32_t length; off_t mmap_off;
+ uint64_t comp_mask; };
+#endif
+
+#ifndef HAVE_IBV_RELAXED_ORDERING
+#define IBV_ACCESS_RELAXED_ORDERING 0
+#endif
+
+struct mlx5_glue {
+ const char *version;
+ int (*fork_init)(void);
+ struct ibv_pd *(*alloc_pd)(struct ibv_context *context);
+ int (*dealloc_pd)(struct ibv_pd *pd);
+ struct ibv_device **(*get_device_list)(int *num_devices);
+ void (*free_device_list)(struct ibv_device **list);
+ struct ibv_context *(*open_device)(struct ibv_device *device);
+ int (*close_device)(struct ibv_context *context);
+ int (*query_device)(struct ibv_context *context,
+ struct ibv_device_attr *device_attr);
+ int (*query_device_ex)(struct ibv_context *context,
+ const struct ibv_query_device_ex_input *input,
+ struct ibv_device_attr_ex *attr);
+ int (*query_rt_values_ex)(struct ibv_context *context,
+ struct ibv_values_ex *values);
+ int (*query_port)(struct ibv_context *context, uint8_t port_num,
+ struct ibv_port_attr *port_attr);
+ struct ibv_comp_channel *(*create_comp_channel)
+ (struct ibv_context *context);
+ int (*destroy_comp_channel)(struct ibv_comp_channel *channel);
+ struct ibv_cq *(*create_cq)(struct ibv_context *context, int cqe,
+ void *cq_context,
+ struct ibv_comp_channel *channel,
+ int comp_vector);
+ int (*destroy_cq)(struct ibv_cq *cq);
+ int (*get_cq_event)(struct ibv_comp_channel *channel,
+ struct ibv_cq **cq, void **cq_context);
+ void (*ack_cq_events)(struct ibv_cq *cq, unsigned int nevents);
+ struct ibv_rwq_ind_table *(*create_rwq_ind_table)
+ (struct ibv_context *context,
+ struct ibv_rwq_ind_table_init_attr *init_attr);
+ int (*destroy_rwq_ind_table)(struct ibv_rwq_ind_table *rwq_ind_table);
+ struct ibv_wq *(*create_wq)(struct ibv_context *context,
+ struct ibv_wq_init_attr *wq_init_attr);
+ int (*destroy_wq)(struct ibv_wq *wq);
+ int (*modify_wq)(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr);
+ struct ibv_flow *(*create_flow)(struct ibv_qp *qp,
+ struct ibv_flow_attr *flow);
+ int (*destroy_flow)(struct ibv_flow *flow_id);
+ int (*destroy_flow_action)(void *action);
+ struct ibv_qp *(*create_qp)(struct ibv_pd *pd,
+ struct ibv_qp_init_attr *qp_init_attr);
+ struct ibv_qp *(*create_qp_ex)
+ (struct ibv_context *context,
+ struct ibv_qp_init_attr_ex *qp_init_attr_ex);
+ int (*destroy_qp)(struct ibv_qp *qp);
+ int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+ int attr_mask);
+ struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr,
+ size_t length, int access);
+ struct ibv_mr *(*alloc_null_mr)(struct ibv_pd *pd);
+ int (*dereg_mr)(struct ibv_mr *mr);
+ struct ibv_counter_set *(*create_counter_set)
+ (struct ibv_context *context,
+ struct ibv_counter_set_init_attr *init_attr);
+ int (*destroy_counter_set)(struct ibv_counter_set *cs);
+ int (*describe_counter_set)
+ (struct ibv_context *context,
+ uint16_t counter_set_id,
+ struct ibv_counter_set_description *cs_desc);
+ int (*query_counter_set)(struct ibv_query_counter_set_attr *query_attr,
+ struct ibv_counter_set_data *cs_data);
+ struct ibv_counters *(*create_counters)
+ (struct ibv_context *context,
+ struct ibv_counters_init_attr *init_attr);
+ int (*destroy_counters)(struct ibv_counters *counters);
+ int (*attach_counters)(struct ibv_counters *counters,
+ struct ibv_counter_attach_attr *attr,
+ struct ibv_flow *flow);
+ int (*query_counters)(struct ibv_counters *counters,
+ uint64_t *counters_value,
+ uint32_t ncounters,
+ uint32_t flags);
+ void (*ack_async_event)(struct ibv_async_event *event);
+ int (*get_async_event)(struct ibv_context *context,
+ struct ibv_async_event *event);
+ const char *(*port_state_str)(enum ibv_port_state port_state);
+ struct ibv_cq *(*cq_ex_to_cq)(struct ibv_cq_ex *cq);
+ void *(*dr_create_flow_action_dest_flow_tbl)(void *tbl);
+ void *(*dr_create_flow_action_dest_port)(void *domain,
+ uint32_t port);
+ void *(*dr_create_flow_action_drop)(void);
+ void *(*dr_create_flow_action_push_vlan)
+ (struct mlx5dv_dr_domain *domain,
+ rte_be32_t vlan_tag);
+ void *(*dr_create_flow_action_pop_vlan)(void);
+ void *(*dr_create_flow_tbl)(void *domain, uint32_t level);
+ int (*dr_destroy_flow_tbl)(void *tbl);
+ void *(*dr_create_domain)(struct ibv_context *ctx,
+ enum mlx5dv_dr_domain_type domain);
+ int (*dr_destroy_domain)(void *domain);
+ int (*dr_sync_domain)(void *domain, uint32_t flags);
+ struct ibv_cq_ex *(*dv_create_cq)
+ (struct ibv_context *context,
+ struct ibv_cq_init_attr_ex *cq_attr,
+ struct mlx5dv_cq_init_attr *mlx5_cq_attr);
+ struct ibv_wq *(*dv_create_wq)
+ (struct ibv_context *context,
+ struct ibv_wq_init_attr *wq_attr,
+ struct mlx5dv_wq_init_attr *mlx5_wq_attr);
+ int (*dv_query_device)(struct ibv_context *ctx_in,
+ struct mlx5dv_context *attrs_out);
+ int (*dv_set_context_attr)(struct ibv_context *ibv_ctx,
+ enum mlx5dv_set_ctx_attr_type type,
+ void *attr);
+ int (*dv_init_obj)(struct mlx5dv_obj *obj, uint64_t obj_type);
+ struct ibv_qp *(*dv_create_qp)
+ (struct ibv_context *context,
+ struct ibv_qp_init_attr_ex *qp_init_attr_ex,
+ struct mlx5dv_qp_init_attr *dv_qp_init_attr);
+ void *(*dv_create_flow_matcher)
+ (struct ibv_context *context,
+ struct mlx5dv_flow_matcher_attr *matcher_attr,
+ void *tbl);
+ void *(*dv_create_flow)(void *matcher, void *match_value,
+ size_t num_actions, void *actions[]);
+ void *(*dv_create_flow_action_counter)(void *obj, uint32_t offset);
+ void *(*dv_create_flow_action_dest_ibv_qp)(void *qp);
+ void *(*dv_create_flow_action_dest_devx_tir)(void *tir);
+ void *(*dv_create_flow_action_modify_header)
+ (struct ibv_context *ctx, enum mlx5dv_flow_table_type ft_type,
+ void *domain, uint64_t flags, size_t actions_sz,
+ uint64_t actions[]);
+ void *(*dv_create_flow_action_packet_reformat)
+ (struct ibv_context *ctx,
+ enum mlx5dv_flow_action_packet_reformat_type reformat_type,
+ enum mlx5dv_flow_table_type ft_type,
+ struct mlx5dv_dr_domain *domain,
+ uint32_t flags, size_t data_sz, void *data);
+ void *(*dv_create_flow_action_tag)(uint32_t tag);
+ void *(*dv_create_flow_action_meter)
+ (struct mlx5dv_dr_flow_meter_attr *attr);
+ int (*dv_modify_flow_action_meter)(void *action,
+ struct mlx5dv_dr_flow_meter_attr *attr, uint64_t modify_bits);
+ void *(*dr_create_flow_action_default_miss)(void);
+ int (*dv_destroy_flow)(void *flow);
+ int (*dv_destroy_flow_matcher)(void *matcher);
+ struct ibv_context *(*dv_open_device)(struct ibv_device *device);
+ struct mlx5dv_var *(*dv_alloc_var)(struct ibv_context *context,
+ uint32_t flags);
+ void (*dv_free_var)(struct mlx5dv_var *var);
+ struct mlx5dv_devx_uar *(*devx_alloc_uar)(struct ibv_context *context,
+ uint32_t flags);
+ void (*devx_free_uar)(struct mlx5dv_devx_uar *devx_uar);
+ struct mlx5dv_devx_obj *(*devx_obj_create)
+ (struct ibv_context *ctx,
+ const void *in, size_t inlen,
+ void *out, size_t outlen);
+ int (*devx_obj_destroy)(struct mlx5dv_devx_obj *obj);
+ int (*devx_obj_query)(struct mlx5dv_devx_obj *obj,
+ const void *in, size_t inlen,
+ void *out, size_t outlen);
+ int (*devx_obj_modify)(struct mlx5dv_devx_obj *obj,
+ const void *in, size_t inlen,
+ void *out, size_t outlen);
+ int (*devx_general_cmd)(struct ibv_context *context,
+ const void *in, size_t inlen,
+ void *out, size_t outlen);
+ struct mlx5dv_devx_cmd_comp *(*devx_create_cmd_comp)
+ (struct ibv_context *context);
+ void (*devx_destroy_cmd_comp)(struct mlx5dv_devx_cmd_comp *cmd_comp);
+ int (*devx_obj_query_async)(struct mlx5dv_devx_obj *obj,
+ const void *in, size_t inlen,
+ size_t outlen, uint64_t wr_id,
+ struct mlx5dv_devx_cmd_comp *cmd_comp);
+ int (*devx_get_async_cmd_comp)(struct mlx5dv_devx_cmd_comp *cmd_comp,
+ struct mlx5dv_devx_async_cmd_hdr *resp,
+ size_t cmd_resp_len);
+ struct mlx5dv_devx_umem *(*devx_umem_reg)(struct ibv_context *context,
+ void *addr, size_t size,
+ uint32_t access);
+ int (*devx_umem_dereg)(struct mlx5dv_devx_umem *dv_devx_umem);
+ int (*devx_qp_query)(struct ibv_qp *qp,
+ const void *in, size_t inlen,
+ void *out, size_t outlen);
+ int (*devx_wq_query)(struct ibv_wq *wq, const void *in, size_t inlen,
+ void *out, size_t outlen);
+ int (*devx_port_query)(struct ibv_context *ctx,
+ uint32_t port_num,
+ struct mlx5_port_info *info);
+ int (*dr_dump_domain)(FILE *file, void *domain);
+ int (*dr_dump_rule)(FILE *file, void *rule);
+ int (*devx_query_eqn)(struct ibv_context *context, uint32_t cpus,
+ uint32_t *eqn);
+ struct mlx5dv_devx_event_channel *(*devx_create_event_channel)
+ (struct ibv_context *context, int flags);
+ void (*devx_destroy_event_channel)
+ (struct mlx5dv_devx_event_channel *event_channel);
+ int (*devx_subscribe_devx_event)
+ (struct mlx5dv_devx_event_channel *event_channel,
+ struct mlx5dv_devx_obj *obj,
+ uint16_t events_sz,
+ uint16_t events_num[],
+ uint64_t cookie);
+ int (*devx_subscribe_devx_event_fd)
+ (struct mlx5dv_devx_event_channel *event_channel,
+ int fd,
+ struct mlx5dv_devx_obj *obj,
+ uint16_t event_num);
+ ssize_t (*devx_get_event)
+ (struct mlx5dv_devx_event_channel *event_channel,
+ struct mlx5dv_devx_async_event_hdr *event_data,
+ size_t event_resp_len);
+ void (*dr_reclaim_domain_memory)(void *domain, uint32_t enable);
+ void (*dr_allow_duplicate_rules)(void *domain, uint32_t allow);
+ struct mlx5dv_pp *(*dv_alloc_pp)(struct ibv_context *context,
+ size_t pp_context_sz,
+ const void *pp_context,
+ uint32_t flags);
+ void (*dv_free_pp)(struct mlx5dv_pp *pp);
+ void *(*dr_create_flow_action_sampler)
+ (struct mlx5dv_dr_flow_sampler_attr *attr);
+ void *(*dr_create_flow_action_dest_array)
+ (void *domain,
+ size_t num_dest,
+ struct mlx5dv_dr_action_dest_attr *dests[]);
+ void *(*dv_create_flow_action_aso)
+ (struct mlx5dv_dr_domain *domain, void *aso_obj,
+ uint32_t offset, uint32_t flags, uint8_t return_reg_c);
+};
+
+extern const struct mlx5_glue *mlx5_glue;
+
+#endif /* MLX5_GLUE_H_ */