On Sun, Jun 03, 2018 at 06:29:13PM +0300, Moti Haimovsky wrote:
> This RFC and patch aim to add support for building mlx5 PMD on 32bit
> archs such as i686.
> The attached patch was not tested on 32bit environments and is given
> here for explanatory purposes.
>
> Prerequisites:
> * RDMA-core with 32bit support libraries.
> * compile and build tools for 32bit systems according to DPDK documentation.
>
> Assumptions:
> * Kernel and drivers are 64bit.
> * Only user programs and support libraries are 32Bit.
>
> Changes proposed in the PMD for adding 32bit support:
> The attached patch provides the changes needed in the PMD in order to support
> 32bit builds with the major changes being:
> * Modifying the UAR access routine to support non-atomic 64bit writes
> according to the mlx5 hardware reference manual.
> * Handling 64bit address fields that are written to the device WQEs.
> * Modify UAR address and size mapping to fit into the 32bit address space.
>
> Concerns:
> * The major concern of this modification is the UAR mapping.
> Current PMD assumes 64bit address and therefore feels free to map 4G size
> UAR to virtual address just above the huge-page addresses which of course
> cannot work for 32bit address space.
> As a workaround I reduced the UAR size to half the original size but
> this of course is not the solution.
> * Will vectorized support work on 32bit builds ?
>
> I am interested in your inputs regarding the concerns listed above,
> especially the UAR mapping issue mentioned there.
>[...]
Hi Moti,
My biggest concern and why such support has been removed [1] is because
Mellanox OFED is not supporting 32bits compilation. As Mellanox OFED is
still supported, we need such guarantee.
On Mellanox website there is no way to download such kind of ISO which
means such support becomes only valid for upstream RDMA-Core library
with almost newer kernels.
If you can find a way to detect the PMD is compiled in front of Mellanox
OFED to forbid such support, it would be great, otherwise I am afraid
such support cannot be added back without creating a lot of issues for
people wanting it on 32bits processors with Mellanox OFED.
Thanks,
[1] https://dpdk.org/browse/dpdk/commit/?id=ebbb81eb27daca0a89ee8f228fcf141d9eb6ef1c
@@ -43,5 +43,6 @@ Multiprocess aware = Y
Other kdrv = Y
ARMv8 = Y
Power8 = Y
+x86-32 = Y
x86-64 = Y
Usage doc = Y
@@ -110,6 +110,28 @@
/** Driver-specific log messages type. */
int mlx5_logtype;
+static rte_spinlock_t mlx5_mmio_spinlock = RTE_SPINLOCK_INITIALIZER;
+
+/**
+ * Provide an emulation for 64 bit store on arch that does not have a 64 bit
+ * atomic store command. This is done using two 32bit store operations in
+ * address ascending order while holding a global spinlock.
+ */
+void
+mlx5_mmio_write64(uint64_t be_val, volatile void *addr)
+{
+ uint32_t first_dword =
+ rte_cpu_to_be_32(rte_be_to_cpu_64(be_val) >> 32);
+ uint32_t second_dword =
+ rte_cpu_to_be_32(rte_be_to_cpu_64(be_val));
+
+ rte_spinlock_lock(&mlx5_mmio_spinlock);
+ rte_write32(first_dword, addr);
+ rte_write32(second_dword,
+ (volatile void *)((volatile char *)addr + 4));
+ rte_spinlock_unlock(&mlx5_mmio_spinlock);
+}
+
/**
* Prepare shared data between primary and secondary process.
*/
@@ -600,9 +622,10 @@
rte_memseg_walk(find_lower_va_bound, &addr);
/* keep distance to hugepages to minimize potential conflicts. */
- addr = RTE_PTR_SUB(addr, MLX5_UAR_OFFSET + MLX5_UAR_SIZE);
+ addr = RTE_PTR_SUB(addr,
+ (uintptr_t)((MLX5_UAR_OFFSET + MLX5_UAR_SIZE) >> 2));
/* anonymous mmap, no real memory consumption. */
- addr = mmap(addr, MLX5_UAR_SIZE,
+ addr = mmap(addr, MLX5_UAR_SIZE >> 2,
PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (addr == MAP_FAILED) {
DRV_LOG(ERR,
@@ -644,7 +667,7 @@
return 0;
}
/* anonymous mmap, no real memory consumption. */
- addr = mmap(priv->uar_base, MLX5_UAR_SIZE,
+ addr = mmap(priv->uar_base, MLX5_UAR_SIZE >> 2,
PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (addr == MAP_FAILED) {
DRV_LOG(ERR, "port %u UAR mmap failed: %p size: %llu",
@@ -643,7 +643,7 @@
doorbell = (uint64_t)doorbell_hi << 32;
doorbell |= rxq->cqn;
rxq->cq_db[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(doorbell_hi);
- rte_write64(rte_cpu_to_be_64(doorbell), cq_db_reg);
+ mlx5_write64(rte_cpu_to_be_64(doorbell), cq_db_reg);
}
/**
@@ -715,8 +715,8 @@
*dseg = (rte_v128u32_t){
rte_cpu_to_be_32(length),
mlx5_tx_mb2mr(txq, buf),
- addr,
- addr >> 32,
+ lower_32_bits(addr),
+ upper_32_bits(addr),
};
++ds;
if (!segs_n)
@@ -754,8 +754,8 @@
*dseg = (rte_v128u32_t){
rte_cpu_to_be_32(length),
mlx5_tx_mb2mr(txq, buf),
- addr,
- addr >> 32,
+ lower_32_bits(addr),
+ upper_32_bits(addr),
};
(*txq->elts)[++elts_head & elts_m] = buf;
if (--segs_n)
@@ -1591,8 +1591,8 @@
*dseg = (rte_v128u32_t) {
rte_cpu_to_be_32(length),
mlx5_tx_mb2mr(txq, buf),
- addr,
- addr >> 32,
+ lower_32_bits(addr),
+ upper_32_bits(addr),
};
mpw.data.raw = (volatile void *)(dseg + 1);
mpw.total_len += (inl_pad + sizeof(*dseg));
@@ -276,7 +276,7 @@
}
/* new address in reserved UAR address space. */
addr = RTE_PTR_ADD(priv->uar_base,
- uar_va & (MLX5_UAR_SIZE - 1));
+ uar_va & (uintptr_t)(MLX5_UAR_SIZE - 1));
if (!already_mapped) {
pages[pages_n++] = uar_va;
/* fixed mmap to specified address in reserved
@@ -13,6 +13,8 @@
#include <assert.h>
#include <errno.h>
+#include <rte_io.h>
+
#include "mlx5_defs.h"
/* Bit-field manipulation. */
@@ -160,4 +162,44 @@
return l + r;
}
+/**
+ * A basic shift-right of a 64- or 32-bit quantity. Use this to suppress
+ * the "right shift count >= width of type" warning when that quantity is
+ * 32-bits.
+ *
+ * @param n
+ * the number we're accessing
+ */
+#define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16))
+
+/**
+ * lower_32_bits - return bits 0-31 of a number
+ *
+ * @param n
+ * the number we're accessing
+ */
+#define lower_32_bits(n) ((uint32_t)(n))
+
+/**
+ * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
+ * 64bit architectures.
+ *
+ * @param ve_val
+ * value to write in big endian format.
+ * @param addr
+ * Address to write to.
+ */
+extern void
+mlx5_mmio_write64(uint64_t be_val, volatile void *addr);
+
+static inline void
+mlx5_write64(uint64_t be_val, volatile void *addr)
+{
+#ifdef RTE_ARCH_64
+ rte_write64(be_val, addr);
+#else
+ mlx5_mmio_write64(be_val, addr);
+#endif
+}
+
#endif /* RTE_PMD_MLX5_UTILS_H_ */