[v6] app/testpmd: add IFPGA AFU register access function

Message ID 1546409941-135237-1-git-send-email-rosen.xu@intel.com
State Changes Requested, archived
Delegated to: Ferruh Yigit
Headers show
Series
  • [v6] app/testpmd: add IFPGA AFU register access function
Related show

Checks

Context Check Description
ci/Intel-compilation success Compilation OK
ci/intel-Performance-Testing success Performance Testing PASS
ci/mellanox-Performance-Testing success Performance Testing PASS
ci/checkpatch success coding style OK

Commit Message

Rosen Xu Jan. 2, 2019, 6:19 a.m.
Currently register read/write of testpmd is only for PCI device,
but more and more IFPGA based AFU devices need this feature to
access registers, this patch will add support for it.

Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Acked-by: Bernard Iremonger <bernard.iremonger@intel.com>

v5 updates:

Comments

Ferruh Yigit Jan. 8, 2019, 3:28 p.m. | #1
On 1/2/2019 6:19 AM, Rosen Xu wrote:
> Currently register read/write of testpmd is only for PCI device,
> but more and more IFPGA based AFU devices need this feature to
> access registers, this patch will add support for it.
> 
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> Acked-by: Bernard Iremonger <bernard.iremonger@intel.com>
> 
> v5 updates:
> ===========
>  - Added Macro to fix compile dependency of ifpga for testpmd
> ---
>  app/test-pmd/config.c  | 253 ++++++++++++++++++++++++++++++++++++++++++++++++-
>  app/test-pmd/testpmd.h |  64 +++++++++++++
>  2 files changed, 315 insertions(+), 2 deletions(-)
> 
> diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
> index b9e5dd9..5600ef5 100644
> --- a/app/test-pmd/config.c
> +++ b/app/test-pmd/config.c
> @@ -866,7 +866,50 @@ void print_valid_ports(void)
>  	printf("Invalid vlan_id %d (must be < 4096)\n", vlan_id);
>  	return 1;
>  }
> +#ifdef RTE_LIBRTE_IFPGA_BUS
> +static int
> +port_reg_off_is_invalid(portid_t port_id, uint32_t reg_off)
> +{
> +	const struct rte_pci_device *pci_dev;
> +	const struct rte_bus *bus;
> +	uint64_t len;
> +	const struct rte_afu_device *afu_dev;
>  
> +	if (reg_off & 0x3) {
> +		printf("Port register offset 0x%X not aligned on a 4-byte "
> +		       "boundary\n",
> +		       (unsigned int)reg_off);
> +		return 1;
> +	}
> +
> +	if (!ports[port_id].dev_info.device) {
> +		printf("Invalid device\n");
> +		return 0;
> +	}
> +
> +	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
> +	if (bus && !strcmp(bus->name, "pci")) {
> +		pci_dev = RTE_DEV_TO_PCI(ports[port_id].dev_info.device);
> +		len = pci_dev->mem_resource[0].len;
> +	} else if (bus && !strcmp(bus->name, "ifpga")) {
> +		afu_dev = RTE_DEV_TO_AFU(ports[port_id].dev_info.device);
> +		len = afu_dev->mem_resource[0].len;
> +	} else {
> +		printf("Not a PCI or AFU device\n");
> +		return 1;
> +	}
> +
> +	if (reg_off >= len) {
> +		printf("Port %d: register offset %u (0x%X) out of port "
> +		       "PCI or AFU device "
> +		       "resource (length=%"PRIu64")\n",
> +		       port_id, (unsigned int)reg_off,
> +			(unsigned int)reg_off, len);
> +		return 1;
> +	}
> +	return 0;
> +}

Do we need to duplicate all function? I think wrapping only following part with
'#ifdef RTE_LIBRTE_IFPGA_BUS' can work:

  	} else if (bus && !strcmp(bus->name, "ifpga")) {
  		afu_dev = RTE_DEV_TO_AFU(ports[port_id].dev_info.device);


<...>

> +void
> +port_reg_set(portid_t port_id, uint32_t reg_off, uint32_t reg_v)
> +{
> +	const struct rte_bus *bus;
> +
> +	if (port_id_is_invalid(port_id, ENABLED_WARN))
> +		return;
> +	if (port_reg_off_is_invalid(port_id, reg_off))
> +		return;
> +
> +	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
> +	if (bus && !strcmp(bus->name, "pci")) {
> +		port_id_pci_reg_write(port_id, reg_off, reg_v);
> +	} else if (bus && !strcmp(bus->name, "ifpga")) {
> +		port_id_afu_reg_write(port_id, reg_off, reg_v);

Same thing for all above functions, instead of duplication all function with
#ifdef, only wrapping "ifpga" related lines should work.

<...>

> @@ -11,6 +11,9 @@
>  #include <rte_bus_pci.h>
>  #include <rte_gro.h>
>  #include <rte_gso.h>
> +#ifdef RTE_LIBRTE_IFPGA_BUS
> +#include <rte_bus_ifpga.h>
> +#endif

This is causing build error with meson, because 'bus_ifpga' is not added as a
dependency to testpmd. Can you please fix it?

Patch

===========
 - Added Macro to fix compile dependency of ifpga for testpmd
---
 app/test-pmd/config.c  | 253 ++++++++++++++++++++++++++++++++++++++++++++++++-
 app/test-pmd/testpmd.h |  64 +++++++++++++
 2 files changed, 315 insertions(+), 2 deletions(-)

diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index b9e5dd9..5600ef5 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -866,7 +866,50 @@  void print_valid_ports(void)
 	printf("Invalid vlan_id %d (must be < 4096)\n", vlan_id);
 	return 1;
 }
+#ifdef RTE_LIBRTE_IFPGA_BUS
+static int
+port_reg_off_is_invalid(portid_t port_id, uint32_t reg_off)
+{
+	const struct rte_pci_device *pci_dev;
+	const struct rte_bus *bus;
+	uint64_t len;
+	const struct rte_afu_device *afu_dev;
 
+	if (reg_off & 0x3) {
+		printf("Port register offset 0x%X not aligned on a 4-byte "
+		       "boundary\n",
+		       (unsigned int)reg_off);
+		return 1;
+	}
+
+	if (!ports[port_id].dev_info.device) {
+		printf("Invalid device\n");
+		return 0;
+	}
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		pci_dev = RTE_DEV_TO_PCI(ports[port_id].dev_info.device);
+		len = pci_dev->mem_resource[0].len;
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		afu_dev = RTE_DEV_TO_AFU(ports[port_id].dev_info.device);
+		len = afu_dev->mem_resource[0].len;
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return 1;
+	}
+
+	if (reg_off >= len) {
+		printf("Port %d: register offset %u (0x%X) out of port "
+		       "PCI or AFU device "
+		       "resource (length=%"PRIu64")\n",
+		       port_id, (unsigned int)reg_off,
+			(unsigned int)reg_off, len);
+		return 1;
+	}
+	return 0;
+}
+#else
 static int
 port_reg_off_is_invalid(portid_t port_id, uint32_t reg_off)
 {
@@ -903,7 +946,7 @@  void print_valid_ports(void)
 	}
 	return 0;
 }
-
+#endif
 static int
 reg_bit_pos_is_invalid(uint8_t bit_pos)
 {
@@ -923,6 +966,212 @@  void print_valid_ports(void)
 	printf("0x%08X (%u)\n", (unsigned)reg_v, (unsigned)reg_v);
 }
 
+#ifdef RTE_LIBRTE_IFPGA_BUS
+void
+port_reg_bit_display(portid_t port_id, uint32_t reg_off, uint8_t bit_x)
+{
+	uint32_t reg_v;
+	const struct rte_bus *bus;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return;
+	if (port_reg_off_is_invalid(port_id, reg_off))
+		return;
+	if (reg_bit_pos_is_invalid(bit_x))
+		return;
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		reg_v = port_id_pci_reg_read(port_id, reg_off);
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		reg_v = port_id_afu_reg_read(port_id, reg_off);
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return;
+	}
+	display_port_and_reg_off(port_id, (unsigned int)reg_off);
+	printf("bit %d=%d\n", bit_x, (int) ((reg_v & (1 << bit_x)) >> bit_x));
+}
+
+void
+port_reg_bit_field_display(portid_t port_id, uint32_t reg_off,
+			   uint8_t bit1_pos, uint8_t bit2_pos)
+{
+	uint32_t reg_v;
+	uint8_t  l_bit;
+	uint8_t  h_bit;
+	const struct rte_bus *bus;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return;
+	if (port_reg_off_is_invalid(port_id, reg_off))
+		return;
+	if (reg_bit_pos_is_invalid(bit1_pos))
+		return;
+	if (reg_bit_pos_is_invalid(bit2_pos))
+		return;
+	if (bit1_pos > bit2_pos)
+		l_bit = bit2_pos, h_bit = bit1_pos;
+	else
+		l_bit = bit1_pos, h_bit = bit2_pos;
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		reg_v = port_id_pci_reg_read(port_id, reg_off);
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		reg_v = port_id_afu_reg_read(port_id, reg_off);
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return;
+	}
+	reg_v >>= l_bit;
+	if (h_bit < 31)
+		reg_v &= ((1 << (h_bit - l_bit + 1)) - 1);
+	display_port_and_reg_off(port_id, (unsigned int)reg_off);
+	printf("bits[%d, %d]=0x%0*X (%u)\n", l_bit, h_bit,
+	       ((h_bit - l_bit) / 4) + 1, (unsigned int)reg_v,
+		(unsigned int)reg_v);
+}
+
+void
+port_reg_display(portid_t port_id, uint32_t reg_off)
+{
+	uint32_t reg_v;
+	const struct rte_bus *bus;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return;
+	if (port_reg_off_is_invalid(port_id, reg_off))
+		return;
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		reg_v = port_id_pci_reg_read(port_id, reg_off);
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		reg_v = port_id_afu_reg_read(port_id, reg_off);
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return;
+	}
+	display_port_reg_value(port_id, reg_off, reg_v);
+}
+
+void
+port_reg_bit_set(portid_t port_id, uint32_t reg_off, uint8_t bit_pos,
+		 uint8_t bit_v)
+{
+	uint32_t reg_v;
+	const struct rte_bus *bus;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return;
+	if (port_reg_off_is_invalid(port_id, reg_off))
+		return;
+	if (reg_bit_pos_is_invalid(bit_pos))
+		return;
+	if (bit_v > 1) {
+		printf("Invalid bit value %d (must be 0 or 1)\n", (int) bit_v);
+		return;
+	}
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		reg_v = port_id_pci_reg_read(port_id, reg_off);
+		if (bit_v == 0)
+			reg_v &= ~(1 << bit_pos);
+		else
+			reg_v |= (1 << bit_pos);
+		port_id_pci_reg_write(port_id, reg_off, reg_v);
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		reg_v = port_id_afu_reg_read(port_id, reg_off);
+		if (bit_v == 0)
+			reg_v &= ~(1 << bit_pos);
+		else
+			reg_v |= (1 << bit_pos);
+		port_id_afu_reg_write(port_id, reg_off, reg_v);
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return;
+	}
+	display_port_reg_value(port_id, reg_off, reg_v);
+}
+
+void
+port_reg_bit_field_set(portid_t port_id, uint32_t reg_off,
+		       uint8_t bit1_pos, uint8_t bit2_pos, uint32_t value)
+{
+	uint32_t max_v;
+	uint32_t reg_v;
+	uint8_t  l_bit;
+	uint8_t  h_bit;
+	const struct rte_bus *bus;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return;
+	if (port_reg_off_is_invalid(port_id, reg_off))
+		return;
+	if (reg_bit_pos_is_invalid(bit1_pos))
+		return;
+	if (reg_bit_pos_is_invalid(bit2_pos))
+		return;
+	if (bit1_pos > bit2_pos)
+		l_bit = bit2_pos, h_bit = bit1_pos;
+	else
+		l_bit = bit1_pos, h_bit = bit2_pos;
+
+	if ((h_bit - l_bit) < 31)
+		max_v = (1 << (h_bit - l_bit + 1)) - 1;
+	else
+		max_v = 0xFFFFFFFF;
+
+	if (value > max_v) {
+		printf("Invalid value %u (0x%x) must be < %u (0x%x)\n",
+				(unsigned int)value, (unsigned int)value,
+				(unsigned int)max_v, (unsigned int)max_v);
+		return;
+	}
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		reg_v = port_id_pci_reg_read(port_id, reg_off);
+		reg_v &= ~(max_v << l_bit); /* Keep unchanged bits */
+		reg_v |= (value << l_bit); /* Set changed bits */
+		port_id_pci_reg_write(port_id, reg_off, reg_v);
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		reg_v = port_id_afu_reg_read(port_id, reg_off);
+		reg_v &= ~(max_v << l_bit); /* Keep unchanged bits */
+		reg_v |= (value << l_bit); /* Set changed bits */
+		port_id_afu_reg_write(port_id, reg_off, reg_v);
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return;
+	}
+	display_port_reg_value(port_id, reg_off, reg_v);
+}
+
+void
+port_reg_set(portid_t port_id, uint32_t reg_off, uint32_t reg_v)
+{
+	const struct rte_bus *bus;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return;
+	if (port_reg_off_is_invalid(port_id, reg_off))
+		return;
+
+	bus = rte_bus_find_by_device(ports[port_id].dev_info.device);
+	if (bus && !strcmp(bus->name, "pci")) {
+		port_id_pci_reg_write(port_id, reg_off, reg_v);
+	} else if (bus && !strcmp(bus->name, "ifpga")) {
+		port_id_afu_reg_write(port_id, reg_off, reg_v);
+	} else {
+		printf("Not a PCI or AFU device\n");
+		return;
+	}
+
+	display_port_reg_value(port_id, reg_off, reg_v);
+}
+#else
 void
 port_reg_bit_display(portid_t port_id, uint32_t reg_off, uint8_t bit_x)
 {
@@ -1058,7 +1307,7 @@  void print_valid_ports(void)
 	port_id_pci_reg_write(port_id, reg_off, reg_v);
 	display_port_reg_value(port_id, reg_off, reg_v);
 }
-
+#endif
 void
 port_mtu_set(portid_t port_id, uint16_t mtu)
 {
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 3ff11e6..49b5de5 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -11,6 +11,9 @@ 
 #include <rte_bus_pci.h>
 #include <rte_gro.h>
 #include <rte_gso.h>
+#ifdef RTE_LIBRTE_IFPGA_BUS
+#include <rte_bus_ifpga.h>
+#endif
 
 #define RTE_PORT_ALL            (~(portid_t)0x0)
 
@@ -671,6 +674,67 @@  struct mplsoudp_decap_conf {
 #define port_id_pci_reg_write(pt_id, reg_off, reg_value) \
 	port_pci_reg_write(&ports[(pt_id)], (reg_off), (reg_value))
 
+#ifdef RTE_LIBRTE_IFPGA_BUS
+/**
+ * Read/Write operations on an AFU register of a port.
+ */
+static inline uint32_t
+port_afu_reg_read(struct rte_port *port, uint32_t reg_off)
+{
+	const struct rte_afu_device *afu_dev;
+	const struct rte_bus *bus;
+	void *reg_addr;
+	uint32_t reg_v;
+
+	if (!port->dev_info.device) {
+		printf("Invalid device\n");
+		return 0;
+	}
+
+	bus = rte_bus_find_by_device(port->dev_info.device);
+	if (bus && !strcmp(bus->name, "ifpga")) {
+		afu_dev = RTE_DEV_TO_AFU(port->dev_info.device);
+	} else {
+		printf("Not an IFPGA AFU device\n");
+		return 0;
+	}
+
+	reg_addr = ((char *)afu_dev->mem_resource[0].addr + reg_off);
+	reg_v = *((volatile uint32_t *)reg_addr);
+	return rte_le_to_cpu_32(reg_v);
+}
+
+#define port_id_afu_reg_read(pt_id, reg_off) \
+	port_afu_reg_read(&ports[(pt_id)], (reg_off))
+
+static inline void
+port_afu_reg_write(struct rte_port *port, uint32_t reg_off, uint32_t reg_v)
+{
+	const struct rte_afu_device *afu_dev;
+	const struct rte_bus *bus;
+	void *reg_addr;
+
+	if (!port->dev_info.device) {
+		printf("Invalid device\n");
+		return;
+	}
+
+	bus = rte_bus_find_by_device(port->dev_info.device);
+	if (bus && !strcmp(bus->name, "ifpga")) {
+		afu_dev = RTE_DEV_TO_AFU(port->dev_info.device);
+	} else {
+		printf("Not an IFPGA AFU device\n");
+		return;
+	}
+
+	reg_addr = ((char *)afu_dev->mem_resource[0].addr + reg_off);
+	*((volatile uint32_t *)reg_addr) = rte_cpu_to_le_32(reg_v);
+}
+
+#define port_id_afu_reg_write(pt_id, reg_off, reg_value) \
+	port_afu_reg_write(&ports[(pt_id)], (reg_off), (reg_value))
+#endif
+
 /* Prototypes */
 unsigned int parse_item_list(char* str, const char* item_name,
 			unsigned int max_items,