[next,2/7] vmxnet3: add support for capability registers
Checks
Commit Message
This patch enhances vmxnet3 to suuport capability registers which
allows it to enable features selectively. The DCR register tracks
the capabilities vmxnet3 device supports. The PTCR register states
the capabilities that the passthrough device supports.
With the help of these registers, vmxnet3 can enable only those
features which the passthrough device supoprts. This allows
smooth trasition to Uniform-Passthrough (UPT) mode if the virtual
nic requests it. If PTCR register returns nothing or error it means
UPT is not being requested and vnic will continue in emulation mode.
Signed-off-by: Ronak Doshi <doshir@vmware.com>
Acked-by: Jochen Behrens <jbehrens@vmware.com>
---
drivers/net/vmxnet3/base/vmxnet3_defs.h | 44 +++++++++++++++++---
drivers/net/vmxnet3/vmxnet3_ethdev.c | 71 +++++++++++++++++++++++++++++++++
drivers/net/vmxnet3/vmxnet3_ethdev.h | 11 +++++
lib/ethdev/rte_ethdev.h | 2 +
4 files changed, 123 insertions(+), 5 deletions(-)
Comments
On 4/12/2023 5:26 PM, Ronak Doshi wrote:
> This patch enhances vmxnet3 to suuport capability registers which
s/suuport/support/
> allows it to enable features selectively. The DCR register tracks
> the capabilities vmxnet3 device supports. The PTCR register states
> the capabilities that the passthrough device supports.
>
> With the help of these registers, vmxnet3 can enable only those
> features which the passthrough device supoprts. This allows
s/supoprts/supports/
> smooth trasition to Uniform-Passthrough (UPT) mode if the virtual
s/trasition/transition/
> nic requests it. If PTCR register returns nothing or error it means
s/nic/NIC/
> UPT is not being requested and vnic will continue in emulation mode.
>
> Signed-off-by: Ronak Doshi <doshir@vmware.com>
> Acked-by: Jochen Behrens <jbehrens@vmware.com>
<...>
> +static int
> +eth_vmxnet3_setup_capabilities(struct vmxnet3_hw *hw,
> + struct rte_eth_dev *eth_dev)
> +{
> + uint32_t dcr, ptcr, value;
> + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
> +
> + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
> + VMXNET3_CMD_GET_MAX_CAPABILITIES);
> + value = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
> + hw->max_capabilities[0] = value;
> + dcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_DCR);
> + hw->DCR_capabilities[0] = dcr;
> + hw->used_DCR_capabilities[0] = 0;
> + ptcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_PTCR);
> + hw->PTCR_capabilities[0] = ptcr;
> + hw->used_PTCR_capabilities[0] = 0;
> +
> + if (hw->uptv2_enabled && !(ptcr & (1 << VMXNET3_DCR_ERROR))) {
> + PMD_DRV_LOG(NOTICE, "UPTv2 enabled");
> + hw->used_PTCR_capabilities[0] = ptcr;
> + } else {
> + /* Use all DCR capabilities, but disable large bar */
> + hw->used_DCR_capabilities[0] = dcr &
> + (~(1UL << VMXNET3_CAP_LARGE_BAR));
> + PMD_DRV_LOG(NOTICE, "UPTv2 disabled");
> + }
> + if (hw->DCR_capabilities[0] & (1UL << VMXNET3_CAP_OOORX_COMP) &&
> + hw->PTCR_capabilities[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) {
> + if (hw->uptv2_enabled) {
> + hw->used_PTCR_capabilities[0] |=
> + (1UL << VMXNET3_CAP_OOORX_COMP);
> + }
> + }
> + if (hw->used_PTCR_capabilities[0]) {
> + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DCR,
> + hw->used_PTCR_capabilities[0]);
> + } else if (hw->used_DCR_capabilities[0]) {
> + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DCR,
> + hw->used_DCR_capabilities[0]);
> + }
> + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
> + dcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
> + hw->used_DCR_capabilities[0] = dcr;
> + PMD_DRV_LOG(NOTICE, "Dev "PCI_PRI_FMT", vmxnet3 v%d, UPT enabled: %s, "
Checkpatch complains about syntax:
HECK:CONCATENATED_STRING: Concatenated strings should use spaces between
elements
#200: FILE: drivers/net/vmxnet3/vmxnet3_ethdev.c:346:
+ PMD_DRV_LOG(NOTICE, "Dev "PCI_PRI_FMT", vmxnet3 v%d, UPT enabled: %s, "
> + "DCR0=0x%08x, used DCR=0x%08x, "
> + "PTCR=0x%08x, used PTCR=0x%08x",
> + pci_dev->addr.domain, pci_dev->addr.bus,
> + pci_dev->addr.devid, pci_dev->addr.function, hw->version,
> + hw->uptv2_enabled ? "true" : "false",
> + hw->DCR_capabilities[0], hw->used_DCR_capabilities[0],
> + hw->PTCR_capabilities[0], hw->used_PTCR_capabilities[0]);
This log level is NOTICE, making it visible by default. And this
function always called when vesion >= 7.
Are you sure to make log this verbose? Or can it be DEBUG level?
<...>
> --- a/lib/ethdev/rte_ethdev.h
> +++ b/lib/ethdev/rte_ethdev.h
> @@ -1606,6 +1606,8 @@ struct rte_eth_conf {
> #define RTE_ETH_DEV_CAPA_FLOW_RULE_KEEP RTE_BIT64(3)
> /** Device supports keeping shared flow objects across restart. */
> #define RTE_ETH_DEV_CAPA_FLOW_SHARED_OBJECT_KEEP RTE_BIT64(4)
> +/**< (vmxnet3) Device supports smart NIC pass through queues */
> +#define RTE_ETH_DEV_CAPA_PASS_THRU RTE_BIT64(5)
'dev_info->dev_capa' is for generic device capabilities, we can't have
device specific capability here.
Can you please describe why it is required to expose this capability to
user, what user should do with this information?
@@ -26,6 +26,12 @@
#define VMXNET3_REG_MACH 0x30 /* MAC Address High */
#define VMXNET3_REG_ICR 0x38 /* Interrupt Cause Register */
#define VMXNET3_REG_ECR 0x40 /* Event Cause Register */
+#define VMXNET3_REG_DCR 0x48 /* Device capability register,
+ * from 0x48 to 0x80
+ */
+#define VMXNET3_REG_PTCR 0x88 /* Passthru capbility register
+ * from 0x88 to 0xb0
+ */
#define VMXNET3_REG_WSAL 0xF00 /* Wireless Shared Address Lo */
#define VMXNET3_REG_WSAH 0xF08 /* Wireless Shared Address Hi */
@@ -103,11 +109,13 @@ typedef enum {
VMXNET3_CMD_GET_CONF_INTR,
VMXNET3_CMD_GET_ADAPTIVE_RING_INFO,
VMXNET3_CMD_GET_TXDATA_DESC_SIZE,
- VMXNET3_CMD_RESERVED5,
- VMXNET3_CMD_RESERVED6,
- VMXNET3_CMD_RESERVED7,
- VMXNET3_CMD_RESERVED8,
- VMXNET3_CMD_GET_MAX_QUEUES_CONF,
+ VMXNET3_CMD_RESERVED5,
+ VMXNET3_CMD_RESERVED6,
+ VMXNET3_CMD_RESERVED7,
+ VMXNET3_CMD_RESERVED8,
+ VMXNET3_CMD_GET_MAX_QUEUES_CONF,
+ VMXNET3_CMD_GET_MAX_CAPABILITIES,
+ VMXNET3_CMD_GET_DCR0_REG,
} Vmxnet3_Cmd;
/* Adaptive Ring Info Flags */
@@ -877,4 +885,30 @@ do {\
#define VMXNET3_DID_PASSTHRU 0xFFFF
+#define VMXNET3_DCR_ERROR 31 /* error when bit 31 of DCR is set */
+#define VMXNET3_CAP_UDP_RSS 0 /* bit 0 of DCR 0 */
+#define VMXNET3_CAP_ESP_RSS_IPV4 1 /* bit 1 of DCR 0 */
+#define VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD 2 /* bit 2 of DCR 0 */
+#define VMXNET3_CAP_GENEVE_TSO 3 /* bit 3 of DCR 0 */
+#define VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD 4 /* bit 4 of DCR 0 */
+#define VMXNET3_CAP_VXLAN_TSO 5 /* bit 5 of DCR 0 */
+#define VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD 6 /* bit 6 of DCR 0 */
+#define VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD 7 /* bit 7 of DCR 0 */
+#define VMXNET3_CAP_PKT_STEERING_IPV4 8 /* bit 8 of DCR 0 */
+#define VMXNET3_CAP_VERSION_4_MAX VMXNET3_CAP_PKT_STEERING_IPV4
+#define VMXNET3_CAP_ESP_RSS_IPV6 9 /* bit 9 of DCR 0 */
+#define VMXNET3_CAP_VERSION_5_MAX VMXNET3_CAP_ESP_RSS_IPV6
+#define VMXNET3_CAP_ESP_OVER_UDP_RSS 10 /* bit 10 of DCR 0 */
+#define VMXNET3_CAP_INNER_RSS 11 /* bit 11 of DCR 0 */
+#define VMXNET3_CAP_INNER_ESP_RSS 12 /* bit 12 of DCR 0 */
+#define VMXNET3_CAP_CRC32_HASH_FUNC 13 /* bit 13 of DCR 0 */
+#define VMXNET3_CAP_VERSION_6_MAX VMXNET3_CAP_CRC32_HASH_FUNC
+#define VMXNET3_CAP_OAM_FILTER 14 /* bit 14 of DCR 0 */
+#define VMXNET3_CAP_ESP_QS 15 /* bit 15 of DCR 0 */
+#define VMXNET3_CAP_LARGE_BAR 16 /* bit 16 of DCR 0 */
+#define VMXNET3_CAP_OOORX_COMP 17 /* bit 17 of DCR 0 */
+#define VMXNET3_CAP_VERSION_7_MAX 18
+/* when new capability is introduced, update VMXNET3_CAP_MAX */
+#define VMXNET3_CAP_MAX VMXNET3_CAP_VERSION_7_MAX
+
#endif /* _VMXNET3_DEFS_H_ */
@@ -299,6 +299,61 @@ eth_vmxnet3_txdata_get(struct vmxnet3_hw *hw)
sizeof(struct Vmxnet3_TxDataDesc) : txdata_desc_size;
}
+static int
+eth_vmxnet3_setup_capabilities(struct vmxnet3_hw *hw,
+ struct rte_eth_dev *eth_dev)
+{
+ uint32_t dcr, ptcr, value;
+ struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+
+ VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
+ VMXNET3_CMD_GET_MAX_CAPABILITIES);
+ value = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
+ hw->max_capabilities[0] = value;
+ dcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_DCR);
+ hw->DCR_capabilities[0] = dcr;
+ hw->used_DCR_capabilities[0] = 0;
+ ptcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_PTCR);
+ hw->PTCR_capabilities[0] = ptcr;
+ hw->used_PTCR_capabilities[0] = 0;
+
+ if (hw->uptv2_enabled && !(ptcr & (1 << VMXNET3_DCR_ERROR))) {
+ PMD_DRV_LOG(NOTICE, "UPTv2 enabled");
+ hw->used_PTCR_capabilities[0] = ptcr;
+ } else {
+ /* Use all DCR capabilities, but disable large bar */
+ hw->used_DCR_capabilities[0] = dcr &
+ (~(1UL << VMXNET3_CAP_LARGE_BAR));
+ PMD_DRV_LOG(NOTICE, "UPTv2 disabled");
+ }
+ if (hw->DCR_capabilities[0] & (1UL << VMXNET3_CAP_OOORX_COMP) &&
+ hw->PTCR_capabilities[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) {
+ if (hw->uptv2_enabled) {
+ hw->used_PTCR_capabilities[0] |=
+ (1UL << VMXNET3_CAP_OOORX_COMP);
+ }
+ }
+ if (hw->used_PTCR_capabilities[0]) {
+ VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DCR,
+ hw->used_PTCR_capabilities[0]);
+ } else if (hw->used_DCR_capabilities[0]) {
+ VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DCR,
+ hw->used_DCR_capabilities[0]);
+ }
+ VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
+ dcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
+ hw->used_DCR_capabilities[0] = dcr;
+ PMD_DRV_LOG(NOTICE, "Dev "PCI_PRI_FMT", vmxnet3 v%d, UPT enabled: %s, "
+ "DCR0=0x%08x, used DCR=0x%08x, "
+ "PTCR=0x%08x, used PTCR=0x%08x",
+ pci_dev->addr.domain, pci_dev->addr.bus,
+ pci_dev->addr.devid, pci_dev->addr.function, hw->version,
+ hw->uptv2_enabled ? "true" : "false",
+ hw->DCR_capabilities[0], hw->used_DCR_capabilities[0],
+ hw->PTCR_capabilities[0], hw->used_PTCR_capabilities[0]);
+ return 0;
+}
+
/*
* It returns 0 on success.
*/
@@ -396,6 +451,11 @@ eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev)
return -EIO;
}
+ if (VMXNET3_VERSION_GE_7(hw)) {
+ /* start with UPTv2 enabled to avoid ESXi issues */
+ hw->uptv2_enabled = TRUE;
+ eth_vmxnet3_setup_capabilities(hw, eth_dev);
+ }
/* Getting MAC Address */
mac_lo = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACL);
mac_hi = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACH);
@@ -1433,6 +1493,17 @@ vmxnet3_dev_info_get(struct rte_eth_dev *dev,
{
struct vmxnet3_hw *hw = dev->data->dev_private;
int queues = 0;
+ uint32_t dcr, ptcr;
+
+ if (VMXNET3_VERSION_GE_7(hw)) {
+ dcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_DCR);
+ if (dcr & (1 << VMXNET3_CAP_LARGE_BAR)) {
+ ptcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_PTCR);
+ if (!(ptcr & (1 << VMXNET3_DCR_ERROR))) {
+ dev_info->dev_capa = RTE_ETH_DEV_CAPA_PASS_THRU;
+ }
+ }
+ }
if (VMXNET3_VERSION_GE_6(hw)) {
VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
@@ -122,6 +122,17 @@ struct vmxnet3_hw {
UPT1_RxStats saved_rx_stats[VMXNET3_EXT_MAX_RX_QUEUES];
UPT1_TxStats snapshot_tx_stats[VMXNET3_MAX_TX_QUEUES];
UPT1_RxStats snapshot_rx_stats[VMXNET3_MAX_RX_QUEUES];
+ /* device capability bit map */
+ uint32_t DCR_capabilities[8];
+ /* pass-through capability bit map */
+ uint32_t PTCR_capabilities[8];
+ /* max number of capabilities */
+ uint32_t max_capabilities[8];
+ /* used device capability bit map */
+ uint32_t used_DCR_capabilities[8];
+ /* used pass-through capability bit map */
+ uint32_t used_PTCR_capabilities[8];
+ bool uptv2_enabled;
};
#define VMXNET3_REV_7 6 /* Vmxnet3 Rev. 7 */
@@ -1606,6 +1606,8 @@ struct rte_eth_conf {
#define RTE_ETH_DEV_CAPA_FLOW_RULE_KEEP RTE_BIT64(3)
/** Device supports keeping shared flow objects across restart. */
#define RTE_ETH_DEV_CAPA_FLOW_SHARED_OBJECT_KEEP RTE_BIT64(4)
+/**< (vmxnet3) Device supports smart NIC pass through queues */
+#define RTE_ETH_DEV_CAPA_PASS_THRU RTE_BIT64(5)
/**@}*/
/*