[RFC,v1] raw/ntb: add support for 6th Gen Intel Xeon

Message ID 20231208080759.3484405-1-junfeng.guo@intel.com (mailing list archive)
State New
Delegated to: Thomas Monjalon
Headers
Series [RFC,v1] raw/ntb: add support for 6th Gen Intel Xeon |

Checks

Context Check Description
ci/checkpatch success coding style OK
ci/loongarch-compilation warning apply patch failure
ci/Intel-compilation success Compilation OK
ci/intel-Testing success Testing PASS
ci/intel-Functional success Functional PASS

Commit Message

Guo, Junfeng Dec. 8, 2023, 8:07 a.m. UTC
  Add support for the 6th Gen Intel Xeon Scalable processors. Note that NTB
devices within the 6th Gen Intel Xeon compliant to PCIe 5.0 spec.

Signed-off-by: Junfeng Guo <junfeng.guo@intel.com>
---
Depends-on: series-30460 \
	("raw/ntb: refactor to support NTB on 5th Gen Intel Xeon")
---
 doc/guides/rawdevs/ntb.rst     |  1 +
 drivers/raw/ntb/ntb.c          |  2 ++
 drivers/raw/ntb/ntb.h          |  1 +
 drivers/raw/ntb/ntb_hw_intel.c | 66 +++++++++++++++++-----------------
 drivers/raw/ntb/ntb_hw_intel.h | 40 ++++++++++++---------
 usertools/dpdk-devbind.py      |  2 +-
 6 files changed, 61 insertions(+), 51 deletions(-)
  

Patch

diff --git a/doc/guides/rawdevs/ntb.rst b/doc/guides/rawdevs/ntb.rst
index ee452af4fd..e663b7a088 100644
--- a/doc/guides/rawdevs/ntb.rst
+++ b/doc/guides/rawdevs/ntb.rst
@@ -153,6 +153,7 @@  Limitation
 
 This PMD is only supported on Intel Xeon Platforms:
 
+- 6th Generation Intel® Xeon® Scalable Processors. (NTB GEN5 device id: 0x0DB4)
 - 5th Generation Intel® Xeon® Scalable Processors. (NTB GEN4 device id: 0x347E)
 - 4th Generation Intel® Xeon® Scalable Processors. (NTB GEN4 device id: 0x347E)
 - 3rd Generation Intel® Xeon® Scalable Processors. (NTB GEN4 device id: 0x347E)
diff --git a/drivers/raw/ntb/ntb.c b/drivers/raw/ntb/ntb.c
index 2a163eff60..f1e48b877d 100644
--- a/drivers/raw/ntb/ntb.c
+++ b/drivers/raw/ntb/ntb.c
@@ -26,6 +26,7 @@ 
 static const struct rte_pci_id pci_id_ntb_map[] = {
 	{ RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_GEN3) },
 	{ RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_GEN4) },
+	{ RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_GEN5) },
 	{ .vendor_id = 0, /* sentinel */ },
 };
 
@@ -1380,6 +1381,7 @@  ntb_init_hw(struct rte_rawdev *dev, struct rte_pci_device *pci_dev)
 	switch (pci_dev->id.device_id) {
 	case NTB_INTEL_DEV_ID_B2B_GEN3:
 	case NTB_INTEL_DEV_ID_B2B_GEN4:
+	case NTB_INTEL_DEV_ID_B2B_GEN5:
 		hw->ntb_ops = &intel_ntb_ops;
 		break;
 	default:
diff --git a/drivers/raw/ntb/ntb.h b/drivers/raw/ntb/ntb.h
index d2e6d566ed..adbc164b48 100644
--- a/drivers/raw/ntb/ntb.h
+++ b/drivers/raw/ntb/ntb.h
@@ -19,6 +19,7 @@  extern int ntb_logtype;
 /* Device IDs */
 #define NTB_INTEL_DEV_ID_B2B_GEN3   0x201C
 #define NTB_INTEL_DEV_ID_B2B_GEN4   0x347E
+#define NTB_INTEL_DEV_ID_B2B_GEN5   0x0DB4
 
 /* Reserved to app to use. */
 #define NTB_SPAD_USER               "spad_user_"
diff --git a/drivers/raw/ntb/ntb_hw_intel.c b/drivers/raw/ntb/ntb_hw_intel.c
index 3e2094437f..76dce5e95f 100644
--- a/drivers/raw/ntb/ntb_hw_intel.c
+++ b/drivers/raw/ntb/ntb_hw_intel.c
@@ -98,7 +98,7 @@  intel_ntb4_check_ppd_for_ICX(struct ntb_hw *hw, uint32_t reg_val)
 }
 
 static int
-intel_ntb4_check_ppd(struct ntb_hw *hw)
+intel_ntb4_5_check_ppd(struct ntb_hw *hw)
 {
 	uint8_t revision_id;
 	uint32_t reg_val;
@@ -112,7 +112,7 @@  intel_ntb4_check_ppd(struct ntb_hw *hw)
 		return -EIO;
 	}
 
-	reg_val = rte_read32(hw->hw_addr + XEON_NTB4_PPD1_OFFSET);
+	reg_val = rte_read32(hw->hw_addr + XEON_NTB4_5_PPD1_OFFSET);
 
 	/* Distinguish HW platform (3rd Gen Xeon) via PCI Revision ID */
 	if (revision_id <= NTB_PCI_DEV_REVISION_ICX_MAX &&
@@ -120,8 +120,8 @@  intel_ntb4_check_ppd(struct ntb_hw *hw)
 		return intel_ntb4_check_ppd_for_ICX(hw, reg_val);
 
 	/* Check connection topo type. Only support B2B. */
-	switch (reg_val & XEON_NTB4_PPD_CONN_MASK) {
-	case XEON_NTB4_PPD_CONN_B2B:
+	switch (reg_val & XEON_NTB4_5_PPD_CONN_MASK) {
+	case XEON_NTB4_5_PPD_CONN_B2B:
 		NTB_LOG(INFO, "Topo B2B (back to back) is using.");
 		break;
 	default:
@@ -130,12 +130,12 @@  intel_ntb4_check_ppd(struct ntb_hw *hw)
 	}
 
 	/* Check device config status. */
-	switch (reg_val & XEON_NTB4_PPD_DEV_MASK) {
-	case XEON_NTB4_PPD_DEV_DSD:
+	switch (reg_val & XEON_NTB4_5_PPD_DEV_MASK) {
+	case XEON_NTB4_5_PPD_DEV_DSD:
 		NTB_LOG(INFO, "DSD, Downstream Device.");
 		hw->topo = NTB_TOPO_B2B_DSD;
 		break;
-	case XEON_NTB4_PPD_DEV_USD:
+	case XEON_NTB4_5_PPD_DEV_USD:
 		NTB_LOG(INFO, "USD, Upstream device.");
 		hw->topo = NTB_TOPO_B2B_USD;
 		break;
@@ -161,9 +161,9 @@  intel_ntb_dev_init(const struct rte_rawdev *dev)
 	if (is_gen3_ntb(hw->pci_dev))
 		/* PPD is in config space for NTB Gen3 */
 		ret = intel_ntb3_check_ppd(hw);
-	else if (is_gen4_ntb(hw->pci_dev))
-		/* PPD is in MMIO for NTB Gen4 */
-		ret = intel_ntb4_check_ppd(hw);
+	else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev))
+		/* PPD is in MMIO for NTB Gen4/5 */
+		ret = intel_ntb4_5_check_ppd(hw);
 	else {
 		NTB_LOG(ERR, "Cannot init device for unsupported device.");
 		return -ENOTSUP;
@@ -264,9 +264,9 @@  intel_ntb_mw_set_trans(const struct rte_rawdev *dev, int mw_idx,
 		base &= ~0xf;
 		limit = base + size;
 		rte_write64(limit, limit_addr);
-	} else if (is_gen4_ntb(hw->pci_dev)) {
+	} else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev)) {
 		/* Set translate base address index register */
-		xlat_off = XEON_NTB4_IM1XBASEIDX_OFFSET +
+		xlat_off = XEON_NTB4_5_IM1XBASEIDX_OFFSET +
 			   mw_idx * XEON_XBASEIDX_INTERVAL;
 		xlat_addr = hw->hw_addr + xlat_off;
 		rte_write16(rte_log2_u64(size), xlat_addr);
@@ -321,8 +321,8 @@  intel_ntb_get_link_status(const struct rte_rawdev *dev)
 			NTB_LOG(ERR, "Unable to get link status.");
 			return -EIO;
 		}
-	} else if (is_gen4_ntb(hw->pci_dev)) {
-		reg_off = XEON_NTB4_LINK_STATUS_OFFSET;
+	} else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev)) {
+		reg_off = XEON_NTB4_5_LINK_STATUS_OFFSET;
 		reg_val = rte_read16(hw->hw_addr + reg_off);
 	} else {
 		NTB_LOG(ERR, "Cannot get link status for unsupported device.");
@@ -368,7 +368,7 @@  intel_ntb_gen3_set_link(const struct ntb_hw *hw, bool up)
 }
 
 static int
-intel_ntb_gen4_set_link(const struct ntb_hw *hw, bool up)
+intel_ntb_gen4_5_set_link(const struct ntb_hw *hw, bool up)
 {
 	uint32_t ntb_ctrl, ppd0;
 	uint16_t link_ctrl;
@@ -380,20 +380,20 @@  intel_ntb_gen4_set_link(const struct ntb_hw *hw, bool up)
 		ntb_ctrl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
 		rte_write32(ntb_ctrl, reg_addr);
 
-		reg_addr = hw->hw_addr + XEON_NTB4_LINK_CTRL_OFFSET;
+		reg_addr = hw->hw_addr + XEON_NTB4_5_LINK_CTRL_OFFSET;
 		link_ctrl = rte_read16(reg_addr);
-		link_ctrl &= ~XEON_NTB4_LINK_CTRL_LINK_DIS;
+		link_ctrl &= ~XEON_NTB4_5_LINK_CTRL_LINK_DIS;
 		rte_write16(link_ctrl, reg_addr);
 
 		/* start link training */
-		reg_addr = hw->hw_addr + XEON_NTB4_PPD0_OFFSET;
+		reg_addr = hw->hw_addr + XEON_NTB4_5_PPD0_OFFSET;
 		ppd0 = rte_read32(reg_addr);
-		ppd0 |= XEON_NTB4_PPD_LINKTRN;
+		ppd0 |= XEON_NTB4_5_PPD_LINKTRN;
 		rte_write32(ppd0, reg_addr);
 
 		/* make sure link training has started */
 		ppd0 = rte_read32(reg_addr);
-		if (!(ppd0 & XEON_NTB4_PPD_LINKTRN)) {
+		if (!(ppd0 & XEON_NTB4_5_PPD_LINKTRN)) {
 			NTB_LOG(ERR, "Link is not training.");
 			return -EINVAL;
 		}
@@ -404,9 +404,9 @@  intel_ntb_gen4_set_link(const struct ntb_hw *hw, bool up)
 		ntb_ctrl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
 		rte_write32(ntb_ctrl, reg_addr);
 
-		reg_addr = hw->hw_addr + XEON_NTB4_LINK_CTRL_OFFSET;
+		reg_addr = hw->hw_addr + XEON_NTB4_5_LINK_CTRL_OFFSET;
 		link_ctrl = rte_read16(reg_addr);
-		link_ctrl |= XEON_NTB4_LINK_CTRL_LINK_DIS;
+		link_ctrl |= XEON_NTB4_5_LINK_CTRL_LINK_DIS;
 		rte_write16(link_ctrl, reg_addr);
 	}
 
@@ -421,8 +421,8 @@  intel_ntb_set_link(const struct rte_rawdev *dev, bool up)
 
 	if (is_gen3_ntb(hw->pci_dev))
 		ret = intel_ntb_gen3_set_link(hw, up);
-	else if (is_gen4_ntb(hw->pci_dev))
-		ret = intel_ntb_gen4_set_link(hw, up);
+	else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev))
+		ret = intel_ntb_gen4_5_set_link(hw, up);
 	else {
 		NTB_LOG(ERR, "Cannot set link for unsupported device.");
 		ret = -ENOTSUP;
@@ -447,8 +447,8 @@  intel_ntb_spad_read(const struct rte_rawdev *dev, int spad, bool peer)
 	if (is_gen3_ntb(hw->pci_dev))
 		reg_off = peer ? XEON_NTB3_B2B_SPAD_OFFSET :
 				XEON_IM_SPAD_OFFSET;
-	else if (is_gen4_ntb(hw->pci_dev))
-		reg_off = peer ? XEON_NTB4_B2B_SPAD_OFFSET :
+	else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev))
+		reg_off = peer ? XEON_NTB4_5_B2B_SPAD_OFFSET :
 				XEON_IM_SPAD_OFFSET;
 	else {
 		NTB_LOG(ERR, "Cannot read spad for unsupported device.");
@@ -477,8 +477,8 @@  intel_ntb_spad_write(const struct rte_rawdev *dev, int spad,
 	if (is_gen3_ntb(hw->pci_dev))
 		reg_off = peer ? XEON_NTB3_B2B_SPAD_OFFSET :
 				XEON_IM_SPAD_OFFSET;
-	else if (is_gen4_ntb(hw->pci_dev))
-		reg_off = peer ? XEON_NTB4_B2B_SPAD_OFFSET :
+	else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev))
+		reg_off = peer ? XEON_NTB4_5_B2B_SPAD_OFFSET :
 				XEON_IM_SPAD_OFFSET;
 	else {
 		NTB_LOG(ERR, "Cannot write spad for unsupported device.");
@@ -516,9 +516,9 @@  intel_ntb_db_clear(const struct rte_rawdev *dev, uint64_t db_bits)
 	db_off = XEON_IM_INT_STATUS_OFFSET;
 	db_addr = hw->hw_addr + db_off;
 
-	if (is_gen4_ntb(hw->pci_dev))
-		rte_write16(XEON_NTB4_SLOTSTS_DLLSCS,
-			    hw->hw_addr + XEON_NTB4_SLOTSTS);
+	if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev))
+		rte_write16(XEON_NTB4_5_SLOTSTS_DLLSCS,
+			    hw->hw_addr + XEON_NTB4_5_SLOTSTS);
 	rte_write64(db_bits, db_addr);
 
 	return 0;
@@ -578,8 +578,8 @@  intel_ntb_vector_bind(const struct rte_rawdev *dev, uint8_t intr, uint8_t msix)
 	/* Bind intr source to msix vector */
 	if (is_gen3_ntb(hw->pci_dev))
 		reg_off = XEON_NTB3_INTVEC_OFFSET;
-	else if (is_gen4_ntb(hw->pci_dev))
-		reg_off = XEON_NTB4_INTVEC_OFFSET;
+	else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev))
+		reg_off = XEON_NTB4_5_INTVEC_OFFSET;
 	else {
 		NTB_LOG(ERR, "Cannot bind vectors for unsupported device.");
 		return -ENOTSUP;
diff --git a/drivers/raw/ntb/ntb_hw_intel.h b/drivers/raw/ntb/ntb_hw_intel.h
index 5a45785b57..404dfcf2fc 100644
--- a/drivers/raw/ntb/ntb_hw_intel.h
+++ b/drivers/raw/ntb/ntb_hw_intel.h
@@ -42,11 +42,11 @@ 
 #define XEON_CORERRSTS_OFFSET		0x0158
 #define XEON_NTB3_LINK_STATUS_OFFSET	0x01a2
 /* Link status and PPD are in MMIO but not config space for Gen4/5 NTB */
-#define XEON_NTB4_PPD0_OFFSET		0xb0d4
-#define XEON_NTB4_PPD1_OFFSET		0xb4c0
-#define XEON_NTB4_LINK_CTRL_OFFSET	0xb050
-#define XEON_NTB4_LINK_STATUS_OFFSET	0xb052
-#define XEON_NTB4_LINK_CTRL_LINK_DIS	0x0010
+#define XEON_NTB4_5_PPD0_OFFSET		0xb0d4
+#define XEON_NTB4_5_PPD1_OFFSET		0xb4c0
+#define XEON_NTB4_5_LINK_CTRL_OFFSET	0xb050
+#define XEON_NTB4_5_LINK_STATUS_OFFSET	0xb052
+#define XEON_NTB4_5_LINK_CTRL_LINK_DIS	0x0010
 
 
 #define XEON_NTBCNTL_OFFSET		0x0000
@@ -56,16 +56,16 @@ 
 #define XEON_IMBAR2XBASE_OFFSET		0x0020		/* SBAR4XLAT */
 #define XEON_IMBAR2XLMT_OFFSET		0x0028		/* SBAR4LMT */
 #define XEON_XBASEIDX_INTERVAL		0x0002
-#define XEON_NTB4_IM1XBASEIDX_OFFSET	0x0074
-#define XEON_NTB4_IM2XBASEIDX_OFFSET	0x0076
+#define XEON_NTB4_5_IM1XBASEIDX_OFFSET	0x0074
+#define XEON_NTB4_5_IM2XBASEIDX_OFFSET	0x0076
 #define XEON_IM_INT_STATUS_OFFSET	0x0040
 #define XEON_IM_INT_DISABLE_OFFSET	0x0048
 #define XEON_IM_SPAD_OFFSET		0x0080		/* SPAD */
 #define XEON_NTB3_B2B_SPAD_OFFSET	0x0180		/* NTB GEN3 B2B SPAD */
-#define XEON_NTB4_B2B_SPAD_OFFSET	0x8080		/* NTB GEN4/5 B2B SPAD */
+#define XEON_NTB4_5_B2B_SPAD_OFFSET	0x8080		/* NTB GEN4/5 B2B SPAD */
 #define XEON_USMEMMISS_OFFSET		0x0070
 #define XEON_NTB3_INTVEC_OFFSET		0x00d0
-#define XEON_NTB4_INTVEC_OFFSET		0x0050
+#define XEON_NTB4_5_INTVEC_OFFSET	0x0050
 #define XEON_IM_DOORBELL_OFFSET		0x0100		/* SDOORBELL0 */
 #define XEON_EMBAR0XBASE_OFFSET		0x4008		/* B2B_XLAT */
 #define XEON_EMBAR1XBASE_OFFSET		0x4010		/* PBAR2XLAT */
@@ -95,15 +95,15 @@ 
 #define XEON_GEN3_NTB4_PPD_DEV_DSD	0x1000
 #define XEON_GEN3_NTB4_PPD_DEV_USD	0x0000
 
-#define XEON_NTB4_PPD_CONN_MASK		0x0700
-#define XEON_NTB4_PPD_CONN_B2B		0x0200
-#define XEON_NTB4_PPD_DEV_MASK		0x4000
-#define XEON_NTB4_PPD_DEV_DSD		0x4000
-#define XEON_NTB4_PPD_DEV_USD		0x0000
+#define XEON_NTB4_5_PPD_CONN_MASK	0x0700
+#define XEON_NTB4_5_PPD_CONN_B2B	0x0200
+#define XEON_NTB4_5_PPD_DEV_MASK	0x4000
+#define XEON_NTB4_5_PPD_DEV_DSD		0x4000
+#define XEON_NTB4_5_PPD_DEV_USD		0x0000
 
-#define XEON_NTB4_PPD_LINKTRN		0x0008
-#define XEON_NTB4_SLOTSTS		0xb05a
-#define XEON_NTB4_SLOTSTS_DLLSCS	0x100
+#define XEON_NTB4_5_PPD_LINKTRN		0x0008
+#define XEON_NTB4_5_SLOTSTS		0xb05a
+#define XEON_NTB4_5_SLOTSTS_DLLSCS	0x100
 
 #define XEON_MW_COUNT			2
 
@@ -129,4 +129,10 @@  is_gen4_ntb(const struct rte_pci_device *pci_dev)
 	return pci_dev->id.device_id == NTB_INTEL_DEV_ID_B2B_GEN4;
 }
 
+static inline bool
+is_gen5_ntb(const struct rte_pci_device *pci_dev)
+{
+	return pci_dev->id.device_id == NTB_INTEL_DEV_ID_B2B_GEN5;
+}
+
 #endif /* _NTB_HW_INTEL_H_ */
diff --git a/usertools/dpdk-devbind.py b/usertools/dpdk-devbind.py
index 230c81c5f3..6cfe2aa7ed 100755
--- a/usertools/dpdk-devbind.py
+++ b/usertools/dpdk-devbind.py
@@ -60,7 +60,7 @@ 
                   'SVendor': None, 'SDevice': None}
 intel_idxd_spr = {'Class': '08', 'Vendor': '8086', 'Device': '0b25',
                   'SVendor': None, 'SDevice': None}
-intel_ntb = {'Class': '06', 'Vendor': '8086', 'Device': '201c,347e',
+intel_ntb = {'Class': '06', 'Vendor': '8086', 'Device': '201c,347e,0db4',
              'SVendor': None, 'SDevice': None}
 
 cnxk_sso = {'Class': '08', 'Vendor': '177d', 'Device': 'a0f9,a0fa',