[v3,4/5] raw/ifpga: support ofs card probe

  PAC N6000 is the first OFS platform, its device id is added to ifpga
device support list.

Previous FPGA platform like Intel PAC N3000 and N5000, FME DFL (Device
Feature List) starts from BAR0 by default, port DFL location is indicated
in PORTn_OFFSET register in FME. In OFS implementation, FME DFL and port
DFL location can be defined individually in PCIe VSEC (Vendor Specific
Extented Capabilities). In this patch, DFL definition is searched in VSEC,
the legacy DFL is used only when DFL VSEC is not present.

In original DFL enumeration process, AFU is expected to locate in port DFL,
but this is not the case in OFS implementation. In this patch, enumeration
can search AFU in any PF/VF which has no FME and port.

Signed-off-by: Wei Huang <wei.huang@intel.com>
---
v2: fix build error in UB2004-32
---
v3: update commit log with Tianfei's comment, treat 7 as special BAR index.
---
 drivers/raw/ifpga/base/ifpga_api.c         |  28 +-
 drivers/raw/ifpga/base/ifpga_defines.h     |   1 +
 drivers/raw/ifpga/base/ifpga_enumerate.c   | 681 ++++++++++++++++++++++-------
 drivers/raw/ifpga/base/ifpga_enumerate.h   |   2 +
 drivers/raw/ifpga/base/ifpga_feature_dev.c |  18 +-
 drivers/raw/ifpga/base/ifpga_hw.h          |   2 +
 drivers/raw/ifpga/base/opae_hw_api.c       |   2 +-
 drivers/raw/ifpga/base/opae_hw_api.h       |   2 +
 drivers/raw/ifpga/ifpga_rawdev.c           |  48 +-
 9 files changed, 592 insertions(+), 192 deletions(-)
  

Hi Wei,

Pls close compile warnings. Thanks a lot.

Thanks,
Rosen

> -----Original Message-----
> From: Huang, Wei <wei.huang@intel.com>
> Sent: Thursday, May 26, 2022 11:33
> To: dev@dpdk.org; thomas@monjalon.net; nipun.gupta@nxp.com;
> hemant.agrawal@nxp.com
> Cc: stable@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Huang, Wei
> <wei.huang@intel.com>
> Subject: [PATCH v3 4/5] raw/ifpga: support ofs card probe
> 
> PAC N6000 is the first OFS platform, its device id is added to ifpga device
> support list.
> 
> Previous FPGA platform like Intel PAC N3000 and N5000, FME DFL (Device
> Feature List) starts from BAR0 by default, port DFL location is indicated in
> PORTn_OFFSET register in FME. In OFS implementation, FME DFL and port
> DFL location can be defined individually in PCIe VSEC (Vendor Specific
> Extented Capabilities). In this patch, DFL definition is searched in VSEC, the
> legacy DFL is used only when DFL VSEC is not present.
> 
> In original DFL enumeration process, AFU is expected to locate in port DFL,
> but this is not the case in OFS implementation. In this patch, enumeration can
> search AFU in any PF/VF which has no FME and port.
> 
> Signed-off-by: Wei Huang <wei.huang@intel.com>
> ---
> v2: fix build error in UB2004-32
> ---
> v3: update commit log with Tianfei's comment, treat 7 as special BAR index.
> ---
>  drivers/raw/ifpga/base/ifpga_api.c         |  28 +-
>  drivers/raw/ifpga/base/ifpga_defines.h     |   1 +
>  drivers/raw/ifpga/base/ifpga_enumerate.c   | 681
> ++++++++++++++++++++++-------
>  drivers/raw/ifpga/base/ifpga_enumerate.h   |   2 +
>  drivers/raw/ifpga/base/ifpga_feature_dev.c |  18 +-
>  drivers/raw/ifpga/base/ifpga_hw.h          |   2 +
>  drivers/raw/ifpga/base/opae_hw_api.c       |   2 +-
>  drivers/raw/ifpga/base/opae_hw_api.h       |   2 +
>  drivers/raw/ifpga/ifpga_rawdev.c           |  48 +-
>  9 files changed, 592 insertions(+), 192 deletions(-)
> 
> diff --git a/drivers/raw/ifpga/base/ifpga_api.c
> b/drivers/raw/ifpga/base/ifpga_api.c
> index 4610ef1..f19cc26 100644
> --- a/drivers/raw/ifpga/base/ifpga_api.c
> +++ b/drivers/raw/ifpga/base/ifpga_api.c
> @@ -13,15 +13,22 @@
>  static int ifpga_acc_get_uuid(struct opae_accelerator *acc,
>  			      struct uuid *uuid)
>  {
> -	struct opae_bridge *br = acc->br;
> -	struct ifpga_port_hw *port;
> +	struct ifpga_afu_info *afu_info = acc->data;
> +	struct opae_reg_region *region;
> +	u64 val = 0;
> 
> -	if (!br || !br->data)
> -		return -EINVAL;
> +	if (!afu_info)
> +		return -ENODEV;
> 
> -	port = br->data;
> +	region = &afu_info->region[0];
> +	if (uuid) {
> +		val = readq(region->addr + sizeof(struct feature_header));
> +		opae_memcpy(uuid->b, &val, sizeof(u64));
> +		val = readq(region->addr + sizeof(struct feature_header) + 8);
> +		opae_memcpy(uuid->b + 8, &val, sizeof(u64));
> +	}
> 
> -	return fpga_get_afu_uuid(port, uuid);
> +	return 0;
>  }
> 
>  static int ifpga_acc_set_irq(struct opae_accelerator *acc, @@ -32,6 +39,9
> @@ static int ifpga_acc_set_irq(struct opae_accelerator *acc,
>  	struct ifpga_port_hw *port;
>  	struct fpga_uafu_irq_set irq_set;
> 
> +	if (!afu_info)
> +		return -ENODEV;
> +
>  	if (!br || !br->data)
>  		return -EINVAL;
> 
> @@ -68,7 +78,7 @@ static int ifpga_acc_get_region_info(struct
> opae_accelerator *acc,
>  	struct ifpga_afu_info *afu_info = acc->data;
> 
>  	if (!afu_info)
> -		return -EINVAL;
> +		return -ENODEV;
> 
>  	if (info->index >= afu_info->num_regions)
>  		return -EINVAL;
> @@ -89,7 +99,7 @@ static int ifpga_acc_read(struct opae_accelerator *acc,
> unsigned int region_idx,
>  	struct opae_reg_region *region;
> 
>  	if (!afu_info)
> -		return -EINVAL;
> +		return -ENODEV;
> 
>  	if (offset + byte <= offset)
>  		return -EINVAL;
> @@ -129,7 +139,7 @@ static int ifpga_acc_write(struct opae_accelerator
> *acc,
>  	struct opae_reg_region *region;
> 
>  	if (!afu_info)
> -		return -EINVAL;
> +		return -ENODEV;
> 
>  	if (offset + byte <= offset)
>  		return -EINVAL;
> diff --git a/drivers/raw/ifpga/base/ifpga_defines.h
> b/drivers/raw/ifpga/base/ifpga_defines.h
> index 8f62033..9a280eb 100644
> --- a/drivers/raw/ifpga/base/ifpga_defines.h
> +++ b/drivers/raw/ifpga/base/ifpga_defines.h
> @@ -73,6 +73,7 @@
>  enum fpga_id_type {
>  	FME_ID,
>  	PORT_ID,
> +	AFU_ID,
>  	FPGA_ID_MAX,
>  };
> 
> diff --git a/drivers/raw/ifpga/base/ifpga_enumerate.c
> b/drivers/raw/ifpga/base/ifpga_enumerate.c
> index 48b8af4..0e09904 100644
> --- a/drivers/raw/ifpga/base/ifpga_enumerate.c
> +++ b/drivers/raw/ifpga/base/ifpga_enumerate.c
> @@ -2,6 +2,10 @@
>   * Copyright(c) 2010-2018 Intel Corporation
>   */
> 
> +#include <fcntl.h>
> +#include <inttypes.h>
> +#include <unistd.h>
> +
>  #include "opae_hw_api.h"
>  #include "ifpga_api.h"
> 
> @@ -9,6 +13,19 @@
>  #include "ifpga_enumerate.h"
>  #include "ifpga_feature_dev.h"
> 
> +struct dfl_fpga_enum_dfl {
> +	u64 start;
> +	u64 len;
> +	void *addr;
> +	TAILQ_ENTRY(dfl_fpga_enum_dfl) node;
> +};
> +
> +TAILQ_HEAD(dfl_fpga_enum_dfls, dfl_fpga_enum_dfl); struct
> +dfl_fpga_enum_info {
> +	struct ifpga_hw *hw;
> +	struct dfl_fpga_enum_dfls dfls;
> +};
> +
>  struct build_feature_devs_info {
>  	struct opae_adapter_data_pci *pci_data;
> 
> @@ -21,7 +38,6 @@ struct build_feature_devs_info {
>  	void *ioaddr;
>  	void *ioend;
>  	uint64_t phys_addr;
> -	int current_bar;
> 
>  	void *pfme_hdr;
> 
> @@ -141,8 +157,10 @@ static u64 feature_id(void __iomem *start)
>  			unsigned int size, unsigned int vec_start,
>  			unsigned int vec_cnt)
>  {
> -	return build_info_add_sub_feature(binfo, start, fid, size, vec_start,
> -			vec_cnt);
> +	if (binfo->current_type != AFU_ID)
> +		return build_info_add_sub_feature(binfo, start, fid, size,
> +			vec_start, vec_cnt);
> +	return 0;
>  }
> 
>  /*
> @@ -152,13 +170,14 @@ static u64 feature_id(void __iomem *start)
>   */
>  static bool feature_is_UAFU(struct build_feature_devs_info *binfo)  {
> -	if (binfo->current_type != PORT_ID)
> -		return false;
> +	if ((binfo->current_type == PORT_ID) ||
> +		(binfo->current_type == AFU_ID))
> +		return true;
> 
> -	return true;
> +	return false;
>  }
> 
> -static int parse_feature_port_uafu(struct build_feature_devs_info *binfo,
> +static int parse_feature_uafu(struct build_feature_devs_info *binfo,
>  				   struct feature_header *hdr)
>  {
>  	u64 id = PORT_FEATURE_ID_UAFU;
> @@ -169,9 +188,19 @@ static int parse_feature_port_uafu(struct
> build_feature_devs_info *binfo,
>  	int ret;
>  	int size;
> 
> +	if (binfo->acc_info) {
> +		dev_info(binfo, "Sub AFU found @ %p.\n", start);
> +		return 0;
> +	}
> +
>  	capability.csr = readq(&port_hdr->capability);
> 
> -	size = capability.mmio_size << 10;
> +	if (binfo->current_type == AFU_ID) {
> +		size = AFU_REGION_SIZE;
> +	} else {
> +		capability.csr = readq(&port_hdr->capability);
> +		size = capability.mmio_size << 10;
> +	}
> 
>  	ret = create_feature_instance(binfo, hdr, id, size, 0, 0);
>  	if (ret)
> @@ -185,44 +214,13 @@ static int parse_feature_port_uafu(struct
> build_feature_devs_info *binfo,
>  	info->region[0].phys_addr = binfo->phys_addr +
>  			(uint8_t *)start - (uint8_t *)binfo->ioaddr;
>  	info->region[0].len = size;
> -	info->num_regions = 1;
> +	info->num_regions = AFU_MAX_REGION;
> 
>  	binfo->acc_info = info;
> 
>  	return ret;
>  }
> 
> -static int parse_feature_afus(struct build_feature_devs_info *binfo,
> -			      struct feature_header *hdr)
> -{
> -	int ret;
> -	struct feature_afu_header *afu_hdr, header;
> -	u8 __iomem *start;
> -	u8 __iomem *end = binfo->ioend;
> -
> -	start = (u8 __iomem *)hdr;
> -	for (; start < end; start += header.next_afu) {
> -		if ((unsigned int)(end - start) <
> -			(unsigned int)(sizeof(*afu_hdr) + sizeof(*hdr)))
> -			return -EINVAL;
> -
> -		hdr = (struct feature_header *)start;
> -		afu_hdr = (struct feature_afu_header *)(hdr + 1);
> -		header.csr = readq(&afu_hdr->csr);
> -
> -		if (feature_is_UAFU(binfo)) {
> -			ret = parse_feature_port_uafu(binfo, hdr);
> -			if (ret)
> -				return ret;
> -		}
> -
> -		if (!header.next_afu)
> -			break;
> -	}
> -
> -	return 0;
> -}
> -
>  /* create and register proper private data */  static int
> build_info_commit_dev(struct build_feature_devs_info *binfo)  { @@ -
> 235,13 +233,9 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
>  	struct ifpga_fme_hw *fme;
>  	struct ifpga_feature *feature;
> 
> -	if (!binfo->fiu)
> -		return 0;
> -
>  	if (binfo->current_type == PORT_ID) {
> -		/* return error if no valid acc info data structure */
> -		if (!info)
> -			return -EFAULT;
> +		if (!binfo->fiu)
> +			return 0;
> 
>  		br = opae_bridge_alloc(hw->adapter->name, &ifpga_br_ops,
>  				       binfo->fiu);
> @@ -254,7 +248,7 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
>  		port = &hw->port[binfo->current_port_id];
>  		feature = get_feature_by_id(&port->feature_list,
>  				PORT_FEATURE_ID_UINT);
> -		if (feature)
> +		if (feature && info)
>  			info->num_irqs = feature->vec_cnt;
> 
>  		acc = opae_accelerator_alloc(hw->adapter->name,
> @@ -264,17 +258,21 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
>  			return -ENOMEM;
>  		}
> 
> +		acc->adapter = hw->adapter;
>  		acc->br = br;
>  		if (hw->adapter->mgr)
>  			acc->mgr = hw->adapter->mgr;
>  		acc->index = br->id;
> 
>  		fme = &hw->fme;
> -		fme->nums_acc_region = info->num_regions;
> +		fme->nums_acc_region = info ? info->num_regions : 0;
> 
>  		opae_adapter_add_acc(hw->adapter, acc);
> 
>  	} else if (binfo->current_type == FME_ID) {
> +		if (!binfo->fiu)
> +			return 0;
> +
>  		mgr = opae_manager_alloc(hw->adapter->name,
> &ifpga_mgr_ops,
>  				&ifpga_mgr_network_ops, binfo->fiu);
>  		if (!mgr)
> @@ -282,6 +280,22 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
> 
>  		mgr->adapter = hw->adapter;
>  		hw->adapter->mgr = mgr;
> +	} else if (binfo->current_type == AFU_ID) {
> +		if (!info)
> +			return -EFAULT;
> +
> +		info->num_irqs = 0;
> +		acc = opae_accelerator_alloc(hw->adapter->name,
> +					&ifpga_acc_ops, info);
> +		if (!acc)
> +			return -ENOMEM;
> +
> +		acc->adapter = hw->adapter;
> +		acc->br = NULL;
> +		acc->mgr = NULL;
> +		acc->index = hw->num_afus++;
> +
> +		opae_adapter_add_acc(hw->adapter, acc);
>  	}
> 
>  	binfo->fiu = NULL;
> @@ -295,11 +309,15 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)  {
>  	int ret;
> 
> +	if ((type == AFU_ID) && (binfo->current_type == PORT_ID))
> +		return 0;
> +
>  	ret = build_info_commit_dev(binfo);
>  	if (ret)
>  		return ret;
> 
>  	binfo->current_type = type;
> +	binfo->acc_info = NULL;
> 
>  	if (type == FME_ID) {
>  		binfo->fiu = &binfo->hw->fme;
> @@ -311,6 +329,41 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
>  	return 0;
>  }
> 
> +static int parse_feature_afus(struct build_feature_devs_info *binfo,
> +			      struct feature_header *hdr)
> +{
> +	int ret;
> +	struct feature_afu_header *afu_hdr, header;
> +	u8 __iomem *start;
> +	u8 __iomem *end = binfo->ioend;
> +
> +	ret = build_info_create_dev(binfo, AFU_ID, 0);
> +	if (ret)
> +		return ret;
> +
> +	start = (u8 __iomem *)hdr;
> +	for (; start < end; start += header.next_afu) {
> +		if ((unsigned int)(end - start) <
> +			(unsigned int)(sizeof(*afu_hdr) + sizeof(*hdr)))
> +			return -EINVAL;
> +
> +		hdr = (struct feature_header *)start;
> +		afu_hdr = (struct feature_afu_header *)(hdr + 1);
> +		header.csr = readq(&afu_hdr->csr);
> +
> +		if (feature_is_UAFU(binfo)) {
> +			ret = parse_feature_uafu(binfo, hdr);
> +			if (ret)
> +				return ret;
> +		}
> +
> +		if (!header.next_afu)
> +			break;
> +	}
> +
> +	return 0;
> +}
> +
>  static int parse_feature_fme(struct build_feature_devs_info *binfo,
>  			     struct feature_header *start)
>  {
> @@ -405,7 +458,7 @@ static int parse_feature_fiu(struct
> build_feature_devs_info *binfo,
>  			if (ret)
>  				return ret;
>  		} else {
> -			dev_info(binfo, "No AFUs detected on Port\n");
> +			dev_info(binfo, "No AFU detected on Port\n");
>  		}
> 
>  		break;
> @@ -426,7 +479,7 @@ static void parse_feature_irqs(struct
> build_feature_devs_info *binfo,
> 
>  	id = feature_id(start);
> 
> -	if (id == PORT_FEATURE_ID_UINT) {
> +	if ((binfo->current_type == PORT_ID) && (id ==
> PORT_FEATURE_ID_UINT))
> +{
>  		struct feature_port_uint *port_uint = start;
>  		struct feature_port_uint_cap uint_cap;
> 
> @@ -437,7 +490,8 @@ static void parse_feature_irqs(struct
> build_feature_devs_info *binfo,
>  		} else {
>  			dev_debug(binfo, "UAFU doesn't support
> interrupt\n");
>  		}
> -	} else if (id == PORT_FEATURE_ID_ERROR) {
> +	} else if ((binfo->current_type == PORT_ID) &&
> +			(id == PORT_FEATURE_ID_ERROR)) {
>  		struct feature_port_error *port_err = start;
>  		struct feature_port_err_capability port_err_cap;
> 
> @@ -449,7 +503,8 @@ static void parse_feature_irqs(struct
> build_feature_devs_info *binfo,
>  			dev_debug(&binfo, "Port error doesn't support
> interrupt\n");
>  		}
> 
> -	} else if (id == FME_FEATURE_ID_GLOBAL_ERR) {
> +	} else if ((binfo->current_type == FME_ID) &&
> +			(id == FME_FEATURE_ID_GLOBAL_ERR)) {
>  		struct feature_fme_err *fme_err = start;
>  		struct feature_fme_error_capability fme_err_cap;
> 
> @@ -497,9 +552,15 @@ static int parse_feature_private(struct
> build_feature_devs_info *binfo,
>  		return parse_feature_fme_private(binfo, hdr);
>  	case PORT_ID:
>  		return parse_feature_port_private(binfo, hdr);
> +	case AFU_ID:
> +		dev_err(binfo, "private feature %x belonging to AFU "
> +			"is not supported yet.\n", header.id);
> +		break;
>  	default:
> -		dev_err(binfo, "private feature %x belonging to AFU %d
> (unknown_type) is not supported yet.\n",
> +		dev_err(binfo, "private feature %x belonging to TYPE %d "
> +			"(unknown_type) is not supported yet.\n",
>  			header.id, binfo->current_type);
> +		break;
>  	}
>  	return 0;
>  }
> @@ -530,32 +591,57 @@ static int parse_feature(struct
> build_feature_devs_info *binfo,
>  	return ret;
>  }
> 
> -static int
> -parse_feature_list(struct build_feature_devs_info *binfo, u8 __iomem
> *start)
> +static int build_info_prepare(struct build_feature_devs_info *binfo,
> +	struct dfl_fpga_enum_dfl *dfl)
>  {
> +	if (!binfo || !dfl)
> +		return -EINVAL;
> +
> +	binfo->ioaddr = dfl->addr;
> +	binfo->ioend = (u8 *)dfl->addr + dfl->len;
> +	binfo->phys_addr = dfl->start;
> +
> +	return 0;
> +}
> +
> +static int parse_feature_list(struct build_feature_devs_info *binfo,
> +	struct dfl_fpga_enum_dfl *dfl)
> +{
> +	u8 *start, *end;
>  	struct feature_header *hdr, header;
> -	u8 __iomem *end = (u8 __iomem *)binfo->ioend;
>  	int ret = 0;
> 
> +	ret = build_info_prepare(binfo, dfl);
> +	if (ret)
> +		return ret;
> +
> +	start = (u8 *)binfo->ioaddr;
> +	end = (u8 *)binfo->ioend;
> +
> +	/* walk through the device feature list via DFH's next DFH pointer. */
>  	for (; start < end; start += header.next_header_offset) {
>  		if ((unsigned int)(end - start) < (unsigned int)sizeof(*hdr)) {
> -			dev_err(binfo, "The region is too small to contain a
> feature.\n");
> -			ret =  -EINVAL;
> +			dev_err(binfo, "The region is too small to "
> +				"contain a feature.\n");
> +			ret = -EINVAL;
>  			break;
>  		}
> 
>  		hdr = (struct feature_header *)start;
> -		header.csr = readq(hdr);
> +		header.csr = opae_readq(hdr);
> 
> -		dev_debug(binfo, "%s: address=0x%p, val=0x%llx,
> header.id=0x%x, header.next_offset=0x%x, header.eol=0x%x,
> header.type=0x%x\n",
> -			__func__, hdr, (unsigned long long)header.csr,
> -			header.id, header.next_header_offset,
> -			header.end_of_list, header.type);
> +		dev_debug(binfo, "%s: address=0x%p, val=0x%"PRIx64", "
> +			"header.id=0x%x, header.next_offset=0x%x, "
> +			"header.eol=0x%x, header.type=0x%x\n",
> +			__func__, hdr, header.csr, header.id,
> +			header.next_header_offset, header.end_of_list,
> +			header.type);
> 
>  		ret = parse_feature(binfo, hdr);
>  		if (ret)
>  			return ret;
> 
> +		/* stop parsing if EOL(End of List) is set or offset is 0 */
>  		if (header.end_of_list || !header.next_header_offset)
>  			break;
>  	}
> @@ -563,82 +649,9 @@ static int parse_feature(struct
> build_feature_devs_info *binfo,
>  	return build_info_commit_dev(binfo);
>  }
> 
> -/* switch the memory mapping to BAR# @bar */ -static int
> parse_switch_to(struct build_feature_devs_info *binfo, int bar) -{
> -	struct opae_adapter_data_pci *pci_data = binfo->pci_data;
> -
> -	if (!pci_data->region[bar].addr)
> -		return -ENOMEM;
> -
> -	binfo->ioaddr = pci_data->region[bar].addr;
> -	binfo->ioend = (u8 __iomem *)binfo->ioaddr + pci_data-
> >region[bar].len;
> -	binfo->phys_addr = pci_data->region[bar].phys_addr;
> -	binfo->current_bar = bar;
> -
> -	return 0;
> -}
> -
> -static int parse_ports_from_fme(struct build_feature_devs_info *binfo) -{
> -	struct feature_fme_header *fme_hdr;
> -	struct feature_fme_port port;
> -	int i = 0, ret = 0;
> -
> -	if (!binfo->pfme_hdr) {
> -		dev_info(binfo,  "VF is detected.\n");
> -		return ret;
> -	}
> -
> -	fme_hdr = binfo->pfme_hdr;
> -
> -	do {
> -		port.csr = readq(&fme_hdr->port[i]);
> -		if (!port.port_implemented)
> -			break;
> -
> -		/* skip port which only could be accessed via VF */
> -		if (port.afu_access_control == FME_AFU_ACCESS_VF)
> -			continue;
> -
> -		ret = parse_switch_to(binfo, port.port_bar);
> -		if (ret)
> -			break;
> -
> -		ret = parse_feature_list(binfo,
> -					 (u8 __iomem *)binfo->ioaddr +
> -					  port.port_offset);
> -		if (ret)
> -			break;
> -	} while (++i < MAX_FPGA_PORT_NUM);
> -
> -	return ret;
> -}
> -
> -static struct build_feature_devs_info * -build_info_alloc_and_init(struct
> ifpga_hw *hw) -{
> -	struct build_feature_devs_info *binfo;
> -
> -	binfo = zmalloc(sizeof(*binfo));
> -	if (!binfo)
> -		return binfo;
> -
> -	binfo->hw = hw;
> -	binfo->pci_data = hw->pci_data;
> -
> -	/* fpga feature list starts from BAR 0 */
> -	if (parse_switch_to(binfo, 0)) {
> -		free(binfo);
> -		return NULL;
> -	}
> -
> -	return binfo;
> -}
> -
>  static void build_info_free(struct build_feature_devs_info *binfo)  {
> -	free(binfo);
> +	opae_free(binfo);
>  }
> 
>  static void ifpga_print_device_feature_list(struct ifpga_hw *hw) @@ -648,6
> +661,11 @@ static void ifpga_print_device_feature_list(struct ifpga_hw *hw)
>  	struct ifpga_feature *feature;
>  	int i;
> 
> +	if (fme->state == IFPGA_FME_UNUSED) {
> +		dev_info(hw, "FME is not present\n");
> +		return;
> +	}
> +
>  	dev_info(hw, "found fme_device, is in PF: %s\n",
>  		 is_ifpga_hw_pf(hw) ? "yes" : "no");
> 
> @@ -685,40 +703,411 @@ static void ifpga_print_device_feature_list(struct
> ifpga_hw *hw)
>  	}
>  }
> 
> -int ifpga_bus_enumerate(struct ifpga_hw *hw)
> +static struct dfl_fpga_enum_info *dfl_fpga_enum_info_alloc(struct
> +ifpga_hw *hw)
>  {
> -	struct build_feature_devs_info *binfo;
> +	struct dfl_fpga_enum_info *info;
> +
> +	info = opae_zmalloc(sizeof(*info));
> +	if (!info)
> +		return NULL;
> +
> +	info->hw = hw;
> +	TAILQ_INIT(&info->dfls);
> +
> +	return info;
> +}
> +
> +static void dfl_fpga_enum_info_free(struct dfl_fpga_enum_info *info) {
> +	struct dfl_fpga_enum_dfl *tmp, *dfl;
> +
> +	if (!info)
> +		return;
> +
> +	/* remove all device feature lists in the list. */
> +	for (dfl = TAILQ_FIRST(&info->dfls);
> +		dfl && (tmp = TAILQ_NEXT(dfl, node), 1);
> +		dfl = tmp) {
> +		TAILQ_REMOVE(&info->dfls, dfl, node);
> +		opae_free(dfl);
> +	}
> +
> +	opae_free(info);
> +}
> +
> +static int dfl_fpga_enum_info_add_dfl(struct dfl_fpga_enum_info *info,
> +	u64 start, u64 len, void *addr)
> +{
> +	struct dfl_fpga_enum_dfl *dfl;
> +
> +	dfl = opae_zmalloc(sizeof(*dfl));
> +	if (!dfl)
> +		return -ENOMEM;
> +
> +	dfl->start = start;
> +	dfl->len = len;
> +	dfl->addr = addr;
> +
> +	TAILQ_INSERT_TAIL(&info->dfls, dfl, node);
> +
> +	return 0;
> +}
> +
> +#define PCI_CFG_SPACE_SIZE	256
> +#define PCI_CFG_SPACE_EXP_SIZE	4096
> +#define PCI_EXT_CAP_ID(header)		(header & 0x0000ffff)
> +#define PCI_EXT_CAP_NEXT(header)	((header >> 20) & 0xffc)
> +
> +static int
> +pci_find_next_ecap(int fd, int start, u32 cap) {
> +	u32 header;
> +	int ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
> +	int pos = PCI_CFG_SPACE_SIZE;
>  	int ret;
> 
> -	binfo = build_info_alloc_and_init(hw);
> +	if (start > 0)
> +		pos = start;
> +
> +	ret = pread(fd, &header, sizeof(header), pos);
> +	if (ret < 0)
> +		return ret;
> +
> +	/*
> +	 * If we have no capabilities, this is indicated by cap ID,
> +	 * cap version and next pointer all being 0.
> +	 */
> +	if (header == 0)
> +		return 0;
> +
> +	while (ttl-- > 0) {
> +		if ((PCI_EXT_CAP_ID(header) == cap) && (pos != start))
> +			return pos;
> +
> +		pos = PCI_EXT_CAP_NEXT(header);
> +		if (pos < PCI_CFG_SPACE_SIZE)
> +			break;
> +		ret = pread(fd, &header, sizeof(header), pos);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +#define PCI_EXT_CAP_ID_VNDR	0x0B
> +#define PCI_VNDR_HEADER		4
> +#define PCI_VNDR_HEADER_ID(x)	((x) & 0xffff)
> +#define PCI_VENDOR_ID_INTEL 0x8086
> +#define PCI_VSEC_ID_INTEL_DFLS 0x43
> +#define PCI_VNDR_DFLS_CNT 0x8
> +#define PCI_VNDR_DFLS_RES 0xc
> +#define PCI_VNDR_DFLS_RES_BAR_MASK GENMASK(2, 0) #define
> +PCI_VNDR_DFLS_RES_OFF_MASK GENMASK(31, 3)
> +
> +static int find_dfls_by_vsec(struct dfl_fpga_enum_info *info) {
> +	struct ifpga_hw *hw;
> +	struct opae_adapter_data_pci *pci_data;
> +	char path[64];
> +	u32 bir, offset, vndr_hdr, i, dfl_cnt, dfl_res;
> +	int fd, ret, dfl_res_off, voff = 0;
> +	u64 start, len;
> +	void *addr;
> +
> +	if (!info || !info->hw)
> +		return -EINVAL;
> +	hw = info->hw;
> +
> +	if (!hw->adapter || !hw->pci_data)
> +		return -EINVAL;
> +	pci_data = hw->pci_data;
> +
> +	ret = snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/config",
> +			hw->adapter->name);
> +	if ((unsigned int)ret >= sizeof(path))
> +		return -EINVAL;
> +
> +	fd = open(path, O_RDWR);
> +	if (fd < 0)
> +		return -EIO;
> +
> +	while ((voff = pci_find_next_ecap(fd, voff,
> +		PCI_EXT_CAP_ID_VNDR))) {
> +		vndr_hdr = 0;
> +		ret = pread(fd, &vndr_hdr, sizeof(vndr_hdr),
> +			voff + PCI_VNDR_HEADER);
> +		if (ret < 0)
> +			return -EIO;
> +		if (PCI_VNDR_HEADER_ID(vndr_hdr) ==
> PCI_VSEC_ID_INTEL_DFLS &&
> +			pci_data->vendor_id == PCI_VENDOR_ID_INTEL)
> +			break;
> +	}
> +
> +	if (!voff) {
> +		dev_debug(hw, "%s no DFL VSEC found\n", __func__);
> +		return -ENODEV;
> +	}
> +
> +	dfl_cnt = 0;
> +	ret = pread(fd, &dfl_cnt, sizeof(dfl_cnt), voff +
> PCI_VNDR_DFLS_CNT);
> +	if (ret < 0)
> +		return -EIO;
> +
> +	dfl_res_off = voff + PCI_VNDR_DFLS_RES;
> +	if (dfl_res_off + (dfl_cnt * sizeof(u32)) > PCI_CFG_SPACE_EXP_SIZE)
> {
> +		dev_err(hw, "%s DFL VSEC too big for PCIe config space\n",
> +			__func__);
> +		return -EINVAL;
> +	}
> +
> +	for (i = 0; i < dfl_cnt; i++, dfl_res_off += sizeof(u32)) {
> +		dfl_res = GENMASK(31, 0);
> +		ret = pread(fd, &dfl_res, sizeof(dfl_res), dfl_res_off);
> +		bir = dfl_res & PCI_VNDR_DFLS_RES_BAR_MASK;
> +		if (bir >= PCI_MAX_RESOURCE) {
> +			dev_err(hw, "%s bad bir number %d\n",
> +				__func__, bir);
> +			return -EINVAL;
> +		}
> +
> +		len = pci_data->region[bir].len;
> +		offset = dfl_res & PCI_VNDR_DFLS_RES_OFF_MASK;
> +		if (offset >= len) {
> +			dev_err(hw, "%s bad offset %u >= %"PRIu64"\n",
> +				__func__, offset, len);
> +			return -EINVAL;
> +		}
> +
> +		dev_debug(hw, "%s BAR %d offset 0x%x\n", __func__, bir,
> offset);
> +		len -= offset;
> +		start = pci_data->region[bir].phys_addr + offset;
> +		addr = pci_data->region[bir].addr + offset;
> +		dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> +	}
> +
> +	return 0;
> +}
> +
> +/* default method of finding dfls starting at offset 0 of bar 0 */
> +static int find_dfls_by_default(struct dfl_fpga_enum_info *info) {
> +	struct ifpga_hw *hw;
> +	struct opae_adapter_data_pci *pci_data;
> +	int port_num, bar, i, ret = 0;
> +	u64 start, len;
> +	void *addr;
> +	u32 offset;
> +	struct feature_header hdr;
> +	struct feature_fme_capability cap;
> +	struct feature_fme_port port;
> +	struct feature_fme_header *fme_hdr;
> +
> +	if (!info || !info->hw)
> +		return -EINVAL;
> +	hw = info->hw;
> +
> +	if (!hw->pci_data)
> +		return -EINVAL;
> +	pci_data = hw->pci_data;
> +
> +	/* start to find Device Feature List from Bar 0 */
> +	addr = pci_data->region[0].addr;
> +	if (!addr)
> +		return -ENOMEM;
> +
> +	/*
> +	 * PF device has FME and Ports/AFUs, and VF device only has one
> +	 * Port/AFU. Check them and add related "Device Feature List" info
> +	 * for the next step enumeration.
> +	 */
> +	hdr.csr = opae_readq(addr);
> +	if ((hdr.type == FEATURE_TYPE_FIU) && (hdr.id ==
> FEATURE_FIU_ID_FME)) {
> +		start = pci_data->region[0].phys_addr;
> +		len = pci_data->region[0].len;
> +		addr = pci_data->region[0].addr;
> +
> +		dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> +
> +		/*
> +		 * find more Device Feature Lists (e.g. Ports) per information
> +		 * indicated by FME module.
> +		 */
> +		fme_hdr = (struct feature_fme_header *)addr;
> +		cap.csr = opae_readq(&fme_hdr->capability);
> +		port_num = (int)cap.num_ports;
> +
> +		dev_info(hw, "port_num = %d\n", port_num);
> +		if (port_num > MAX_FPGA_PORT_NUM)
> +			port_num = MAX_FPGA_PORT_NUM;
> +
> +		for (i = 0; i < port_num; i++) {
> +			port.csr = opae_readq(&fme_hdr->port[i]);
> +
> +			/* skip ports which are not implemented. */
> +			if (!port.port_implemented)
> +				continue;
> +
> +			/* skip port which only could be accessed via VF */
> +			if (port.afu_access_control == FME_AFU_ACCESS_VF)
> +				continue;
> +
> +			/*
> +			 * add Port's Device Feature List information for next
> +			 * step enumeration.
> +			 */
> +			bar = (int)port.port_bar;
> +			offset = port.port_offset;
> +			if (bar == FME_PORT_OFST_BAR_SKIP) {
> +				continue;
> +			} else if (bar >= PCI_MAX_RESOURCE) {
> +				dev_err(hw, "bad BAR %d for port %d\n", bar,
> i);
> +				ret = -EINVAL;
> +				break;
> +			} else {
> +				dev_info(hw, "BAR %d offset %u\n", bar,
> offset);
> +			}
> +
> +			len = pci_data->region[bar].len;
> +			if (offset >= len) {
> +				dev_warn(hw, "bad port
> offset %u >= %pa\n",
> +					 offset, &len);
> +				continue;
> +			}
> +
> +			len -= offset;
> +			start = pci_data->region[bar].phys_addr + offset;
> +			addr = pci_data->region[bar].addr + offset;
> +			dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> +		}
> +	} else if ((hdr.type == FEATURE_TYPE_FIU) &&
> +		(hdr.id == FEATURE_FIU_ID_PORT)) {
> +		start = pci_data->region[0].phys_addr;
> +		len = pci_data->region[0].len;
> +		addr = pci_data->region[0].addr;
> +
> +		dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> +	} else if (hdr.type == FEATURE_TYPE_AFU) {
> +		start = pci_data->region[0].phys_addr;
> +		len = pci_data->region[0].len;
> +		addr = pci_data->region[0].addr;
> +
> +		dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> +	} else {
> +		dev_info(hw, "Unknown feature type 0x%x id 0x%x\n",
> +			 hdr.type, hdr.id);
> +		ret = -ENODEV;
> +	}
> +
> +	return ret;
> +}
> +
> +static int dfl_fpga_feature_devs_enumerate(struct dfl_fpga_enum_info
> +*info) {
> +	struct build_feature_devs_info *binfo;
> +	struct dfl_fpga_enum_dfl *dfl;
> +	int ret = 0;
> +
> +	if (!info || !info->hw)
> +		return -EINVAL;
> +
> +	/* create and init build info for enumeration */
> +	binfo = opae_zmalloc(sizeof(*binfo));
>  	if (!binfo)
>  		return -ENOMEM;
> 
> -	ret = parse_feature_list(binfo, binfo->ioaddr);
> +	binfo->hw = info->hw;
> +	binfo->pci_data = info->hw->pci_data;
> +
> +	/*
> +	 * start enumeration for all feature devices based on Device Feature
> +	 * Lists.
> +	 */
> +	TAILQ_FOREACH(dfl, &info->dfls, node) {
> +		ret = parse_feature_list(binfo, dfl);
> +		if (ret)
> +			break;
> +	}
> +
> +	build_info_free(binfo);
> +
> +	return ret;
> +}
> +
> +int ifpga_bus_enumerate(struct ifpga_hw *hw) {
> +	struct dfl_fpga_enum_info *info;
> +	int ret;
> +
> +	/* allocate enumeration info */
> +	info = dfl_fpga_enum_info_alloc(hw);
> +	if (!info)
> +		return -ENOMEM;
> +
> +	ret = find_dfls_by_vsec(info);
> +	if (ret < 0)
> +		ret = find_dfls_by_default(info);
> +
>  	if (ret)
>  		goto exit;
> 
> -	ret = parse_ports_from_fme(binfo);
> -	if (ret)
> +	/* start enumeration with prepared enumeration information */
> +	ret = dfl_fpga_feature_devs_enumerate(info);
> +	if (ret < 0) {
> +		dev_err(hw, "Enumeration failure\n");
>  		goto exit;
> +	}
> 
>  	ifpga_print_device_feature_list(hw);
> 
>  exit:
> -	build_info_free(binfo);
> +	dfl_fpga_enum_info_free(info);
> +
>  	return ret;
>  }
> 
> -int ifpga_bus_init(struct ifpga_hw *hw)
> +static void ifpga_print_acc_list(struct opae_adapter *adapter)
>  {
> +	struct opae_accelerator *acc;
> +	struct ifpga_afu_info *info;
> +	struct uuid guid;
> +	char buf[48];
>  	int i;
> +
> +	opae_adapter_for_each_acc(adapter, acc) {
> +		info = acc->data;
> +		if (!info)
> +			continue;
> +		acc->ops->get_uuid(acc, &guid);
> +		i = sprintf(buf, "%02x%02x%02x%02x-",
> +			guid.b[15], guid.b[14], guid.b[13], guid.b[12]);
> +		i += sprintf(buf+i, "%02x%02x-", guid.b[11], guid.b[10]);
> +		i += sprintf(buf+i, "%02x%02x-", guid.b[9], guid.b[8]);
> +		i += sprintf(buf+i, "%02x%02x-", guid.b[7], guid.b[6]);
> +		sprintf(buf+i, "%02x%02x%02x%02x%02x%02x",
> +			guid.b[5], guid.b[4], guid.b[3],
> +			guid.b[2], guid.b[1], guid.b[0]);
> +		dev_info(hw, "AFU(%s-%d)@%p: len:0x%"PRIx64",
> guid:%s\n",
> +			acc->name, acc->index, info->region[0].addr,
> +			info->region[0].len, buf);
> +	}
> +}
> +
> +int ifpga_bus_init(struct ifpga_hw *hw) {
> +	int i, ret = 0;
>  	struct ifpga_port_hw *port;
> 
> -	fme_hw_init(&hw->fme);
> +	ret = fme_hw_init(&hw->fme);
> +	if (ret)
> +		return ret;
> +
>  	for (i = 0; i < MAX_FPGA_PORT_NUM; i++) {
>  		port = &hw->port[i];
>  		port_hw_init(port);
>  	}
> +	ifpga_print_acc_list(hw->adapter);
> 
>  	return 0;
>  }
> diff --git a/drivers/raw/ifpga/base/ifpga_enumerate.h
> b/drivers/raw/ifpga/base/ifpga_enumerate.h
> index 95ed594..e6b04f0 100644
> --- a/drivers/raw/ifpga/base/ifpga_enumerate.h
> +++ b/drivers/raw/ifpga/base/ifpga_enumerate.h
> @@ -5,6 +5,8 @@
>  #ifndef _IFPGA_ENUMERATE_H_
>  #define _IFPGA_ENUMERATE_H_
> 
> +#define FME_PORT_OFST_BAR_SKIP  7
> +
>  int ifpga_bus_init(struct ifpga_hw *hw);  int ifpga_bus_uinit(struct ifpga_hw
> *hw);  int ifpga_bus_enumerate(struct ifpga_hw *hw); diff --git
> a/drivers/raw/ifpga/base/ifpga_feature_dev.c
> b/drivers/raw/ifpga/base/ifpga_feature_dev.c
> index 0813513..dbecc7b 100644
> --- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
> +++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
> @@ -59,7 +59,7 @@ int __fpga_port_disable(struct ifpga_port_hw *port)
>  	if (fpga_wait_register_field(port_sftrst_ack, control,
>  				     &port_hdr->control, RST_POLL_TIMEOUT,
>  				     RST_POLL_INVL)) {
> -		dev_err(port, "timeout, fail to reset device\n");
> +		dev_err(port, "timeout, fail to reset FIM port\n");
>  		return -ETIMEDOUT;
>  	}
> 
> @@ -277,10 +277,11 @@ static void feature_uinit(struct ifpga_feature_list
> *list)
>  	struct ifpga_feature *feature;
> 
>  	TAILQ_FOREACH(feature, list, next) {
> -		if (feature->state != IFPGA_FEATURE_ATTACHED)
> +		if (feature->state != IFPGA_FEATURE_INITED)
>  			continue;
>  		if (feature->ops && feature->ops->uinit)
>  			feature->ops->uinit(feature);
> +		feature->state = IFPGA_FEATURE_ATTACHED;
>  	}
>  }
> 
> @@ -301,6 +302,9 @@ static int feature_init(struct feature_driver *drv,
>  					ret = feature->ops->init(feature);
>  					if (ret)
>  						goto error;
> +					else
> +						feature->state =
> +
> 	IFPGA_FEATURE_INITED;
>  				}
>  			}
>  		}
> @@ -315,14 +319,8 @@ static int feature_init(struct feature_driver *drv,
> 
>  int fme_hw_init(struct ifpga_fme_hw *fme)  {
> -	int ret;
> -
> -	if (fme->state != IFPGA_FME_IMPLEMENTED)
> -		return -ENODEV;
> -
> -	ret = feature_init(fme_feature_drvs, &fme->feature_list);
> -	if (ret)
> -		return ret;
> +	if (fme->state == IFPGA_FME_IMPLEMENTED)
> +		return feature_init(fme_feature_drvs, &fme->feature_list);
> 
>  	return 0;
>  }
> diff --git a/drivers/raw/ifpga/base/ifpga_hw.h
> b/drivers/raw/ifpga/base/ifpga_hw.h
> index ed5edc6..4d56deb 100644
> --- a/drivers/raw/ifpga/base/ifpga_hw.h
> +++ b/drivers/raw/ifpga/base/ifpga_hw.h
> @@ -15,6 +15,7 @@
>  enum ifpga_feature_state {
>  	IFPGA_FEATURE_UNUSED = 0,
>  	IFPGA_FEATURE_ATTACHED,
> +	IFPGA_FEATURE_INITED
>  };
> 
>  enum feature_type {
> @@ -134,6 +135,7 @@ struct ifpga_hw {
> 
>  	struct ifpga_fme_hw fme;
>  	struct ifpga_port_hw port[MAX_FPGA_PORT_NUM];
> +	int num_afus;
>  };
> 
>  static inline bool is_ifpga_hw_pf(struct ifpga_hw *hw) diff --git
> a/drivers/raw/ifpga/base/opae_hw_api.c
> b/drivers/raw/ifpga/base/opae_hw_api.c
> index 11c9887..87256fc 100644
> --- a/drivers/raw/ifpga/base/opae_hw_api.c
> +++ b/drivers/raw/ifpga/base/opae_hw_api.c
> @@ -177,7 +177,7 @@ int opae_acc_get_region_info(struct
> opae_accelerator *acc,  int opae_acc_set_irq(struct opae_accelerator *acc,
>  		     u32 start, u32 count, s32 evtfds[])  {
> -	if (!acc || !acc->data)
> +	if (!acc)
>  		return -EINVAL;
> 
>  	if (start + count <= start)
> diff --git a/drivers/raw/ifpga/base/opae_hw_api.h
> b/drivers/raw/ifpga/base/opae_hw_api.h
> index 7e04b56..fd40e09 100644
> --- a/drivers/raw/ifpga/base/opae_hw_api.h
> +++ b/drivers/raw/ifpga/base/opae_hw_api.h
> @@ -143,6 +143,7 @@ struct opae_accelerator {
>  	TAILQ_ENTRY(opae_accelerator) node;
>  	const char *name;
>  	int index;
> +	struct opae_adapter *adapter;
>  	struct opae_bridge *br;
>  	struct opae_manager *mgr;
>  	struct opae_accelerator_ops *ops;
> @@ -240,6 +241,7 @@ struct opae_adapter_data {
> 
>  struct opae_reg_region {
>  	u64 phys_addr;
> +#define AFU_REGION_SIZE  0x8000
>  	u64 len;
>  	u8 *addr;
>  };
> diff --git a/drivers/raw/ifpga/ifpga_rawdev.c
> b/drivers/raw/ifpga/ifpga_rawdev.c
> index 94df56c..ceb18ae 100644
> --- a/drivers/raw/ifpga/ifpga_rawdev.c
> +++ b/drivers/raw/ifpga/ifpga_rawdev.c
> @@ -47,11 +47,13 @@
>  #define PCIE_DEVICE_ID_PF_INT_6_X    0xBCC0
>  #define PCIE_DEVICE_ID_PF_DSC_1_X    0x09C4
>  #define PCIE_DEVICE_ID_PAC_N3000     0x0B30
> +#define PCIE_DEVICE_ID_PAC_N6000     0xBCCE
>  /* VF Device */
>  #define PCIE_DEVICE_ID_VF_INT_5_X    0xBCBF
>  #define PCIE_DEVICE_ID_VF_INT_6_X    0xBCC1
>  #define PCIE_DEVICE_ID_VF_DSC_1_X    0x09C5
>  #define PCIE_DEVICE_ID_VF_PAC_N3000  0x0B31
> +#define PCIE_DEVICE_ID_VF_PAC_N6000  0xBCCF
>  #define RTE_MAX_RAW_DEVICE           10
> 
>  static const struct rte_pci_id pci_ifpga_map[] = { @@ -63,6 +65,8 @@
>  	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_VF_DSC_1_X) },
>  	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_PAC_N3000),},
>  	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_VF_PAC_N3000),},
> +	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_PAC_N6000),},
> +	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_VF_PAC_N6000),},
>  	{ .vendor_id = 0, /* sentinel */ },
>  };
> 
> @@ -110,6 +114,7 @@ struct ifpga_rawdev *
> 
>  	return IFPGA_RAWDEV_NUM;
>  }
> +
>  static struct ifpga_rawdev *
>  ifpga_rawdev_allocate(struct rte_rawdev *rawdev)  { @@ -365,7 +370,7
> @@ static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev)
>  		return -ENODEV;
> 
>  	mgr = opae_adapter_get_mgr(adapter);
> -	if (!mgr)
> +	if (!mgr || !mgr->sensor_list)
>  		return -ENODEV;
> 
>  	opae_mgr_for_each_sensor(mgr, sensor) { @@ -377,7 +382,7 @@
> static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev)
>  			goto fail;
> 
>  		if (value == 0xdeadbeef) {
> -			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s
> value %x\n",
> +			IFPGA_RAWDEV_PMD_DEBUG("dev_id %d
> sensor %s value %x\n",
>  					raw_dev->dev_id, sensor->name,
> value);
>  			continue;
>  		}
> @@ -893,7 +898,7 @@ static int set_surprise_link_check_aer(  {
>  	struct opae_adapter *adapter;
>  	struct opae_manager *mgr;
> -	struct opae_board_info *info;
> +	struct opae_board_info *info = NULL;
>  	struct rte_afu_pr_conf *afu_pr_conf;
>  	int ret;
>  	struct uuid uuid;
> @@ -921,17 +926,14 @@ static int set_surprise_link_check_aer(
>  	}
> 
>  	mgr = opae_adapter_get_mgr(adapter);
> -	if (!mgr) {
> -		IFPGA_RAWDEV_PMD_ERR("opae_manager of
> opae_adapter is NULL");
> -		return -1;
> -	}
> -
> -	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
> -		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info
> fail!");
> -		return -1;
> +	if (mgr) {
> +		if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
> +			IFPGA_RAWDEV_PMD_ERR("ifpga manager
> get_board_info fail!");
> +			return -1;
> +		}
>  	}
> 
> -	if (info->lightweight) {
> +	if (info && info->lightweight) {
>  		/* set uuid to all 0, when fpga is lightweight image */
>  		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0,
> sizeof(u64));
>  		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0,
> sizeof(u64)); @@ -953,7 +955,7 @@ static int set_surprise_link_check_aer(
>  			__func__,
>  			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
>  			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
> -		}
> +	}
>  	return 0;
>  }
> 
> @@ -1592,7 +1594,7 @@ static int fme_clean_fme_error(struct
> opae_manager *mgr)
>  	ret = opae_adapter_init(adapter, pci_dev->device.name, data);
>  	if (ret) {
>  		ret = -ENOMEM;
> -		goto free_adapter_data;
> +		goto cleanup;
>  	}
> 
>  	rawdev->dev_ops = &ifpga_rawdev_ops;
> @@ -1602,29 +1604,23 @@ static int fme_clean_fme_error(struct
> opae_manager *mgr)
>  	/* must enumerate the adapter before use it */
>  	ret = opae_adapter_enumerate(adapter);
>  	if (ret)
> -		goto free_adapter_data;
> +		goto cleanup;
> 
>  	/* get opae_manager to rawdev */
>  	mgr = opae_adapter_get_mgr(adapter);
>  	if (mgr) {
> -		/* PF function */
> -		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
> +		ret = ifpga_register_msix_irq(dev, 0, IFPGA_FME_IRQ, 0, 0,
> +				fme_interrupt_handler, "fme_irq", mgr);
> +		if (ret)
> +			goto cleanup;
>  	}
> 
> -	ret = ifpga_register_msix_irq(dev, 0, IFPGA_FME_IRQ, 0, 0,
> -			fme_interrupt_handler, "fme_irq", mgr);
> -	if (ret)
> -		goto free_adapter_data;
> -
>  	ret = ifpga_monitor_start_func(dev);
>  	if (ret)
> -		goto free_adapter_data;
> +		goto cleanup;
> 
>  	return ret;
> 
> -free_adapter_data:
> -	if (data)
> -		opae_adapter_data_free(data);
>  cleanup:
>  	if (rawdev)
>  		rte_rawdev_pmd_release(rawdev);
> --
> 1.8.3.1
  

> -----Original Message-----
> From: Huang, Wei <wei.huang@intel.com>
> Sent: Thursday, May 26, 2022 11:33 AM
> To: dev@dpdk.org; thomas@monjalon.net; nipun.gupta@nxp.com;
> hemant.agrawal@nxp.com
> Cc: stable@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Huang, Wei
> <wei.huang@intel.com>
> Subject: [PATCH v3 4/5] raw/ifpga: support ofs card probe
> 
> PAC N6000 is the first OFS platform, its device id is added to ifpga device support
> list.
> 
> Previous FPGA platform like Intel PAC N3000 and N5000, FME DFL (Device
> Feature List) starts from BAR0 by default, port DFL location is indicated in
> PORTn_OFFSET register in FME. In OFS implementation, FME DFL and port DFL
> location can be defined individually in PCIe VSEC (Vendor Specific Extented
> Capabilities). In this patch, DFL definition is searched in VSEC, the legacy DFL is
> used only when DFL VSEC is not present.
> 
> In original DFL enumeration process, AFU is expected to locate in port DFL, but
> this is not the case in OFS implementation. In this patch, enumeration can search
> AFU in any PF/VF which has no FME and port.
> 
> Signed-off-by: Wei Huang <wei.huang@intel.com>
> ---
> v2: fix build error in UB2004-32
> ---
> v3: update commit log with Tianfei's comment, treat 7 as special BAR index.
> ---
>  drivers/raw/ifpga/base/ifpga_api.c         |  28 +-
>  drivers/raw/ifpga/base/ifpga_defines.h     |   1 +
>  drivers/raw/ifpga/base/ifpga_enumerate.c   | 681 ++++++++++++++++++++++--
> -----
>  drivers/raw/ifpga/base/ifpga_enumerate.h   |   2 +
>  drivers/raw/ifpga/base/ifpga_feature_dev.c |  18 +-
>  drivers/raw/ifpga/base/ifpga_hw.h          |   2 +
>  drivers/raw/ifpga/base/opae_hw_api.c       |   2 +-
>  drivers/raw/ifpga/base/opae_hw_api.h       |   2 +
>  drivers/raw/ifpga/ifpga_rawdev.c           |  48 +-
>  9 files changed, 592 insertions(+), 192 deletions(-)
> 
> diff --git a/drivers/raw/ifpga/base/ifpga_api.c
> b/drivers/raw/ifpga/base/ifpga_api.c
> index 4610ef1..f19cc26 100644
> --- a/drivers/raw/ifpga/base/ifpga_api.c
> +++ b/drivers/raw/ifpga/base/ifpga_api.c
> @@ -13,15 +13,22 @@
>  static int ifpga_acc_get_uuid(struct opae_accelerator *acc,
>  			      struct uuid *uuid)
>  {
> -	struct opae_bridge *br = acc->br;
> -	struct ifpga_port_hw *port;
> +	struct ifpga_afu_info *afu_info = acc->data;
> +	struct opae_reg_region *region;
> +	u64 val = 0;
> 
> -	if (!br || !br->data)
> -		return -EINVAL;
> +	if (!afu_info)
> +		return -ENODEV;
> 
> -	port = br->data;
> +	region = &afu_info->region[0];
> +	if (uuid) {
> +		val = readq(region->addr + sizeof(struct feature_header));
> +		opae_memcpy(uuid->b, &val, sizeof(u64));
> +		val = readq(region->addr + sizeof(struct feature_header) + 8);
> +		opae_memcpy(uuid->b + 8, &val, sizeof(u64));
> +	}
> 
> -	return fpga_get_afu_uuid(port, uuid);
> +	return 0;
>  }
> 
>  static int ifpga_acc_set_irq(struct opae_accelerator *acc, @@ -32,6 +39,9 @@
> static int ifpga_acc_set_irq(struct opae_accelerator *acc,
>  	struct ifpga_port_hw *port;
>  	struct fpga_uafu_irq_set irq_set;
> 
> +	if (!afu_info)
> +		return -ENODEV;
> +
>  	if (!br || !br->data)
>  		return -EINVAL;
> 
> @@ -68,7 +78,7 @@ static int ifpga_acc_get_region_info(struct
> opae_accelerator *acc,
>  	struct ifpga_afu_info *afu_info = acc->data;
> 
>  	if (!afu_info)
> -		return -EINVAL;
> +		return -ENODEV;
> 
>  	if (info->index >= afu_info->num_regions)
>  		return -EINVAL;
> @@ -89,7 +99,7 @@ static int ifpga_acc_read(struct opae_accelerator *acc,
> unsigned int region_idx,
>  	struct opae_reg_region *region;
> 
>  	if (!afu_info)
> -		return -EINVAL;
> +		return -ENODEV;
> 
>  	if (offset + byte <= offset)
>  		return -EINVAL;
> @@ -129,7 +139,7 @@ static int ifpga_acc_write(struct opae_accelerator *acc,
>  	struct opae_reg_region *region;
> 
>  	if (!afu_info)
> -		return -EINVAL;
> +		return -ENODEV;
> 
>  	if (offset + byte <= offset)
>  		return -EINVAL;
> diff --git a/drivers/raw/ifpga/base/ifpga_defines.h
> b/drivers/raw/ifpga/base/ifpga_defines.h
> index 8f62033..9a280eb 100644
> --- a/drivers/raw/ifpga/base/ifpga_defines.h
> +++ b/drivers/raw/ifpga/base/ifpga_defines.h
> @@ -73,6 +73,7 @@
>  enum fpga_id_type {
>  	FME_ID,
>  	PORT_ID,
> +	AFU_ID,
>  	FPGA_ID_MAX,
>  };
> 
> diff --git a/drivers/raw/ifpga/base/ifpga_enumerate.c
> b/drivers/raw/ifpga/base/ifpga_enumerate.c
> index 48b8af4..0e09904 100644
> --- a/drivers/raw/ifpga/base/ifpga_enumerate.c
> +++ b/drivers/raw/ifpga/base/ifpga_enumerate.c
> @@ -2,6 +2,10 @@
>   * Copyright(c) 2010-2018 Intel Corporation
>   */
> 
> +#include <fcntl.h>
> +#include <inttypes.h>
> +#include <unistd.h>
> +
>  #include "opae_hw_api.h"
>  #include "ifpga_api.h"
> 
> @@ -9,6 +13,19 @@
>  #include "ifpga_enumerate.h"
>  #include "ifpga_feature_dev.h"
> 
> +struct dfl_fpga_enum_dfl {
> +	u64 start;
> +	u64 len;
> +	void *addr;
> +	TAILQ_ENTRY(dfl_fpga_enum_dfl) node;
> +};
> +
> +TAILQ_HEAD(dfl_fpga_enum_dfls, dfl_fpga_enum_dfl); struct
> +dfl_fpga_enum_info {
> +	struct ifpga_hw *hw;
> +	struct dfl_fpga_enum_dfls dfls;
> +};
> +
>  struct build_feature_devs_info {
>  	struct opae_adapter_data_pci *pci_data;
> 
> @@ -21,7 +38,6 @@ struct build_feature_devs_info {
>  	void *ioaddr;
>  	void *ioend;
>  	uint64_t phys_addr;
> -	int current_bar;
> 
>  	void *pfme_hdr;
> 
> @@ -141,8 +157,10 @@ static u64 feature_id(void __iomem *start)
>  			unsigned int size, unsigned int vec_start,
>  			unsigned int vec_cnt)
>  {
> -	return build_info_add_sub_feature(binfo, start, fid, size, vec_start,
> -			vec_cnt);
> +	if (binfo->current_type != AFU_ID)
> +		return build_info_add_sub_feature(binfo, start, fid, size,
> +			vec_start, vec_cnt);
> +	return 0;
>  }
> 
>  /*
> @@ -152,13 +170,14 @@ static u64 feature_id(void __iomem *start)
>   */
>  static bool feature_is_UAFU(struct build_feature_devs_info *binfo)  {
> -	if (binfo->current_type != PORT_ID)
> -		return false;
> +	if ((binfo->current_type == PORT_ID) ||
> +		(binfo->current_type == AFU_ID))
> +		return true;
> 
> -	return true;
> +	return false;
>  }
> 
> -static int parse_feature_port_uafu(struct build_feature_devs_info *binfo,
> +static int parse_feature_uafu(struct build_feature_devs_info *binfo,
>  				   struct feature_header *hdr)
>  {
>  	u64 id = PORT_FEATURE_ID_UAFU;
> @@ -169,9 +188,19 @@ static int parse_feature_port_uafu(struct
> build_feature_devs_info *binfo,
>  	int ret;
>  	int size;
> 
> +	if (binfo->acc_info) {
> +		dev_info(binfo, "Sub AFU found @ %p.\n", start);
> +		return 0;
> +	}
> +
>  	capability.csr = readq(&port_hdr->capability);
> 
> -	size = capability.mmio_size << 10;
> +	if (binfo->current_type == AFU_ID) {
> +		size = AFU_REGION_SIZE;
> +	} else {
> +		capability.csr = readq(&port_hdr->capability);
> +		size = capability.mmio_size << 10;
> +	}
> 
>  	ret = create_feature_instance(binfo, hdr, id, size, 0, 0);
>  	if (ret)
> @@ -185,44 +214,13 @@ static int parse_feature_port_uafu(struct
> build_feature_devs_info *binfo,
>  	info->region[0].phys_addr = binfo->phys_addr +
>  			(uint8_t *)start - (uint8_t *)binfo->ioaddr;
>  	info->region[0].len = size;
> -	info->num_regions = 1;
> +	info->num_regions = AFU_MAX_REGION;
> 
>  	binfo->acc_info = info;
> 
>  	return ret;
>  }
> 
> -static int parse_feature_afus(struct build_feature_devs_info *binfo,
> -			      struct feature_header *hdr)
> -{
> -	int ret;
> -	struct feature_afu_header *afu_hdr, header;
> -	u8 __iomem *start;
> -	u8 __iomem *end = binfo->ioend;
> -
> -	start = (u8 __iomem *)hdr;
> -	for (; start < end; start += header.next_afu) {
> -		if ((unsigned int)(end - start) <
> -			(unsigned int)(sizeof(*afu_hdr) + sizeof(*hdr)))
> -			return -EINVAL;
> -
> -		hdr = (struct feature_header *)start;
> -		afu_hdr = (struct feature_afu_header *)(hdr + 1);
> -		header.csr = readq(&afu_hdr->csr);
> -
> -		if (feature_is_UAFU(binfo)) {
> -			ret = parse_feature_port_uafu(binfo, hdr);
> -			if (ret)
> -				return ret;
> -		}
> -
> -		if (!header.next_afu)
> -			break;
> -	}
> -
> -	return 0;
> -}
> -
>  /* create and register proper private data */  static int
> build_info_commit_dev(struct build_feature_devs_info *binfo)  { @@ -235,13
> +233,9 @@ static int build_info_commit_dev(struct build_feature_devs_info
> *binfo)
>  	struct ifpga_fme_hw *fme;
>  	struct ifpga_feature *feature;
> 
> -	if (!binfo->fiu)
> -		return 0;
> -
>  	if (binfo->current_type == PORT_ID) {
> -		/* return error if no valid acc info data structure */
> -		if (!info)
> -			return -EFAULT;
> +		if (!binfo->fiu)
> +			return 0;
> 
>  		br = opae_bridge_alloc(hw->adapter->name, &ifpga_br_ops,
>  				       binfo->fiu);
> @@ -254,7 +248,7 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
>  		port = &hw->port[binfo->current_port_id];
>  		feature = get_feature_by_id(&port->feature_list,
>  				PORT_FEATURE_ID_UINT);
> -		if (feature)
> +		if (feature && info)
>  			info->num_irqs = feature->vec_cnt;
> 
>  		acc = opae_accelerator_alloc(hw->adapter->name,
> @@ -264,17 +258,21 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
>  			return -ENOMEM;
>  		}
> 
> +		acc->adapter = hw->adapter;
>  		acc->br = br;
>  		if (hw->adapter->mgr)
>  			acc->mgr = hw->adapter->mgr;
>  		acc->index = br->id;
> 
>  		fme = &hw->fme;
> -		fme->nums_acc_region = info->num_regions;
> +		fme->nums_acc_region = info ? info->num_regions : 0;
> 
>  		opae_adapter_add_acc(hw->adapter, acc);
> 
>  	} else if (binfo->current_type == FME_ID) {
> +		if (!binfo->fiu)
> +			return 0;
> +
>  		mgr = opae_manager_alloc(hw->adapter->name,
> &ifpga_mgr_ops,
>  				&ifpga_mgr_network_ops, binfo->fiu);
>  		if (!mgr)
> @@ -282,6 +280,22 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
> 
>  		mgr->adapter = hw->adapter;
>  		hw->adapter->mgr = mgr;
> +	} else if (binfo->current_type == AFU_ID) {
> +		if (!info)
> +			return -EFAULT;
> +
> +		info->num_irqs = 0;
> +		acc = opae_accelerator_alloc(hw->adapter->name,
> +					&ifpga_acc_ops, info);
> +		if (!acc)
> +			return -ENOMEM;
> +
> +		acc->adapter = hw->adapter;
> +		acc->br = NULL;
> +		acc->mgr = NULL;
> +		acc->index = hw->num_afus++;
> +
> +		opae_adapter_add_acc(hw->adapter, acc);
>  	}
> 
>  	binfo->fiu = NULL;
> @@ -295,11 +309,15 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)  {
>  	int ret;
> 
> +	if ((type == AFU_ID) && (binfo->current_type == PORT_ID))
> +		return 0;
> +
>  	ret = build_info_commit_dev(binfo);
>  	if (ret)
>  		return ret;
> 
>  	binfo->current_type = type;
> +	binfo->acc_info = NULL;
> 
>  	if (type == FME_ID) {
>  		binfo->fiu = &binfo->hw->fme;
> @@ -311,6 +329,41 @@ static int build_info_commit_dev(struct
> build_feature_devs_info *binfo)
>  	return 0;
>  }
> 
> +static int parse_feature_afus(struct build_feature_devs_info *binfo,
> +			      struct feature_header *hdr)
> +{
> +	int ret;
> +	struct feature_afu_header *afu_hdr, header;
> +	u8 __iomem *start;
> +	u8 __iomem *end = binfo->ioend;
> +
> +	ret = build_info_create_dev(binfo, AFU_ID, 0);
> +	if (ret)
> +		return ret;
> +
> +	start = (u8 __iomem *)hdr;
> +	for (; start < end; start += header.next_afu) {
> +		if ((unsigned int)(end - start) <
> +			(unsigned int)(sizeof(*afu_hdr) + sizeof(*hdr)))
> +			return -EINVAL;
> +
> +		hdr = (struct feature_header *)start;
> +		afu_hdr = (struct feature_afu_header *)(hdr + 1);
> +		header.csr = readq(&afu_hdr->csr);
> +
> +		if (feature_is_UAFU(binfo)) {
> +			ret = parse_feature_uafu(binfo, hdr);
> +			if (ret)
> +				return ret;
> +		}
> +
> +		if (!header.next_afu)
> +			break;
> +	}
> +
> +	return 0;
> +}
> +
>  static int parse_feature_fme(struct build_feature_devs_info *binfo,
>  			     struct feature_header *start)
>  {
> @@ -405,7 +458,7 @@ static int parse_feature_fiu(struct
> build_feature_devs_info *binfo,
>  			if (ret)
>  				return ret;
>  		} else {
> -			dev_info(binfo, "No AFUs detected on Port\n");
> +			dev_info(binfo, "No AFU detected on Port\n");
>  		}
> 
>  		break;
> @@ -426,7 +479,7 @@ static void parse_feature_irqs(struct
> build_feature_devs_info *binfo,
> 
>  	id = feature_id(start);
> 
> -	if (id == PORT_FEATURE_ID_UINT) {
> +	if ((binfo->current_type == PORT_ID) && (id ==
> PORT_FEATURE_ID_UINT))
> +{
>  		struct feature_port_uint *port_uint = start;
>  		struct feature_port_uint_cap uint_cap;
> 
> @@ -437,7 +490,8 @@ static void parse_feature_irqs(struct
> build_feature_devs_info *binfo,
>  		} else {
>  			dev_debug(binfo, "UAFU doesn't support interrupt\n");
>  		}
> -	} else if (id == PORT_FEATURE_ID_ERROR) {
> +	} else if ((binfo->current_type == PORT_ID) &&
> +			(id == PORT_FEATURE_ID_ERROR)) {
>  		struct feature_port_error *port_err = start;
>  		struct feature_port_err_capability port_err_cap;
> 
> @@ -449,7 +503,8 @@ static void parse_feature_irqs(struct
> build_feature_devs_info *binfo,
>  			dev_debug(&binfo, "Port error doesn't support
> interrupt\n");
>  		}
> 
> -	} else if (id == FME_FEATURE_ID_GLOBAL_ERR) {
> +	} else if ((binfo->current_type == FME_ID) &&
> +			(id == FME_FEATURE_ID_GLOBAL_ERR)) {
>  		struct feature_fme_err *fme_err = start;
>  		struct feature_fme_error_capability fme_err_cap;
> 
> @@ -497,9 +552,15 @@ static int parse_feature_private(struct
> build_feature_devs_info *binfo,
>  		return parse_feature_fme_private(binfo, hdr);
>  	case PORT_ID:
>  		return parse_feature_port_private(binfo, hdr);
> +	case AFU_ID:
> +		dev_err(binfo, "private feature %x belonging to AFU "
> +			"is not supported yet.\n", header.id);
> +		break;
>  	default:
> -		dev_err(binfo, "private feature %x belonging to AFU %d
> (unknown_type) is not supported yet.\n",
> +		dev_err(binfo, "private feature %x belonging to TYPE %d "
> +			"(unknown_type) is not supported yet.\n",
>  			header.id, binfo->current_type);
> +		break;
>  	}
>  	return 0;
>  }
> @@ -530,32 +591,57 @@ static int parse_feature(struct
> build_feature_devs_info *binfo,
>  	return ret;
>  }
> 
> -static int
> -parse_feature_list(struct build_feature_devs_info *binfo, u8 __iomem *start)
> +static int build_info_prepare(struct build_feature_devs_info *binfo,
> +	struct dfl_fpga_enum_dfl *dfl)
>  {
> +	if (!binfo || !dfl)
> +		return -EINVAL;
> +
> +	binfo->ioaddr = dfl->addr;
> +	binfo->ioend = (u8 *)dfl->addr + dfl->len;
> +	binfo->phys_addr = dfl->start;
> +
> +	return 0;
> +}
> +
> +static int parse_feature_list(struct build_feature_devs_info *binfo,
> +	struct dfl_fpga_enum_dfl *dfl)
> +{
> +	u8 *start, *end;
>  	struct feature_header *hdr, header;
> -	u8 __iomem *end = (u8 __iomem *)binfo->ioend;
>  	int ret = 0;
> 
> +	ret = build_info_prepare(binfo, dfl);
> +	if (ret)
> +		return ret;
> +
> +	start = (u8 *)binfo->ioaddr;
> +	end = (u8 *)binfo->ioend;
> +
> +	/* walk through the device feature list via DFH's next DFH pointer. */
>  	for (; start < end; start += header.next_header_offset) {
>  		if ((unsigned int)(end - start) < (unsigned int)sizeof(*hdr)) {
> -			dev_err(binfo, "The region is too small to contain a
> feature.\n");
> -			ret =  -EINVAL;
> +			dev_err(binfo, "The region is too small to "
> +				"contain a feature.\n");
> +			ret = -EINVAL;
>  			break;
>  		}
> 
>  		hdr = (struct feature_header *)start;
> -		header.csr = readq(hdr);
> +		header.csr = opae_readq(hdr);
> 
> -		dev_debug(binfo, "%s: address=0x%p, val=0x%llx,
> header.id=0x%x, header.next_offset=0x%x, header.eol=0x%x,
> header.type=0x%x\n",
> -			__func__, hdr, (unsigned long long)header.csr,
> -			header.id, header.next_header_offset,
> -			header.end_of_list, header.type);
> +		dev_debug(binfo, "%s: address=0x%p, val=0x%"PRIx64", "
> +			"header.id=0x%x, header.next_offset=0x%x, "
> +			"header.eol=0x%x, header.type=0x%x\n",
> +			__func__, hdr, header.csr, header.id,
> +			header.next_header_offset, header.end_of_list,
> +			header.type);
> 
>  		ret = parse_feature(binfo, hdr);
>  		if (ret)
>  			return ret;
> 
> +		/* stop parsing if EOL(End of List) is set or offset is 0 */
>  		if (header.end_of_list || !header.next_header_offset)
>  			break;
>  	}
> @@ -563,82 +649,9 @@ static int parse_feature(struct build_feature_devs_info
> *binfo,
>  	return build_info_commit_dev(binfo);
>  }
> 
> -/* switch the memory mapping to BAR# @bar */ -static int
> parse_switch_to(struct build_feature_devs_info *binfo, int bar) -{
> -	struct opae_adapter_data_pci *pci_data = binfo->pci_data;
> -
> -	if (!pci_data->region[bar].addr)
> -		return -ENOMEM;
> -
> -	binfo->ioaddr = pci_data->region[bar].addr;
> -	binfo->ioend = (u8 __iomem *)binfo->ioaddr + pci_data-
> >region[bar].len;
> -	binfo->phys_addr = pci_data->region[bar].phys_addr;
> -	binfo->current_bar = bar;
> -
> -	return 0;
> -}
> -
> -static int parse_ports_from_fme(struct build_feature_devs_info *binfo) -{
> -	struct feature_fme_header *fme_hdr;
> -	struct feature_fme_port port;
> -	int i = 0, ret = 0;
> -
> -	if (!binfo->pfme_hdr) {
> -		dev_info(binfo,  "VF is detected.\n");
> -		return ret;
> -	}
> -
> -	fme_hdr = binfo->pfme_hdr;
> -
> -	do {
> -		port.csr = readq(&fme_hdr->port[i]);
> -		if (!port.port_implemented)
> -			break;
> -
> -		/* skip port which only could be accessed via VF */
> -		if (port.afu_access_control == FME_AFU_ACCESS_VF)
> -			continue;
> -
> -		ret = parse_switch_to(binfo, port.port_bar);
> -		if (ret)
> -			break;
> -
> -		ret = parse_feature_list(binfo,
> -					 (u8 __iomem *)binfo->ioaddr +
> -					  port.port_offset);
> -		if (ret)
> -			break;
> -	} while (++i < MAX_FPGA_PORT_NUM);
> -
> -	return ret;
> -}
> -
> -static struct build_feature_devs_info * -build_info_alloc_and_init(struct
> ifpga_hw *hw) -{
> -	struct build_feature_devs_info *binfo;
> -
> -	binfo = zmalloc(sizeof(*binfo));
> -	if (!binfo)
> -		return binfo;
> -
> -	binfo->hw = hw;
> -	binfo->pci_data = hw->pci_data;
> -
> -	/* fpga feature list starts from BAR 0 */
> -	if (parse_switch_to(binfo, 0)) {
> -		free(binfo);
> -		return NULL;
> -	}
> -
> -	return binfo;
> -}
> -
>  static void build_info_free(struct build_feature_devs_info *binfo)  {
> -	free(binfo);
> +	opae_free(binfo);
>  }
> 
>  static void ifpga_print_device_feature_list(struct ifpga_hw *hw) @@ -648,6
> +661,11 @@ static void ifpga_print_device_feature_list(struct ifpga_hw *hw)
>  	struct ifpga_feature *feature;
>  	int i;
> 
> +	if (fme->state == IFPGA_FME_UNUSED) {
> +		dev_info(hw, "FME is not present\n");
> +		return;
> +	}
> +
>  	dev_info(hw, "found fme_device, is in PF: %s\n",
>  		 is_ifpga_hw_pf(hw) ? "yes" : "no");
> 
> @@ -685,40 +703,411 @@ static void ifpga_print_device_feature_list(struct
> ifpga_hw *hw)
>  	}
>  }
> 
> -int ifpga_bus_enumerate(struct ifpga_hw *hw)
> +static struct dfl_fpga_enum_info *dfl_fpga_enum_info_alloc(struct
> +ifpga_hw *hw)
>  {
> -	struct build_feature_devs_info *binfo;
> +	struct dfl_fpga_enum_info *info;
> +
> +	info = opae_zmalloc(sizeof(*info));
> +	if (!info)
> +		return NULL;
> +
> +	info->hw = hw;
> +	TAILQ_INIT(&info->dfls);
> +
> +	return info;
> +}
> +
> +static void dfl_fpga_enum_info_free(struct dfl_fpga_enum_info *info) {
> +	struct dfl_fpga_enum_dfl *tmp, *dfl;
> +
> +	if (!info)
> +		return;
> +
> +	/* remove all device feature lists in the list. */
> +	for (dfl = TAILQ_FIRST(&info->dfls);
> +		dfl && (tmp = TAILQ_NEXT(dfl, node), 1);
> +		dfl = tmp) {
> +		TAILQ_REMOVE(&info->dfls, dfl, node);
> +		opae_free(dfl);
> +	}
> +
> +	opae_free(info);
> +}
> +
> +static int dfl_fpga_enum_info_add_dfl(struct dfl_fpga_enum_info *info,
> +	u64 start, u64 len, void *addr)
> +{
> +	struct dfl_fpga_enum_dfl *dfl;
> +
> +	dfl = opae_zmalloc(sizeof(*dfl));
> +	if (!dfl)
> +		return -ENOMEM;
> +
> +	dfl->start = start;
> +	dfl->len = len;
> +	dfl->addr = addr;
> +
> +	TAILQ_INSERT_TAIL(&info->dfls, dfl, node);
> +
> +	return 0;
> +}
> +
> +#define PCI_CFG_SPACE_SIZE	256
> +#define PCI_CFG_SPACE_EXP_SIZE	4096
> +#define PCI_EXT_CAP_ID(header)		(header & 0x0000ffff)
> +#define PCI_EXT_CAP_NEXT(header)	((header >> 20) & 0xffc)
> +
> +static int
> +pci_find_next_ecap(int fd, int start, u32 cap) {
> +	u32 header;
> +	int ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
> +	int pos = PCI_CFG_SPACE_SIZE;
>  	int ret;
> 
> -	binfo = build_info_alloc_and_init(hw);
> +	if (start > 0)
> +		pos = start;
> +
> +	ret = pread(fd, &header, sizeof(header), pos);
> +	if (ret < 0)
> +		return ret;
> +
> +	/*
> +	 * If we have no capabilities, this is indicated by cap ID,
> +	 * cap version and next pointer all being 0.
> +	 */
> +	if (header == 0)
> +		return 0;
> +
> +	while (ttl-- > 0) {
> +		if ((PCI_EXT_CAP_ID(header) == cap) && (pos != start))
> +			return pos;
> +
> +		pos = PCI_EXT_CAP_NEXT(header);
> +		if (pos < PCI_CFG_SPACE_SIZE)
> +			break;
> +		ret = pread(fd, &header, sizeof(header), pos);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +#define PCI_EXT_CAP_ID_VNDR	0x0B
> +#define PCI_VNDR_HEADER		4
> +#define PCI_VNDR_HEADER_ID(x)	((x) & 0xffff)
> +#define PCI_VENDOR_ID_INTEL 0x8086
> +#define PCI_VSEC_ID_INTEL_DFLS 0x43
> +#define PCI_VNDR_DFLS_CNT 0x8
> +#define PCI_VNDR_DFLS_RES 0xc
> +#define PCI_VNDR_DFLS_RES_BAR_MASK GENMASK(2, 0) #define
> +PCI_VNDR_DFLS_RES_OFF_MASK GENMASK(31, 3)
> +
> +static int find_dfls_by_vsec(struct dfl_fpga_enum_info *info) {
> +	struct ifpga_hw *hw;
> +	struct opae_adapter_data_pci *pci_data;
> +	char path[64];
> +	u32 bir, offset, vndr_hdr, i, dfl_cnt, dfl_res;
> +	int fd, ret, dfl_res_off, voff = 0;
> +	u64 start, len;
> +	void *addr;
> +
> +	if (!info || !info->hw)
> +		return -EINVAL;
> +	hw = info->hw;
> +
> +	if (!hw->adapter || !hw->pci_data)
> +		return -EINVAL;
> +	pci_data = hw->pci_data;
> +
> +	ret = snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/config",
> +			hw->adapter->name);
> +	if ((unsigned int)ret >= sizeof(path))
> +		return -EINVAL;
> +
> +	fd = open(path, O_RDWR);
> +	if (fd < 0)
> +		return -EIO;
> +
> +	while ((voff = pci_find_next_ecap(fd, voff,
> +		PCI_EXT_CAP_ID_VNDR))) {
> +		vndr_hdr = 0;
> +		ret = pread(fd, &vndr_hdr, sizeof(vndr_hdr),
> +			voff + PCI_VNDR_HEADER);
> +		if (ret < 0)
> +			return -EIO;
> +		if (PCI_VNDR_HEADER_ID(vndr_hdr) ==
> PCI_VSEC_ID_INTEL_DFLS &&
> +			pci_data->vendor_id == PCI_VENDOR_ID_INTEL)
> +			break;
> +	}
> +
> +	if (!voff) {
> +		dev_debug(hw, "%s no DFL VSEC found\n", __func__);
> +		return -ENODEV;
> +	}
> +
> +	dfl_cnt = 0;
> +	ret = pread(fd, &dfl_cnt, sizeof(dfl_cnt), voff + PCI_VNDR_DFLS_CNT);
> +	if (ret < 0)
> +		return -EIO;
> +
> +	dfl_res_off = voff + PCI_VNDR_DFLS_RES;
> +	if (dfl_res_off + (dfl_cnt * sizeof(u32)) > PCI_CFG_SPACE_EXP_SIZE) {
> +		dev_err(hw, "%s DFL VSEC too big for PCIe config space\n",
> +			__func__);
> +		return -EINVAL;
> +	}
> +
> +	for (i = 0; i < dfl_cnt; i++, dfl_res_off += sizeof(u32)) {
> +		dfl_res = GENMASK(31, 0);
> +		ret = pread(fd, &dfl_res, sizeof(dfl_res), dfl_res_off);
> +		bir = dfl_res & PCI_VNDR_DFLS_RES_BAR_MASK;
> +		if (bir >= PCI_MAX_RESOURCE) {
> +			dev_err(hw, "%s bad bir number %d\n",
> +				__func__, bir);
> +			return -EINVAL;
> +		}
> +
> +		len = pci_data->region[bir].len;
> +		offset = dfl_res & PCI_VNDR_DFLS_RES_OFF_MASK;
> +		if (offset >= len) {
> +			dev_err(hw, "%s bad offset %u >= %"PRIu64"\n",
> +				__func__, offset, len);
> +			return -EINVAL;
> +		}
> +
> +		dev_debug(hw, "%s BAR %d offset 0x%x\n", __func__, bir,
> offset);
> +		len -= offset;
> +		start = pci_data->region[bir].phys_addr + offset;
> +		addr = pci_data->region[bir].addr + offset;
> +		dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> +	}
> +
> +	return 0;
> +}
> +
> +/* default method of finding dfls starting at offset 0 of bar 0 */
> +static int find_dfls_by_default(struct dfl_fpga_enum_info *info) {
> +	struct ifpga_hw *hw;
> +	struct opae_adapter_data_pci *pci_data;
> +	int port_num, bar, i, ret = 0;
> +	u64 start, len;
> +	void *addr;
> +	u32 offset;
> +	struct feature_header hdr;
> +	struct feature_fme_capability cap;
> +	struct feature_fme_port port;
> +	struct feature_fme_header *fme_hdr;
> +
> +	if (!info || !info->hw)
> +		return -EINVAL;
> +	hw = info->hw;
> +
> +	if (!hw->pci_data)
> +		return -EINVAL;
> +	pci_data = hw->pci_data;
> +
> +	/* start to find Device Feature List from Bar 0 */
> +	addr = pci_data->region[0].addr;
> +	if (!addr)
> +		return -ENOMEM;
> +
> +	/*
> +	 * PF device has FME and Ports/AFUs, and VF device only has one
> +	 * Port/AFU. Check them and add related "Device Feature List" info
> +	 * for the next step enumeration.
> +	 */
> +	hdr.csr = opae_readq(addr);
> +	if ((hdr.type == FEATURE_TYPE_FIU) && (hdr.id ==
> FEATURE_FIU_ID_FME)) {
> +		start = pci_data->region[0].phys_addr;
> +		len = pci_data->region[0].len;
> +		addr = pci_data->region[0].addr;
> +
> +		dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> +
> +		/*
> +		 * find more Device Feature Lists (e.g. Ports) per information
> +		 * indicated by FME module.
> +		 */
> +		fme_hdr = (struct feature_fme_header *)addr;
> +		cap.csr = opae_readq(&fme_hdr->capability);
> +		port_num = (int)cap.num_ports;
> +
> +		dev_info(hw, "port_num = %d\n", port_num);
> +		if (port_num > MAX_FPGA_PORT_NUM)
> +			port_num = MAX_FPGA_PORT_NUM;
> +
> +		for (i = 0; i < port_num; i++) {
> +			port.csr = opae_readq(&fme_hdr->port[i]);
> +
> +			/* skip ports which are not implemented. */
> +			if (!port.port_implemented)
> +				continue;
> +
> +			/* skip port which only could be accessed via VF */
> +			if (port.afu_access_control == FME_AFU_ACCESS_VF)
> +				continue;
> +
> +			/*
> +			 * add Port's Device Feature List information for next
> +			 * step enumeration.
> +			 */
> +			bar = (int)port.port_bar;
> +			offset = port.port_offset;
> +			if (bar == FME_PORT_OFST_BAR_SKIP) {
> +				continue;
> +			} else if (bar >= PCI_MAX_RESOURCE) {
> +				dev_err(hw, "bad BAR %d for port %d\n", bar,
> i);
> +				ret = -EINVAL;
> +				break;
> +			} else {
> +				dev_info(hw, "BAR %d offset %u\n", bar,
> offset);
> +			}
> +
> +			len = pci_data->region[bar].len;
> +			if (offset >= len) {
> +				dev_warn(hw, "bad port offset %u >= %pa\n",
> +					 offset, &len);
> +				continue;
> +			}
> +
> +			len -= offset;
> +			start = pci_data->region[bar].phys_addr + offset;
> +			addr = pci_data->region[bar].addr + offset;
> +			dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> +		}
> +	} else if ((hdr.type == FEATURE_TYPE_FIU) &&
> +		(hdr.id == FEATURE_FIU_ID_PORT)) {
> +		start = pci_data->region[0].phys_addr;
> +		len = pci_data->region[0].len;
> +		addr = pci_data->region[0].addr;
> +
> +		dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> +	} else if (hdr.type == FEATURE_TYPE_AFU) {
> +		start = pci_data->region[0].phys_addr;
> +		len = pci_data->region[0].len;
> +		addr = pci_data->region[0].addr;
> +
> +		dfl_fpga_enum_info_add_dfl(info, start, len, addr);
> +	} else {
> +		dev_info(hw, "Unknown feature type 0x%x id 0x%x\n",
> +			 hdr.type, hdr.id);
> +		ret = -ENODEV;
> +	}
> +
> +	return ret;
> +}
> +
> +static int dfl_fpga_feature_devs_enumerate(struct dfl_fpga_enum_info
> +*info) {
> +	struct build_feature_devs_info *binfo;
> +	struct dfl_fpga_enum_dfl *dfl;
> +	int ret = 0;
> +
> +	if (!info || !info->hw)
> +		return -EINVAL;
> +
> +	/* create and init build info for enumeration */
> +	binfo = opae_zmalloc(sizeof(*binfo));
>  	if (!binfo)
>  		return -ENOMEM;
> 
> -	ret = parse_feature_list(binfo, binfo->ioaddr);
> +	binfo->hw = info->hw;
> +	binfo->pci_data = info->hw->pci_data;
> +
> +	/*
> +	 * start enumeration for all feature devices based on Device Feature
> +	 * Lists.
> +	 */
> +	TAILQ_FOREACH(dfl, &info->dfls, node) {
> +		ret = parse_feature_list(binfo, dfl);
> +		if (ret)
> +			break;
> +	}
> +
> +	build_info_free(binfo);
> +
> +	return ret;
> +}
> +
> +int ifpga_bus_enumerate(struct ifpga_hw *hw) {
> +	struct dfl_fpga_enum_info *info;
> +	int ret;
> +
> +	/* allocate enumeration info */
> +	info = dfl_fpga_enum_info_alloc(hw);
> +	if (!info)
> +		return -ENOMEM;
> +
> +	ret = find_dfls_by_vsec(info);
> +	if (ret < 0)
> +		ret = find_dfls_by_default(info);
> +
>  	if (ret)
>  		goto exit;
> 
> -	ret = parse_ports_from_fme(binfo);
> -	if (ret)
> +	/* start enumeration with prepared enumeration information */
> +	ret = dfl_fpga_feature_devs_enumerate(info);
> +	if (ret < 0) {
> +		dev_err(hw, "Enumeration failure\n");
>  		goto exit;
> +	}
> 
>  	ifpga_print_device_feature_list(hw);
> 
>  exit:
> -	build_info_free(binfo);
> +	dfl_fpga_enum_info_free(info);
> +
>  	return ret;
>  }
> 
> -int ifpga_bus_init(struct ifpga_hw *hw)
> +static void ifpga_print_acc_list(struct opae_adapter *adapter)
>  {
> +	struct opae_accelerator *acc;
> +	struct ifpga_afu_info *info;
> +	struct uuid guid;
> +	char buf[48];
>  	int i;
> +
> +	opae_adapter_for_each_acc(adapter, acc) {
> +		info = acc->data;
> +		if (!info)
> +			continue;
> +		acc->ops->get_uuid(acc, &guid);
> +		i = sprintf(buf, "%02x%02x%02x%02x-",
> +			guid.b[15], guid.b[14], guid.b[13], guid.b[12]);
> +		i += sprintf(buf+i, "%02x%02x-", guid.b[11], guid.b[10]);
> +		i += sprintf(buf+i, "%02x%02x-", guid.b[9], guid.b[8]);
> +		i += sprintf(buf+i, "%02x%02x-", guid.b[7], guid.b[6]);
> +		sprintf(buf+i, "%02x%02x%02x%02x%02x%02x",
> +			guid.b[5], guid.b[4], guid.b[3],
> +			guid.b[2], guid.b[1], guid.b[0]);
> +		dev_info(hw, "AFU(%s-%d)@%p: len:0x%"PRIx64", guid:%s\n",
> +			acc->name, acc->index, info->region[0].addr,
> +			info->region[0].len, buf);
> +	}
> +}
> +
> +int ifpga_bus_init(struct ifpga_hw *hw) {
> +	int i, ret = 0;
>  	struct ifpga_port_hw *port;
> 
> -	fme_hw_init(&hw->fme);
> +	ret = fme_hw_init(&hw->fme);
> +	if (ret)
> +		return ret;
> +
>  	for (i = 0; i < MAX_FPGA_PORT_NUM; i++) {
>  		port = &hw->port[i];
>  		port_hw_init(port);
>  	}
> +	ifpga_print_acc_list(hw->adapter);
> 
>  	return 0;
>  }
> diff --git a/drivers/raw/ifpga/base/ifpga_enumerate.h
> b/drivers/raw/ifpga/base/ifpga_enumerate.h
> index 95ed594..e6b04f0 100644
> --- a/drivers/raw/ifpga/base/ifpga_enumerate.h
> +++ b/drivers/raw/ifpga/base/ifpga_enumerate.h
> @@ -5,6 +5,8 @@
>  #ifndef _IFPGA_ENUMERATE_H_
>  #define _IFPGA_ENUMERATE_H_
> 
> +#define FME_PORT_OFST_BAR_SKIP  7
> +
>  int ifpga_bus_init(struct ifpga_hw *hw);  int ifpga_bus_uinit(struct ifpga_hw
> *hw);  int ifpga_bus_enumerate(struct ifpga_hw *hw); diff --git
> a/drivers/raw/ifpga/base/ifpga_feature_dev.c
> b/drivers/raw/ifpga/base/ifpga_feature_dev.c
> index 0813513..dbecc7b 100644
> --- a/drivers/raw/ifpga/base/ifpga_feature_dev.c
> +++ b/drivers/raw/ifpga/base/ifpga_feature_dev.c
> @@ -59,7 +59,7 @@ int __fpga_port_disable(struct ifpga_port_hw *port)
>  	if (fpga_wait_register_field(port_sftrst_ack, control,
>  				     &port_hdr->control, RST_POLL_TIMEOUT,
>  				     RST_POLL_INVL)) {
> -		dev_err(port, "timeout, fail to reset device\n");
> +		dev_err(port, "timeout, fail to reset FIM port\n");
>  		return -ETIMEDOUT;
>  	}
> 
> @@ -277,10 +277,11 @@ static void feature_uinit(struct ifpga_feature_list
> *list)
>  	struct ifpga_feature *feature;
> 
>  	TAILQ_FOREACH(feature, list, next) {
> -		if (feature->state != IFPGA_FEATURE_ATTACHED)
> +		if (feature->state != IFPGA_FEATURE_INITED)
>  			continue;
>  		if (feature->ops && feature->ops->uinit)
>  			feature->ops->uinit(feature);
> +		feature->state = IFPGA_FEATURE_ATTACHED;
>  	}
>  }
> 
> @@ -301,6 +302,9 @@ static int feature_init(struct feature_driver *drv,
>  					ret = feature->ops->init(feature);
>  					if (ret)
>  						goto error;
> +					else
> +						feature->state =
> +
> 	IFPGA_FEATURE_INITED;
>  				}
>  			}
>  		}
> @@ -315,14 +319,8 @@ static int feature_init(struct feature_driver *drv,
> 
>  int fme_hw_init(struct ifpga_fme_hw *fme)  {
> -	int ret;
> -
> -	if (fme->state != IFPGA_FME_IMPLEMENTED)
> -		return -ENODEV;
> -
> -	ret = feature_init(fme_feature_drvs, &fme->feature_list);
> -	if (ret)
> -		return ret;
> +	if (fme->state == IFPGA_FME_IMPLEMENTED)
> +		return feature_init(fme_feature_drvs, &fme->feature_list);
> 
>  	return 0;
>  }
> diff --git a/drivers/raw/ifpga/base/ifpga_hw.h
> b/drivers/raw/ifpga/base/ifpga_hw.h
> index ed5edc6..4d56deb 100644
> --- a/drivers/raw/ifpga/base/ifpga_hw.h
> +++ b/drivers/raw/ifpga/base/ifpga_hw.h
> @@ -15,6 +15,7 @@
>  enum ifpga_feature_state {
>  	IFPGA_FEATURE_UNUSED = 0,
>  	IFPGA_FEATURE_ATTACHED,
> +	IFPGA_FEATURE_INITED
>  };
> 
>  enum feature_type {
> @@ -134,6 +135,7 @@ struct ifpga_hw {
> 
>  	struct ifpga_fme_hw fme;
>  	struct ifpga_port_hw port[MAX_FPGA_PORT_NUM];
> +	int num_afus;
>  };
> 
>  static inline bool is_ifpga_hw_pf(struct ifpga_hw *hw) diff --git
> a/drivers/raw/ifpga/base/opae_hw_api.c
> b/drivers/raw/ifpga/base/opae_hw_api.c
> index 11c9887..87256fc 100644
> --- a/drivers/raw/ifpga/base/opae_hw_api.c
> +++ b/drivers/raw/ifpga/base/opae_hw_api.c
> @@ -177,7 +177,7 @@ int opae_acc_get_region_info(struct opae_accelerator
> *acc,  int opae_acc_set_irq(struct opae_accelerator *acc,
>  		     u32 start, u32 count, s32 evtfds[])  {
> -	if (!acc || !acc->data)
> +	if (!acc)
>  		return -EINVAL;
> 
>  	if (start + count <= start)
> diff --git a/drivers/raw/ifpga/base/opae_hw_api.h
> b/drivers/raw/ifpga/base/opae_hw_api.h
> index 7e04b56..fd40e09 100644
> --- a/drivers/raw/ifpga/base/opae_hw_api.h
> +++ b/drivers/raw/ifpga/base/opae_hw_api.h
> @@ -143,6 +143,7 @@ struct opae_accelerator {
>  	TAILQ_ENTRY(opae_accelerator) node;
>  	const char *name;
>  	int index;
> +	struct opae_adapter *adapter;
>  	struct opae_bridge *br;
>  	struct opae_manager *mgr;
>  	struct opae_accelerator_ops *ops;
> @@ -240,6 +241,7 @@ struct opae_adapter_data {
> 
>  struct opae_reg_region {
>  	u64 phys_addr;
> +#define AFU_REGION_SIZE  0x8000
>  	u64 len;
>  	u8 *addr;
>  };
> diff --git a/drivers/raw/ifpga/ifpga_rawdev.c
> b/drivers/raw/ifpga/ifpga_rawdev.c
> index 94df56c..ceb18ae 100644
> --- a/drivers/raw/ifpga/ifpga_rawdev.c
> +++ b/drivers/raw/ifpga/ifpga_rawdev.c
> @@ -47,11 +47,13 @@
>  #define PCIE_DEVICE_ID_PF_INT_6_X    0xBCC0
>  #define PCIE_DEVICE_ID_PF_DSC_1_X    0x09C4
>  #define PCIE_DEVICE_ID_PAC_N3000     0x0B30
> +#define PCIE_DEVICE_ID_PAC_N6000     0xBCCE
>  /* VF Device */
>  #define PCIE_DEVICE_ID_VF_INT_5_X    0xBCBF
>  #define PCIE_DEVICE_ID_VF_INT_6_X    0xBCC1
>  #define PCIE_DEVICE_ID_VF_DSC_1_X    0x09C5
>  #define PCIE_DEVICE_ID_VF_PAC_N3000  0x0B31
> +#define PCIE_DEVICE_ID_VF_PAC_N6000  0xBCCF
>  #define RTE_MAX_RAW_DEVICE           10
> 
>  static const struct rte_pci_id pci_ifpga_map[] = { @@ -63,6 +65,8 @@
>  	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_VF_DSC_1_X) },
>  	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_PAC_N3000),},
>  	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_VF_PAC_N3000),},
> +	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_PAC_N6000),},
> +	{ RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_VF_PAC_N6000),},
>  	{ .vendor_id = 0, /* sentinel */ },
>  };
> 
> @@ -110,6 +114,7 @@ struct ifpga_rawdev *
> 
>  	return IFPGA_RAWDEV_NUM;
>  }
> +
>  static struct ifpga_rawdev *
>  ifpga_rawdev_allocate(struct rte_rawdev *rawdev)  { @@ -365,7 +370,7 @@
> static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev)
>  		return -ENODEV;
> 
>  	mgr = opae_adapter_get_mgr(adapter);
> -	if (!mgr)
> +	if (!mgr || !mgr->sensor_list)
>  		return -ENODEV;
> 
>  	opae_mgr_for_each_sensor(mgr, sensor) { @@ -377,7 +382,7 @@
> static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev)
>  			goto fail;
> 
>  		if (value == 0xdeadbeef) {
> -			IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value
> %x\n",
> +			IFPGA_RAWDEV_PMD_DEBUG("dev_id %d sensor %s
> value %x\n",
>  					raw_dev->dev_id, sensor->name,
> value);
>  			continue;
>  		}
> @@ -893,7 +898,7 @@ static int set_surprise_link_check_aer(  {
>  	struct opae_adapter *adapter;
>  	struct opae_manager *mgr;
> -	struct opae_board_info *info;
> +	struct opae_board_info *info = NULL;
>  	struct rte_afu_pr_conf *afu_pr_conf;
>  	int ret;
>  	struct uuid uuid;
> @@ -921,17 +926,14 @@ static int set_surprise_link_check_aer(
>  	}
> 
>  	mgr = opae_adapter_get_mgr(adapter);
> -	if (!mgr) {
> -		IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is
> NULL");
> -		return -1;
> -	}
> -
> -	if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
> -		IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info
> fail!");
> -		return -1;
> +	if (mgr) {
> +		if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
> +			IFPGA_RAWDEV_PMD_ERR("ifpga manager
> get_board_info fail!");
> +			return -1;
> +		}
>  	}
> 
> -	if (info->lightweight) {
> +	if (info && info->lightweight) {
>  		/* set uuid to all 0, when fpga is lightweight image */
>  		memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
>  		memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
> @@ -953,7 +955,7 @@ static int set_surprise_link_check_aer(
>  			__func__,
>  			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
>  			(unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
> -		}
> +	}
>  	return 0;
>  }
> 
> @@ -1592,7 +1594,7 @@ static int fme_clean_fme_error(struct opae_manager
> *mgr)
>  	ret = opae_adapter_init(adapter, pci_dev->device.name, data);
>  	if (ret) {
>  		ret = -ENOMEM;
> -		goto free_adapter_data;
> +		goto cleanup;
>  	}
> 
>  	rawdev->dev_ops = &ifpga_rawdev_ops;
> @@ -1602,29 +1604,23 @@ static int fme_clean_fme_error(struct
> opae_manager *mgr)
>  	/* must enumerate the adapter before use it */
>  	ret = opae_adapter_enumerate(adapter);
>  	if (ret)
> -		goto free_adapter_data;
> +		goto cleanup;
> 
>  	/* get opae_manager to rawdev */
>  	mgr = opae_adapter_get_mgr(adapter);
>  	if (mgr) {
> -		/* PF function */
> -		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
> +		ret = ifpga_register_msix_irq(dev, 0, IFPGA_FME_IRQ, 0, 0,
> +				fme_interrupt_handler, "fme_irq", mgr);
> +		if (ret)
> +			goto cleanup;
>  	}
> 
> -	ret = ifpga_register_msix_irq(dev, 0, IFPGA_FME_IRQ, 0, 0,
> -			fme_interrupt_handler, "fme_irq", mgr);
> -	if (ret)
> -		goto free_adapter_data;
> -
>  	ret = ifpga_monitor_start_func(dev);
>  	if (ret)
> -		goto free_adapter_data;
> +		goto cleanup;
> 
>  	return ret;
> 
> -free_adapter_data:
> -	if (data)
> -		opae_adapter_data_free(data);
>  cleanup:
>  	if (rawdev)
>  		rte_rawdev_pmd_release(rawdev);
> --
> 1.8.3.1

It looks good for me, you can add:
Acked-by: Tianfei Zhang <tianfei.zhang@intel.com>