[dpdk-dev,v10,3/3] iFPGA: Add Intel FPGA BUS Rawdev Driver
Checks
Commit Message
From: Rosen Xu <rosen.xu@intel.com>
Add Intel FPGA BUS Rawdev Driver which is based on
librte_rawdev library.
Signed-off-by: Rosen Xu <rosen.xu@intel.com>
Signed-off-by: Yanglong Wu <yanglong.wu@intel.com>
Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
Acked-by: Shreyansh Jain <shreyansh.jain@nxp.com>
---
MAINTAINERS | 3 +
config/common_base | 5 +
doc/guides/rawdevs/ifpga_rawdev.rst | 112 ++++
doc/guides/rawdevs/index.rst | 1 +
doc/guides/rel_notes/release_18_05.rst | 8 +
drivers/raw/Makefile | 1 +
drivers/raw/ifpga_rawdev/Makefile | 36 ++
drivers/raw/ifpga_rawdev/ifpga_rawdev.c | 608 +++++++++++++++++++++
drivers/raw/ifpga_rawdev/ifpga_rawdev.h | 37 ++
drivers/raw/ifpga_rawdev/meson.build | 15 +
.../ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map | 4 +
drivers/raw/meson.build | 2 +-
mk/rte.app.mk | 4 +-
13 files changed, 834 insertions(+), 2 deletions(-)
create mode 100644 doc/guides/rawdevs/ifpga_rawdev.rst
create mode 100644 drivers/raw/ifpga_rawdev/Makefile
create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.c
create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.h
create mode 100644 drivers/raw/ifpga_rawdev/meson.build
create mode 100644 drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
Comments
09/05/2018 09:43, Xu, Rosen:
> From: Rosen Xu <rosen.xu@intel.com>
>
> Add Intel FPGA BUS Rawdev Driver which is based on
> librte_rawdev library.
>
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> Signed-off-by: Yanglong Wu <yanglong.wu@intel.com>
> Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
> Acked-by: Shreyansh Jain <shreyansh.jain@nxp.com>
I have a compilation error:
drivers/raw/ifpga_rawdev/base/ifpga_fme_pr.c:10:15: error:
instruction requires: AVX-512 ISA
because of vmovdqu64:
#if defined(RTE_ARCH_X86_64)
static inline void copy512(const void *src, void *dst)
{
asm volatile("vmovdqu64 (%0), %%zmm0;"
"vmovntdq %%zmm0, (%1);"
:
: "r"(src), "r"(dst));
}
#else
static inline void copy512(const void *src, void *dst)
{
UNUSED(src);
UNUSED(dst);
WARN_ON(1);
}
#endif
I suggest to fix it quickly without waiting a v11 with this:
static inline void copy512(const void *src, void *dst)
{
#ifdef CC_SUPPORT_AVX512F
asm volatile("vmovdqu64 (%0), %%zmm0;"
"vmovntdq %%zmm0, (%1);"
:
: "r"(src), "r"(dst));
#else
UNUSED(src);
UNUSED(dst);
WARN_ON(1);
#endif
}
It does not make any runtime detection, but it's better than previously.
> -----Original Message-----
> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> Sent: Wednesday, May 9, 2018 10:47 PM
> To: Xu, Rosen <rosen.xu@intel.com>
> Cc: dev@dpdk.org; Zhang, Roy Fan <roy.fan.zhang@intel.com>; Doherty,
> Declan <declan.doherty@intel.com>; Richardson, Bruce
> <bruce.richardson@intel.com>; shreyansh.jain@nxp.com; Yigit, Ferruh
> <ferruh.yigit@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>;
> Liu, Song <song.liu@intel.com>; Wu, Hao <hao.wu@intel.com>;
> gaetan.rivet@6wind.com; Wu, Yanglong <yanglong.wu@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS Rawdev
> Driver
>
> 09/05/2018 09:43, Xu, Rosen:
> > From: Rosen Xu <rosen.xu@intel.com>
> >
> > Add Intel FPGA BUS Rawdev Driver which is based on librte_rawdev
> > library.
> >
> > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > Signed-off-by: Yanglong Wu <yanglong.wu@intel.com>
> > Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
> > Acked-by: Shreyansh Jain <shreyansh.jain@nxp.com>
>
> I have a compilation error:
> drivers/raw/ifpga_rawdev/base/ifpga_fme_pr.c:10:15: error:
> instruction requires: AVX-512 ISA
>
> because of vmovdqu64:
>
> #if defined(RTE_ARCH_X86_64)
> static inline void copy512(const void *src, void *dst) {
> asm volatile("vmovdqu64 (%0), %%zmm0;"
> "vmovntdq %%zmm0, (%1);"
> :
> : "r"(src), "r"(dst));
> }
> #else
> static inline void copy512(const void *src, void *dst) {
> UNUSED(src);
> UNUSED(dst);
> WARN_ON(1);
> }
> #endif
>
> I suggest to fix it quickly without waiting a v11 with this:
>
> static inline void copy512(const void *src, void *dst) { #ifdef
> CC_SUPPORT_AVX512F
> asm volatile("vmovdqu64 (%0), %%zmm0;"
> "vmovntdq %%zmm0, (%1);"
> :
> : "r"(src), "r"(dst));
> #else
> UNUSED(src);
> UNUSED(dst);
> WARN_ON(1);
> #endif
> }
>
> It does not make any runtime detection, but it's better than previously.
>
Which linux distribution are you use? We can compile it on Ubuntu 16.04 and RHEL 7.4.
We will fix it on V11.
On Wed, May 09, 2018 at 04:33:45PM +0100, Zhang, Tianfei wrote:
>
>
> > -----Original Message-----
> > From: Thomas Monjalon [mailto:thomas@monjalon.net]
> > Sent: Wednesday, May 9, 2018 10:47 PM
> > To: Xu, Rosen <rosen.xu@intel.com>
> > Cc: dev@dpdk.org; Zhang, Roy Fan <roy.fan.zhang@intel.com>; Doherty,
> > Declan <declan.doherty@intel.com>; Richardson, Bruce
> > <bruce.richardson@intel.com>; shreyansh.jain@nxp.com; Yigit, Ferruh
> > <ferruh.yigit@intel.com>; Ananyev, Konstantin
> > <konstantin.ananyev@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>;
> > Liu, Song <song.liu@intel.com>; Wu, Hao <hao.wu@intel.com>;
> > gaetan.rivet@6wind.com; Wu, Yanglong <yanglong.wu@intel.com>
> > Subject: Re: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS Rawdev
> > Driver
> >
> > 09/05/2018 09:43, Xu, Rosen:
> > > From: Rosen Xu <rosen.xu@intel.com>
> > >
> > > Add Intel FPGA BUS Rawdev Driver which is based on librte_rawdev
> > > library.
> > >
> > > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > > Signed-off-by: Yanglong Wu <yanglong.wu@intel.com>
> > > Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
> > > Acked-by: Shreyansh Jain <shreyansh.jain@nxp.com>
> >
> > I have a compilation error:
> > drivers/raw/ifpga_rawdev/base/ifpga_fme_pr.c:10:15: error:
> > instruction requires: AVX-512 ISA
> >
> > because of vmovdqu64:
> >
> > #if defined(RTE_ARCH_X86_64)
> > static inline void copy512(const void *src, void *dst) {
> > asm volatile("vmovdqu64 (%0), %%zmm0;"
> > "vmovntdq %%zmm0, (%1);"
> > :
> > : "r"(src), "r"(dst));
> > }
> > #else
> > static inline void copy512(const void *src, void *dst) {
> > UNUSED(src);
> > UNUSED(dst);
> > WARN_ON(1);
> > }
> > #endif
> >
> > I suggest to fix it quickly without waiting a v11 with this:
> >
> > static inline void copy512(const void *src, void *dst) { #ifdef
> > CC_SUPPORT_AVX512F
> > asm volatile("vmovdqu64 (%0), %%zmm0;"
> > "vmovntdq %%zmm0, (%1);"
> > :
> > : "r"(src), "r"(dst));
> > #else
> > UNUSED(src);
> > UNUSED(dst);
> > WARN_ON(1);
> > #endif
> > }
> >
> > It does not make any runtime detection, but it's better than previously.
> >
>
> Which linux distribution are you use? We can compile it on Ubuntu 16.04 and RHEL 7.4.
> We will fix it on V11.
>
This shows up with meson builds for non-avx-512 architectures using clang.
The makefile has the following snippet that doesn't have an equivalent in
the meson.build file.
26 ifeq ($(CONFIG_RTE_TOOLCHAIN_CLANG),y)$
27 >-------CFLAGS_ifpga_fme_pr.o += -march=knl$
28 endif$
However, it does also bring up the questions as to the unconditional use of
AVX-512 code? What happens if this code is run on a system without AVX-512
support?
Regards,
/Bruce
> -----Original Message-----
> From: Richardson, Bruce
> Sent: Wednesday, May 9, 2018 11:37 PM
> To: Zhang, Tianfei <tianfei.zhang@intel.com>
> Cc: Thomas Monjalon <thomas@monjalon.net>; Xu, Rosen
> <rosen.xu@intel.com>; dev@dpdk.org; Zhang, Roy Fan
> <roy.fan.zhang@intel.com>; Doherty, Declan <declan.doherty@intel.com>;
> shreyansh.jain@nxp.com; Yigit, Ferruh <ferruh.yigit@intel.com>; Ananyev,
> Konstantin <konstantin.ananyev@intel.com>; Liu, Song
> <song.liu@intel.com>; Wu, Hao <hao.wu@intel.com>;
> gaetan.rivet@6wind.com; Wu, Yanglong <yanglong.wu@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS Rawdev
> Driver
>
> On Wed, May 09, 2018 at 04:33:45PM +0100, Zhang, Tianfei wrote:
> >
> >
> > > -----Original Message-----
> > > From: Thomas Monjalon [mailto:thomas@monjalon.net]
> > > Sent: Wednesday, May 9, 2018 10:47 PM
> > > To: Xu, Rosen <rosen.xu@intel.com>
> > > Cc: dev@dpdk.org; Zhang, Roy Fan <roy.fan.zhang@intel.com>; Doherty,
> > > Declan <declan.doherty@intel.com>; Richardson, Bruce
> > > <bruce.richardson@intel.com>; shreyansh.jain@nxp.com; Yigit, Ferruh
> > > <ferruh.yigit@intel.com>; Ananyev, Konstantin
> > > <konstantin.ananyev@intel.com>; Zhang, Tianfei
> > > <tianfei.zhang@intel.com>; Liu, Song <song.liu@intel.com>; Wu, Hao
> > > <hao.wu@intel.com>; gaetan.rivet@6wind.com; Wu, Yanglong
> > > <yanglong.wu@intel.com>
> > > Subject: Re: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS
> > > Rawdev Driver
> > >
> > > 09/05/2018 09:43, Xu, Rosen:
> > > > From: Rosen Xu <rosen.xu@intel.com>
> > > >
> > > > Add Intel FPGA BUS Rawdev Driver which is based on librte_rawdev
> > > > library.
> > > >
> > > > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > > > Signed-off-by: Yanglong Wu <yanglong.wu@intel.com>
> > > > Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
> > > > Acked-by: Shreyansh Jain <shreyansh.jain@nxp.com>
> > >
> > > I have a compilation error:
> > > drivers/raw/ifpga_rawdev/base/ifpga_fme_pr.c:10:15: error:
> > > instruction requires: AVX-512 ISA
> > >
> > > because of vmovdqu64:
> > >
> > > #if defined(RTE_ARCH_X86_64)
> > > static inline void copy512(const void *src, void *dst) {
> > > asm volatile("vmovdqu64 (%0), %%zmm0;"
> > > "vmovntdq %%zmm0, (%1);"
> > > :
> > > : "r"(src), "r"(dst));
> > > }
> > > #else
> > > static inline void copy512(const void *src, void *dst) {
> > > UNUSED(src);
> > > UNUSED(dst);
> > > WARN_ON(1);
> > > }
> > > #endif
> > >
> > > I suggest to fix it quickly without waiting a v11 with this:
> > >
> > > static inline void copy512(const void *src, void *dst) { #ifdef
> > > CC_SUPPORT_AVX512F
> > > asm volatile("vmovdqu64 (%0), %%zmm0;"
> > > "vmovntdq %%zmm0, (%1);"
> > > :
> > > : "r"(src), "r"(dst));
> > > #else
> > > UNUSED(src);
> > > UNUSED(dst);
> > > WARN_ON(1);
> > > #endif
> > > }
> > >
> > > It does not make any runtime detection, but it's better than previously.
> > >
> >
> > Which linux distribution are you use? We can compile it on Ubuntu 16.04
> and RHEL 7.4.
> > We will fix it on V11.
> >
>
> This shows up with meson builds for non-avx-512 architectures using clang.
> The makefile has the following snippet that doesn't have an equivalent in the
> meson.build file.
>
> 26 ifeq ($(CONFIG_RTE_TOOLCHAIN_CLANG),y)$
> 27 >-------CFLAGS_ifpga_fme_pr.o += -march=knl$
> 28 endif$
>
> However, it does also bring up the questions as to the unconditional use of
> AVX-512 code? What happens if this code is run on a system without
> AVX-512 support?
This function is for SKY MCP platform (Xeon + FPGA integrated chip) which can accelerate the PR (partial reconfiguration ) function on FPGA.
So the SKY support the AVX-512 instruction set.
Hi, Rosen
Few comments below.
Thanks
Jingjing
[...]
> +static int
> +ifpga_rawdev_start(struct rte_rawdev *dev)
> +{
> + int ret = 0;
> + struct opae_adapter *adapter;
> +
> + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> +
> + RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
> +
> + adapter = ifpga_rawdev_get_priv(dev);
> + if (!adapter)
> + return -ENODEV;
> +
Set dev->started?
> + return ret;
> +}
[...]
> +
> +static const struct rte_rawdev_ops ifpga_rawdev_ops = {
> + .dev_info_get = ifpga_rawdev_info_get,
> + .dev_configure = NULL,
If go the declaration of rte_rawdev_configure, you will see "This function must be invoked first before any other function in the API."
So I think we need to function for it, even it does nothing.
[...]
> +static struct rte_pci_driver rte_ifpga_rawdev_pmd = {
> + .id_table = pci_ifpga_map,
> + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
Is RTE_PCI_DRV_INTR_LSC supported?
[...]
> +static struct rte_vdev_driver ifpga_cfg_driver = {
> + .probe = ifpga_cfg_probe,
> + .remove = ifpga_cfg_remove,
> +};
> +
> +RTE_PMD_REGISTER_VDEV(net_ifpga_cfg, ifpga_cfg_driver);
I think prefix net_ would mean the device is net device (eth_dev)? How about to change the prefix to raw_?
> +RTE_PMD_REGISTER_ALIAS(net_ifpga_cfg, ifpga_cfg);
> +RTE_PMD_REGISTER_PARAM_STRING(net_ifpga_cfg,
> + "bdf=<string> "
ifpga=<string>?
> + "port=<int> "
> + "afu_bts=<path>");
> +
Hi Jingjing,
> -----Original Message-----
> From: Wu, Jingjing
> Sent: Thursday, May 10, 2018 17:21
> To: Xu, Rosen <rosen.xu@intel.com>; dev@dpdk.org; thomas@monjalon.net
> Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Roy Fan
> <roy.fan.zhang@intel.com>; Doherty, Declan <declan.doherty@intel.com>;
> Richardson, Bruce <bruce.richardson@intel.com>; shreyansh.jain@nxp.com;
> Yigit, Ferruh <ferruh.yigit@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>;
> Liu, Song <song.liu@intel.com>; Wu, Hao <hao.wu@intel.com>;
> gaetan.rivet@6wind.com; Wu, Yanglong <yanglong.wu@intel.com>
> Subject: RE: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS Rawdev
> Driver
>
> Hi, Rosen
>
> Few comments below.
Thanks a lot.
> Thanks
> Jingjing
>
> [...]
>
> > +static int
> > +ifpga_rawdev_start(struct rte_rawdev *dev) {
> > + int ret = 0;
> > + struct opae_adapter *adapter;
> > +
> > + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> > +
> > + RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
> > +
> > + adapter = ifpga_rawdev_get_priv(dev);
> > + if (!adapter)
> > + return -ENODEV;
> > +
> Set dev->started?
Rawdev lib enable it.
> > + return ret;
> > +}
>
>
> [...]
>
> > +
> > +static const struct rte_rawdev_ops ifpga_rawdev_ops = {
> > + .dev_info_get = ifpga_rawdev_info_get,
> > + .dev_configure = NULL,
> If go the declaration of rte_rawdev_configure, you will see "This function
> must be invoked first before any other function in the API."
> So I think we need to function for it, even it does nothing.
For other Rawdev Drivers, no used function pointer is initialized with NULL.
>
> [...]
>
> > +static struct rte_pci_driver rte_ifpga_rawdev_pmd = {
> > + .id_table = pci_ifpga_map,
> > + .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
> RTE_PCI_DRV_INTR_LSC,
> Is RTE_PCI_DRV_INTR_LSC supported?
>
> [...]
>
> > +static struct rte_vdev_driver ifpga_cfg_driver = {
> > + .probe = ifpga_cfg_probe,
> > + .remove = ifpga_cfg_remove,
> > +};
> > +
> > +RTE_PMD_REGISTER_VDEV(net_ifpga_cfg, ifpga_cfg_driver);
> I think prefix net_ would mean the device is net device (eth_dev)? How about
> to change the prefix to raw_?
I have fixed it both in code and document.
> > +RTE_PMD_REGISTER_ALIAS(net_ifpga_cfg, ifpga_cfg);
> > +RTE_PMD_REGISTER_PARAM_STRING(net_ifpga_cfg,
> > + "bdf=<string> "
> ifpga=<string>?
> > + "port=<int> "
> > + "afu_bts=<path>");
> > +
Hi Thomas and Bruce,
To improve PR speed we involved AVX-512, but AVX-512 is not available in all IA platforms,
besides this after we have tested several times, using AVX-512 only improve 1 ms.
So this patch we don't use it.
> -----Original Message-----
> From: Zhang, Tianfei
> Sent: Wednesday, May 09, 2018 23:58
> To: Richardson, Bruce <bruce.richardson@intel.com>
> Cc: Thomas Monjalon <thomas@monjalon.net>; Xu, Rosen
> <rosen.xu@intel.com>; dev@dpdk.org; Zhang, Roy Fan
> <roy.fan.zhang@intel.com>; Doherty, Declan <declan.doherty@intel.com>;
> shreyansh.jain@nxp.com; Yigit, Ferruh <ferruh.yigit@intel.com>; Ananyev,
> Konstantin <konstantin.ananyev@intel.com>; Liu, Song <song.liu@intel.com>;
> Wu, Hao <hao.wu@intel.com>; gaetan.rivet@6wind.com; Wu, Yanglong
> <yanglong.wu@intel.com>
> Subject: RE: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS Rawdev
> Driver
>
>
>
> > -----Original Message-----
> > From: Richardson, Bruce
> > Sent: Wednesday, May 9, 2018 11:37 PM
> > To: Zhang, Tianfei <tianfei.zhang@intel.com>
> > Cc: Thomas Monjalon <thomas@monjalon.net>; Xu, Rosen
> > <rosen.xu@intel.com>; dev@dpdk.org; Zhang, Roy Fan
> > <roy.fan.zhang@intel.com>; Doherty, Declan <declan.doherty@intel.com>;
> > shreyansh.jain@nxp.com; Yigit, Ferruh <ferruh.yigit@intel.com>;
> > Ananyev, Konstantin <konstantin.ananyev@intel.com>; Liu, Song
> > <song.liu@intel.com>; Wu, Hao <hao.wu@intel.com>;
> > gaetan.rivet@6wind.com; Wu, Yanglong <yanglong.wu@intel.com>
> > Subject: Re: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS
> > Rawdev Driver
> >
> > On Wed, May 09, 2018 at 04:33:45PM +0100, Zhang, Tianfei wrote:
> > >
> > >
> > > > -----Original Message-----
> > > > From: Thomas Monjalon [mailto:thomas@monjalon.net]
> > > > Sent: Wednesday, May 9, 2018 10:47 PM
> > > > To: Xu, Rosen <rosen.xu@intel.com>
> > > > Cc: dev@dpdk.org; Zhang, Roy Fan <roy.fan.zhang@intel.com>;
> > > > Doherty, Declan <declan.doherty@intel.com>; Richardson, Bruce
> > > > <bruce.richardson@intel.com>; shreyansh.jain@nxp.com; Yigit,
> > > > Ferruh <ferruh.yigit@intel.com>; Ananyev, Konstantin
> > > > <konstantin.ananyev@intel.com>; Zhang, Tianfei
> > > > <tianfei.zhang@intel.com>; Liu, Song <song.liu@intel.com>; Wu, Hao
> > > > <hao.wu@intel.com>; gaetan.rivet@6wind.com; Wu, Yanglong
> > > > <yanglong.wu@intel.com>
> > > > Subject: Re: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS
> > > > Rawdev Driver
> > > >
> > > > 09/05/2018 09:43, Xu, Rosen:
> > > > > From: Rosen Xu <rosen.xu@intel.com>
> > > > >
> > > > > Add Intel FPGA BUS Rawdev Driver which is based on librte_rawdev
> > > > > library.
> > > > >
> > > > > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > > > > Signed-off-by: Yanglong Wu <yanglong.wu@intel.com>
> > > > > Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
> > > > > Acked-by: Shreyansh Jain <shreyansh.jain@nxp.com>
> > > >
> > > > I have a compilation error:
> > > > drivers/raw/ifpga_rawdev/base/ifpga_fme_pr.c:10:15: error:
> > > > instruction requires: AVX-512 ISA
> > > >
> > > > because of vmovdqu64:
> > > >
> > > > #if defined(RTE_ARCH_X86_64)
> > > > static inline void copy512(const void *src, void *dst) {
> > > > asm volatile("vmovdqu64 (%0), %%zmm0;"
> > > > "vmovntdq %%zmm0, (%1);"
> > > > :
> > > > : "r"(src), "r"(dst)); } #else static inline void
> > > > copy512(const void *src, void *dst) {
> > > > UNUSED(src);
> > > > UNUSED(dst);
> > > > WARN_ON(1);
> > > > }
> > > > #endif
> > > >
> > > > I suggest to fix it quickly without waiting a v11 with this:
> > > >
> > > > static inline void copy512(const void *src, void *dst) { #ifdef
> > > > CC_SUPPORT_AVX512F
> > > > asm volatile("vmovdqu64 (%0), %%zmm0;"
> > > > "vmovntdq %%zmm0, (%1);"
> > > > :
> > > > : "r"(src), "r"(dst)); #else
> > > > UNUSED(src);
> > > > UNUSED(dst);
> > > > WARN_ON(1);
> > > > #endif
> > > > }
> > > >
> > > > It does not make any runtime detection, but it's better than previously.
> > > >
> > >
> > > Which linux distribution are you use? We can compile it on Ubuntu
> > > 16.04
> > and RHEL 7.4.
> > > We will fix it on V11.
> > >
> >
> > This shows up with meson builds for non-avx-512 architectures using clang.
> > The makefile has the following snippet that doesn't have an equivalent
> > in the meson.build file.
> >
> > 26 ifeq ($(CONFIG_RTE_TOOLCHAIN_CLANG),y)$
> > 27 >-------CFLAGS_ifpga_fme_pr.o += -march=knl$
> > 28 endif$
> >
> > However, it does also bring up the questions as to the unconditional
> > use of
> > AVX-512 code? What happens if this code is run on a system without
> > AVX-512 support?
>
> This function is for SKY MCP platform (Xeon + FPGA integrated chip) which
> can accelerate the PR (partial reconfiguration ) function on FPGA.
> So the SKY support the AVX-512 instruction set.
Hi Rosen:
After read this patch, I have below suggestion
1. rte_ifgpa_device is not necessary, since we already have ifgpa bus, it manages all devices on that bus, we don't need another abstraction to manage afu devices, and user can add to or remove from the bus.
2. rename rte_afu_device to rte_ifpga_device , we follow the name rule.
3. data structure looks like below.
Struct rte_ifpga_deivce {
rte_device device /* backing device */
Rte_raw_device *device /* link to raw pci device */
Struct Rte_afu_info info; /* afu information */
}
struct rte_afu_info {
struct rte_ifpga_device *ifpga_dev; /**< Point ifpga device */
struct rte_afu_id id; /**< AFU id within FPGA. */
uint32_t num_region; /**< number of regions found */
....
...
}
Regards
Qi
> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Xu, Rosen
> Sent: Wednesday, May 9, 2018 3:43 PM
> To: dev@dpdk.org; thomas@monjalon.net
> Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Roy Fan
> <roy.fan.zhang@intel.com>; Doherty, Declan <declan.doherty@intel.com>;
> Richardson, Bruce <bruce.richardson@intel.com>; shreyansh.jain@nxp.com;
> Yigit, Ferruh <ferruh.yigit@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>;
> Liu, Song <song.liu@intel.com>; Wu, Hao <hao.wu@intel.com>;
> gaetan.rivet@6wind.com; Wu, Yanglong <yanglong.wu@intel.com>
> Subject: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS Rawdev Driver
>
> From: Rosen Xu <rosen.xu@intel.com>
>
> Add Intel FPGA BUS Rawdev Driver which is based on librte_rawdev library.
>
> Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> Signed-off-by: Yanglong Wu <yanglong.wu@intel.com>
> Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
> Acked-by: Shreyansh Jain <shreyansh.jain@nxp.com>
> ---
> MAINTAINERS | 3 +
> config/common_base | 5 +
> doc/guides/rawdevs/ifpga_rawdev.rst | 112 ++++
> doc/guides/rawdevs/index.rst | 1 +
> doc/guides/rel_notes/release_18_05.rst | 8 +
> drivers/raw/Makefile | 1 +
> drivers/raw/ifpga_rawdev/Makefile | 36 ++
> drivers/raw/ifpga_rawdev/ifpga_rawdev.c | 608
> +++++++++++++++++++++
> drivers/raw/ifpga_rawdev/ifpga_rawdev.h | 37 ++
> drivers/raw/ifpga_rawdev/meson.build | 15 +
> .../ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map | 4 +
> drivers/raw/meson.build | 2 +-
> mk/rte.app.mk | 4 +-
> 13 files changed, 834 insertions(+), 2 deletions(-) create mode 100644
> doc/guides/rawdevs/ifpga_rawdev.rst
> create mode 100644 drivers/raw/ifpga_rawdev/Makefile create mode
> 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> create mode 100644 drivers/raw/ifpga_rawdev/meson.build
> create mode 100644
> drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 9c27d5a..a93ce9f 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -851,8 +851,11 @@ Rawdev Drivers
> --------------
>
> Intel FPGA
> +M: Rosen Xu <rosen.xu@intel.com>
> M: Tianfei zhang <tianfei.zhang@intel.com>
> +F: drivers/raw/ifpga_rawdev/
> F: drivers/raw/ifpga_rawdev/base/
> +F: doc/guides/rawdevs/ifpga_rawdev.rst
>
> NXP DPAA2 QDMA
> M: Nipun Gupta <nipun.gupta@nxp.com>
> diff --git a/config/common_base b/config/common_base index
> 1440316..017a15a 100644
> --- a/config/common_base
> +++ b/config/common_base
> @@ -633,6 +633,11 @@
> CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV=n
> CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
>
> #
> +# Compile PMD for Intel FPGA raw device #
> +CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
> +
> +#
> # Compile librte_ring
> #
> CONFIG_RTE_LIBRTE_RING=y
> diff --git a/doc/guides/rawdevs/ifpga_rawdev.rst
> b/doc/guides/rawdevs/ifpga_rawdev.rst
> new file mode 100644
> index 0000000..37ae4cc
> --- /dev/null
> +++ b/doc/guides/rawdevs/ifpga_rawdev.rst
> @@ -0,0 +1,112 @@
> +.. SPDX-License-Identifier: BSD-3-Clause
> + Copyright(c) 2018 Intel Corporation.
> +
> +IFPGA Rawdev Driver
> +======================
> +
> +FPGA is used more and more widely in Cloud and NFV, one primary reason
> +is that FPGA not only provides ASIC performance but also it's more
> +flexible than ASIC.
> +
> +FPGA uses Partial Reconfigure (PR) Parts of Bit Stream to achieve its
> +flexibility. That means one FPGA Device Bit Stream is divided into many
> +Parts of Bit Stream(each Part of Bit Stream is defined as
> +AFU-Accelerated Function Unit), and each AFU is a hardware acceleration
> +unit which can be dynamically reloaded respectively.
> +
> +By PR (Partial Reconfiguration) AFUs, one FPGA resources can be
> +time-shared by different users. FPGA hot upgrade and fault tolerance can be
> provided easily.
> +
> +The SW IFPGA Rawdev Driver (**ifpga_rawdev**) provides a Rawdev driver
> +that utilizes Intel FPGA Software Stack OPAE(Open Programmable
> +Acceleration
> +Engine) for FPGA management.
> +
> +Implementation details
> +----------------------
> +
> +Each instance of IFPGA Rawdev Driver is probed by Intel FpgaDev. In
> +coordination with OPAE share code IFPGA Rawdev Driver provides common
> +FPGA management ops for FPGA operation, OPAE provides all following
> operations:
> +- FPGA PR (Partial Reconfiguration) management
> +- FPGA AFUs Identifying
> +- FPGA Thermal Management
> +- FPGA Power Management
> +- FPGA Performance reporting
> +- FPGA Remote Debug
> +
> +All configuration parameters are taken by vdev_ifpga_cfg driver.
> +Besides configuration, vdev_ifpga_cfg driver also hot plugs in IFPGA Bus.
> +
> +All of the AFUs of one FPGA may share same PCI BDF and AFUs scan depend
> +on IFPGA Rawdev Driver so IFPGA Bus takes AFU device scan and AFU drivers
> probe.
> +All AFU device driver bind to AFU device by its UUID (Universally
> +Unique Identifier).
> +
> +To avoid unnecessary code duplication and ensure maximum performance,
> +handling of AFU devices is left to different PMDs; all the design as
> +summarized by the following block diagram::
> +
> + +---------------------------------------------------------------+
> + | Application(s)
> |
> + +----------------------------.----------------------------------+
> + |
> + |
> + +----------------------------'----------------------------------+
> + | DPDK Framework (APIs)
> |
> + +----------|------------|--------.---------------------|--------+
> + / \ |
> + / \ |
> + +-------'-------+ +-------'-------+ +--------'--------+
> + | Eth PMD | | Crypto PMD | |
> |
> + +-------.-------+ +-------.-------+ | |
> + | | |
> |
> + | | |
> |
> + +-------'-------+ +-------'-------+ | IFPGA |
> + | Eth AFU Dev | |Crypto AFU Dev | | Rawdev Driver
> |
> + +-------.-------+ +-------.-------+ |(OPAE Share Code)|
> + | | |
> |
> + | | Rawdev |
> |
> + +-------'------------------'-------+ Ops | |
> + | IFPGA Bus | -------->|
> |
> + +-----------------.----------------+ +--------.--------+
> + | |
> + Hot-plugin -->| |
> + | |
> + +-----------------'------------------+ +--------'--------+
> + | vdev_ifpga_cfg driver | | Intel FpgaDev
> |
> + +------------------------------------+ +-----------------+
> +
> +Build options
> +-------------
> +
> +- ``CONFIG_RTE_LIBRTE_IFPGA_BUS`` (default ``y``)
> +
> + Toggle compilation of IFPGA Bus library.
> +
> +- ``CONFIG_RTE_LIBRTE_IFPGA_RAWDEV`` (default ``y``)
> +
> + Toggle compilation of the ``ifpga_rawdev`` driver.
> +
> +Run-time parameters
> +-------------------
> +
> +This driver is invoked automatically in systems added with Intel FPGA,
> +but PR and IFPGA Bus scan is trigged by command line using ``--vdev
> +'net_ifpga_cfg`` EAL option.
> +
> +The following device parameters are supported:
> +
> +- ``ifpga`` [string]
> +
> + Provide a specific Intel FPGA device PCI BDF. Can be provided
> + multiple times for additional instances.
> +
> +- ``port`` [int]
> +
> + Each FPGA can provide many channels to PR AFU by software, each
> + channels is identified by this parameter.
> +
> +- ``afu_bts`` [string]
> +
> + If null, the AFU Bit Stream has been PR in FPGA, if not forces PR and
> + identifies AFU Bit Stream file.
> diff --git a/doc/guides/rawdevs/index.rst b/doc/guides/rawdevs/index.rst
> index 7769083..7c3bd95 100644
> --- a/doc/guides/rawdevs/index.rst
> +++ b/doc/guides/rawdevs/index.rst
> @@ -13,3 +13,4 @@ application through rawdev API.
>
> dpaa2_cmdif
> dpaa2_qdma
> + ifpga_rawdev
> diff --git a/doc/guides/rel_notes/release_18_05.rst
> b/doc/guides/rel_notes/release_18_05.rst
> index 265950a..f5241a1 100644
> --- a/doc/guides/rel_notes/release_18_05.rst
> +++ b/doc/guides/rel_notes/release_18_05.rst
> @@ -189,6 +189,14 @@ New Features
> the DPDK framework. It provides Intel FPGA Partial Bit Stream
> AFU(Accelerated
> Function Unit) scan and drivers prove.
>
> +* **Added IFPGA(Intel FPGA) Rawdev Driver.**
> +
> + Added a new Rawdev driver called IFPGA(Intel FPGA) Rawdev Driver,
> + which cooperates with OPAE(Open Programmable Acceleration Engine)
> + share code provides common FPGA management ops for FPGA operation.
> +
> + See the :doc:`../rawdevs/ifpga_rawdev` programmer's guide for more
> details.
> +
> API Changes
> -----------
>
> diff --git a/drivers/raw/Makefile b/drivers/raw/Makefile index
> 2eb2787..8e29b4a 100644
> --- a/drivers/raw/Makefile
> +++ b/drivers/raw/Makefile
> @@ -9,5 +9,6 @@ ifeq
> ($(CONFIG_RTE_EAL_VFIO)$(CONFIG_RTE_LIBRTE_FSLMC_BUS),yy)
> DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV) += dpaa2_cmdif
> DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += dpaa2_qdma
> endif
> +DIRS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += ifpga_rawdev
>
> include $(RTE_SDK)/mk/rte.subdir.mk
> diff --git a/drivers/raw/ifpga_rawdev/Makefile
> b/drivers/raw/ifpga_rawdev/Makefile
> new file mode 100644
> index 0000000..f3b9d5e
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/Makefile
> @@ -0,0 +1,36 @@
> +# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Intel
> +Corporation
> +
> +include $(RTE_SDK)/mk/rte.vars.mk
> +
> +#
> +# library name
> +#
> +LIB = librte_pmd_ifpga_rawdev.a
> +
> +CFLAGS += -DALLOW_EXPERIMENTAL_API
> +CFLAGS += -O3
> +CFLAGS += $(WERROR_FLAGS)
> +CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga CFLAGS +=
> +-I$(RTE_SDK)/drivers/raw/ifpga_rawdev
> +LDLIBS += -lrte_eal
> +LDLIBS += -lrte_rawdev
> +LDLIBS += -lrte_bus_vdev
> +LDLIBS += -lrte_kvargs
> +LDLIBS += -lrte_bus_pci
> +LDLIBS += -lrte_bus_ifpga
> +
> +EXPORT_MAP := rte_pmd_ifpga_rawdev_version.map
> +
> +LIBABIVER := 1
> +
> +VPATH += $(SRCDIR)/base
> +
> +include $(RTE_SDK)/drivers/raw/ifpga_rawdev/base/Makefile
> +
> +#
> +# all source are stored in SRCS-y
> +#
> +SRCS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += ifpga_rawdev.c
> +
> +include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> new file mode 100644
> index 0000000..32e811c
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> @@ -0,0 +1,608 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2010-2018 Intel Corporation */
> +
> +#include <string.h>
> +#include <dirent.h>
> +#include <sys/stat.h>
> +#include <unistd.h>
> +#include <sys/types.h>
> +#include <fcntl.h>
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_eal_memconfig.h>
> +#include <rte_malloc.h>
> +#include <rte_devargs.h>
> +#include <rte_memcpy.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +#include <rte_kvargs.h>
> +#include <rte_alarm.h>
> +
> +#include <rte_errno.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_eal.h>
> +#include <rte_common.h>
> +#include <rte_bus_vdev.h>
> +
> +#include "base/opae_hw_api.h"
> +#include "rte_rawdev.h"
> +#include "rte_rawdev_pmd.h"
> +#include "rte_bus_ifpga.h"
> +#include "ifpga_common.h"
> +#include "ifpga_logs.h"
> +#include "ifpga_rawdev.h"
> +
> +int ifpga_rawdev_logtype;
> +
> +#define PCI_VENDOR_ID_INTEL 0x8086
> +/* PCI Device ID */
> +#define PCIE_DEVICE_ID_PF_INT_5_X 0xBCBD
> +#define PCIE_DEVICE_ID_PF_INT_6_X 0xBCC0
> +#define PCIE_DEVICE_ID_PF_DSC_1_X 0x09C4
> +/* VF Device */
> +#define PCIE_DEVICE_ID_VF_INT_5_X 0xBCBF
> +#define PCIE_DEVICE_ID_VF_INT_6_X 0xBCC1
> +#define PCIE_DEVICE_ID_VF_DSC_1_X 0x09C5
> +#define RTE_MAX_RAW_DEVICE 10
> +
> +static const struct rte_pci_id pci_ifpga_map[] = {
> + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_PF_INT_5_X) },
> + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_VF_INT_5_X) },
> + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_PF_INT_6_X) },
> + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_VF_INT_6_X) },
> + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_PF_DSC_1_X) },
> + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> PCIE_DEVICE_ID_VF_DSC_1_X) },
> + { .vendor_id = 0, /* sentinel */ },
> +};
> +
> +static int
> +ifpga_fill_afu_dev(struct opae_accelerator *acc,
> + struct rte_afu_device *afu_dev)
> +{
> + struct rte_mem_resource *res = afu_dev->mem_resource;
> + struct opae_acc_region_info region_info;
> + struct opae_acc_info info;
> + unsigned long i;
> + int ret;
> +
> + ret = opae_acc_get_info(acc, &info);
> + if (ret)
> + return ret;
> +
> + if (info.num_regions > PCI_MAX_RESOURCE)
> + return -EFAULT;
> +
> + afu_dev->num_region = info.num_regions;
> +
> + for (i = 0; i < info.num_regions; i++) {
> + region_info.index = i;
> + ret = opae_acc_get_region_info(acc, ®ion_info);
> + if (ret)
> + return ret;
> +
> + if ((region_info.flags & ACC_REGION_MMIO) &&
> + (region_info.flags & ACC_REGION_READ) &&
> + (region_info.flags & ACC_REGION_WRITE)) {
> + res[i].phys_addr = region_info.phys_addr;
> + res[i].len = region_info.len;
> + res[i].addr = region_info.addr;
> + } else
> + return -EFAULT;
> + }
> +
> + return 0;
> +}
> +
> +static void
> +ifpga_rawdev_info_get(struct rte_rawdev *dev,
> + rte_rawdev_obj_t dev_info)
> +{
> + struct opae_adapter *adapter;
> + struct opae_accelerator *acc;
> + struct rte_afu_device *afu_dev;
> +
> + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> +
> + if (!dev_info) {
> + IFPGA_RAWDEV_PMD_ERR("Invalid request");
> + return;
> + }
> +
> + adapter = ifpga_rawdev_get_priv(dev);
> + if (!adapter)
> + return;
> +
> + afu_dev = dev_info;
> + afu_dev->rawdev = dev;
> +
> + /* find opae_accelerator and fill info into afu_device */
> + opae_adapter_for_each_acc(adapter, acc) {
> + if (acc->index != afu_dev->id.port)
> + continue;
> +
> + if (ifpga_fill_afu_dev(acc, afu_dev)) {
> + IFPGA_RAWDEV_PMD_ERR("cannot get info\n");
> + return;
> + }
> + }
> +}
> +
> +static int
> +ifpga_rawdev_start(struct rte_rawdev *dev) {
> + int ret = 0;
> + struct opae_adapter *adapter;
> +
> + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> +
> + RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
> +
> + adapter = ifpga_rawdev_get_priv(dev);
> + if (!adapter)
> + return -ENODEV;
> +
> + return ret;
> +}
> +
> +static void
> +ifpga_rawdev_stop(struct rte_rawdev *dev) {
> + dev->started = 0;
> +}
> +
> +static int
> +ifpga_rawdev_close(struct rte_rawdev *dev) {
> + return dev ? 0:1;
> +}
> +
> +static int
> +ifpga_rawdev_reset(struct rte_rawdev *dev) {
> + return dev ? 0:1;
> +}
> +
> +static int
> +fpga_pr(struct rte_rawdev *raw_dev, u32 port_id, u64 *buffer, u32 size,
> + u64 *status)
> +{
> +
> + struct opae_adapter *adapter;
> + struct opae_manager *mgr;
> + struct opae_accelerator *acc;
> + struct opae_bridge *br;
> + int ret;
> +
> + adapter = ifpga_rawdev_get_priv(raw_dev);
> + if (!adapter)
> + return -ENODEV;
> +
> + mgr = opae_adapter_get_mgr(adapter);
> + if (!mgr)
> + return -ENODEV;
> +
> + acc = opae_adapter_get_acc(adapter, port_id);
> + if (!acc)
> + return -ENODEV;
> +
> + br = opae_acc_get_br(acc);
> + if (!br)
> + return -ENODEV;
> +
> + ret = opae_manager_flash(mgr, port_id, buffer, size, status);
> + if (ret) {
> + IFPGA_RAWDEV_PMD_ERR("%s pr error %d\n", __func__, ret);
> + return ret;
> + }
> +
> + ret = opae_bridge_reset(br);
> + if (ret) {
> + IFPGA_RAWDEV_PMD_ERR("%s reset port:%d error %d\n",
> + __func__, port_id, ret);
> + return ret;
> + }
> +
> + return ret;
> +}
> +
> +static int
> +rte_fpga_do_pr(struct rte_rawdev *rawdev, int port_id,
> + const char *file_name)
> +{
> + struct stat file_stat;
> + int file_fd;
> + int ret = 0;
> + u32 buffer_size;
> + void *buffer;
> + u64 pr_error;
> +
> + if (!file_name)
> + return -EINVAL;
> +
> + file_fd = open(file_name, O_RDONLY);
> + if (file_fd < 0) {
> + IFPGA_RAWDEV_PMD_ERR("%s: open file error: %s\n",
> + __func__, file_name);
> + IFPGA_RAWDEV_PMD_ERR("Message : %s\n", strerror(errno));
> + return -EINVAL;
> + }
> + ret = stat(file_name, &file_stat);
> + if (ret) {
> + IFPGA_RAWDEV_PMD_ERR("stat on bitstream file failed: %s\n",
> + file_name);
> + return -EINVAL;
> + }
> + buffer_size = file_stat.st_size;
> + IFPGA_RAWDEV_PMD_INFO("bitstream file size: %u\n", buffer_size);
> + buffer = rte_malloc(NULL, buffer_size, 0);
> + if (!buffer) {
> + ret = -ENOMEM;
> + goto close_fd;
> + }
> +
> + /*read the raw data*/
> + if (buffer_size != read(file_fd, (void *)buffer, buffer_size)) {
> + ret = -EINVAL;
> + goto free_buffer;
> + }
> +
> + /*do PR now*/
> + ret = fpga_pr(rawdev, port_id, buffer, buffer_size, &pr_error);
> + IFPGA_RAWDEV_PMD_INFO("downloading to device port %d....%s.\n",
> port_id,
> + ret ? "failed" : "success");
> + if (ret) {
> + ret = -EINVAL;
> + goto free_buffer;
> + }
> +
> +free_buffer:
> + if (buffer)
> + rte_free(buffer);
> +close_fd:
> + close(file_fd);
> + file_fd = 0;
> + return ret;
> +}
> +
> +static int
> +ifpga_rawdev_pr(struct rte_rawdev *dev,
> + rte_rawdev_obj_t pr_conf)
> +{
> + struct opae_adapter *adapter;
> + struct rte_afu_pr_conf *afu_pr_conf;
> + int ret;
> + struct uuid uuid;
> + struct opae_accelerator *acc;
> +
> + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> +
> + adapter = ifpga_rawdev_get_priv(dev);
> + if (!adapter)
> + return -ENODEV;
> +
> + if (!pr_conf)
> + return -EINVAL;
> +
> + afu_pr_conf = pr_conf;
> +
> + if (afu_pr_conf->pr_enable) {
> + ret = rte_fpga_do_pr(dev,
> + afu_pr_conf->afu_id.port,
> + afu_pr_conf->bs_path);
> + if (ret) {
> + IFPGA_RAWDEV_PMD_ERR("do pr error %d\n", ret);
> + return ret;
> + }
> + }
> +
> + acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
> + if (!acc)
> + return -ENODEV;
> +
> + ret = opae_acc_get_uuid(acc, &uuid);
> + if (ret)
> + return ret;
> +
> + memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
> + memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
> +
> + IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
> __func__,
> + (u64)afu_pr_conf->afu_id.uuid.uuid_low,
> + (u64)afu_pr_conf->afu_id.uuid.uuid_high);
> +
> + return 0;
> +}
> +
> +static const struct rte_rawdev_ops ifpga_rawdev_ops = {
> + .dev_info_get = ifpga_rawdev_info_get,
> + .dev_configure = NULL,
> + .dev_start = ifpga_rawdev_start,
> + .dev_stop = ifpga_rawdev_stop,
> + .dev_close = ifpga_rawdev_close,
> + .dev_reset = ifpga_rawdev_reset,
> +
> + .queue_def_conf = NULL,
> + .queue_setup = NULL,
> + .queue_release = NULL,
> +
> + .attr_get = NULL,
> + .attr_set = NULL,
> +
> + .enqueue_bufs = NULL,
> + .dequeue_bufs = NULL,
> +
> + .dump = NULL,
> +
> + .xstats_get = NULL,
> + .xstats_get_names = NULL,
> + .xstats_get_by_name = NULL,
> + .xstats_reset = NULL,
> +
> + .firmware_status_get = NULL,
> + .firmware_version_get = NULL,
> + .firmware_load = ifpga_rawdev_pr,
> + .firmware_unload = NULL,
> +
> + .dev_selftest = NULL,
> +};
> +
> +static int
> +ifpga_rawdev_create(struct rte_pci_device *pci_dev,
> + int socket_id)
> +{
> + int ret = 0;
> + struct rte_rawdev *rawdev = NULL;
> + struct opae_adapter *adapter = NULL;
> + struct opae_manager *mgr = NULL;
> + struct opae_adapter_data_pci *data = NULL;
> + char name[RTE_RAWDEV_NAME_MAX_LEN];
> + int i;
> +
> + if (!pci_dev) {
> + IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
> + ret = -EINVAL;
> + goto cleanup;
> + }
> +
> + memset(name, 0, sizeof(name));
> + snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
> + pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
> +
> + IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name,
> +rte_socket_id());
> +
> + /* Allocate device structure */
> + rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct opae_adapter),
> + socket_id);
> + if (rawdev == NULL) {
> + IFPGA_RAWDEV_PMD_ERR("Unable to allocate rawdevice");
> + ret = -EINVAL;
> + goto cleanup;
> + }
> +
> + /* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
> + data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
> + if (!data) {
> + ret = -ENOMEM;
> + goto cleanup;
> + }
> +
> + /* init opae_adapter_data_pci for device specific information */
> + for (i = 0; i < PCI_MAX_RESOURCE; i++) {
> + data->region[i].phys_addr = pci_dev->mem_resource[i].phys_addr;
> + data->region[i].len = pci_dev->mem_resource[i].len;
> + data->region[i].addr = pci_dev->mem_resource[i].addr;
> + }
> + data->device_id = pci_dev->id.device_id;
> + data->vendor_id = pci_dev->id.vendor_id;
> +
> + /* create a opae_adapter based on above device data */
> + adapter = opae_adapter_alloc(pci_dev->device.name, data);
> + if (!adapter) {
> + ret = -ENOMEM;
> + goto free_adapter_data;
> + }
> +
> + rawdev->dev_ops = &ifpga_rawdev_ops;
> + rawdev->device = &pci_dev->device;
> + rawdev->driver_name = pci_dev->device.driver->name;
> +
> + rawdev->dev_private = adapter;
> +
> + /* must enumerate the adapter before use it */
> + ret = opae_adapter_enumerate(adapter);
> + if (ret)
> + goto free_adapter;
> +
> + /* get opae_manager to rawdev */
> + mgr = opae_adapter_get_mgr(adapter);
> + if (mgr) {
> + /* PF function */
> + IFPGA_RAWDEV_PMD_INFO("this is a PF function");
> + }
> +
> + return ret;
> +
> +free_adapter:
> + if (adapter)
> + opae_adapter_free(adapter);
> +free_adapter_data:
> + if (data)
> + opae_adapter_data_free(data);
> +cleanup:
> + if (rawdev)
> + rte_rawdev_pmd_release(rawdev);
> +
> + return ret;
> +}
> +
> +static int
> +ifpga_rawdev_destroy(struct rte_pci_device *pci_dev) {
> + int ret;
> + struct rte_rawdev *rawdev;
> + char name[RTE_RAWDEV_NAME_MAX_LEN];
> + struct opae_adapter *adapter;
> +
> + if (!pci_dev) {
> + IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
> + ret = -EINVAL;
> + return ret;
> + }
> +
> + memset(name, 0, sizeof(name));
> + snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
> + pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
> +
> + IFPGA_RAWDEV_PMD_INFO("Closing %s on NUMA node %d",
> + name, rte_socket_id());
> +
> + rawdev = rte_rawdev_pmd_get_named_dev(name);
> + if (!rawdev) {
> + IFPGA_RAWDEV_PMD_ERR("Invalid device name (%s)", name);
> + return -EINVAL;
> + }
> +
> + adapter = ifpga_rawdev_get_priv(rawdev);
> + if (!adapter)
> + return -ENODEV;
> +
> + opae_adapter_data_free(adapter->data);
> + opae_adapter_free(adapter);
> +
> + /* rte_rawdev_close is called by pmd_release */
> + ret = rte_rawdev_pmd_release(rawdev);
> + if (ret)
> + IFPGA_RAWDEV_PMD_DEBUG("Device cleanup failed");
> +
> + return ret;
> +}
> +
> +static int
> +ifpga_rawdev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
> + struct rte_pci_device *pci_dev)
> +{
> +
> + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> + return ifpga_rawdev_create(pci_dev, rte_socket_id()); }
> +
> +static int
> +ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev) {
> + return ifpga_rawdev_destroy(pci_dev);
> +}
> +
> +static struct rte_pci_driver rte_ifpga_rawdev_pmd = {
> + .id_table = pci_ifpga_map,
> + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
> + .probe = ifpga_rawdev_pci_probe,
> + .remove = ifpga_rawdev_pci_remove,
> +};
> +
> +RTE_PMD_REGISTER_PCI(ifpga_rawdev_pci_driver, rte_ifpga_rawdev_pmd);
> +RTE_PMD_REGISTER_PCI_TABLE(ifpga_rawdev_pci_driver,
> +rte_ifpga_rawdev_pmd);
> +RTE_PMD_REGISTER_KMOD_DEP(ifpga_rawdev_pci_driver, "* igb_uio |
> +uio_pci_generic | vfio-pci");
> +
> +RTE_INIT(ifpga_rawdev_init_log);
> +static void
> +ifpga_rawdev_init_log(void)
> +{
> + ifpga_rawdev_logtype = rte_log_register("driver.raw.init");
> + if (ifpga_rawdev_logtype >= 0)
> + rte_log_set_level(ifpga_rawdev_logtype, RTE_LOG_NOTICE); }
> +
> +static const char * const valid_args[] = {
> +#define IFPGA_ARG_NAME "ifpga"
> + IFPGA_ARG_NAME,
> +#define IFPGA_ARG_PORT "port"
> + IFPGA_ARG_PORT,
> +#define IFPGA_AFU_BTS "afu_bts"
> + IFPGA_AFU_BTS,
> + NULL
> +};
> +
> +static int
> +ifpga_cfg_probe(struct rte_vdev_device *dev) {
> + struct rte_devargs *devargs;
> + struct rte_kvargs *kvlist = NULL;
> + int port;
> + char *name = NULL;
> + char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
> +
> + devargs = dev->device.devargs;
> +
> + kvlist = rte_kvargs_parse(devargs->args, valid_args);
> + if (!kvlist) {
> + IFPGA_RAWDEV_PMD_LOG(ERR, "error when parsing param");
> + goto end;
> + }
> +
> + if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
> + if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
> + &rte_ifpga_get_string_arg, &name) < 0) {
> + IFPGA_RAWDEV_PMD_ERR("error to parse %s",
> + IFPGA_ARG_NAME);
> + goto end;
> + }
> + } else {
> + IFPGA_RAWDEV_PMD_ERR("arg %s is mandatory for ifpga bus",
> + IFPGA_ARG_NAME);
> + goto end;
> + }
> +
> + if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) {
> + if (rte_kvargs_process(kvlist,
> + IFPGA_ARG_PORT,
> + &rte_ifpga_get_integer32_arg,
> + &port) < 0) {
> + IFPGA_RAWDEV_PMD_ERR("error to parse %s",
> + IFPGA_ARG_PORT);
> + goto end;
> + }
> + } else {
> + IFPGA_RAWDEV_PMD_ERR("arg %s is mandatory for ifpga bus",
> + IFPGA_ARG_PORT);
> + goto end;
> + }
> +
> + memset(dev_name, 0, sizeof(dev_name));
> + snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
> + port, name);
> +
> + rte_eal_hotplug_add(RTE_STR(IFPGA_BUS_NAME),
> + dev_name, devargs->args);
> +end:
> + if (kvlist)
> + rte_kvargs_free(kvlist);
> + if (name)
> + free(name);
> +
> + return 0;
> +}
> +
> +static int
> +ifpga_cfg_remove(struct rte_vdev_device *vdev) {
> + IFPGA_RAWDEV_PMD_INFO("Remove ifpga_cfg %p",
> + vdev);
> +
> + return 0;
> +}
> +
> +static struct rte_vdev_driver ifpga_cfg_driver = {
> + .probe = ifpga_cfg_probe,
> + .remove = ifpga_cfg_remove,
> +};
> +
> +RTE_PMD_REGISTER_VDEV(net_ifpga_cfg, ifpga_cfg_driver);
> +RTE_PMD_REGISTER_ALIAS(net_ifpga_cfg, ifpga_cfg);
> +RTE_PMD_REGISTER_PARAM_STRING(net_ifpga_cfg,
> + "bdf=<string> "
> + "port=<int> "
> + "afu_bts=<path>");
> +
> diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> new file mode 100644
> index 0000000..c7759b8
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> @@ -0,0 +1,37 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2010-2018 Intel Corporation */
> +
> +#ifndef _IFPGA_RAWDEV_H_
> +#define _IFPGA_RAWDEV_H_
> +
> +extern int ifpga_rawdev_logtype;
> +
> +#define IFPGA_RAWDEV_PMD_LOG(level, fmt, args...) \
> + rte_log(RTE_LOG_ ## level, ifpga_rawdev_logtype, "ifgpa: " fmt, \
> + ##args)
> +
> +#define IFPGA_RAWDEV_PMD_FUNC_TRACE()
> IFPGA_RAWDEV_PMD_LOG(DEBUG, ">>")
> +
> +#define IFPGA_RAWDEV_PMD_DEBUG(fmt, args...) \
> + IFPGA_RAWDEV_PMD_LOG(DEBUG, fmt, ## args) #define
> +IFPGA_RAWDEV_PMD_INFO(fmt, args...) \
> + IFPGA_RAWDEV_PMD_LOG(INFO, fmt, ## args) #define
> +IFPGA_RAWDEV_PMD_ERR(fmt, args...) \
> + IFPGA_RAWDEV_PMD_LOG(ERR, fmt, ## args) #define
> +IFPGA_RAWDEV_PMD_WARN(fmt, args...) \
> + IFPGA_RAWDEV_PMD_LOG(WARNING, fmt, ## args)
> +
> +enum ifpga_rawdev_device_state {
> + IFPGA_IDLE,
> + IFPGA_READY,
> + IFPGA_ERROR
> +};
> +
> +static inline struct opae_adapter *
> +ifpga_rawdev_get_priv(const struct rte_rawdev *rawdev) {
> + return rawdev->dev_private;
> +}
> +
> +#endif /* _IFPGA_RAWDEV_H_ */
> diff --git a/drivers/raw/ifpga_rawdev/meson.build
> b/drivers/raw/ifpga_rawdev/meson.build
> new file mode 100644
> index 0000000..6725687
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/meson.build
> @@ -0,0 +1,15 @@
> +# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Intel
> +Corporation
> +
> +version = 1
> +
> +subdir('base')
> +objs = [base_objs]
> +
> +deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
> + 'bus_vdev', 'bus_ifpga']
> +sources = files('ifpga_rawdev.c')
> +
> +includes += include_directories('base')
> +
> +allow_experimental_apis = true
> diff --git a/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> b/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> new file mode 100644
> index 0000000..9b9ab1a
> --- /dev/null
> +++ b/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> @@ -0,0 +1,4 @@
> +DPDK_18.05 {
> +
> + local: *;
> +};
> diff --git a/drivers/raw/meson.build b/drivers/raw/meson.build index
> 34dfac4..a61cdcc 100644
> --- a/drivers/raw/meson.build
> +++ b/drivers/raw/meson.build
> @@ -1,7 +1,7 @@
> # SPDX-License-Identifier: BSD-3-Clause # Copyright 2018 NXP
>
> -drivers = ['skeleton_rawdev', 'dpaa2_cmdif', 'dpaa2_qdma']
> +drivers = ['skeleton_rawdev', 'dpaa2_cmdif', 'dpaa2_qdma',
> +'ifpga_rawdev']
> std_deps = ['rawdev']
> config_flag_fmt = 'RTE_LIBRTE_PMD_@0@_RAWDEV'
> driver_name_fmt = 'rte_pmd_@0@'
> diff --git a/mk/rte.app.mk b/mk/rte.app.mk index 3861e1a..acc0ec3 100644
> --- a/mk/rte.app.mk
> +++ b/mk/rte.app.mk
> @@ -256,9 +256,11 @@
> _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV) +=
> -lrte_pmd_dpaa2_cmdif
> _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) +=
> -lrte_pmd_dpaa2_qdma endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
> _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += -lrte_bus_ifpga
> +ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) +=
> -lrte_pmd_ifpga_rawdev
> +endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
> endif # CONFIG_RTE_LIBRTE_RAWDEV
>
> -
> endif # !CONFIG_RTE_BUILD_SHARED_LIBS
>
> _LDLIBS-y += --no-whole-archive
> --
> 1.8.3.1
Hi Qi,
> -----Original Message-----
> From: Zhang, Qi Z
> Sent: Thursday, May 10, 2018 22:24
> To: Xu, Rosen <rosen.xu@intel.com>; dev@dpdk.org; thomas@monjalon.net
> Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Roy Fan
> <roy.fan.zhang@intel.com>; Doherty, Declan <declan.doherty@intel.com>;
> Richardson, Bruce <bruce.richardson@intel.com>; shreyansh.jain@nxp.com;
> Yigit, Ferruh <ferruh.yigit@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>;
> Liu, Song <song.liu@intel.com>; Wu, Hao <hao.wu@intel.com>;
> gaetan.rivet@6wind.com; Wu, Yanglong <yanglong.wu@intel.com>
> Subject: RE: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS Rawdev
> Driver
>
> Hi Rosen:
>
> After read this patch, I have below suggestion
>
> 1. rte_ifgpa_device is not necessary, since we already have ifgpa bus,
> it manages all devices on that bus, we don't need another abstraction to
> manage afu devices, and user can add to or remove from the bus.
Remove no used definition of rte_ifgpa_device
> 2. rename rte_afu_device to rte_ifpga_device , we follow the name
> rule.
AFU is partial bit stream of FPGA, so AFU and FPGA are different definition.
But we define global variables: ifpga_afu_dev_list
> 3. data structure looks like below.
>
> Struct rte_ifpga_deivce {
> rte_device device /* backing device */
> Rte_raw_device *device /* link to raw pci device */
> Struct Rte_afu_info info; /* afu information */
> }
>
> struct rte_afu_info {
> struct rte_ifpga_device *ifpga_dev; /**< Point ifpga device */
> struct rte_afu_id id; /**< AFU id within FPGA. */
> uint32_t num_region; /**< number of regions found */
> ....
> ...
> }
For there is not ifpga definition in our patch, so I still keep the definition of rte_afu_dev.
Is it ok?
> Regards
> Qi
>
> > -----Original Message-----
> > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Xu, Rosen
> > Sent: Wednesday, May 9, 2018 3:43 PM
> > To: dev@dpdk.org; thomas@monjalon.net
> > Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Roy Fan
> > <roy.fan.zhang@intel.com>; Doherty, Declan <declan.doherty@intel.com>;
> > Richardson, Bruce <bruce.richardson@intel.com>;
> > shreyansh.jain@nxp.com; Yigit, Ferruh <ferruh.yigit@intel.com>;
> > Ananyev, Konstantin <konstantin.ananyev@intel.com>; Zhang, Tianfei
> > <tianfei.zhang@intel.com>; Liu, Song <song.liu@intel.com>; Wu, Hao
> > <hao.wu@intel.com>; gaetan.rivet@6wind.com; Wu, Yanglong
> > <yanglong.wu@intel.com>
> > Subject: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS Rawdev
> > Driver
> >
> > From: Rosen Xu <rosen.xu@intel.com>
> >
> > Add Intel FPGA BUS Rawdev Driver which is based on librte_rawdev library.
> >
> > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > Signed-off-by: Yanglong Wu <yanglong.wu@intel.com>
> > Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
> > Acked-by: Shreyansh Jain <shreyansh.jain@nxp.com>
> > ---
> > MAINTAINERS | 3 +
> > config/common_base | 5 +
> > doc/guides/rawdevs/ifpga_rawdev.rst | 112 ++++
> > doc/guides/rawdevs/index.rst | 1 +
> > doc/guides/rel_notes/release_18_05.rst | 8 +
> > drivers/raw/Makefile | 1 +
> > drivers/raw/ifpga_rawdev/Makefile | 36 ++
> > drivers/raw/ifpga_rawdev/ifpga_rawdev.c | 608
> > +++++++++++++++++++++
> > drivers/raw/ifpga_rawdev/ifpga_rawdev.h | 37 ++
> > drivers/raw/ifpga_rawdev/meson.build | 15 +
> > .../ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map | 4 +
> > drivers/raw/meson.build | 2 +-
> > mk/rte.app.mk | 4 +-
> > 13 files changed, 834 insertions(+), 2 deletions(-) create mode
> > 100644 doc/guides/rawdevs/ifpga_rawdev.rst
> > create mode 100644 drivers/raw/ifpga_rawdev/Makefile create mode
> > 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> > create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> > create mode 100644 drivers/raw/ifpga_rawdev/meson.build
> > create mode 100644
> > drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> >
> > diff --git a/MAINTAINERS b/MAINTAINERS index 9c27d5a..a93ce9f 100644
> > --- a/MAINTAINERS
> > +++ b/MAINTAINERS
> > @@ -851,8 +851,11 @@ Rawdev Drivers
> > --------------
> >
> > Intel FPGA
> > +M: Rosen Xu <rosen.xu@intel.com>
> > M: Tianfei zhang <tianfei.zhang@intel.com>
> > +F: drivers/raw/ifpga_rawdev/
> > F: drivers/raw/ifpga_rawdev/base/
> > +F: doc/guides/rawdevs/ifpga_rawdev.rst
> >
> > NXP DPAA2 QDMA
> > M: Nipun Gupta <nipun.gupta@nxp.com>
> > diff --git a/config/common_base b/config/common_base index
> > 1440316..017a15a 100644
> > --- a/config/common_base
> > +++ b/config/common_base
> > @@ -633,6 +633,11 @@
> > CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV=n
> > CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
> >
> > #
> > +# Compile PMD for Intel FPGA raw device #
> > +CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
> > +
> > +#
> > # Compile librte_ring
> > #
> > CONFIG_RTE_LIBRTE_RING=y
> > diff --git a/doc/guides/rawdevs/ifpga_rawdev.rst
> > b/doc/guides/rawdevs/ifpga_rawdev.rst
> > new file mode 100644
> > index 0000000..37ae4cc
> > --- /dev/null
> > +++ b/doc/guides/rawdevs/ifpga_rawdev.rst
> > @@ -0,0 +1,112 @@
> > +.. SPDX-License-Identifier: BSD-3-Clause
> > + Copyright(c) 2018 Intel Corporation.
> > +
> > +IFPGA Rawdev Driver
> > +======================
> > +
> > +FPGA is used more and more widely in Cloud and NFV, one primary
> > +reason is that FPGA not only provides ASIC performance but also it's
> > +more flexible than ASIC.
> > +
> > +FPGA uses Partial Reconfigure (PR) Parts of Bit Stream to achieve its
> > +flexibility. That means one FPGA Device Bit Stream is divided into
> > +many Parts of Bit Stream(each Part of Bit Stream is defined as
> > +AFU-Accelerated Function Unit), and each AFU is a hardware
> > +acceleration unit which can be dynamically reloaded respectively.
> > +
> > +By PR (Partial Reconfiguration) AFUs, one FPGA resources can be
> > +time-shared by different users. FPGA hot upgrade and fault tolerance
> > +can be
> > provided easily.
> > +
> > +The SW IFPGA Rawdev Driver (**ifpga_rawdev**) provides a Rawdev
> > +driver that utilizes Intel FPGA Software Stack OPAE(Open Programmable
> > +Acceleration
> > +Engine) for FPGA management.
> > +
> > +Implementation details
> > +----------------------
> > +
> > +Each instance of IFPGA Rawdev Driver is probed by Intel FpgaDev. In
> > +coordination with OPAE share code IFPGA Rawdev Driver provides
> common
> > +FPGA management ops for FPGA operation, OPAE provides all following
> > operations:
> > +- FPGA PR (Partial Reconfiguration) management
> > +- FPGA AFUs Identifying
> > +- FPGA Thermal Management
> > +- FPGA Power Management
> > +- FPGA Performance reporting
> > +- FPGA Remote Debug
> > +
> > +All configuration parameters are taken by vdev_ifpga_cfg driver.
> > +Besides configuration, vdev_ifpga_cfg driver also hot plugs in IFPGA Bus.
> > +
> > +All of the AFUs of one FPGA may share same PCI BDF and AFUs scan
> > +depend on IFPGA Rawdev Driver so IFPGA Bus takes AFU device scan and
> > +AFU drivers
> > probe.
> > +All AFU device driver bind to AFU device by its UUID (Universally
> > +Unique Identifier).
> > +
> > +To avoid unnecessary code duplication and ensure maximum
> performance,
> > +handling of AFU devices is left to different PMDs; all the design as
> > +summarized by the following block diagram::
> > +
> > + +---------------------------------------------------------------+
> > + | Application(s)
> > |
> > + +----------------------------.----------------------------------+
> > + |
> > + |
> > + +----------------------------'----------------------------------+
> > + | DPDK Framework (APIs)
> > |
> > + +----------|------------|--------.---------------------|--------+
> > + / \ |
> > + / \ |
> > + +-------'-------+ +-------'-------+ +--------'--------+
> > + | Eth PMD | | Crypto PMD | |
> > |
> > + +-------.-------+ +-------.-------+ | |
> > + | | |
> > |
> > + | | |
> > |
> > + +-------'-------+ +-------'-------+ | IFPGA |
> > + | Eth AFU Dev | |Crypto AFU Dev | | Rawdev Driver
> > |
> > + +-------.-------+ +-------.-------+ |(OPAE Share Code)|
> > + | | |
> > |
> > + | | Rawdev |
> > |
> > + +-------'------------------'-------+ Ops | |
> > + | IFPGA Bus | -------->|
> > |
> > + +-----------------.----------------+ +--------.--------+
> > + | |
> > + Hot-plugin -->| |
> > + | |
> > + +-----------------'------------------+ +--------'--------+
> > + | vdev_ifpga_cfg driver | | Intel FpgaDev
> > |
> > + +------------------------------------+ +-----------------+
> > +
> > +Build options
> > +-------------
> > +
> > +- ``CONFIG_RTE_LIBRTE_IFPGA_BUS`` (default ``y``)
> > +
> > + Toggle compilation of IFPGA Bus library.
> > +
> > +- ``CONFIG_RTE_LIBRTE_IFPGA_RAWDEV`` (default ``y``)
> > +
> > + Toggle compilation of the ``ifpga_rawdev`` driver.
> > +
> > +Run-time parameters
> > +-------------------
> > +
> > +This driver is invoked automatically in systems added with Intel
> > +FPGA, but PR and IFPGA Bus scan is trigged by command line using
> > +``--vdev 'net_ifpga_cfg`` EAL option.
> > +
> > +The following device parameters are supported:
> > +
> > +- ``ifpga`` [string]
> > +
> > + Provide a specific Intel FPGA device PCI BDF. Can be provided
> > + multiple times for additional instances.
> > +
> > +- ``port`` [int]
> > +
> > + Each FPGA can provide many channels to PR AFU by software, each
> > + channels is identified by this parameter.
> > +
> > +- ``afu_bts`` [string]
> > +
> > + If null, the AFU Bit Stream has been PR in FPGA, if not forces PR
> > + and identifies AFU Bit Stream file.
> > diff --git a/doc/guides/rawdevs/index.rst
> > b/doc/guides/rawdevs/index.rst index 7769083..7c3bd95 100644
> > --- a/doc/guides/rawdevs/index.rst
> > +++ b/doc/guides/rawdevs/index.rst
> > @@ -13,3 +13,4 @@ application through rawdev API.
> >
> > dpaa2_cmdif
> > dpaa2_qdma
> > + ifpga_rawdev
> > diff --git a/doc/guides/rel_notes/release_18_05.rst
> > b/doc/guides/rel_notes/release_18_05.rst
> > index 265950a..f5241a1 100644
> > --- a/doc/guides/rel_notes/release_18_05.rst
> > +++ b/doc/guides/rel_notes/release_18_05.rst
> > @@ -189,6 +189,14 @@ New Features
> > the DPDK framework. It provides Intel FPGA Partial Bit Stream
> > AFU(Accelerated
> > Function Unit) scan and drivers prove.
> >
> > +* **Added IFPGA(Intel FPGA) Rawdev Driver.**
> > +
> > + Added a new Rawdev driver called IFPGA(Intel FPGA) Rawdev Driver,
> > + which cooperates with OPAE(Open Programmable Acceleration Engine)
> > + share code provides common FPGA management ops for FPGA operation.
> > +
> > + See the :doc:`../rawdevs/ifpga_rawdev` programmer's guide for more
> > details.
> > +
> > API Changes
> > -----------
> >
> > diff --git a/drivers/raw/Makefile b/drivers/raw/Makefile index
> > 2eb2787..8e29b4a 100644
> > --- a/drivers/raw/Makefile
> > +++ b/drivers/raw/Makefile
> > @@ -9,5 +9,6 @@ ifeq
> > ($(CONFIG_RTE_EAL_VFIO)$(CONFIG_RTE_LIBRTE_FSLMC_BUS),yy)
> > DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV) +=
> dpaa2_cmdif
> > DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) +=
> dpaa2_qdma endif
> > +DIRS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += ifpga_rawdev
> >
> > include $(RTE_SDK)/mk/rte.subdir.mk
> > diff --git a/drivers/raw/ifpga_rawdev/Makefile
> > b/drivers/raw/ifpga_rawdev/Makefile
> > new file mode 100644
> > index 0000000..f3b9d5e
> > --- /dev/null
> > +++ b/drivers/raw/ifpga_rawdev/Makefile
> > @@ -0,0 +1,36 @@
> > +# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Intel
> > +Corporation
> > +
> > +include $(RTE_SDK)/mk/rte.vars.mk
> > +
> > +#
> > +# library name
> > +#
> > +LIB = librte_pmd_ifpga_rawdev.a
> > +
> > +CFLAGS += -DALLOW_EXPERIMENTAL_API
> > +CFLAGS += -O3
> > +CFLAGS += $(WERROR_FLAGS)
> > +CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga CFLAGS +=
> > +-I$(RTE_SDK)/drivers/raw/ifpga_rawdev
> > +LDLIBS += -lrte_eal
> > +LDLIBS += -lrte_rawdev
> > +LDLIBS += -lrte_bus_vdev
> > +LDLIBS += -lrte_kvargs
> > +LDLIBS += -lrte_bus_pci
> > +LDLIBS += -lrte_bus_ifpga
> > +
> > +EXPORT_MAP := rte_pmd_ifpga_rawdev_version.map
> > +
> > +LIBABIVER := 1
> > +
> > +VPATH += $(SRCDIR)/base
> > +
> > +include $(RTE_SDK)/drivers/raw/ifpga_rawdev/base/Makefile
> > +
> > +#
> > +# all source are stored in SRCS-y
> > +#
> > +SRCS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += ifpga_rawdev.c
> > +
> > +include $(RTE_SDK)/mk/rte.lib.mk
> > diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> > b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> > new file mode 100644
> > index 0000000..32e811c
> > --- /dev/null
> > +++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> > @@ -0,0 +1,608 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2010-2018 Intel Corporation */
> > +
> > +#include <string.h>
> > +#include <dirent.h>
> > +#include <sys/stat.h>
> > +#include <unistd.h>
> > +#include <sys/types.h>
> > +#include <fcntl.h>
> > +#include <rte_log.h>
> > +#include <rte_bus.h>
> > +#include <rte_eal_memconfig.h>
> > +#include <rte_malloc.h>
> > +#include <rte_devargs.h>
> > +#include <rte_memcpy.h>
> > +#include <rte_pci.h>
> > +#include <rte_bus_pci.h>
> > +#include <rte_kvargs.h>
> > +#include <rte_alarm.h>
> > +
> > +#include <rte_errno.h>
> > +#include <rte_per_lcore.h>
> > +#include <rte_memory.h>
> > +#include <rte_memzone.h>
> > +#include <rte_eal.h>
> > +#include <rte_common.h>
> > +#include <rte_bus_vdev.h>
> > +
> > +#include "base/opae_hw_api.h"
> > +#include "rte_rawdev.h"
> > +#include "rte_rawdev_pmd.h"
> > +#include "rte_bus_ifpga.h"
> > +#include "ifpga_common.h"
> > +#include "ifpga_logs.h"
> > +#include "ifpga_rawdev.h"
> > +
> > +int ifpga_rawdev_logtype;
> > +
> > +#define PCI_VENDOR_ID_INTEL 0x8086
> > +/* PCI Device ID */
> > +#define PCIE_DEVICE_ID_PF_INT_5_X 0xBCBD
> > +#define PCIE_DEVICE_ID_PF_INT_6_X 0xBCC0
> > +#define PCIE_DEVICE_ID_PF_DSC_1_X 0x09C4
> > +/* VF Device */
> > +#define PCIE_DEVICE_ID_VF_INT_5_X 0xBCBF
> > +#define PCIE_DEVICE_ID_VF_INT_6_X 0xBCC1
> > +#define PCIE_DEVICE_ID_VF_DSC_1_X 0x09C5
> > +#define RTE_MAX_RAW_DEVICE 10
> > +
> > +static const struct rte_pci_id pci_ifpga_map[] = {
> > + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> > PCIE_DEVICE_ID_PF_INT_5_X) },
> > + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> > PCIE_DEVICE_ID_VF_INT_5_X) },
> > + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> > PCIE_DEVICE_ID_PF_INT_6_X) },
> > + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> > PCIE_DEVICE_ID_VF_INT_6_X) },
> > + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> > PCIE_DEVICE_ID_PF_DSC_1_X) },
> > + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> > PCIE_DEVICE_ID_VF_DSC_1_X) },
> > + { .vendor_id = 0, /* sentinel */ },
> > +};
> > +
> > +static int
> > +ifpga_fill_afu_dev(struct opae_accelerator *acc,
> > + struct rte_afu_device *afu_dev)
> > +{
> > + struct rte_mem_resource *res = afu_dev->mem_resource;
> > + struct opae_acc_region_info region_info;
> > + struct opae_acc_info info;
> > + unsigned long i;
> > + int ret;
> > +
> > + ret = opae_acc_get_info(acc, &info);
> > + if (ret)
> > + return ret;
> > +
> > + if (info.num_regions > PCI_MAX_RESOURCE)
> > + return -EFAULT;
> > +
> > + afu_dev->num_region = info.num_regions;
> > +
> > + for (i = 0; i < info.num_regions; i++) {
> > + region_info.index = i;
> > + ret = opae_acc_get_region_info(acc, ®ion_info);
> > + if (ret)
> > + return ret;
> > +
> > + if ((region_info.flags & ACC_REGION_MMIO) &&
> > + (region_info.flags & ACC_REGION_READ) &&
> > + (region_info.flags & ACC_REGION_WRITE)) {
> > + res[i].phys_addr = region_info.phys_addr;
> > + res[i].len = region_info.len;
> > + res[i].addr = region_info.addr;
> > + } else
> > + return -EFAULT;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +static void
> > +ifpga_rawdev_info_get(struct rte_rawdev *dev,
> > + rte_rawdev_obj_t dev_info)
> > +{
> > + struct opae_adapter *adapter;
> > + struct opae_accelerator *acc;
> > + struct rte_afu_device *afu_dev;
> > +
> > + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> > +
> > + if (!dev_info) {
> > + IFPGA_RAWDEV_PMD_ERR("Invalid request");
> > + return;
> > + }
> > +
> > + adapter = ifpga_rawdev_get_priv(dev);
> > + if (!adapter)
> > + return;
> > +
> > + afu_dev = dev_info;
> > + afu_dev->rawdev = dev;
> > +
> > + /* find opae_accelerator and fill info into afu_device */
> > + opae_adapter_for_each_acc(adapter, acc) {
> > + if (acc->index != afu_dev->id.port)
> > + continue;
> > +
> > + if (ifpga_fill_afu_dev(acc, afu_dev)) {
> > + IFPGA_RAWDEV_PMD_ERR("cannot get info\n");
> > + return;
> > + }
> > + }
> > +}
> > +
> > +static int
> > +ifpga_rawdev_start(struct rte_rawdev *dev) {
> > + int ret = 0;
> > + struct opae_adapter *adapter;
> > +
> > + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> > +
> > + RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
> > +
> > + adapter = ifpga_rawdev_get_priv(dev);
> > + if (!adapter)
> > + return -ENODEV;
> > +
> > + return ret;
> > +}
> > +
> > +static void
> > +ifpga_rawdev_stop(struct rte_rawdev *dev) {
> > + dev->started = 0;
> > +}
> > +
> > +static int
> > +ifpga_rawdev_close(struct rte_rawdev *dev) {
> > + return dev ? 0:1;
> > +}
> > +
> > +static int
> > +ifpga_rawdev_reset(struct rte_rawdev *dev) {
> > + return dev ? 0:1;
> > +}
> > +
> > +static int
> > +fpga_pr(struct rte_rawdev *raw_dev, u32 port_id, u64 *buffer, u32 size,
> > + u64 *status)
> > +{
> > +
> > + struct opae_adapter *adapter;
> > + struct opae_manager *mgr;
> > + struct opae_accelerator *acc;
> > + struct opae_bridge *br;
> > + int ret;
> > +
> > + adapter = ifpga_rawdev_get_priv(raw_dev);
> > + if (!adapter)
> > + return -ENODEV;
> > +
> > + mgr = opae_adapter_get_mgr(adapter);
> > + if (!mgr)
> > + return -ENODEV;
> > +
> > + acc = opae_adapter_get_acc(adapter, port_id);
> > + if (!acc)
> > + return -ENODEV;
> > +
> > + br = opae_acc_get_br(acc);
> > + if (!br)
> > + return -ENODEV;
> > +
> > + ret = opae_manager_flash(mgr, port_id, buffer, size, status);
> > + if (ret) {
> > + IFPGA_RAWDEV_PMD_ERR("%s pr error %d\n", __func__,
> ret);
> > + return ret;
> > + }
> > +
> > + ret = opae_bridge_reset(br);
> > + if (ret) {
> > + IFPGA_RAWDEV_PMD_ERR("%s reset port:%d error %d\n",
> > + __func__, port_id, ret);
> > + return ret;
> > + }
> > +
> > + return ret;
> > +}
> > +
> > +static int
> > +rte_fpga_do_pr(struct rte_rawdev *rawdev, int port_id,
> > + const char *file_name)
> > +{
> > + struct stat file_stat;
> > + int file_fd;
> > + int ret = 0;
> > + u32 buffer_size;
> > + void *buffer;
> > + u64 pr_error;
> > +
> > + if (!file_name)
> > + return -EINVAL;
> > +
> > + file_fd = open(file_name, O_RDONLY);
> > + if (file_fd < 0) {
> > + IFPGA_RAWDEV_PMD_ERR("%s: open file error: %s\n",
> > + __func__, file_name);
> > + IFPGA_RAWDEV_PMD_ERR("Message : %s\n",
> strerror(errno));
> > + return -EINVAL;
> > + }
> > + ret = stat(file_name, &file_stat);
> > + if (ret) {
> > + IFPGA_RAWDEV_PMD_ERR("stat on bitstream file
> failed: %s\n",
> > + file_name);
> > + return -EINVAL;
> > + }
> > + buffer_size = file_stat.st_size;
> > + IFPGA_RAWDEV_PMD_INFO("bitstream file size: %u\n", buffer_size);
> > + buffer = rte_malloc(NULL, buffer_size, 0);
> > + if (!buffer) {
> > + ret = -ENOMEM;
> > + goto close_fd;
> > + }
> > +
> > + /*read the raw data*/
> > + if (buffer_size != read(file_fd, (void *)buffer, buffer_size)) {
> > + ret = -EINVAL;
> > + goto free_buffer;
> > + }
> > +
> > + /*do PR now*/
> > + ret = fpga_pr(rawdev, port_id, buffer, buffer_size, &pr_error);
> > + IFPGA_RAWDEV_PMD_INFO("downloading to device
> port %d....%s.\n",
> > port_id,
> > + ret ? "failed" : "success");
> > + if (ret) {
> > + ret = -EINVAL;
> > + goto free_buffer;
> > + }
> > +
> > +free_buffer:
> > + if (buffer)
> > + rte_free(buffer);
> > +close_fd:
> > + close(file_fd);
> > + file_fd = 0;
> > + return ret;
> > +}
> > +
> > +static int
> > +ifpga_rawdev_pr(struct rte_rawdev *dev,
> > + rte_rawdev_obj_t pr_conf)
> > +{
> > + struct opae_adapter *adapter;
> > + struct rte_afu_pr_conf *afu_pr_conf;
> > + int ret;
> > + struct uuid uuid;
> > + struct opae_accelerator *acc;
> > +
> > + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> > +
> > + adapter = ifpga_rawdev_get_priv(dev);
> > + if (!adapter)
> > + return -ENODEV;
> > +
> > + if (!pr_conf)
> > + return -EINVAL;
> > +
> > + afu_pr_conf = pr_conf;
> > +
> > + if (afu_pr_conf->pr_enable) {
> > + ret = rte_fpga_do_pr(dev,
> > + afu_pr_conf->afu_id.port,
> > + afu_pr_conf->bs_path);
> > + if (ret) {
> > + IFPGA_RAWDEV_PMD_ERR("do pr error %d\n", ret);
> > + return ret;
> > + }
> > + }
> > +
> > + acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
> > + if (!acc)
> > + return -ENODEV;
> > +
> > + ret = opae_acc_get_uuid(acc, &uuid);
> > + if (ret)
> > + return ret;
> > +
> > + memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
> > + memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
> > +sizeof(u64));
> > +
> > + IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
> > __func__,
> > + (u64)afu_pr_conf->afu_id.uuid.uuid_low,
> > + (u64)afu_pr_conf->afu_id.uuid.uuid_high);
> > +
> > + return 0;
> > +}
> > +
> > +static const struct rte_rawdev_ops ifpga_rawdev_ops = {
> > + .dev_info_get = ifpga_rawdev_info_get,
> > + .dev_configure = NULL,
> > + .dev_start = ifpga_rawdev_start,
> > + .dev_stop = ifpga_rawdev_stop,
> > + .dev_close = ifpga_rawdev_close,
> > + .dev_reset = ifpga_rawdev_reset,
> > +
> > + .queue_def_conf = NULL,
> > + .queue_setup = NULL,
> > + .queue_release = NULL,
> > +
> > + .attr_get = NULL,
> > + .attr_set = NULL,
> > +
> > + .enqueue_bufs = NULL,
> > + .dequeue_bufs = NULL,
> > +
> > + .dump = NULL,
> > +
> > + .xstats_get = NULL,
> > + .xstats_get_names = NULL,
> > + .xstats_get_by_name = NULL,
> > + .xstats_reset = NULL,
> > +
> > + .firmware_status_get = NULL,
> > + .firmware_version_get = NULL,
> > + .firmware_load = ifpga_rawdev_pr,
> > + .firmware_unload = NULL,
> > +
> > + .dev_selftest = NULL,
> > +};
> > +
> > +static int
> > +ifpga_rawdev_create(struct rte_pci_device *pci_dev,
> > + int socket_id)
> > +{
> > + int ret = 0;
> > + struct rte_rawdev *rawdev = NULL;
> > + struct opae_adapter *adapter = NULL;
> > + struct opae_manager *mgr = NULL;
> > + struct opae_adapter_data_pci *data = NULL;
> > + char name[RTE_RAWDEV_NAME_MAX_LEN];
> > + int i;
> > +
> > + if (!pci_dev) {
> > + IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
> > + ret = -EINVAL;
> > + goto cleanup;
> > + }
> > +
> > + memset(name, 0, sizeof(name));
> > + snprintf(name, RTE_RAWDEV_NAME_MAX_LEN,
> "IFPGA:%x:%02x.%x",
> > + pci_dev->addr.bus, pci_dev->addr.devid, pci_dev-
> >addr.function);
> > +
> > + IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name,
> > +rte_socket_id());
> > +
> > + /* Allocate device structure */
> > + rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct
> opae_adapter),
> > + socket_id);
> > + if (rawdev == NULL) {
> > + IFPGA_RAWDEV_PMD_ERR("Unable to allocate rawdevice");
> > + ret = -EINVAL;
> > + goto cleanup;
> > + }
> > +
> > + /* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API
> */
> > + data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
> > + if (!data) {
> > + ret = -ENOMEM;
> > + goto cleanup;
> > + }
> > +
> > + /* init opae_adapter_data_pci for device specific information */
> > + for (i = 0; i < PCI_MAX_RESOURCE; i++) {
> > + data->region[i].phys_addr = pci_dev-
> >mem_resource[i].phys_addr;
> > + data->region[i].len = pci_dev->mem_resource[i].len;
> > + data->region[i].addr = pci_dev->mem_resource[i].addr;
> > + }
> > + data->device_id = pci_dev->id.device_id;
> > + data->vendor_id = pci_dev->id.vendor_id;
> > +
> > + /* create a opae_adapter based on above device data */
> > + adapter = opae_adapter_alloc(pci_dev->device.name, data);
> > + if (!adapter) {
> > + ret = -ENOMEM;
> > + goto free_adapter_data;
> > + }
> > +
> > + rawdev->dev_ops = &ifpga_rawdev_ops;
> > + rawdev->device = &pci_dev->device;
> > + rawdev->driver_name = pci_dev->device.driver->name;
> > +
> > + rawdev->dev_private = adapter;
> > +
> > + /* must enumerate the adapter before use it */
> > + ret = opae_adapter_enumerate(adapter);
> > + if (ret)
> > + goto free_adapter;
> > +
> > + /* get opae_manager to rawdev */
> > + mgr = opae_adapter_get_mgr(adapter);
> > + if (mgr) {
> > + /* PF function */
> > + IFPGA_RAWDEV_PMD_INFO("this is a PF function");
> > + }
> > +
> > + return ret;
> > +
> > +free_adapter:
> > + if (adapter)
> > + opae_adapter_free(adapter);
> > +free_adapter_data:
> > + if (data)
> > + opae_adapter_data_free(data);
> > +cleanup:
> > + if (rawdev)
> > + rte_rawdev_pmd_release(rawdev);
> > +
> > + return ret;
> > +}
> > +
> > +static int
> > +ifpga_rawdev_destroy(struct rte_pci_device *pci_dev) {
> > + int ret;
> > + struct rte_rawdev *rawdev;
> > + char name[RTE_RAWDEV_NAME_MAX_LEN];
> > + struct opae_adapter *adapter;
> > +
> > + if (!pci_dev) {
> > + IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
> > + ret = -EINVAL;
> > + return ret;
> > + }
> > +
> > + memset(name, 0, sizeof(name));
> > + snprintf(name, RTE_RAWDEV_NAME_MAX_LEN,
> "IFPGA:%x:%02x.%x",
> > + pci_dev->addr.bus, pci_dev->addr.devid, pci_dev-
> >addr.function);
> > +
> > + IFPGA_RAWDEV_PMD_INFO("Closing %s on NUMA node %d",
> > + name, rte_socket_id());
> > +
> > + rawdev = rte_rawdev_pmd_get_named_dev(name);
> > + if (!rawdev) {
> > + IFPGA_RAWDEV_PMD_ERR("Invalid device name (%s)",
> name);
> > + return -EINVAL;
> > + }
> > +
> > + adapter = ifpga_rawdev_get_priv(rawdev);
> > + if (!adapter)
> > + return -ENODEV;
> > +
> > + opae_adapter_data_free(adapter->data);
> > + opae_adapter_free(adapter);
> > +
> > + /* rte_rawdev_close is called by pmd_release */
> > + ret = rte_rawdev_pmd_release(rawdev);
> > + if (ret)
> > + IFPGA_RAWDEV_PMD_DEBUG("Device cleanup failed");
> > +
> > + return ret;
> > +}
> > +
> > +static int
> > +ifpga_rawdev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
> > + struct rte_pci_device *pci_dev)
> > +{
> > +
> > + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> > + return ifpga_rawdev_create(pci_dev, rte_socket_id()); }
> > +
> > +static int
> > +ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev) {
> > + return ifpga_rawdev_destroy(pci_dev); }
> > +
> > +static struct rte_pci_driver rte_ifpga_rawdev_pmd = {
> > + .id_table = pci_ifpga_map,
> > + .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
> RTE_PCI_DRV_INTR_LSC,
> > + .probe = ifpga_rawdev_pci_probe,
> > + .remove = ifpga_rawdev_pci_remove,
> > +};
> > +
> > +RTE_PMD_REGISTER_PCI(ifpga_rawdev_pci_driver,
> rte_ifpga_rawdev_pmd);
> > +RTE_PMD_REGISTER_PCI_TABLE(ifpga_rawdev_pci_driver,
> > +rte_ifpga_rawdev_pmd);
> > +RTE_PMD_REGISTER_KMOD_DEP(ifpga_rawdev_pci_driver, "* igb_uio |
> > +uio_pci_generic | vfio-pci");
> > +
> > +RTE_INIT(ifpga_rawdev_init_log);
> > +static void
> > +ifpga_rawdev_init_log(void)
> > +{
> > + ifpga_rawdev_logtype = rte_log_register("driver.raw.init");
> > + if (ifpga_rawdev_logtype >= 0)
> > + rte_log_set_level(ifpga_rawdev_logtype, RTE_LOG_NOTICE); }
> > +
> > +static const char * const valid_args[] = {
> > +#define IFPGA_ARG_NAME "ifpga"
> > + IFPGA_ARG_NAME,
> > +#define IFPGA_ARG_PORT "port"
> > + IFPGA_ARG_PORT,
> > +#define IFPGA_AFU_BTS "afu_bts"
> > + IFPGA_AFU_BTS,
> > + NULL
> > +};
> > +
> > +static int
> > +ifpga_cfg_probe(struct rte_vdev_device *dev) {
> > + struct rte_devargs *devargs;
> > + struct rte_kvargs *kvlist = NULL;
> > + int port;
> > + char *name = NULL;
> > + char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
> > +
> > + devargs = dev->device.devargs;
> > +
> > + kvlist = rte_kvargs_parse(devargs->args, valid_args);
> > + if (!kvlist) {
> > + IFPGA_RAWDEV_PMD_LOG(ERR, "error when parsing
> param");
> > + goto end;
> > + }
> > +
> > + if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
> > + if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
> > + &rte_ifpga_get_string_arg, &name) < 0) {
> > + IFPGA_RAWDEV_PMD_ERR("error to parse %s",
> > + IFPGA_ARG_NAME);
> > + goto end;
> > + }
> > + } else {
> > + IFPGA_RAWDEV_PMD_ERR("arg %s is mandatory for ifpga
> bus",
> > + IFPGA_ARG_NAME);
> > + goto end;
> > + }
> > +
> > + if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) {
> > + if (rte_kvargs_process(kvlist,
> > + IFPGA_ARG_PORT,
> > + &rte_ifpga_get_integer32_arg,
> > + &port) < 0) {
> > + IFPGA_RAWDEV_PMD_ERR("error to parse %s",
> > + IFPGA_ARG_PORT);
> > + goto end;
> > + }
> > + } else {
> > + IFPGA_RAWDEV_PMD_ERR("arg %s is mandatory for ifpga
> bus",
> > + IFPGA_ARG_PORT);
> > + goto end;
> > + }
> > +
> > + memset(dev_name, 0, sizeof(dev_name));
> > + snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
> > + port, name);
> > +
> > + rte_eal_hotplug_add(RTE_STR(IFPGA_BUS_NAME),
> > + dev_name, devargs->args);
> > +end:
> > + if (kvlist)
> > + rte_kvargs_free(kvlist);
> > + if (name)
> > + free(name);
> > +
> > + return 0;
> > +}
> > +
> > +static int
> > +ifpga_cfg_remove(struct rte_vdev_device *vdev) {
> > + IFPGA_RAWDEV_PMD_INFO("Remove ifpga_cfg %p",
> > + vdev);
> > +
> > + return 0;
> > +}
> > +
> > +static struct rte_vdev_driver ifpga_cfg_driver = {
> > + .probe = ifpga_cfg_probe,
> > + .remove = ifpga_cfg_remove,
> > +};
> > +
> > +RTE_PMD_REGISTER_VDEV(net_ifpga_cfg, ifpga_cfg_driver);
> > +RTE_PMD_REGISTER_ALIAS(net_ifpga_cfg, ifpga_cfg);
> > +RTE_PMD_REGISTER_PARAM_STRING(net_ifpga_cfg,
> > + "bdf=<string> "
> > + "port=<int> "
> > + "afu_bts=<path>");
> > +
> > diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> > b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> > new file mode 100644
> > index 0000000..c7759b8
> > --- /dev/null
> > +++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> > @@ -0,0 +1,37 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(c) 2010-2018 Intel Corporation */
> > +
> > +#ifndef _IFPGA_RAWDEV_H_
> > +#define _IFPGA_RAWDEV_H_
> > +
> > +extern int ifpga_rawdev_logtype;
> > +
> > +#define IFPGA_RAWDEV_PMD_LOG(level, fmt, args...) \
> > + rte_log(RTE_LOG_ ## level, ifpga_rawdev_logtype, "ifgpa: " fmt, \
> > + ##args)
> > +
> > +#define IFPGA_RAWDEV_PMD_FUNC_TRACE()
> > IFPGA_RAWDEV_PMD_LOG(DEBUG, ">>")
> > +
> > +#define IFPGA_RAWDEV_PMD_DEBUG(fmt, args...) \
> > + IFPGA_RAWDEV_PMD_LOG(DEBUG, fmt, ## args) #define
> > +IFPGA_RAWDEV_PMD_INFO(fmt, args...) \
> > + IFPGA_RAWDEV_PMD_LOG(INFO, fmt, ## args) #define
> > +IFPGA_RAWDEV_PMD_ERR(fmt, args...) \
> > + IFPGA_RAWDEV_PMD_LOG(ERR, fmt, ## args) #define
> > +IFPGA_RAWDEV_PMD_WARN(fmt, args...) \
> > + IFPGA_RAWDEV_PMD_LOG(WARNING, fmt, ## args)
> > +
> > +enum ifpga_rawdev_device_state {
> > + IFPGA_IDLE,
> > + IFPGA_READY,
> > + IFPGA_ERROR
> > +};
> > +
> > +static inline struct opae_adapter *
> > +ifpga_rawdev_get_priv(const struct rte_rawdev *rawdev) {
> > + return rawdev->dev_private;
> > +}
> > +
> > +#endif /* _IFPGA_RAWDEV_H_ */
> > diff --git a/drivers/raw/ifpga_rawdev/meson.build
> > b/drivers/raw/ifpga_rawdev/meson.build
> > new file mode 100644
> > index 0000000..6725687
> > --- /dev/null
> > +++ b/drivers/raw/ifpga_rawdev/meson.build
> > @@ -0,0 +1,15 @@
> > +# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Intel
> > +Corporation
> > +
> > +version = 1
> > +
> > +subdir('base')
> > +objs = [base_objs]
> > +
> > +deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
> > + 'bus_vdev', 'bus_ifpga']
> > +sources = files('ifpga_rawdev.c')
> > +
> > +includes += include_directories('base')
> > +
> > +allow_experimental_apis = true
> > diff --git a/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> > b/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> > new file mode 100644
> > index 0000000..9b9ab1a
> > --- /dev/null
> > +++ b/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> > @@ -0,0 +1,4 @@
> > +DPDK_18.05 {
> > +
> > + local: *;
> > +};
> > diff --git a/drivers/raw/meson.build b/drivers/raw/meson.build index
> > 34dfac4..a61cdcc 100644
> > --- a/drivers/raw/meson.build
> > +++ b/drivers/raw/meson.build
> > @@ -1,7 +1,7 @@
> > # SPDX-License-Identifier: BSD-3-Clause # Copyright 2018 NXP
> >
> > -drivers = ['skeleton_rawdev', 'dpaa2_cmdif', 'dpaa2_qdma']
> > +drivers = ['skeleton_rawdev', 'dpaa2_cmdif', 'dpaa2_qdma',
> > +'ifpga_rawdev']
> > std_deps = ['rawdev']
> > config_flag_fmt = 'RTE_LIBRTE_PMD_@0@_RAWDEV'
> > driver_name_fmt = 'rte_pmd_@0@'
> > diff --git a/mk/rte.app.mk b/mk/rte.app.mk index 3861e1a..acc0ec3
> > 100644
> > --- a/mk/rte.app.mk
> > +++ b/mk/rte.app.mk
> > @@ -256,9 +256,11 @@
> > _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV) +=
> > -lrte_pmd_dpaa2_cmdif
> > _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) +=
> > -lrte_pmd_dpaa2_qdma endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
> > _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += -lrte_bus_ifpga
> > +ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
> > +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) +=
> > -lrte_pmd_ifpga_rawdev
> > +endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
> > endif # CONFIG_RTE_LIBRTE_RAWDEV
> >
> > -
> > endif # !CONFIG_RTE_BUILD_SHARED_LIBS
> >
> > _LDLIBS-y += --no-whole-archive
> > --
> > 1.8.3.1
Hi Jingjing,
After we have aligned this morning, I will reply your email again.
> -----Original Message-----
> From: Wu, Jingjing
> Sent: Thursday, May 10, 2018 17:21
> To: Xu, Rosen <rosen.xu@intel.com>; dev@dpdk.org; thomas@monjalon.net
> Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Roy Fan
> <roy.fan.zhang@intel.com>; Doherty, Declan <declan.doherty@intel.com>;
> Richardson, Bruce <bruce.richardson@intel.com>; shreyansh.jain@nxp.com;
> Yigit, Ferruh <ferruh.yigit@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>;
> Liu, Song <song.liu@intel.com>; Wu, Hao <hao.wu@intel.com>;
> gaetan.rivet@6wind.com; Wu, Yanglong <yanglong.wu@intel.com>
> Subject: RE: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS Rawdev
> Driver
>
> Hi, Rosen
>
> Few comments below.
>
> Thanks
> Jingjing
>
> [...]
>
> > +static int
> > +ifpga_rawdev_start(struct rte_rawdev *dev) {
> > + int ret = 0;
> > + struct opae_adapter *adapter;
> > +
> > + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> > +
> > + RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
> > +
> > + adapter = ifpga_rawdev_get_priv(dev);
> > + if (!adapter)
> > + return -ENODEV;
> > +
> Set dev->started?
> > + return ret;
> > +}
Fixed.
>
> [...]
>
> > +
> > +static const struct rte_rawdev_ops ifpga_rawdev_ops = {
> > + .dev_info_get = ifpga_rawdev_info_get,
> > + .dev_configure = NULL,
> If go the declaration of rte_rawdev_configure, you will see "This function
> must be invoked first before any other function in the API."
> So I think we need to function for it, even it does nothing.
Yes, the rte_rawdev_configure function should not NULL.
I have fill it.
>
> [...]
>
> > +static struct rte_pci_driver rte_ifpga_rawdev_pmd = {
> > + .id_table = pci_ifpga_map,
> > + .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
> RTE_PCI_DRV_INTR_LSC,
> Is RTE_PCI_DRV_INTR_LSC supported?
Not supported, and I removed it.
> [...]
>
> > +static struct rte_vdev_driver ifpga_cfg_driver = {
> > + .probe = ifpga_cfg_probe,
> > + .remove = ifpga_cfg_remove,
> > +};
> > +
> > +RTE_PMD_REGISTER_VDEV(net_ifpga_cfg, ifpga_cfg_driver);
> I think prefix net_ would mean the device is net device (eth_dev)? How about
> to change the prefix to raw_?
>
> > +RTE_PMD_REGISTER_ALIAS(net_ifpga_cfg, ifpga_cfg);
> > +RTE_PMD_REGISTER_PARAM_STRING(net_ifpga_cfg,
> > + "bdf=<string> "
> ifpga=<string>?
Fixed.
> > + "port=<int> "
> > + "afu_bts=<path>");
> > +
> -----Original Message-----
> From: Xu, Rosen
> Sent: Friday, May 11, 2018 11:17 AM
> To: Zhang, Qi Z <qi.z.zhang@intel.com>; dev@dpdk.org;
> thomas@monjalon.net
> Cc: Zhang, Roy Fan <roy.fan.zhang@intel.com>; Doherty, Declan
> <declan.doherty@intel.com>; Richardson, Bruce
> <bruce.richardson@intel.com>; shreyansh.jain@nxp.com; Yigit, Ferruh
> <ferruh.yigit@intel.com>; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; Zhang, Tianfei <tianfei.zhang@intel.com>;
> Liu, Song <song.liu@intel.com>; Wu, Hao <hao.wu@intel.com>;
> gaetan.rivet@6wind.com; Wu, Yanglong <yanglong.wu@intel.com>
> Subject: RE: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS Rawdev
> Driver
>
> Hi Qi,
>
> > -----Original Message-----
> > From: Zhang, Qi Z
> > Sent: Thursday, May 10, 2018 22:24
> > To: Xu, Rosen <rosen.xu@intel.com>; dev@dpdk.org;
> thomas@monjalon.net
> > Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Roy Fan
> > <roy.fan.zhang@intel.com>; Doherty, Declan <declan.doherty@intel.com>;
> > Richardson, Bruce <bruce.richardson@intel.com>;
> > shreyansh.jain@nxp.com; Yigit, Ferruh <ferruh.yigit@intel.com>;
> > Ananyev, Konstantin <konstantin.ananyev@intel.com>; Zhang, Tianfei
> > <tianfei.zhang@intel.com>; Liu, Song <song.liu@intel.com>; Wu, Hao
> > <hao.wu@intel.com>; gaetan.rivet@6wind.com; Wu, Yanglong
> > <yanglong.wu@intel.com>
> > Subject: RE: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS
> > Rawdev Driver
> >
> > Hi Rosen:
> >
> > After read this patch, I have below suggestion
> >
> > 1. rte_ifgpa_device is not necessary, since we already have ifgpa
> > bus, it manages all devices on that bus, we don't need another
> > abstraction to manage afu devices, and user can add to or remove from the
> bus.
>
> Remove no used definition of rte_ifgpa_device
OK
>
> > 2. rename rte_afu_device to rte_ifpga_device , we follow the name
> > rule.
>
> AFU is partial bit stream of FPGA, so AFU and FPGA are different definition.
> But we define global variables: ifpga_afu_dev_list
I think ifpga is an abstract bus, we could have rte_ifpga_device as an abstract device type,
afu is the physical resource the abstract device link to,
Name rte_afu_device on ifpga bus looks not consistent to me.
Regards
Qi
>
> > 3. data structure looks like below.
> >
> > Struct rte_ifpga_deivce {
> > rte_device device /* backing device */
> > Rte_raw_device *device /* link to raw pci device */
> > Struct Rte_afu_info info; /* afu information */
> > }
> >
> > struct rte_afu_info {
> > struct rte_ifpga_device *ifpga_dev; /**< Point ifpga device */
> > struct rte_afu_id id; /**< AFU id within FPGA.
> */
> > uint32_t num_region; /**< number of regions found */
> > ....
> > ...
> > }
>
> For there is not ifpga definition in our patch, so I still keep the definition of
> rte_afu_dev.
> Is it ok?
>
> > Regards
> > Qi
> >
> > > -----Original Message-----
> > > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Xu, Rosen
> > > Sent: Wednesday, May 9, 2018 3:43 PM
> > > To: dev@dpdk.org; thomas@monjalon.net
> > > Cc: Xu, Rosen <rosen.xu@intel.com>; Zhang, Roy Fan
> > > <roy.fan.zhang@intel.com>; Doherty, Declan
> > > <declan.doherty@intel.com>; Richardson, Bruce
> > > <bruce.richardson@intel.com>; shreyansh.jain@nxp.com; Yigit, Ferruh
> > > <ferruh.yigit@intel.com>; Ananyev, Konstantin
> > > <konstantin.ananyev@intel.com>; Zhang, Tianfei
> > > <tianfei.zhang@intel.com>; Liu, Song <song.liu@intel.com>; Wu, Hao
> > > <hao.wu@intel.com>; gaetan.rivet@6wind.com; Wu, Yanglong
> > > <yanglong.wu@intel.com>
> > > Subject: [dpdk-dev] [PATCH v10 3/3] iFPGA: Add Intel FPGA BUS Rawdev
> > > Driver
> > >
> > > From: Rosen Xu <rosen.xu@intel.com>
> > >
> > > Add Intel FPGA BUS Rawdev Driver which is based on librte_rawdev library.
> > >
> > > Signed-off-by: Rosen Xu <rosen.xu@intel.com>
> > > Signed-off-by: Yanglong Wu <yanglong.wu@intel.com>
> > > Signed-off-by: Tianfei Zhang <tianfei.zhang@intel.com>
> > > Acked-by: Shreyansh Jain <shreyansh.jain@nxp.com>
> > > ---
> > > MAINTAINERS | 3 +
> > > config/common_base | 5 +
> > > doc/guides/rawdevs/ifpga_rawdev.rst | 112 ++++
> > > doc/guides/rawdevs/index.rst | 1 +
> > > doc/guides/rel_notes/release_18_05.rst | 8 +
> > > drivers/raw/Makefile | 1 +
> > > drivers/raw/ifpga_rawdev/Makefile | 36 ++
> > > drivers/raw/ifpga_rawdev/ifpga_rawdev.c | 608
> > > +++++++++++++++++++++
> > > drivers/raw/ifpga_rawdev/ifpga_rawdev.h | 37 ++
> > > drivers/raw/ifpga_rawdev/meson.build | 15 +
> > > .../ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map | 4 +
> > > drivers/raw/meson.build | 2 +-
> > > mk/rte.app.mk | 4 +-
> > > 13 files changed, 834 insertions(+), 2 deletions(-) create mode
> > > 100644 doc/guides/rawdevs/ifpga_rawdev.rst
> > > create mode 100644 drivers/raw/ifpga_rawdev/Makefile create mode
> > > 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> > > create mode 100644 drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> > > create mode 100644 drivers/raw/ifpga_rawdev/meson.build
> > > create mode 100644
> > > drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> > >
> > > diff --git a/MAINTAINERS b/MAINTAINERS index 9c27d5a..a93ce9f 100644
> > > --- a/MAINTAINERS
> > > +++ b/MAINTAINERS
> > > @@ -851,8 +851,11 @@ Rawdev Drivers
> > > --------------
> > >
> > > Intel FPGA
> > > +M: Rosen Xu <rosen.xu@intel.com>
> > > M: Tianfei zhang <tianfei.zhang@intel.com>
> > > +F: drivers/raw/ifpga_rawdev/
> > > F: drivers/raw/ifpga_rawdev/base/
> > > +F: doc/guides/rawdevs/ifpga_rawdev.rst
> > >
> > > NXP DPAA2 QDMA
> > > M: Nipun Gupta <nipun.gupta@nxp.com> diff --git
> > > a/config/common_base b/config/common_base index 1440316..017a15a
> > > 100644
> > > --- a/config/common_base
> > > +++ b/config/common_base
> > > @@ -633,6 +633,11 @@
> > > CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV=n
> > > CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
> > >
> > > #
> > > +# Compile PMD for Intel FPGA raw device #
> > > +CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
> > > +
> > > +#
> > > # Compile librte_ring
> > > #
> > > CONFIG_RTE_LIBRTE_RING=y
> > > diff --git a/doc/guides/rawdevs/ifpga_rawdev.rst
> > > b/doc/guides/rawdevs/ifpga_rawdev.rst
> > > new file mode 100644
> > > index 0000000..37ae4cc
> > > --- /dev/null
> > > +++ b/doc/guides/rawdevs/ifpga_rawdev.rst
> > > @@ -0,0 +1,112 @@
> > > +.. SPDX-License-Identifier: BSD-3-Clause
> > > + Copyright(c) 2018 Intel Corporation.
> > > +
> > > +IFPGA Rawdev Driver
> > > +======================
> > > +
> > > +FPGA is used more and more widely in Cloud and NFV, one primary
> > > +reason is that FPGA not only provides ASIC performance but also
> > > +it's more flexible than ASIC.
> > > +
> > > +FPGA uses Partial Reconfigure (PR) Parts of Bit Stream to achieve
> > > +its flexibility. That means one FPGA Device Bit Stream is divided
> > > +into many Parts of Bit Stream(each Part of Bit Stream is defined as
> > > +AFU-Accelerated Function Unit), and each AFU is a hardware
> > > +acceleration unit which can be dynamically reloaded respectively.
> > > +
> > > +By PR (Partial Reconfiguration) AFUs, one FPGA resources can be
> > > +time-shared by different users. FPGA hot upgrade and fault
> > > +tolerance can be
> > > provided easily.
> > > +
> > > +The SW IFPGA Rawdev Driver (**ifpga_rawdev**) provides a Rawdev
> > > +driver that utilizes Intel FPGA Software Stack OPAE(Open
> > > +Programmable Acceleration
> > > +Engine) for FPGA management.
> > > +
> > > +Implementation details
> > > +----------------------
> > > +
> > > +Each instance of IFPGA Rawdev Driver is probed by Intel FpgaDev. In
> > > +coordination with OPAE share code IFPGA Rawdev Driver provides
> > common
> > > +FPGA management ops for FPGA operation, OPAE provides all following
> > > operations:
> > > +- FPGA PR (Partial Reconfiguration) management
> > > +- FPGA AFUs Identifying
> > > +- FPGA Thermal Management
> > > +- FPGA Power Management
> > > +- FPGA Performance reporting
> > > +- FPGA Remote Debug
> > > +
> > > +All configuration parameters are taken by vdev_ifpga_cfg driver.
> > > +Besides configuration, vdev_ifpga_cfg driver also hot plugs in IFPGA Bus.
> > > +
> > > +All of the AFUs of one FPGA may share same PCI BDF and AFUs scan
> > > +depend on IFPGA Rawdev Driver so IFPGA Bus takes AFU device scan
> > > +and AFU drivers
> > > probe.
> > > +All AFU device driver bind to AFU device by its UUID (Universally
> > > +Unique Identifier).
> > > +
> > > +To avoid unnecessary code duplication and ensure maximum
> > performance,
> > > +handling of AFU devices is left to different PMDs; all the design
> > > +as summarized by the following block diagram::
> > > +
> > > + +---------------------------------------------------------------+
> > > + | Application(s)
> > > |
> > > + +----------------------------.----------------------------------+
> > > + |
> > > + |
> > > + +----------------------------'----------------------------------+
> > > + | DPDK Framework (APIs)
> > > |
> > > + +----------|------------|--------.---------------------|--------+
> > > + / \ |
> > > + / \ |
> > > + +-------'-------+ +-------'-------+ +--------'--------+
> > > + | Eth PMD | | Crypto PMD | |
> > > |
> > > + +-------.-------+ +-------.-------+ | |
> > > + | | |
> > > |
> > > + | | |
> > > |
> > > + +-------'-------+ +-------'-------+ | IFPGA |
> > > + | Eth AFU Dev | |Crypto AFU Dev | | Rawdev
> Driver
> > > |
> > > + +-------.-------+ +-------.-------+ |(OPAE Share Code)|
> > > + | | |
> > > |
> > > + | | Rawdev |
> > > |
> > > + +-------'------------------'-------+ Ops | |
> > > + | IFPGA Bus | -------->|
> > > |
> > > + +-----------------.----------------+ +--------.--------+
> > > + | |
> > > + Hot-plugin -->| |
> > > + | |
> > > + +-----------------'------------------+ +--------'--------+
> > > + | vdev_ifpga_cfg driver | | Intel
> FpgaDev
> > > |
> > > + +------------------------------------+ +-----------------+
> > > +
> > > +Build options
> > > +-------------
> > > +
> > > +- ``CONFIG_RTE_LIBRTE_IFPGA_BUS`` (default ``y``)
> > > +
> > > + Toggle compilation of IFPGA Bus library.
> > > +
> > > +- ``CONFIG_RTE_LIBRTE_IFPGA_RAWDEV`` (default ``y``)
> > > +
> > > + Toggle compilation of the ``ifpga_rawdev`` driver.
> > > +
> > > +Run-time parameters
> > > +-------------------
> > > +
> > > +This driver is invoked automatically in systems added with Intel
> > > +FPGA, but PR and IFPGA Bus scan is trigged by command line using
> > > +``--vdev 'net_ifpga_cfg`` EAL option.
> > > +
> > > +The following device parameters are supported:
> > > +
> > > +- ``ifpga`` [string]
> > > +
> > > + Provide a specific Intel FPGA device PCI BDF. Can be provided
> > > + multiple times for additional instances.
> > > +
> > > +- ``port`` [int]
> > > +
> > > + Each FPGA can provide many channels to PR AFU by software, each
> > > + channels is identified by this parameter.
> > > +
> > > +- ``afu_bts`` [string]
> > > +
> > > + If null, the AFU Bit Stream has been PR in FPGA, if not forces PR
> > > + and identifies AFU Bit Stream file.
> > > diff --git a/doc/guides/rawdevs/index.rst
> > > b/doc/guides/rawdevs/index.rst index 7769083..7c3bd95 100644
> > > --- a/doc/guides/rawdevs/index.rst
> > > +++ b/doc/guides/rawdevs/index.rst
> > > @@ -13,3 +13,4 @@ application through rawdev API.
> > >
> > > dpaa2_cmdif
> > > dpaa2_qdma
> > > + ifpga_rawdev
> > > diff --git a/doc/guides/rel_notes/release_18_05.rst
> > > b/doc/guides/rel_notes/release_18_05.rst
> > > index 265950a..f5241a1 100644
> > > --- a/doc/guides/rel_notes/release_18_05.rst
> > > +++ b/doc/guides/rel_notes/release_18_05.rst
> > > @@ -189,6 +189,14 @@ New Features
> > > the DPDK framework. It provides Intel FPGA Partial Bit Stream
> > > AFU(Accelerated
> > > Function Unit) scan and drivers prove.
> > >
> > > +* **Added IFPGA(Intel FPGA) Rawdev Driver.**
> > > +
> > > + Added a new Rawdev driver called IFPGA(Intel FPGA) Rawdev Driver,
> > > + which cooperates with OPAE(Open Programmable Acceleration Engine)
> > > + share code provides common FPGA management ops for FPGA
> operation.
> > > +
> > > + See the :doc:`../rawdevs/ifpga_rawdev` programmer's guide for
> > > + more
> > > details.
> > > +
> > > API Changes
> > > -----------
> > >
> > > diff --git a/drivers/raw/Makefile b/drivers/raw/Makefile index
> > > 2eb2787..8e29b4a 100644
> > > --- a/drivers/raw/Makefile
> > > +++ b/drivers/raw/Makefile
> > > @@ -9,5 +9,6 @@ ifeq
> > > ($(CONFIG_RTE_EAL_VFIO)$(CONFIG_RTE_LIBRTE_FSLMC_BUS),yy)
> > > DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV) +=
> > dpaa2_cmdif
> > > DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) +=
> > dpaa2_qdma endif
> > > +DIRS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += ifpga_rawdev
> > >
> > > include $(RTE_SDK)/mk/rte.subdir.mk diff --git
> > > a/drivers/raw/ifpga_rawdev/Makefile
> > > b/drivers/raw/ifpga_rawdev/Makefile
> > > new file mode 100644
> > > index 0000000..f3b9d5e
> > > --- /dev/null
> > > +++ b/drivers/raw/ifpga_rawdev/Makefile
> > > @@ -0,0 +1,36 @@
> > > +# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Intel
> > > +Corporation
> > > +
> > > +include $(RTE_SDK)/mk/rte.vars.mk
> > > +
> > > +#
> > > +# library name
> > > +#
> > > +LIB = librte_pmd_ifpga_rawdev.a
> > > +
> > > +CFLAGS += -DALLOW_EXPERIMENTAL_API
> > > +CFLAGS += -O3
> > > +CFLAGS += $(WERROR_FLAGS)
> > > +CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga CFLAGS +=
> > > +-I$(RTE_SDK)/drivers/raw/ifpga_rawdev
> > > +LDLIBS += -lrte_eal
> > > +LDLIBS += -lrte_rawdev
> > > +LDLIBS += -lrte_bus_vdev
> > > +LDLIBS += -lrte_kvargs
> > > +LDLIBS += -lrte_bus_pci
> > > +LDLIBS += -lrte_bus_ifpga
> > > +
> > > +EXPORT_MAP := rte_pmd_ifpga_rawdev_version.map
> > > +
> > > +LIBABIVER := 1
> > > +
> > > +VPATH += $(SRCDIR)/base
> > > +
> > > +include $(RTE_SDK)/drivers/raw/ifpga_rawdev/base/Makefile
> > > +
> > > +#
> > > +# all source are stored in SRCS-y
> > > +#
> > > +SRCS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += ifpga_rawdev.c
> > > +
> > > +include $(RTE_SDK)/mk/rte.lib.mk
> > > diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> > > b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> > > new file mode 100644
> > > index 0000000..32e811c
> > > --- /dev/null
> > > +++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.c
> > > @@ -0,0 +1,608 @@
> > > +/* SPDX-License-Identifier: BSD-3-Clause
> > > + * Copyright(c) 2010-2018 Intel Corporation */
> > > +
> > > +#include <string.h>
> > > +#include <dirent.h>
> > > +#include <sys/stat.h>
> > > +#include <unistd.h>
> > > +#include <sys/types.h>
> > > +#include <fcntl.h>
> > > +#include <rte_log.h>
> > > +#include <rte_bus.h>
> > > +#include <rte_eal_memconfig.h>
> > > +#include <rte_malloc.h>
> > > +#include <rte_devargs.h>
> > > +#include <rte_memcpy.h>
> > > +#include <rte_pci.h>
> > > +#include <rte_bus_pci.h>
> > > +#include <rte_kvargs.h>
> > > +#include <rte_alarm.h>
> > > +
> > > +#include <rte_errno.h>
> > > +#include <rte_per_lcore.h>
> > > +#include <rte_memory.h>
> > > +#include <rte_memzone.h>
> > > +#include <rte_eal.h>
> > > +#include <rte_common.h>
> > > +#include <rte_bus_vdev.h>
> > > +
> > > +#include "base/opae_hw_api.h"
> > > +#include "rte_rawdev.h"
> > > +#include "rte_rawdev_pmd.h"
> > > +#include "rte_bus_ifpga.h"
> > > +#include "ifpga_common.h"
> > > +#include "ifpga_logs.h"
> > > +#include "ifpga_rawdev.h"
> > > +
> > > +int ifpga_rawdev_logtype;
> > > +
> > > +#define PCI_VENDOR_ID_INTEL 0x8086
> > > +/* PCI Device ID */
> > > +#define PCIE_DEVICE_ID_PF_INT_5_X 0xBCBD
> > > +#define PCIE_DEVICE_ID_PF_INT_6_X 0xBCC0
> > > +#define PCIE_DEVICE_ID_PF_DSC_1_X 0x09C4
> > > +/* VF Device */
> > > +#define PCIE_DEVICE_ID_VF_INT_5_X 0xBCBF
> > > +#define PCIE_DEVICE_ID_VF_INT_6_X 0xBCC1
> > > +#define PCIE_DEVICE_ID_VF_DSC_1_X 0x09C5
> > > +#define RTE_MAX_RAW_DEVICE 10
> > > +
> > > +static const struct rte_pci_id pci_ifpga_map[] = {
> > > + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> > > PCIE_DEVICE_ID_PF_INT_5_X) },
> > > + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> > > PCIE_DEVICE_ID_VF_INT_5_X) },
> > > + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> > > PCIE_DEVICE_ID_PF_INT_6_X) },
> > > + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> > > PCIE_DEVICE_ID_VF_INT_6_X) },
> > > + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> > > PCIE_DEVICE_ID_PF_DSC_1_X) },
> > > + { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL,
> > > PCIE_DEVICE_ID_VF_DSC_1_X) },
> > > + { .vendor_id = 0, /* sentinel */ }, };
> > > +
> > > +static int
> > > +ifpga_fill_afu_dev(struct opae_accelerator *acc,
> > > + struct rte_afu_device *afu_dev)
> > > +{
> > > + struct rte_mem_resource *res = afu_dev->mem_resource;
> > > + struct opae_acc_region_info region_info;
> > > + struct opae_acc_info info;
> > > + unsigned long i;
> > > + int ret;
> > > +
> > > + ret = opae_acc_get_info(acc, &info);
> > > + if (ret)
> > > + return ret;
> > > +
> > > + if (info.num_regions > PCI_MAX_RESOURCE)
> > > + return -EFAULT;
> > > +
> > > + afu_dev->num_region = info.num_regions;
> > > +
> > > + for (i = 0; i < info.num_regions; i++) {
> > > + region_info.index = i;
> > > + ret = opae_acc_get_region_info(acc, ®ion_info);
> > > + if (ret)
> > > + return ret;
> > > +
> > > + if ((region_info.flags & ACC_REGION_MMIO) &&
> > > + (region_info.flags & ACC_REGION_READ) &&
> > > + (region_info.flags & ACC_REGION_WRITE)) {
> > > + res[i].phys_addr = region_info.phys_addr;
> > > + res[i].len = region_info.len;
> > > + res[i].addr = region_info.addr;
> > > + } else
> > > + return -EFAULT;
> > > + }
> > > +
> > > + return 0;
> > > +}
> > > +
> > > +static void
> > > +ifpga_rawdev_info_get(struct rte_rawdev *dev,
> > > + rte_rawdev_obj_t dev_info) {
> > > + struct opae_adapter *adapter;
> > > + struct opae_accelerator *acc;
> > > + struct rte_afu_device *afu_dev;
> > > +
> > > + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> > > +
> > > + if (!dev_info) {
> > > + IFPGA_RAWDEV_PMD_ERR("Invalid request");
> > > + return;
> > > + }
> > > +
> > > + adapter = ifpga_rawdev_get_priv(dev);
> > > + if (!adapter)
> > > + return;
> > > +
> > > + afu_dev = dev_info;
> > > + afu_dev->rawdev = dev;
> > > +
> > > + /* find opae_accelerator and fill info into afu_device */
> > > + opae_adapter_for_each_acc(adapter, acc) {
> > > + if (acc->index != afu_dev->id.port)
> > > + continue;
> > > +
> > > + if (ifpga_fill_afu_dev(acc, afu_dev)) {
> > > + IFPGA_RAWDEV_PMD_ERR("cannot get info\n");
> > > + return;
> > > + }
> > > + }
> > > +}
> > > +
> > > +static int
> > > +ifpga_rawdev_start(struct rte_rawdev *dev) {
> > > + int ret = 0;
> > > + struct opae_adapter *adapter;
> > > +
> > > + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> > > +
> > > + RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
> > > +
> > > + adapter = ifpga_rawdev_get_priv(dev);
> > > + if (!adapter)
> > > + return -ENODEV;
> > > +
> > > + return ret;
> > > +}
> > > +
> > > +static void
> > > +ifpga_rawdev_stop(struct rte_rawdev *dev) {
> > > + dev->started = 0;
> > > +}
> > > +
> > > +static int
> > > +ifpga_rawdev_close(struct rte_rawdev *dev) {
> > > + return dev ? 0:1;
> > > +}
> > > +
> > > +static int
> > > +ifpga_rawdev_reset(struct rte_rawdev *dev) {
> > > + return dev ? 0:1;
> > > +}
> > > +
> > > +static int
> > > +fpga_pr(struct rte_rawdev *raw_dev, u32 port_id, u64 *buffer, u32 size,
> > > + u64 *status)
> > > +{
> > > +
> > > + struct opae_adapter *adapter;
> > > + struct opae_manager *mgr;
> > > + struct opae_accelerator *acc;
> > > + struct opae_bridge *br;
> > > + int ret;
> > > +
> > > + adapter = ifpga_rawdev_get_priv(raw_dev);
> > > + if (!adapter)
> > > + return -ENODEV;
> > > +
> > > + mgr = opae_adapter_get_mgr(adapter);
> > > + if (!mgr)
> > > + return -ENODEV;
> > > +
> > > + acc = opae_adapter_get_acc(adapter, port_id);
> > > + if (!acc)
> > > + return -ENODEV;
> > > +
> > > + br = opae_acc_get_br(acc);
> > > + if (!br)
> > > + return -ENODEV;
> > > +
> > > + ret = opae_manager_flash(mgr, port_id, buffer, size, status);
> > > + if (ret) {
> > > + IFPGA_RAWDEV_PMD_ERR("%s pr error %d\n", __func__,
> > ret);
> > > + return ret;
> > > + }
> > > +
> > > + ret = opae_bridge_reset(br);
> > > + if (ret) {
> > > + IFPGA_RAWDEV_PMD_ERR("%s reset port:%d error %d\n",
> > > + __func__, port_id, ret);
> > > + return ret;
> > > + }
> > > +
> > > + return ret;
> > > +}
> > > +
> > > +static int
> > > +rte_fpga_do_pr(struct rte_rawdev *rawdev, int port_id,
> > > + const char *file_name)
> > > +{
> > > + struct stat file_stat;
> > > + int file_fd;
> > > + int ret = 0;
> > > + u32 buffer_size;
> > > + void *buffer;
> > > + u64 pr_error;
> > > +
> > > + if (!file_name)
> > > + return -EINVAL;
> > > +
> > > + file_fd = open(file_name, O_RDONLY);
> > > + if (file_fd < 0) {
> > > + IFPGA_RAWDEV_PMD_ERR("%s: open file error: %s\n",
> > > + __func__, file_name);
> > > + IFPGA_RAWDEV_PMD_ERR("Message : %s\n",
> > strerror(errno));
> > > + return -EINVAL;
> > > + }
> > > + ret = stat(file_name, &file_stat);
> > > + if (ret) {
> > > + IFPGA_RAWDEV_PMD_ERR("stat on bitstream file
> > failed: %s\n",
> > > + file_name);
> > > + return -EINVAL;
> > > + }
> > > + buffer_size = file_stat.st_size;
> > > + IFPGA_RAWDEV_PMD_INFO("bitstream file size: %u\n", buffer_size);
> > > + buffer = rte_malloc(NULL, buffer_size, 0);
> > > + if (!buffer) {
> > > + ret = -ENOMEM;
> > > + goto close_fd;
> > > + }
> > > +
> > > + /*read the raw data*/
> > > + if (buffer_size != read(file_fd, (void *)buffer, buffer_size)) {
> > > + ret = -EINVAL;
> > > + goto free_buffer;
> > > + }
> > > +
> > > + /*do PR now*/
> > > + ret = fpga_pr(rawdev, port_id, buffer, buffer_size, &pr_error);
> > > + IFPGA_RAWDEV_PMD_INFO("downloading to device
> > port %d....%s.\n",
> > > port_id,
> > > + ret ? "failed" : "success");
> > > + if (ret) {
> > > + ret = -EINVAL;
> > > + goto free_buffer;
> > > + }
> > > +
> > > +free_buffer:
> > > + if (buffer)
> > > + rte_free(buffer);
> > > +close_fd:
> > > + close(file_fd);
> > > + file_fd = 0;
> > > + return ret;
> > > +}
> > > +
> > > +static int
> > > +ifpga_rawdev_pr(struct rte_rawdev *dev,
> > > + rte_rawdev_obj_t pr_conf)
> > > +{
> > > + struct opae_adapter *adapter;
> > > + struct rte_afu_pr_conf *afu_pr_conf;
> > > + int ret;
> > > + struct uuid uuid;
> > > + struct opae_accelerator *acc;
> > > +
> > > + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> > > +
> > > + adapter = ifpga_rawdev_get_priv(dev);
> > > + if (!adapter)
> > > + return -ENODEV;
> > > +
> > > + if (!pr_conf)
> > > + return -EINVAL;
> > > +
> > > + afu_pr_conf = pr_conf;
> > > +
> > > + if (afu_pr_conf->pr_enable) {
> > > + ret = rte_fpga_do_pr(dev,
> > > + afu_pr_conf->afu_id.port,
> > > + afu_pr_conf->bs_path);
> > > + if (ret) {
> > > + IFPGA_RAWDEV_PMD_ERR("do pr error %d\n", ret);
> > > + return ret;
> > > + }
> > > + }
> > > +
> > > + acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
> > > + if (!acc)
> > > + return -ENODEV;
> > > +
> > > + ret = opae_acc_get_uuid(acc, &uuid);
> > > + if (ret)
> > > + return ret;
> > > +
> > > + memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
> > > + memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
> > > +sizeof(u64));
> > > +
> > > + IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
> > > __func__,
> > > + (u64)afu_pr_conf->afu_id.uuid.uuid_low,
> > > + (u64)afu_pr_conf->afu_id.uuid.uuid_high);
> > > +
> > > + return 0;
> > > +}
> > > +
> > > +static const struct rte_rawdev_ops ifpga_rawdev_ops = {
> > > + .dev_info_get = ifpga_rawdev_info_get,
> > > + .dev_configure = NULL,
> > > + .dev_start = ifpga_rawdev_start,
> > > + .dev_stop = ifpga_rawdev_stop,
> > > + .dev_close = ifpga_rawdev_close,
> > > + .dev_reset = ifpga_rawdev_reset,
> > > +
> > > + .queue_def_conf = NULL,
> > > + .queue_setup = NULL,
> > > + .queue_release = NULL,
> > > +
> > > + .attr_get = NULL,
> > > + .attr_set = NULL,
> > > +
> > > + .enqueue_bufs = NULL,
> > > + .dequeue_bufs = NULL,
> > > +
> > > + .dump = NULL,
> > > +
> > > + .xstats_get = NULL,
> > > + .xstats_get_names = NULL,
> > > + .xstats_get_by_name = NULL,
> > > + .xstats_reset = NULL,
> > > +
> > > + .firmware_status_get = NULL,
> > > + .firmware_version_get = NULL,
> > > + .firmware_load = ifpga_rawdev_pr,
> > > + .firmware_unload = NULL,
> > > +
> > > + .dev_selftest = NULL,
> > > +};
> > > +
> > > +static int
> > > +ifpga_rawdev_create(struct rte_pci_device *pci_dev,
> > > + int socket_id)
> > > +{
> > > + int ret = 0;
> > > + struct rte_rawdev *rawdev = NULL;
> > > + struct opae_adapter *adapter = NULL;
> > > + struct opae_manager *mgr = NULL;
> > > + struct opae_adapter_data_pci *data = NULL;
> > > + char name[RTE_RAWDEV_NAME_MAX_LEN];
> > > + int i;
> > > +
> > > + if (!pci_dev) {
> > > + IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
> > > + ret = -EINVAL;
> > > + goto cleanup;
> > > + }
> > > +
> > > + memset(name, 0, sizeof(name));
> > > + snprintf(name, RTE_RAWDEV_NAME_MAX_LEN,
> > "IFPGA:%x:%02x.%x",
> > > + pci_dev->addr.bus, pci_dev->addr.devid, pci_dev-
> > >addr.function);
> > > +
> > > + IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name,
> > > +rte_socket_id());
> > > +
> > > + /* Allocate device structure */
> > > + rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct
> > opae_adapter),
> > > + socket_id);
> > > + if (rawdev == NULL) {
> > > + IFPGA_RAWDEV_PMD_ERR("Unable to allocate rawdevice");
> > > + ret = -EINVAL;
> > > + goto cleanup;
> > > + }
> > > +
> > > + /* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API
> > */
> > > + data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
> > > + if (!data) {
> > > + ret = -ENOMEM;
> > > + goto cleanup;
> > > + }
> > > +
> > > + /* init opae_adapter_data_pci for device specific information */
> > > + for (i = 0; i < PCI_MAX_RESOURCE; i++) {
> > > + data->region[i].phys_addr = pci_dev-
> > >mem_resource[i].phys_addr;
> > > + data->region[i].len = pci_dev->mem_resource[i].len;
> > > + data->region[i].addr = pci_dev->mem_resource[i].addr;
> > > + }
> > > + data->device_id = pci_dev->id.device_id;
> > > + data->vendor_id = pci_dev->id.vendor_id;
> > > +
> > > + /* create a opae_adapter based on above device data */
> > > + adapter = opae_adapter_alloc(pci_dev->device.name, data);
> > > + if (!adapter) {
> > > + ret = -ENOMEM;
> > > + goto free_adapter_data;
> > > + }
> > > +
> > > + rawdev->dev_ops = &ifpga_rawdev_ops;
> > > + rawdev->device = &pci_dev->device;
> > > + rawdev->driver_name = pci_dev->device.driver->name;
> > > +
> > > + rawdev->dev_private = adapter;
> > > +
> > > + /* must enumerate the adapter before use it */
> > > + ret = opae_adapter_enumerate(adapter);
> > > + if (ret)
> > > + goto free_adapter;
> > > +
> > > + /* get opae_manager to rawdev */
> > > + mgr = opae_adapter_get_mgr(adapter);
> > > + if (mgr) {
> > > + /* PF function */
> > > + IFPGA_RAWDEV_PMD_INFO("this is a PF function");
> > > + }
> > > +
> > > + return ret;
> > > +
> > > +free_adapter:
> > > + if (adapter)
> > > + opae_adapter_free(adapter);
> > > +free_adapter_data:
> > > + if (data)
> > > + opae_adapter_data_free(data);
> > > +cleanup:
> > > + if (rawdev)
> > > + rte_rawdev_pmd_release(rawdev);
> > > +
> > > + return ret;
> > > +}
> > > +
> > > +static int
> > > +ifpga_rawdev_destroy(struct rte_pci_device *pci_dev) {
> > > + int ret;
> > > + struct rte_rawdev *rawdev;
> > > + char name[RTE_RAWDEV_NAME_MAX_LEN];
> > > + struct opae_adapter *adapter;
> > > +
> > > + if (!pci_dev) {
> > > + IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
> > > + ret = -EINVAL;
> > > + return ret;
> > > + }
> > > +
> > > + memset(name, 0, sizeof(name));
> > > + snprintf(name, RTE_RAWDEV_NAME_MAX_LEN,
> > "IFPGA:%x:%02x.%x",
> > > + pci_dev->addr.bus, pci_dev->addr.devid, pci_dev-
> > >addr.function);
> > > +
> > > + IFPGA_RAWDEV_PMD_INFO("Closing %s on NUMA node %d",
> > > + name, rte_socket_id());
> > > +
> > > + rawdev = rte_rawdev_pmd_get_named_dev(name);
> > > + if (!rawdev) {
> > > + IFPGA_RAWDEV_PMD_ERR("Invalid device name (%s)",
> > name);
> > > + return -EINVAL;
> > > + }
> > > +
> > > + adapter = ifpga_rawdev_get_priv(rawdev);
> > > + if (!adapter)
> > > + return -ENODEV;
> > > +
> > > + opae_adapter_data_free(adapter->data);
> > > + opae_adapter_free(adapter);
> > > +
> > > + /* rte_rawdev_close is called by pmd_release */
> > > + ret = rte_rawdev_pmd_release(rawdev);
> > > + if (ret)
> > > + IFPGA_RAWDEV_PMD_DEBUG("Device cleanup failed");
> > > +
> > > + return ret;
> > > +}
> > > +
> > > +static int
> > > +ifpga_rawdev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
> > > + struct rte_pci_device *pci_dev)
> > > +{
> > > +
> > > + IFPGA_RAWDEV_PMD_FUNC_TRACE();
> > > + return ifpga_rawdev_create(pci_dev, rte_socket_id()); }
> > > +
> > > +static int
> > > +ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev) {
> > > + return ifpga_rawdev_destroy(pci_dev); }
> > > +
> > > +static struct rte_pci_driver rte_ifpga_rawdev_pmd = {
> > > + .id_table = pci_ifpga_map,
> > > + .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
> > RTE_PCI_DRV_INTR_LSC,
> > > + .probe = ifpga_rawdev_pci_probe,
> > > + .remove = ifpga_rawdev_pci_remove,
> > > +};
> > > +
> > > +RTE_PMD_REGISTER_PCI(ifpga_rawdev_pci_driver,
> > rte_ifpga_rawdev_pmd);
> > > +RTE_PMD_REGISTER_PCI_TABLE(ifpga_rawdev_pci_driver,
> > > +rte_ifpga_rawdev_pmd);
> > > +RTE_PMD_REGISTER_KMOD_DEP(ifpga_rawdev_pci_driver, "* igb_uio |
> > > +uio_pci_generic | vfio-pci");
> > > +
> > > +RTE_INIT(ifpga_rawdev_init_log);
> > > +static void
> > > +ifpga_rawdev_init_log(void)
> > > +{
> > > + ifpga_rawdev_logtype = rte_log_register("driver.raw.init");
> > > + if (ifpga_rawdev_logtype >= 0)
> > > + rte_log_set_level(ifpga_rawdev_logtype, RTE_LOG_NOTICE); }
> > > +
> > > +static const char * const valid_args[] = {
> > > +#define IFPGA_ARG_NAME "ifpga"
> > > + IFPGA_ARG_NAME,
> > > +#define IFPGA_ARG_PORT "port"
> > > + IFPGA_ARG_PORT,
> > > +#define IFPGA_AFU_BTS "afu_bts"
> > > + IFPGA_AFU_BTS,
> > > + NULL
> > > +};
> > > +
> > > +static int
> > > +ifpga_cfg_probe(struct rte_vdev_device *dev) {
> > > + struct rte_devargs *devargs;
> > > + struct rte_kvargs *kvlist = NULL;
> > > + int port;
> > > + char *name = NULL;
> > > + char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
> > > +
> > > + devargs = dev->device.devargs;
> > > +
> > > + kvlist = rte_kvargs_parse(devargs->args, valid_args);
> > > + if (!kvlist) {
> > > + IFPGA_RAWDEV_PMD_LOG(ERR, "error when parsing
> > param");
> > > + goto end;
> > > + }
> > > +
> > > + if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
> > > + if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
> > > + &rte_ifpga_get_string_arg, &name) < 0) {
> > > + IFPGA_RAWDEV_PMD_ERR("error to parse %s",
> > > + IFPGA_ARG_NAME);
> > > + goto end;
> > > + }
> > > + } else {
> > > + IFPGA_RAWDEV_PMD_ERR("arg %s is mandatory for ifpga
> > bus",
> > > + IFPGA_ARG_NAME);
> > > + goto end;
> > > + }
> > > +
> > > + if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) {
> > > + if (rte_kvargs_process(kvlist,
> > > + IFPGA_ARG_PORT,
> > > + &rte_ifpga_get_integer32_arg,
> > > + &port) < 0) {
> > > + IFPGA_RAWDEV_PMD_ERR("error to parse %s",
> > > + IFPGA_ARG_PORT);
> > > + goto end;
> > > + }
> > > + } else {
> > > + IFPGA_RAWDEV_PMD_ERR("arg %s is mandatory for ifpga
> > bus",
> > > + IFPGA_ARG_PORT);
> > > + goto end;
> > > + }
> > > +
> > > + memset(dev_name, 0, sizeof(dev_name));
> > > + snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
> > > + port, name);
> > > +
> > > + rte_eal_hotplug_add(RTE_STR(IFPGA_BUS_NAME),
> > > + dev_name, devargs->args);
> > > +end:
> > > + if (kvlist)
> > > + rte_kvargs_free(kvlist);
> > > + if (name)
> > > + free(name);
> > > +
> > > + return 0;
> > > +}
> > > +
> > > +static int
> > > +ifpga_cfg_remove(struct rte_vdev_device *vdev) {
> > > + IFPGA_RAWDEV_PMD_INFO("Remove ifpga_cfg %p",
> > > + vdev);
> > > +
> > > + return 0;
> > > +}
> > > +
> > > +static struct rte_vdev_driver ifpga_cfg_driver = {
> > > + .probe = ifpga_cfg_probe,
> > > + .remove = ifpga_cfg_remove,
> > > +};
> > > +
> > > +RTE_PMD_REGISTER_VDEV(net_ifpga_cfg, ifpga_cfg_driver);
> > > +RTE_PMD_REGISTER_ALIAS(net_ifpga_cfg, ifpga_cfg);
> > > +RTE_PMD_REGISTER_PARAM_STRING(net_ifpga_cfg,
> > > + "bdf=<string> "
> > > + "port=<int> "
> > > + "afu_bts=<path>");
> > > +
> > > diff --git a/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> > > b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> > > new file mode 100644
> > > index 0000000..c7759b8
> > > --- /dev/null
> > > +++ b/drivers/raw/ifpga_rawdev/ifpga_rawdev.h
> > > @@ -0,0 +1,37 @@
> > > +/* SPDX-License-Identifier: BSD-3-Clause
> > > + * Copyright(c) 2010-2018 Intel Corporation */
> > > +
> > > +#ifndef _IFPGA_RAWDEV_H_
> > > +#define _IFPGA_RAWDEV_H_
> > > +
> > > +extern int ifpga_rawdev_logtype;
> > > +
> > > +#define IFPGA_RAWDEV_PMD_LOG(level, fmt, args...) \
> > > + rte_log(RTE_LOG_ ## level, ifpga_rawdev_logtype, "ifgpa: " fmt, \
> > > + ##args)
> > > +
> > > +#define IFPGA_RAWDEV_PMD_FUNC_TRACE()
> > > IFPGA_RAWDEV_PMD_LOG(DEBUG, ">>")
> > > +
> > > +#define IFPGA_RAWDEV_PMD_DEBUG(fmt, args...) \
> > > + IFPGA_RAWDEV_PMD_LOG(DEBUG, fmt, ## args) #define
> > > +IFPGA_RAWDEV_PMD_INFO(fmt, args...) \
> > > + IFPGA_RAWDEV_PMD_LOG(INFO, fmt, ## args) #define
> > > +IFPGA_RAWDEV_PMD_ERR(fmt, args...) \
> > > + IFPGA_RAWDEV_PMD_LOG(ERR, fmt, ## args) #define
> > > +IFPGA_RAWDEV_PMD_WARN(fmt, args...) \
> > > + IFPGA_RAWDEV_PMD_LOG(WARNING, fmt, ## args)
> > > +
> > > +enum ifpga_rawdev_device_state {
> > > + IFPGA_IDLE,
> > > + IFPGA_READY,
> > > + IFPGA_ERROR
> > > +};
> > > +
> > > +static inline struct opae_adapter * ifpga_rawdev_get_priv(const
> > > +struct rte_rawdev *rawdev) {
> > > + return rawdev->dev_private;
> > > +}
> > > +
> > > +#endif /* _IFPGA_RAWDEV_H_ */
> > > diff --git a/drivers/raw/ifpga_rawdev/meson.build
> > > b/drivers/raw/ifpga_rawdev/meson.build
> > > new file mode 100644
> > > index 0000000..6725687
> > > --- /dev/null
> > > +++ b/drivers/raw/ifpga_rawdev/meson.build
> > > @@ -0,0 +1,15 @@
> > > +# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Intel
> > > +Corporation
> > > +
> > > +version = 1
> > > +
> > > +subdir('base')
> > > +objs = [base_objs]
> > > +
> > > +deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
> > > + 'bus_vdev', 'bus_ifpga']
> > > +sources = files('ifpga_rawdev.c')
> > > +
> > > +includes += include_directories('base')
> > > +
> > > +allow_experimental_apis = true
> > > diff --git
> > > a/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> > > b/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> > > new file mode 100644
> > > index 0000000..9b9ab1a
> > > --- /dev/null
> > > +++ b/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map
> > > @@ -0,0 +1,4 @@
> > > +DPDK_18.05 {
> > > +
> > > + local: *;
> > > +};
> > > diff --git a/drivers/raw/meson.build b/drivers/raw/meson.build index
> > > 34dfac4..a61cdcc 100644
> > > --- a/drivers/raw/meson.build
> > > +++ b/drivers/raw/meson.build
> > > @@ -1,7 +1,7 @@
> > > # SPDX-License-Identifier: BSD-3-Clause # Copyright 2018 NXP
> > >
> > > -drivers = ['skeleton_rawdev', 'dpaa2_cmdif', 'dpaa2_qdma']
> > > +drivers = ['skeleton_rawdev', 'dpaa2_cmdif', 'dpaa2_qdma',
> > > +'ifpga_rawdev']
> > > std_deps = ['rawdev']
> > > config_flag_fmt = 'RTE_LIBRTE_PMD_@0@_RAWDEV'
> > > driver_name_fmt = 'rte_pmd_@0@'
> > > diff --git a/mk/rte.app.mk b/mk/rte.app.mk index 3861e1a..acc0ec3
> > > 100644
> > > --- a/mk/rte.app.mk
> > > +++ b/mk/rte.app.mk
> > > @@ -256,9 +256,11 @@
> > > _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV) +=
> > > -lrte_pmd_dpaa2_cmdif
> > > _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) +=
> > > -lrte_pmd_dpaa2_qdma endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
> > > _LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += -lrte_bus_ifpga
> > > +ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
> > > +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) +=
> > > -lrte_pmd_ifpga_rawdev
> > > +endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
> > > endif # CONFIG_RTE_LIBRTE_RAWDEV
> > >
> > > -
> > > endif # !CONFIG_RTE_BUILD_SHARED_LIBS
> > >
> > > _LDLIBS-y += --no-whole-archive
> > > --
> > > 1.8.3.1
@@ -851,8 +851,11 @@ Rawdev Drivers
--------------
Intel FPGA
+M: Rosen Xu <rosen.xu@intel.com>
M: Tianfei zhang <tianfei.zhang@intel.com>
+F: drivers/raw/ifpga_rawdev/
F: drivers/raw/ifpga_rawdev/base/
+F: doc/guides/rawdevs/ifpga_rawdev.rst
NXP DPAA2 QDMA
M: Nipun Gupta <nipun.gupta@nxp.com>
@@ -633,6 +633,11 @@ CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV=n
CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV=n
#
+# Compile PMD for Intel FPGA raw device
+#
+CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV=y
+
+#
# Compile librte_ring
#
CONFIG_RTE_LIBRTE_RING=y
new file mode 100644
@@ -0,0 +1,112 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright(c) 2018 Intel Corporation.
+
+IFPGA Rawdev Driver
+======================
+
+FPGA is used more and more widely in Cloud and NFV, one primary reason is
+that FPGA not only provides ASIC performance but also it's more flexible
+than ASIC.
+
+FPGA uses Partial Reconfigure (PR) Parts of Bit Stream to achieve its
+flexibility. That means one FPGA Device Bit Stream is divided into many Parts
+of Bit Stream(each Part of Bit Stream is defined as AFU-Accelerated Function
+Unit), and each AFU is a hardware acceleration unit which can be dynamically
+reloaded respectively.
+
+By PR (Partial Reconfiguration) AFUs, one FPGA resources can be time-shared by
+different users. FPGA hot upgrade and fault tolerance can be provided easily.
+
+The SW IFPGA Rawdev Driver (**ifpga_rawdev**) provides a Rawdev driver
+that utilizes Intel FPGA Software Stack OPAE(Open Programmable Acceleration
+Engine) for FPGA management.
+
+Implementation details
+----------------------
+
+Each instance of IFPGA Rawdev Driver is probed by Intel FpgaDev. In coordination
+with OPAE share code IFPGA Rawdev Driver provides common FPGA management ops
+for FPGA operation, OPAE provides all following operations:
+- FPGA PR (Partial Reconfiguration) management
+- FPGA AFUs Identifying
+- FPGA Thermal Management
+- FPGA Power Management
+- FPGA Performance reporting
+- FPGA Remote Debug
+
+All configuration parameters are taken by vdev_ifpga_cfg driver. Besides
+configuration, vdev_ifpga_cfg driver also hot plugs in IFPGA Bus.
+
+All of the AFUs of one FPGA may share same PCI BDF and AFUs scan depend on
+IFPGA Rawdev Driver so IFPGA Bus takes AFU device scan and AFU drivers probe.
+All AFU device driver bind to AFU device by its UUID (Universally Unique
+Identifier).
+
+To avoid unnecessary code duplication and ensure maximum performance,
+handling of AFU devices is left to different PMDs; all the design as
+summarized by the following block diagram::
+
+ +---------------------------------------------------------------+
+ | Application(s) |
+ +----------------------------.----------------------------------+
+ |
+ |
+ +----------------------------'----------------------------------+
+ | DPDK Framework (APIs) |
+ +----------|------------|--------.---------------------|--------+
+ / \ |
+ / \ |
+ +-------'-------+ +-------'-------+ +--------'--------+
+ | Eth PMD | | Crypto PMD | | |
+ +-------.-------+ +-------.-------+ | |
+ | | | |
+ | | | |
+ +-------'-------+ +-------'-------+ | IFPGA |
+ | Eth AFU Dev | |Crypto AFU Dev | | Rawdev Driver |
+ +-------.-------+ +-------.-------+ |(OPAE Share Code)|
+ | | | |
+ | | Rawdev | |
+ +-------'------------------'-------+ Ops | |
+ | IFPGA Bus | -------->| |
+ +-----------------.----------------+ +--------.--------+
+ | |
+ Hot-plugin -->| |
+ | |
+ +-----------------'------------------+ +--------'--------+
+ | vdev_ifpga_cfg driver | | Intel FpgaDev |
+ +------------------------------------+ +-----------------+
+
+Build options
+-------------
+
+- ``CONFIG_RTE_LIBRTE_IFPGA_BUS`` (default ``y``)
+
+ Toggle compilation of IFPGA Bus library.
+
+- ``CONFIG_RTE_LIBRTE_IFPGA_RAWDEV`` (default ``y``)
+
+ Toggle compilation of the ``ifpga_rawdev`` driver.
+
+Run-time parameters
+-------------------
+
+This driver is invoked automatically in systems added with Intel FPGA,
+but PR and IFPGA Bus scan is trigged by command line using
+``--vdev 'net_ifpga_cfg`` EAL option.
+
+The following device parameters are supported:
+
+- ``ifpga`` [string]
+
+ Provide a specific Intel FPGA device PCI BDF. Can be provided multiple
+ times for additional instances.
+
+- ``port`` [int]
+
+ Each FPGA can provide many channels to PR AFU by software, each channels
+ is identified by this parameter.
+
+- ``afu_bts`` [string]
+
+ If null, the AFU Bit Stream has been PR in FPGA, if not forces PR and
+ identifies AFU Bit Stream file.
@@ -13,3 +13,4 @@ application through rawdev API.
dpaa2_cmdif
dpaa2_qdma
+ ifpga_rawdev
@@ -189,6 +189,14 @@ New Features
the DPDK framework. It provides Intel FPGA Partial Bit Stream AFU(Accelerated
Function Unit) scan and drivers prove.
+* **Added IFPGA(Intel FPGA) Rawdev Driver.**
+
+ Added a new Rawdev driver called IFPGA(Intel FPGA) Rawdev Driver, which cooperates
+ with OPAE(Open Programmable Acceleration Engine) share code provides common FPGA
+ management ops for FPGA operation.
+
+ See the :doc:`../rawdevs/ifpga_rawdev` programmer's guide for more details.
+
API Changes
-----------
@@ -9,5 +9,6 @@ ifeq ($(CONFIG_RTE_EAL_VFIO)$(CONFIG_RTE_LIBRTE_FSLMC_BUS),yy)
DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV) += dpaa2_cmdif
DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += dpaa2_qdma
endif
+DIRS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += ifpga_rawdev
include $(RTE_SDK)/mk/rte.subdir.mk
new file mode 100644
@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_pmd_ifpga_rawdev.a
+
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -I$(RTE_SDK)/drivers/bus/ifpga
+CFLAGS += -I$(RTE_SDK)/drivers/raw/ifpga_rawdev
+LDLIBS += -lrte_eal
+LDLIBS += -lrte_rawdev
+LDLIBS += -lrte_bus_vdev
+LDLIBS += -lrte_kvargs
+LDLIBS += -lrte_bus_pci
+LDLIBS += -lrte_bus_ifpga
+
+EXPORT_MAP := rte_pmd_ifpga_rawdev_version.map
+
+LIBABIVER := 1
+
+VPATH += $(SRCDIR)/base
+
+include $(RTE_SDK)/drivers/raw/ifpga_rawdev/base/Makefile
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += ifpga_rawdev.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
new file mode 100644
@@ -0,0 +1,608 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#include <string.h>
+#include <dirent.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_eal_memconfig.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
+#include <rte_kvargs.h>
+#include <rte_alarm.h>
+
+#include <rte_errno.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_eal.h>
+#include <rte_common.h>
+#include <rte_bus_vdev.h>
+
+#include "base/opae_hw_api.h"
+#include "rte_rawdev.h"
+#include "rte_rawdev_pmd.h"
+#include "rte_bus_ifpga.h"
+#include "ifpga_common.h"
+#include "ifpga_logs.h"
+#include "ifpga_rawdev.h"
+
+int ifpga_rawdev_logtype;
+
+#define PCI_VENDOR_ID_INTEL 0x8086
+/* PCI Device ID */
+#define PCIE_DEVICE_ID_PF_INT_5_X 0xBCBD
+#define PCIE_DEVICE_ID_PF_INT_6_X 0xBCC0
+#define PCIE_DEVICE_ID_PF_DSC_1_X 0x09C4
+/* VF Device */
+#define PCIE_DEVICE_ID_VF_INT_5_X 0xBCBF
+#define PCIE_DEVICE_ID_VF_INT_6_X 0xBCC1
+#define PCIE_DEVICE_ID_VF_DSC_1_X 0x09C5
+#define RTE_MAX_RAW_DEVICE 10
+
+static const struct rte_pci_id pci_ifpga_map[] = {
+ { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_PF_INT_5_X) },
+ { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_VF_INT_5_X) },
+ { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_PF_INT_6_X) },
+ { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_VF_INT_6_X) },
+ { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_PF_DSC_1_X) },
+ { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_VF_DSC_1_X) },
+ { .vendor_id = 0, /* sentinel */ },
+};
+
+static int
+ifpga_fill_afu_dev(struct opae_accelerator *acc,
+ struct rte_afu_device *afu_dev)
+{
+ struct rte_mem_resource *res = afu_dev->mem_resource;
+ struct opae_acc_region_info region_info;
+ struct opae_acc_info info;
+ unsigned long i;
+ int ret;
+
+ ret = opae_acc_get_info(acc, &info);
+ if (ret)
+ return ret;
+
+ if (info.num_regions > PCI_MAX_RESOURCE)
+ return -EFAULT;
+
+ afu_dev->num_region = info.num_regions;
+
+ for (i = 0; i < info.num_regions; i++) {
+ region_info.index = i;
+ ret = opae_acc_get_region_info(acc, ®ion_info);
+ if (ret)
+ return ret;
+
+ if ((region_info.flags & ACC_REGION_MMIO) &&
+ (region_info.flags & ACC_REGION_READ) &&
+ (region_info.flags & ACC_REGION_WRITE)) {
+ res[i].phys_addr = region_info.phys_addr;
+ res[i].len = region_info.len;
+ res[i].addr = region_info.addr;
+ } else
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+static void
+ifpga_rawdev_info_get(struct rte_rawdev *dev,
+ rte_rawdev_obj_t dev_info)
+{
+ struct opae_adapter *adapter;
+ struct opae_accelerator *acc;
+ struct rte_afu_device *afu_dev;
+
+ IFPGA_RAWDEV_PMD_FUNC_TRACE();
+
+ if (!dev_info) {
+ IFPGA_RAWDEV_PMD_ERR("Invalid request");
+ return;
+ }
+
+ adapter = ifpga_rawdev_get_priv(dev);
+ if (!adapter)
+ return;
+
+ afu_dev = dev_info;
+ afu_dev->rawdev = dev;
+
+ /* find opae_accelerator and fill info into afu_device */
+ opae_adapter_for_each_acc(adapter, acc) {
+ if (acc->index != afu_dev->id.port)
+ continue;
+
+ if (ifpga_fill_afu_dev(acc, afu_dev)) {
+ IFPGA_RAWDEV_PMD_ERR("cannot get info\n");
+ return;
+ }
+ }
+}
+
+static int
+ifpga_rawdev_start(struct rte_rawdev *dev)
+{
+ int ret = 0;
+ struct opae_adapter *adapter;
+
+ IFPGA_RAWDEV_PMD_FUNC_TRACE();
+
+ RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
+
+ adapter = ifpga_rawdev_get_priv(dev);
+ if (!adapter)
+ return -ENODEV;
+
+ return ret;
+}
+
+static void
+ifpga_rawdev_stop(struct rte_rawdev *dev)
+{
+ dev->started = 0;
+}
+
+static int
+ifpga_rawdev_close(struct rte_rawdev *dev)
+{
+ return dev ? 0:1;
+}
+
+static int
+ifpga_rawdev_reset(struct rte_rawdev *dev)
+{
+ return dev ? 0:1;
+}
+
+static int
+fpga_pr(struct rte_rawdev *raw_dev, u32 port_id, u64 *buffer, u32 size,
+ u64 *status)
+{
+
+ struct opae_adapter *adapter;
+ struct opae_manager *mgr;
+ struct opae_accelerator *acc;
+ struct opae_bridge *br;
+ int ret;
+
+ adapter = ifpga_rawdev_get_priv(raw_dev);
+ if (!adapter)
+ return -ENODEV;
+
+ mgr = opae_adapter_get_mgr(adapter);
+ if (!mgr)
+ return -ENODEV;
+
+ acc = opae_adapter_get_acc(adapter, port_id);
+ if (!acc)
+ return -ENODEV;
+
+ br = opae_acc_get_br(acc);
+ if (!br)
+ return -ENODEV;
+
+ ret = opae_manager_flash(mgr, port_id, buffer, size, status);
+ if (ret) {
+ IFPGA_RAWDEV_PMD_ERR("%s pr error %d\n", __func__, ret);
+ return ret;
+ }
+
+ ret = opae_bridge_reset(br);
+ if (ret) {
+ IFPGA_RAWDEV_PMD_ERR("%s reset port:%d error %d\n",
+ __func__, port_id, ret);
+ return ret;
+ }
+
+ return ret;
+}
+
+static int
+rte_fpga_do_pr(struct rte_rawdev *rawdev, int port_id,
+ const char *file_name)
+{
+ struct stat file_stat;
+ int file_fd;
+ int ret = 0;
+ u32 buffer_size;
+ void *buffer;
+ u64 pr_error;
+
+ if (!file_name)
+ return -EINVAL;
+
+ file_fd = open(file_name, O_RDONLY);
+ if (file_fd < 0) {
+ IFPGA_RAWDEV_PMD_ERR("%s: open file error: %s\n",
+ __func__, file_name);
+ IFPGA_RAWDEV_PMD_ERR("Message : %s\n", strerror(errno));
+ return -EINVAL;
+ }
+ ret = stat(file_name, &file_stat);
+ if (ret) {
+ IFPGA_RAWDEV_PMD_ERR("stat on bitstream file failed: %s\n",
+ file_name);
+ return -EINVAL;
+ }
+ buffer_size = file_stat.st_size;
+ IFPGA_RAWDEV_PMD_INFO("bitstream file size: %u\n", buffer_size);
+ buffer = rte_malloc(NULL, buffer_size, 0);
+ if (!buffer) {
+ ret = -ENOMEM;
+ goto close_fd;
+ }
+
+ /*read the raw data*/
+ if (buffer_size != read(file_fd, (void *)buffer, buffer_size)) {
+ ret = -EINVAL;
+ goto free_buffer;
+ }
+
+ /*do PR now*/
+ ret = fpga_pr(rawdev, port_id, buffer, buffer_size, &pr_error);
+ IFPGA_RAWDEV_PMD_INFO("downloading to device port %d....%s.\n", port_id,
+ ret ? "failed" : "success");
+ if (ret) {
+ ret = -EINVAL;
+ goto free_buffer;
+ }
+
+free_buffer:
+ if (buffer)
+ rte_free(buffer);
+close_fd:
+ close(file_fd);
+ file_fd = 0;
+ return ret;
+}
+
+static int
+ifpga_rawdev_pr(struct rte_rawdev *dev,
+ rte_rawdev_obj_t pr_conf)
+{
+ struct opae_adapter *adapter;
+ struct rte_afu_pr_conf *afu_pr_conf;
+ int ret;
+ struct uuid uuid;
+ struct opae_accelerator *acc;
+
+ IFPGA_RAWDEV_PMD_FUNC_TRACE();
+
+ adapter = ifpga_rawdev_get_priv(dev);
+ if (!adapter)
+ return -ENODEV;
+
+ if (!pr_conf)
+ return -EINVAL;
+
+ afu_pr_conf = pr_conf;
+
+ if (afu_pr_conf->pr_enable) {
+ ret = rte_fpga_do_pr(dev,
+ afu_pr_conf->afu_id.port,
+ afu_pr_conf->bs_path);
+ if (ret) {
+ IFPGA_RAWDEV_PMD_ERR("do pr error %d\n", ret);
+ return ret;
+ }
+ }
+
+ acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
+ if (!acc)
+ return -ENODEV;
+
+ ret = opae_acc_get_uuid(acc, &uuid);
+ if (ret)
+ return ret;
+
+ memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b, sizeof(u64));
+ memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8, sizeof(u64));
+
+ IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n", __func__,
+ (u64)afu_pr_conf->afu_id.uuid.uuid_low,
+ (u64)afu_pr_conf->afu_id.uuid.uuid_high);
+
+ return 0;
+}
+
+static const struct rte_rawdev_ops ifpga_rawdev_ops = {
+ .dev_info_get = ifpga_rawdev_info_get,
+ .dev_configure = NULL,
+ .dev_start = ifpga_rawdev_start,
+ .dev_stop = ifpga_rawdev_stop,
+ .dev_close = ifpga_rawdev_close,
+ .dev_reset = ifpga_rawdev_reset,
+
+ .queue_def_conf = NULL,
+ .queue_setup = NULL,
+ .queue_release = NULL,
+
+ .attr_get = NULL,
+ .attr_set = NULL,
+
+ .enqueue_bufs = NULL,
+ .dequeue_bufs = NULL,
+
+ .dump = NULL,
+
+ .xstats_get = NULL,
+ .xstats_get_names = NULL,
+ .xstats_get_by_name = NULL,
+ .xstats_reset = NULL,
+
+ .firmware_status_get = NULL,
+ .firmware_version_get = NULL,
+ .firmware_load = ifpga_rawdev_pr,
+ .firmware_unload = NULL,
+
+ .dev_selftest = NULL,
+};
+
+static int
+ifpga_rawdev_create(struct rte_pci_device *pci_dev,
+ int socket_id)
+{
+ int ret = 0;
+ struct rte_rawdev *rawdev = NULL;
+ struct opae_adapter *adapter = NULL;
+ struct opae_manager *mgr = NULL;
+ struct opae_adapter_data_pci *data = NULL;
+ char name[RTE_RAWDEV_NAME_MAX_LEN];
+ int i;
+
+ if (!pci_dev) {
+ IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
+ ret = -EINVAL;
+ goto cleanup;
+ }
+
+ memset(name, 0, sizeof(name));
+ snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+ pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
+
+ IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
+
+ /* Allocate device structure */
+ rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct opae_adapter),
+ socket_id);
+ if (rawdev == NULL) {
+ IFPGA_RAWDEV_PMD_ERR("Unable to allocate rawdevice");
+ ret = -EINVAL;
+ goto cleanup;
+ }
+
+ /* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
+ data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
+ if (!data) {
+ ret = -ENOMEM;
+ goto cleanup;
+ }
+
+ /* init opae_adapter_data_pci for device specific information */
+ for (i = 0; i < PCI_MAX_RESOURCE; i++) {
+ data->region[i].phys_addr = pci_dev->mem_resource[i].phys_addr;
+ data->region[i].len = pci_dev->mem_resource[i].len;
+ data->region[i].addr = pci_dev->mem_resource[i].addr;
+ }
+ data->device_id = pci_dev->id.device_id;
+ data->vendor_id = pci_dev->id.vendor_id;
+
+ /* create a opae_adapter based on above device data */
+ adapter = opae_adapter_alloc(pci_dev->device.name, data);
+ if (!adapter) {
+ ret = -ENOMEM;
+ goto free_adapter_data;
+ }
+
+ rawdev->dev_ops = &ifpga_rawdev_ops;
+ rawdev->device = &pci_dev->device;
+ rawdev->driver_name = pci_dev->device.driver->name;
+
+ rawdev->dev_private = adapter;
+
+ /* must enumerate the adapter before use it */
+ ret = opae_adapter_enumerate(adapter);
+ if (ret)
+ goto free_adapter;
+
+ /* get opae_manager to rawdev */
+ mgr = opae_adapter_get_mgr(adapter);
+ if (mgr) {
+ /* PF function */
+ IFPGA_RAWDEV_PMD_INFO("this is a PF function");
+ }
+
+ return ret;
+
+free_adapter:
+ if (adapter)
+ opae_adapter_free(adapter);
+free_adapter_data:
+ if (data)
+ opae_adapter_data_free(data);
+cleanup:
+ if (rawdev)
+ rte_rawdev_pmd_release(rawdev);
+
+ return ret;
+}
+
+static int
+ifpga_rawdev_destroy(struct rte_pci_device *pci_dev)
+{
+ int ret;
+ struct rte_rawdev *rawdev;
+ char name[RTE_RAWDEV_NAME_MAX_LEN];
+ struct opae_adapter *adapter;
+
+ if (!pci_dev) {
+ IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
+ ret = -EINVAL;
+ return ret;
+ }
+
+ memset(name, 0, sizeof(name));
+ snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%x:%02x.%x",
+ pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
+
+ IFPGA_RAWDEV_PMD_INFO("Closing %s on NUMA node %d",
+ name, rte_socket_id());
+
+ rawdev = rte_rawdev_pmd_get_named_dev(name);
+ if (!rawdev) {
+ IFPGA_RAWDEV_PMD_ERR("Invalid device name (%s)", name);
+ return -EINVAL;
+ }
+
+ adapter = ifpga_rawdev_get_priv(rawdev);
+ if (!adapter)
+ return -ENODEV;
+
+ opae_adapter_data_free(adapter->data);
+ opae_adapter_free(adapter);
+
+ /* rte_rawdev_close is called by pmd_release */
+ ret = rte_rawdev_pmd_release(rawdev);
+ if (ret)
+ IFPGA_RAWDEV_PMD_DEBUG("Device cleanup failed");
+
+ return ret;
+}
+
+static int
+ifpga_rawdev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+ struct rte_pci_device *pci_dev)
+{
+
+ IFPGA_RAWDEV_PMD_FUNC_TRACE();
+ return ifpga_rawdev_create(pci_dev, rte_socket_id());
+}
+
+static int
+ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
+{
+ return ifpga_rawdev_destroy(pci_dev);
+}
+
+static struct rte_pci_driver rte_ifpga_rawdev_pmd = {
+ .id_table = pci_ifpga_map,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
+ .probe = ifpga_rawdev_pci_probe,
+ .remove = ifpga_rawdev_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(ifpga_rawdev_pci_driver, rte_ifpga_rawdev_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(ifpga_rawdev_pci_driver, rte_ifpga_rawdev_pmd);
+RTE_PMD_REGISTER_KMOD_DEP(ifpga_rawdev_pci_driver, "* igb_uio | uio_pci_generic | vfio-pci");
+
+RTE_INIT(ifpga_rawdev_init_log);
+static void
+ifpga_rawdev_init_log(void)
+{
+ ifpga_rawdev_logtype = rte_log_register("driver.raw.init");
+ if (ifpga_rawdev_logtype >= 0)
+ rte_log_set_level(ifpga_rawdev_logtype, RTE_LOG_NOTICE);
+}
+
+static const char * const valid_args[] = {
+#define IFPGA_ARG_NAME "ifpga"
+ IFPGA_ARG_NAME,
+#define IFPGA_ARG_PORT "port"
+ IFPGA_ARG_PORT,
+#define IFPGA_AFU_BTS "afu_bts"
+ IFPGA_AFU_BTS,
+ NULL
+};
+
+static int
+ifpga_cfg_probe(struct rte_vdev_device *dev)
+{
+ struct rte_devargs *devargs;
+ struct rte_kvargs *kvlist = NULL;
+ int port;
+ char *name = NULL;
+ char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
+
+ devargs = dev->device.devargs;
+
+ kvlist = rte_kvargs_parse(devargs->args, valid_args);
+ if (!kvlist) {
+ IFPGA_RAWDEV_PMD_LOG(ERR, "error when parsing param");
+ goto end;
+ }
+
+ if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
+ if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
+ &rte_ifpga_get_string_arg, &name) < 0) {
+ IFPGA_RAWDEV_PMD_ERR("error to parse %s",
+ IFPGA_ARG_NAME);
+ goto end;
+ }
+ } else {
+ IFPGA_RAWDEV_PMD_ERR("arg %s is mandatory for ifpga bus",
+ IFPGA_ARG_NAME);
+ goto end;
+ }
+
+ if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) {
+ if (rte_kvargs_process(kvlist,
+ IFPGA_ARG_PORT,
+ &rte_ifpga_get_integer32_arg,
+ &port) < 0) {
+ IFPGA_RAWDEV_PMD_ERR("error to parse %s",
+ IFPGA_ARG_PORT);
+ goto end;
+ }
+ } else {
+ IFPGA_RAWDEV_PMD_ERR("arg %s is mandatory for ifpga bus",
+ IFPGA_ARG_PORT);
+ goto end;
+ }
+
+ memset(dev_name, 0, sizeof(dev_name));
+ snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
+ port, name);
+
+ rte_eal_hotplug_add(RTE_STR(IFPGA_BUS_NAME),
+ dev_name, devargs->args);
+end:
+ if (kvlist)
+ rte_kvargs_free(kvlist);
+ if (name)
+ free(name);
+
+ return 0;
+}
+
+static int
+ifpga_cfg_remove(struct rte_vdev_device *vdev)
+{
+ IFPGA_RAWDEV_PMD_INFO("Remove ifpga_cfg %p",
+ vdev);
+
+ return 0;
+}
+
+static struct rte_vdev_driver ifpga_cfg_driver = {
+ .probe = ifpga_cfg_probe,
+ .remove = ifpga_cfg_remove,
+};
+
+RTE_PMD_REGISTER_VDEV(net_ifpga_cfg, ifpga_cfg_driver);
+RTE_PMD_REGISTER_ALIAS(net_ifpga_cfg, ifpga_cfg);
+RTE_PMD_REGISTER_PARAM_STRING(net_ifpga_cfg,
+ "bdf=<string> "
+ "port=<int> "
+ "afu_bts=<path>");
+
new file mode 100644
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2018 Intel Corporation
+ */
+
+#ifndef _IFPGA_RAWDEV_H_
+#define _IFPGA_RAWDEV_H_
+
+extern int ifpga_rawdev_logtype;
+
+#define IFPGA_RAWDEV_PMD_LOG(level, fmt, args...) \
+ rte_log(RTE_LOG_ ## level, ifpga_rawdev_logtype, "ifgpa: " fmt, \
+ ##args)
+
+#define IFPGA_RAWDEV_PMD_FUNC_TRACE() IFPGA_RAWDEV_PMD_LOG(DEBUG, ">>")
+
+#define IFPGA_RAWDEV_PMD_DEBUG(fmt, args...) \
+ IFPGA_RAWDEV_PMD_LOG(DEBUG, fmt, ## args)
+#define IFPGA_RAWDEV_PMD_INFO(fmt, args...) \
+ IFPGA_RAWDEV_PMD_LOG(INFO, fmt, ## args)
+#define IFPGA_RAWDEV_PMD_ERR(fmt, args...) \
+ IFPGA_RAWDEV_PMD_LOG(ERR, fmt, ## args)
+#define IFPGA_RAWDEV_PMD_WARN(fmt, args...) \
+ IFPGA_RAWDEV_PMD_LOG(WARNING, fmt, ## args)
+
+enum ifpga_rawdev_device_state {
+ IFPGA_IDLE,
+ IFPGA_READY,
+ IFPGA_ERROR
+};
+
+static inline struct opae_adapter *
+ifpga_rawdev_get_priv(const struct rte_rawdev *rawdev)
+{
+ return rawdev->dev_private;
+}
+
+#endif /* _IFPGA_RAWDEV_H_ */
new file mode 100644
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+version = 1
+
+subdir('base')
+objs = [base_objs]
+
+deps += ['rawdev', 'pci', 'bus_pci', 'kvargs',
+ 'bus_vdev', 'bus_ifpga']
+sources = files('ifpga_rawdev.c')
+
+includes += include_directories('base')
+
+allow_experimental_apis = true
new file mode 100644
@@ -0,0 +1,4 @@
+DPDK_18.05 {
+
+ local: *;
+};
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright 2018 NXP
-drivers = ['skeleton_rawdev', 'dpaa2_cmdif', 'dpaa2_qdma']
+drivers = ['skeleton_rawdev', 'dpaa2_cmdif', 'dpaa2_qdma', 'ifpga_rawdev']
std_deps = ['rawdev']
config_flag_fmt = 'RTE_LIBRTE_PMD_@0@_RAWDEV'
driver_name_fmt = 'rte_pmd_@0@'
@@ -256,9 +256,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_CMDIF_RAWDEV) += -lrte_pmd_dpaa2_cmdif
_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += -lrte_pmd_dpaa2_qdma
endif # CONFIG_RTE_LIBRTE_FSLMC_BUS
_LDLIBS-$(CONFIG_RTE_LIBRTE_IFPGA_BUS) += -lrte_bus_ifpga
+ifeq ($(CONFIG_RTE_LIBRTE_IFPGA_BUS),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += -lrte_pmd_ifpga_rawdev
+endif # CONFIG_RTE_LIBRTE_IFPGA_BUS
endif # CONFIG_RTE_LIBRTE_RAWDEV
-
endif # !CONFIG_RTE_BUILD_SHARED_LIBS
_LDLIBS-y += --no-whole-archive