[dpdk-dev,1/4] pci: allow access to PCI config space
Commit Message
From: Stephen Hemminger <shemming@brocade.com>
Some drivers need ability to access PCI config (for example for power
management). This adds an abstraction to do this; only implemented
on Linux, but should be possible on BSD.
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
lib/librte_eal/common/include/rte_pci.h | 28 +++++++++++++++
lib/librte_eal/linuxapp/eal/eal_pci.c | 48 +++++++++++++++++++++++++
lib/librte_eal/linuxapp/eal/eal_pci_init.h | 11 ++++++
lib/librte_eal/linuxapp/eal/eal_pci_uio.c | 14 ++++++++
lib/librte_eal/linuxapp/eal/eal_pci_vfio.c | 16 +++++++++
lib/librte_eal/linuxapp/eal/rte_eal_version.map | 2 ++
6 files changed, 119 insertions(+)
Comments
On Thu, May 07, 2015 at 04:25:32PM -0700, Stephen Hemminger wrote:
> From: Stephen Hemminger <shemming@brocade.com>
>
> Some drivers need ability to access PCI config (for example for power
> management). This adds an abstraction to do this; only implemented
> on Linux, but should be possible on BSD.
>
> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> ---
> lib/librte_eal/common/include/rte_pci.h | 28 +++++++++++++++
> lib/librte_eal/linuxapp/eal/eal_pci.c | 48 +++++++++++++++++++++++++
> lib/librte_eal/linuxapp/eal/eal_pci_init.h | 11 ++++++
> lib/librte_eal/linuxapp/eal/eal_pci_uio.c | 14 ++++++++
> lib/librte_eal/linuxapp/eal/eal_pci_vfio.c | 16 +++++++++
> lib/librte_eal/linuxapp/eal/rte_eal_version.map | 2 ++
> 6 files changed, 119 insertions(+)
>
> diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h
> index 223d3cd..cea982a 100644
> --- a/lib/librte_eal/common/include/rte_pci.h
> +++ b/lib/librte_eal/common/include/rte_pci.h
> @@ -393,6 +393,34 @@ void rte_eal_pci_register(struct rte_pci_driver *driver);
> */
> void rte_eal_pci_unregister(struct rte_pci_driver *driver);
>
> +/**
> + * Read PCI config space.
> + *
> + * @param device
> + * A pointer to a rte_pci_device structure describing the device
> + * to use
> + * @param buf
> + * A data buffer where the bytes should be read into
> + * @param size
> + * The length of the data buffer.
> + */
> +int rte_eal_pci_read_config(const struct rte_pci_device *device,
> + void *buf, size_t len, off_t offset);
> +
> +/**
> + * Write PCI config space.
> + *
> + * @param device
> + * A pointer to a rte_pci_device structure describing the device
> + * to use
> + * @param buf
> + * A data buffer containing the bytes should be written
> + * @param size
> + * The length of the data buffer.
> + */
> +int rte_eal_pci_write_config(const struct rte_pci_device *device,
> + const void *buf, size_t len, off_t offset);
> +
I still think this needs a BSD implementation before we pull the whole thing in.
Only partially implementing infrastructure like this is bad practice, and will
lead to complicated build procedures (i.e. developers will require institutional
knoweldge to know that bnx2x can't build on BSD because someone still needs to
implement these functions on bsd. Even having them just return -ENOTSUPP would
be preferable, so that you got a proper run time error, rather than a build
break, though I don't think thats even necessecary, as pci passthrough is
possible in project like bhyve (implying user space pci access)
Neil
> #ifdef __cplusplus
> }
> #endif
> diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c
> index d2adc66..6d79a08 100644
> --- a/lib/librte_eal/linuxapp/eal/eal_pci.c
> +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c
> @@ -756,6 +756,54 @@ rte_eal_pci_close_one_driver(struct rte_pci_driver *dr __rte_unused,
> }
> #endif /* RTE_LIBRTE_EAL_HOTPLUG */
>
> +/* Read PCI config space. */
> +int rte_eal_pci_read_config(const struct rte_pci_device *device,
> + void *buf, size_t len, off_t offset)
> +{
> + const struct rte_intr_handle *intr_handle = &device->intr_handle;
> +
> + switch (intr_handle->type) {
> + case RTE_INTR_HANDLE_UIO:
> + return pci_uio_read_config(intr_handle, buf, len, offset);
> +
> +#ifdef VFIO_PRESENT
> + case RTE_INTR_HANDLE_VFIO_MSIX:
> + case RTE_INTR_HANDLE_VFIO_MSI:
> + case RTE_INTR_HANDLE_VFIO_LEGACY:
> + return pci_vfio_read_config(intr_handle, buf, len, offset);
> +#endif
> + default:
> + RTE_LOG(ERR, EAL,
> + "Unknown handle type of fd %d\n",
> + intr_handle->fd);
> + return -1;
> + }
> +}
> +
> +/* Write PCI config space. */
> +int rte_eal_pci_write_config(const struct rte_pci_device *device,
> + const void *buf, size_t len, off_t offset)
> +{
> + const struct rte_intr_handle *intr_handle = &device->intr_handle;
> +
> + switch (intr_handle->type) {
> + case RTE_INTR_HANDLE_UIO:
> + return pci_uio_write_config(intr_handle, buf, len, offset);
> +
> +#ifdef VFIO_PRESENT
> + case RTE_INTR_HANDLE_VFIO_MSIX:
> + case RTE_INTR_HANDLE_VFIO_MSI:
> + case RTE_INTR_HANDLE_VFIO_LEGACY:
> + return pci_vfio_write_config(intr_handle, buf, len, offset);
> +#endif
> + default:
> + RTE_LOG(ERR, EAL,
> + "Unknown handle type of fd %d\n",
> + intr_handle->fd);
> + return -1;
> + }
> +}
> +
> /* Init the PCI EAL subsystem */
> int
> rte_eal_pci_init(void)
> diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_init.h b/lib/librte_eal/linuxapp/eal/eal_pci_init.h
> index aa7b755..c28e5b0 100644
> --- a/lib/librte_eal/linuxapp/eal/eal_pci_init.h
> +++ b/lib/librte_eal/linuxapp/eal/eal_pci_init.h
> @@ -68,6 +68,11 @@ void *pci_find_max_end_va(void);
> void *pci_map_resource(void *requested_addr, int fd, off_t offset,
> size_t size, int additional_flags);
>
> +int pci_uio_read_config(const struct rte_intr_handle *intr_handle,
> + void *buf, size_t len, off_t offs);
> +int pci_uio_write_config(const struct rte_intr_handle *intr_handle,
> + const void *buf, size_t len, off_t offs);
> +
> /* map IGB_UIO resource prototype */
> int pci_uio_map_resource(struct rte_pci_device *dev);
>
> @@ -86,6 +91,12 @@ int pci_vfio_enable(void);
> int pci_vfio_is_enabled(void);
> int pci_vfio_mp_sync_setup(void);
>
> +/* access config space */
> +int pci_vfio_read_config(const struct rte_intr_handle *intr_handle,
> + void *buf, size_t len, off_t offs);
> +int pci_vfio_write_config(const struct rte_intr_handle *intr_handle,
> + const void *buf, size_t len, off_t offs);
> +
> /* map VFIO resource prototype */
> int pci_vfio_map_resource(struct rte_pci_device *dev);
> int pci_vfio_get_group_fd(int iommu_group_fd);
> diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
> index 2d1c69b..bbc59ed 100644
> --- a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
> +++ b/lib/librte_eal/linuxapp/eal/eal_pci_uio.c
> @@ -58,6 +58,20 @@ EAL_REGISTER_TAILQ(rte_uio_tailq)
>
> #define OFF_MAX ((uint64_t)(off_t)-1)
>
> +int
> +pci_uio_read_config(const struct rte_intr_handle *intr_handle,
> + void *buf, size_t len, off_t offset)
> +{
> + return pread(intr_handle->uio_cfg_fd, buf, len, offset);
> +}
> +
> +int
> +pci_uio_write_config(const struct rte_intr_handle *intr_handle,
> + const void *buf, size_t len, off_t offset)
> +{
> + return pwrite(intr_handle->uio_cfg_fd, buf, len, offset);
> +}
> +
> static int
> pci_uio_set_bus_master(int dev_fd)
> {
> diff --git a/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c b/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c
> index aea1fb1..092a369 100644
> --- a/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c
> +++ b/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c
> @@ -77,6 +77,22 @@ EAL_REGISTER_TAILQ(rte_vfio_tailq)
> /* per-process VFIO config */
> static struct vfio_config vfio_cfg;
>
> +int
> +pci_vfio_read_config(const struct rte_intr_handle *intr_handle,
> + void *buf, size_t len, off_t offs)
> +{
> + return pread64(intr_handle->vfio_dev_fd, buf, len,
> + VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + offs);
> +}
> +
> +int
> +pci_vfio_write_config(const struct rte_intr_handle *intr_handle,
> + const void *buf, size_t len, off_t offs)
> +{
> + return pwrite64(intr_handle->vfio_dev_fd, buf, len,
> + VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + offs);
> +}
> +
> /* get PCI BAR number where MSI-X interrupts are */
> static int
> pci_vfio_get_msix_bar(int fd, int *msix_bar, uint32_t *msix_table_offset,
> diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
> index 7e850a9..494aae0 100644
> --- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
> +++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
> @@ -42,9 +42,11 @@ DPDK_2.0 {
> rte_eal_pci_dump;
> rte_eal_pci_probe;
> rte_eal_pci_probe_one;
> + rte_eal_pci_read_config;
> rte_eal_pci_register;
> rte_eal_pci_scan;
> rte_eal_pci_unregister;
> + rte_eal_pci_write_config;
> rte_eal_process_type;
> rte_eal_remote_launch;
> rte_eal_tailq_lookup;
> --
> 2.1.4
>
>
Ok will stub out the pci_config stuff for BSD.
But I don't have time or resources to do real BSD support.
Also, the whole bnx2x driver loads firmware and that probably has dependencies
that are different on BSD
On Mon, May 11, 2015 at 08:23:59AM -0700, Stephen Hemminger wrote:
> Ok will stub out the pci_config stuff for BSD.
> But I don't have time or resources to do real BSD support.
>
> Also, the whole bnx2x driver loads firmware and that probably has dependencies
> that are different on BSD
>
Thats a fair point. Is the implication here that, even with the stub functions
for pci config space, bnx2x won't build on BSD?
Neil
On Mon, 11 May 2015 11:37:08 -0400
Neil Horman <nhorman@tuxdriver.com> wrote:
> On Mon, May 11, 2015 at 08:23:59AM -0700, Stephen Hemminger wrote:
> > Ok will stub out the pci_config stuff for BSD.
> > But I don't have time or resources to do real BSD support.
> >
> > Also, the whole bnx2x driver loads firmware and that probably has dependencies
> > that are different on BSD
> >
> Thats a fair point. Is the implication here that, even with the stub functions
> for pci config space, bnx2x won't build on BSD?
>
> Neil
>
It will build but not run since it looks for firmware in /lib/firmware
@@ -393,6 +393,34 @@ void rte_eal_pci_register(struct rte_pci_driver *driver);
*/
void rte_eal_pci_unregister(struct rte_pci_driver *driver);
+/**
+ * Read PCI config space.
+ *
+ * @param device
+ * A pointer to a rte_pci_device structure describing the device
+ * to use
+ * @param buf
+ * A data buffer where the bytes should be read into
+ * @param size
+ * The length of the data buffer.
+ */
+int rte_eal_pci_read_config(const struct rte_pci_device *device,
+ void *buf, size_t len, off_t offset);
+
+/**
+ * Write PCI config space.
+ *
+ * @param device
+ * A pointer to a rte_pci_device structure describing the device
+ * to use
+ * @param buf
+ * A data buffer containing the bytes should be written
+ * @param size
+ * The length of the data buffer.
+ */
+int rte_eal_pci_write_config(const struct rte_pci_device *device,
+ const void *buf, size_t len, off_t offset);
+
#ifdef __cplusplus
}
#endif
@@ -756,6 +756,54 @@ rte_eal_pci_close_one_driver(struct rte_pci_driver *dr __rte_unused,
}
#endif /* RTE_LIBRTE_EAL_HOTPLUG */
+/* Read PCI config space. */
+int rte_eal_pci_read_config(const struct rte_pci_device *device,
+ void *buf, size_t len, off_t offset)
+{
+ const struct rte_intr_handle *intr_handle = &device->intr_handle;
+
+ switch (intr_handle->type) {
+ case RTE_INTR_HANDLE_UIO:
+ return pci_uio_read_config(intr_handle, buf, len, offset);
+
+#ifdef VFIO_PRESENT
+ case RTE_INTR_HANDLE_VFIO_MSIX:
+ case RTE_INTR_HANDLE_VFIO_MSI:
+ case RTE_INTR_HANDLE_VFIO_LEGACY:
+ return pci_vfio_read_config(intr_handle, buf, len, offset);
+#endif
+ default:
+ RTE_LOG(ERR, EAL,
+ "Unknown handle type of fd %d\n",
+ intr_handle->fd);
+ return -1;
+ }
+}
+
+/* Write PCI config space. */
+int rte_eal_pci_write_config(const struct rte_pci_device *device,
+ const void *buf, size_t len, off_t offset)
+{
+ const struct rte_intr_handle *intr_handle = &device->intr_handle;
+
+ switch (intr_handle->type) {
+ case RTE_INTR_HANDLE_UIO:
+ return pci_uio_write_config(intr_handle, buf, len, offset);
+
+#ifdef VFIO_PRESENT
+ case RTE_INTR_HANDLE_VFIO_MSIX:
+ case RTE_INTR_HANDLE_VFIO_MSI:
+ case RTE_INTR_HANDLE_VFIO_LEGACY:
+ return pci_vfio_write_config(intr_handle, buf, len, offset);
+#endif
+ default:
+ RTE_LOG(ERR, EAL,
+ "Unknown handle type of fd %d\n",
+ intr_handle->fd);
+ return -1;
+ }
+}
+
/* Init the PCI EAL subsystem */
int
rte_eal_pci_init(void)
@@ -68,6 +68,11 @@ void *pci_find_max_end_va(void);
void *pci_map_resource(void *requested_addr, int fd, off_t offset,
size_t size, int additional_flags);
+int pci_uio_read_config(const struct rte_intr_handle *intr_handle,
+ void *buf, size_t len, off_t offs);
+int pci_uio_write_config(const struct rte_intr_handle *intr_handle,
+ const void *buf, size_t len, off_t offs);
+
/* map IGB_UIO resource prototype */
int pci_uio_map_resource(struct rte_pci_device *dev);
@@ -86,6 +91,12 @@ int pci_vfio_enable(void);
int pci_vfio_is_enabled(void);
int pci_vfio_mp_sync_setup(void);
+/* access config space */
+int pci_vfio_read_config(const struct rte_intr_handle *intr_handle,
+ void *buf, size_t len, off_t offs);
+int pci_vfio_write_config(const struct rte_intr_handle *intr_handle,
+ const void *buf, size_t len, off_t offs);
+
/* map VFIO resource prototype */
int pci_vfio_map_resource(struct rte_pci_device *dev);
int pci_vfio_get_group_fd(int iommu_group_fd);
@@ -58,6 +58,20 @@ EAL_REGISTER_TAILQ(rte_uio_tailq)
#define OFF_MAX ((uint64_t)(off_t)-1)
+int
+pci_uio_read_config(const struct rte_intr_handle *intr_handle,
+ void *buf, size_t len, off_t offset)
+{
+ return pread(intr_handle->uio_cfg_fd, buf, len, offset);
+}
+
+int
+pci_uio_write_config(const struct rte_intr_handle *intr_handle,
+ const void *buf, size_t len, off_t offset)
+{
+ return pwrite(intr_handle->uio_cfg_fd, buf, len, offset);
+}
+
static int
pci_uio_set_bus_master(int dev_fd)
{
@@ -77,6 +77,22 @@ EAL_REGISTER_TAILQ(rte_vfio_tailq)
/* per-process VFIO config */
static struct vfio_config vfio_cfg;
+int
+pci_vfio_read_config(const struct rte_intr_handle *intr_handle,
+ void *buf, size_t len, off_t offs)
+{
+ return pread64(intr_handle->vfio_dev_fd, buf, len,
+ VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + offs);
+}
+
+int
+pci_vfio_write_config(const struct rte_intr_handle *intr_handle,
+ const void *buf, size_t len, off_t offs)
+{
+ return pwrite64(intr_handle->vfio_dev_fd, buf, len,
+ VFIO_GET_REGION_ADDR(VFIO_PCI_CONFIG_REGION_INDEX) + offs);
+}
+
/* get PCI BAR number where MSI-X interrupts are */
static int
pci_vfio_get_msix_bar(int fd, int *msix_bar, uint32_t *msix_table_offset,
@@ -42,9 +42,11 @@ DPDK_2.0 {
rte_eal_pci_dump;
rte_eal_pci_probe;
rte_eal_pci_probe_one;
+ rte_eal_pci_read_config;
rte_eal_pci_register;
rte_eal_pci_scan;
rte_eal_pci_unregister;
+ rte_eal_pci_write_config;
rte_eal_process_type;
rte_eal_remote_launch;
rte_eal_tailq_lookup;