diff mbox

[dpdk-dev,v8,01/14] eal_pci: Add flag to hold kernel driver type

Message ID 1424060073-23484-2-git-send-email-mukawa@igel.co.jp (mailing list archive)
State Superseded, archived
Headers show

Commit Message

Tetsuya Mukawa Feb. 16, 2015, 4:14 a.m. UTC
From: Michael Qiu <michael.qiu@intel.com>

Currently, dpdk has no ability to know which type of driver(
vfio-pci/igb_uio/uio_pci_generic) the device used. It only can
check whether vfio is enabled or not staticly.

It really useful to have the flag, becasue different type need to
handle differently in runtime. For example, pci memory map,
pot hotplug, and so on.

This patch add a flag field for pci device to solve above issue.

Signed-off-by: Michael Qiu <michael.qiu@intel.com>
Signed-off-by: Tetsuya Mukawa <mukawa@igel.co.jp>
---
 lib/librte_eal/common/include/rte_pci.h |  8 +++++
 lib/librte_eal/linuxapp/eal/eal_pci.c   | 53 +++++++++++++++++++++++++++++++--
 2 files changed, 59 insertions(+), 2 deletions(-)

Comments

Iremonger, Bernard Feb. 16, 2015, 4:14 p.m. UTC | #1
> -----Original Message-----
> From: Tetsuya Mukawa [mailto:mukawa@igel.co.jp]
> Sent: Monday, February 16, 2015 4:14 AM
> To: dev@dpdk.org
> Cc: Qiu, Michael; Iremonger, Bernard; Tetsuya Mukawa
> Subject: [PATCH v8 01/14] eal_pci: Add flag to hold kernel driver type
> 
> From: Michael Qiu <michael.qiu@intel.com>
> 
> Currently, dpdk has no ability to know which type of driver(
> vfio-pci/igb_uio/uio_pci_generic) the device used. It only can check whether vfio is enabled or not
> staticly.
> 
> It really useful to have the flag, becasue different type need to handle differently in runtime. For
> example, pci memory map, pot hotplug, and so on.
> 
> This patch add a flag field for pci device to solve above issue.
> 
> Signed-off-by: Michael Qiu <michael.qiu@intel.com>
> Signed-off-by: Tetsuya Mukawa <mukawa@igel.co.jp>

Acked-by: Bernard Iremonger <bernard.iremonger@intel.com>
Thomas Monjalon Feb. 17, 2015, 12:12 a.m. UTC | #2
2015-02-16 13:14, Tetsuya Mukawa:
> From: Michael Qiu <michael.qiu@intel.com>
> 
> Currently, dpdk has no ability to know which type of driver(
> vfio-pci/igb_uio/uio_pci_generic) the device used. It only can
> check whether vfio is enabled or not staticly.
> 
> It really useful to have the flag, becasue different type need to
> handle differently in runtime. For example, pci memory map,
> pot hotplug, and so on.
> 
> This patch add a flag field for pci device to solve above issue.
> 
> Signed-off-by: Michael Qiu <michael.qiu@intel.com>
> Signed-off-by: Tetsuya Mukawa <mukawa@igel.co.jp>
> ---
>  lib/librte_eal/common/include/rte_pci.h |  8 +++++
>  lib/librte_eal/linuxapp/eal/eal_pci.c   | 53 +++++++++++++++++++++++++++++++--
>  2 files changed, 59 insertions(+), 2 deletions(-)
> 
> diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h
> index 66ed793..7b48b55 100644
> --- a/lib/librte_eal/common/include/rte_pci.h
> +++ b/lib/librte_eal/common/include/rte_pci.h
> @@ -139,6 +139,13 @@ struct rte_pci_addr {
>  
>  struct rte_devargs;
>  
> +enum rte_pt_driver {
> +	RTE_PT_UNKNOWN		= 0,
> +	RTE_PT_IGB_UIO		= 1,
> +	RTE_PT_VFIO		= 2,
> +	RTE_PT_UIO_GENERIC	= 3,
> +};

What means PT?

> +
>  /**
>   * A structure describing a PCI device.
>   */
> @@ -152,6 +159,7 @@ struct rte_pci_device {
>  	uint16_t max_vfs;                       /**< sriov enable if not zero */
>  	int numa_node;                          /**< NUMA node connection */
>  	struct rte_devargs *devargs;            /**< Device user arguments */
> +	enum rte_pt_driver pt_driver;		/**< Driver of passthrough */

I'm not sure passtrough is the right word for this kind of driver.
What about "kernel driver"?

>  };
>  
>  /** Any PCI device identifier (vendor, device, ...) */
> diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c
> index 15db9c4..e760452 100644
> --- a/lib/librte_eal/linuxapp/eal/eal_pci.c
> +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c
> @@ -97,6 +97,35 @@ error:
>  	return -1;
>  }
>  
> +static int
> +pci_get_kernel_driver_by_path(const char *filename, char *dri_name)
> +{
> +	int count;
> +	char path[PATH_MAX];
> +	char *name;
> +
> +	if (!filename || !dri_name)
> +		return -1;
> +
> +	count = readlink(filename, path, PATH_MAX);
> +	if (count >= PATH_MAX)
> +		return -1;
> +
> +	/* For device does not have a driver */
> +	if (count < 0)
> +		return 1;
> +
> +	path[count] = '\0';
> +
> +	name = strrchr(path, '/');
> +	if (name) {
> +		strncpy(dri_name, name + 1, strlen(name + 1) + 1);
> +		return 0;
> +	}
> +
> +	return -1;
> +}
> +
>  void *
>  pci_find_max_end_va(void)
>  {
> @@ -222,11 +251,12 @@ pci_scan_one(const char *dirname, uint16_t domain, uint8_t bus,
>  	char filename[PATH_MAX];
>  	unsigned long tmp;
>  	struct rte_pci_device *dev;
> +	char driver[PATH_MAX];
> +	int ret;
>  
>  	dev = malloc(sizeof(*dev));
> -	if (dev == NULL) {
> +	if (dev == NULL)
>  		return -1;
> -	}
>  
>  	memset(dev, 0, sizeof(*dev));
>  	dev->addr.domain = domain;
> @@ -305,6 +335,25 @@ pci_scan_one(const char *dirname, uint16_t domain, uint8_t bus,
>  		return -1;
>  	}
>  
> +	/* parse driver */
> +	snprintf(filename, sizeof(filename), "%s/driver", dirname);
> +	ret = pci_get_kernel_driver_by_path(filename, driver);
> +	if (!ret) {
> +		if (!strcmp(driver, "vfio-pci"))
> +			dev->pt_driver = RTE_PT_VFIO;
> +		else if (!strcmp(driver, "igb_uio"))
> +			dev->pt_driver = RTE_PT_IGB_UIO;
> +		else if (!strcmp(driver, "uio_pci_generic"))
> +			dev->pt_driver = RTE_PT_UIO_GENERIC;
> +		else
> +			dev->pt_driver = RTE_PT_UNKNOWN;
> +	} else if (ret < 0) {
> +		RTE_LOG(ERR, EAL, "Fail to get kernel driver\n");
> +		free(dev);
> +		return -1;
> +	} else
> +		dev->pt_driver = RTE_PT_UNKNOWN;

Why not considering a NONE value? Example: for virtio with port I/O.

> +
>  	/* device is valid, add in list (sorted) */
>  	if (TAILQ_EMPTY(&pci_device_list)) {
>  		TAILQ_INSERT_TAIL(&pci_device_list, dev, next);
>
Michael Qiu Feb. 17, 2015, 3:09 a.m. UTC | #3
On 2/17/2015 8:13 AM, Thomas Monjalon wrote:
> 2015-02-16 13:14, Tetsuya Mukawa:
>> From: Michael Qiu <michael.qiu@intel.com>
>>
>> Currently, dpdk has no ability to know which type of driver(
>> vfio-pci/igb_uio/uio_pci_generic) the device used. It only can
>> check whether vfio is enabled or not staticly.
>>
>> It really useful to have the flag, becasue different type need to
>> handle differently in runtime. For example, pci memory map,
>> pot hotplug, and so on.
>>
>> This patch add a flag field for pci device to solve above issue.
>>
>> Signed-off-by: Michael Qiu <michael.qiu@intel.com>
>> Signed-off-by: Tetsuya Mukawa <mukawa@igel.co.jp>
>> ---
>>  lib/librte_eal/common/include/rte_pci.h |  8 +++++
>>  lib/librte_eal/linuxapp/eal/eal_pci.c   | 53 +++++++++++++++++++++++++++++++--
>>  2 files changed, 59 insertions(+), 2 deletions(-)
>>
>> diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h
>> index 66ed793..7b48b55 100644
>> --- a/lib/librte_eal/common/include/rte_pci.h
>> +++ b/lib/librte_eal/common/include/rte_pci.h
>> @@ -139,6 +139,13 @@ struct rte_pci_addr {
>>  
>>  struct rte_devargs;
>>  
>> +enum rte_pt_driver {
>> +	RTE_PT_UNKNOWN		= 0,
>> +	RTE_PT_IGB_UIO		= 1,
>> +	RTE_PT_VFIO		= 2,
>> +	RTE_PT_UIO_GENERIC	= 3,
>> +};
> What means PT?
>
>> +
>>  /**
>>   * A structure describing a PCI device.
>>   */
>> @@ -152,6 +159,7 @@ struct rte_pci_device {
>>  	uint16_t max_vfs;                       /**< sriov enable if not zero */
>>  	int numa_node;                          /**< NUMA node connection */
>>  	struct rte_devargs *devargs;            /**< Device user arguments */
>> +	enum rte_pt_driver pt_driver;		/**< Driver of passthrough */
> I'm not sure passtrough is the right word for this kind of driver.
> What about "kernel driver"?

We only care vfio and uio driver, here pass through means pass the
devices from kernel to userspace

If use "kernel driver"or "kn_driver", there will be some issue that we
set to "UNKNOWN" when the devices use the native linux kernel driver.


>
>>  };
>>  
>>  /** Any PCI device identifier (vendor, device, ...) */
>> diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c
>> index 15db9c4..e760452 100644
>> --- a/lib/librte_eal/linuxapp/eal/eal_pci.c
>> +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c
>> @@ -97,6 +97,35 @@ error:
>>  	return -1;
>>  }
>>  
>> +static int
>> +pci_get_kernel_driver_by_path(const char *filename, char *dri_name)
>> +{
>> +	int count;
>> +	char path[PATH_MAX];
>> +	char *name;
>> +
>> +	if (!filename || !dri_name)
>> +		return -1;
>> +
>> +	count = readlink(filename, path, PATH_MAX);
>> +	if (count >= PATH_MAX)
>> +		return -1;
>> +
>> +	/* For device does not have a driver */
>> +	if (count < 0)
>> +		return 1;
>> +
>> +	path[count] = '\0';
>> +
>> +	name = strrchr(path, '/');
>> +	if (name) {
>> +		strncpy(dri_name, name + 1, strlen(name + 1) + 1);
>> +		return 0;
>> +	}
>> +
>> +	return -1;
>> +}
>> +
>>  void *
>>  pci_find_max_end_va(void)
>>  {
>> @@ -222,11 +251,12 @@ pci_scan_one(const char *dirname, uint16_t domain, uint8_t bus,
>>  	char filename[PATH_MAX];
>>  	unsigned long tmp;
>>  	struct rte_pci_device *dev;
>> +	char driver[PATH_MAX];
>> +	int ret;
>>  
>>  	dev = malloc(sizeof(*dev));
>> -	if (dev == NULL) {
>> +	if (dev == NULL)
>>  		return -1;
>> -	}
>>  
>>  	memset(dev, 0, sizeof(*dev));
>>  	dev->addr.domain = domain;
>> @@ -305,6 +335,25 @@ pci_scan_one(const char *dirname, uint16_t domain, uint8_t bus,
>>  		return -1;
>>  	}
>>  
>> +	/* parse driver */
>> +	snprintf(filename, sizeof(filename), "%s/driver", dirname);
>> +	ret = pci_get_kernel_driver_by_path(filename, driver);
>> +	if (!ret) {
>> +		if (!strcmp(driver, "vfio-pci"))
>> +			dev->pt_driver = RTE_PT_VFIO;
>> +		else if (!strcmp(driver, "igb_uio"))
>> +			dev->pt_driver = RTE_PT_IGB_UIO;
>> +		else if (!strcmp(driver, "uio_pci_generic"))
>> +			dev->pt_driver = RTE_PT_UIO_GENERIC;
>> +		else
>> +			dev->pt_driver = RTE_PT_UNKNOWN;
>> +	} else if (ret < 0) {
>> +		RTE_LOG(ERR, EAL, "Fail to get kernel driver\n");
>> +		free(dev);
>> +		return -1;
>> +	} else
>> +		dev->pt_driver = RTE_PT_UNKNOWN;
> Why not considering a NONE value? Example: for virtio with port I/O.

Why NONE? Do you mean, leave it unset? RTE_PT_UNKNOWN equals to zero,
when code runs here means the devices does not have a driver or not bind
to any driver.

For virtio with port I/O, as I know it will use the kernel driver
virtio-pci. So it will not run here.

Thanks,
Michael
>
>> +
>>  	/* device is valid, add in list (sorted) */
>>  	if (TAILQ_EMPTY(&pci_device_list)) {
>>  		TAILQ_INSERT_TAIL(&pci_device_list, dev, next);
>>
>
>
Tetsuya Mukawa Feb. 20, 2015, 6:39 a.m. UTC | #4
This patch series adds a dynamic port hotplug framework to DPDK.
With the patches, DPDK apps can attach or detach ports at runtime.

The basic concept of the port hotplug is like followings.
- DPDK apps must have responsibility to manage ports.
  DPDK apps only know which ports are attached or detached at the moment.
  The port hotplug framework is implemented to allow DPDK apps to manage ports.
  For example, when DPDK apps call port attach function, attached port number
  will be returned. Also, DPDK apps can detach port by port number.
- Kernel support is needed for attaching or detaching physical device ports.
  To attach a new physical device port, the device will be recognized by
  userspace directly I/O framework in kernel at first. Then DPDK apps can
  call the port hotplug functions to attach ports.
  For detaching, steps are vice versa.
- Before detach ports, ports must be stopped and closed.
  DPDK application must call rte_eth_dev_stop() and rte_eth_dev_close() before
  detaching ports. These function will call finalization codes of PMDs.
  But so far, no PMD frees all resources allocated by initialization.
  It means PMDs are needed to be fixed to support the port hotplug.
  'RTE_PCI_DRV_DETACHABLE' is a new flag indicating a PMD supports detaching.
  Without this flag, detaching will be failed.
- Mustn't affect legacy DPDK apps.
  No DPDK EAL behavior is changed, if the port hotplug functions are't called.
  So all legacy DPDK apps can still work without modifications.

And a few limitations.
- The port hotplug functions are not thread safe.
  DPDK apps should handle it.
- Only support Linux and igb_uio so far.
  BSD and VFIO is not supported. I will send VFIO patches at least, but I don't
  have a plan to submit BSD patch so far.


Here is port hotplug APIs.
-------------------------------------------------------------------------------
/**
 * Attach a new device.
 *
 * @param devargs
 *   A pointer to a strings array describing the new device
 *   to be attached. The strings should be a pci address like
 *   '0000:01:00.0' or virtual device name like 'eth_pcap0'.
 * @param port_id
 *  A pointer to a port identifier actually attached.
 * @return
 *  0 on success and port_id is filled, negative on error
 */
int rte_eal_dev_attach(const char *devargs, uint8_t *port_id);

/**
 * Detach a device.
 *
 * @param port_id
 *   The port identifier of the device to detach.
 * @param addr
 *  A pointer to a device name actually detached.
 * @return
 *  0 on success and devname is filled, negative on error
 */
int rte_eal_dev_detach(uint8_t port_id, char *devname);
-------------------------------------------------------------------------------

This patch series are for DPDK EAL. To use port hotplug function by DPDK apps,
each PMD should be fixed to support 'RTE_PCI_DRV_DETACHABLE' flag. Please check
a patch for pcap PMD.

Also, please check testpmd patch. It will show you how to fix your legacy
applications to support port hotplug feature.

PATCH v10 changes
 - Add comments.
 - Chagne order of version.map.
 - Fix comment of "rte_ethdev.h".
   (Thanks to Thomas Monjalon)
 - Add size parameter to rte_eth_dev_create_unique_device_name().
   (Thanks to Iremonger, Bernard)

PATCH v9 changes
 - Fix commit title.
 - Fix commit log.
 - Fix comments.
 - Define CONFIG_RTE_LIBRTE_EAL_HOTPLUG at the top of this patch series.
 - DEV_INVALID/VALID are removed.
 - DEV_DISCONNECTED is replaced by DEV_DETACHED.
 - DEV_CONNECTED is replaced by DEV_ATTACHED.
 - rte_eth_dev_allocate_new_port() is renamed to
   rte_eth_dev_find_free_port().
 - rte_eth_dev_validate_port() is renamed to rte_eth_dev_is_valid_port().
 - rte_eth_dev_is_valid_port() is changed not to handle log toggle.
 - eal_compare_pci_addr() is replaced by rte_eal_compare_pci_addr().
 - rte_eth_dev_free() is replaced by rte_eth_dev_release_port().
 - Add a function to create a unique device name.
 - Change parameter of pci_devuninit_t and rte_eth_dev_uninit.
 - Remove code that initiaize callback of ethdev from
   rte_eth_dev_uninit().
 - Remove pci_unmap_device(). It will be implemented in later patch.
 - rte_eth_dev_check_detachable() is replaced by
   rte_eth_dev_is_detachable().
 - strncpy() is replaced by strcpy().
 - Implement pci_unmap_device() in this patch.
 - Remove "rte_dev_hotplug.h".
 - Remove needless "#ifdef".
 - Remove RTE_EAL_INVOKE_TYPE_PROBE/CLOSE.
 - RTE_ETH_DEV_PHYSICAL is replaced by RTE_ETH_DEV_PCI.
 - Use strcmp() instead of strncmp().
 - Remove RTE_EAL_INVOKE_TYPE_PROBE/CLOSE.
   (Thanks to Thomas Monjalon)
 - Change definition of rte_dev_uninit_t.
   (Thanks to Thomas Monjalon and Maxime Leroy)
 - Add missing symbol in version map.
   (Thanks to Nail Horman)

PATCH v8 changes
 - Fix Makefile and add version map file.
 - Add missing symbol in version map.
 - Fix pci_scan_one() to update sysfs values.
   (Thanks to Qiu, Michael and Iremonger, Bernard)
 - NONE_TRACE is replaced by NO_TRACE.
 - Fix typo.
 - Add size parameter to rte_eth_dev_save().
   (Thanks to Iremonger, Bernard)

PATCH v7 changes
 - Add a new section to programmer's guide.
   (Thanks to Iremonger, Bernard)
 - Fix port checking implementation of star_port().
 - Fix typo of warning messages.
 - Add pt_driver checking to rte_eth_dev_check_detachable().
   (Thanks to Qiu, Michael)

PATCH v6 changes
 - Fix rte_eth_dev_uninit() to handle a return value of uninit
   function of PMD. To do this, below changes also be applied.
   - Fix a parameter of rte_eth_dev_free().
   - Use rte_eth_dev structure as the paramter of rte_eth_dev_free().

PATCH v5 changes
 - Add runtime check passthrough driver type, like vfio-pci, igb_uio
   and uio_pci_generic.
   This was done by Qiu, Michael. Thanks a lot.
 - Change function names like below.
   - rte_eal_dev_find_and_invoke() to rte_eal_vdev_find_and_invoke().
   - rte_eal_dev_invoke() to rte_eal_vdev_invoke().
 - Add code to handle a return value of rte_eal_devargs_remove().
 - Fix pci address format in rte_eal_dev_detach().
 - Remove RTE_EAL_INVOKE_TYPE_UNKNOWN, because it's unused.
 - Change function definition of rte_eal_devargs_remove().
 - Fix pci_unmap_device() to check pt_driver.
 - Fix return value of below functions.
   - rte_eth_dev_get_changed_port().
   - rte_eth_dev_get_port_by_addr().
 - Change paramters of rte_eth_dev_validate_port() to cleanup code.
 - Fix pci_scan_one to handle pt_driver correctly.
   (Thanks to Qiu, Michael for above suggestions)

PATCH v4 changes
 - Merge patches to review easier.
 - Fix indent of 'if' statement.
 - Fix calculation method of eal_compare_pci_addr().
 - Fix header file declaration.
 - Add header file to determine if hotplug can be enabled.
   (Thanks to Qiu, Michael)
 - Use braces with 'for' loop.
 - Add parameter checking.
 - Fix sanity check code
 - Fix comments of rte_eth_dev_type.
 - Change function names.
   (Thanks to Iremonger, Bernard)

PATCH v3 changes:
 - Fix enum definition used in rte_ethdev.c.
   (Thanks to Zhang, Helin)

PATCH v2 changes:
 - Replace rte_eal_dev_attach_pdev(), rte_eal_dev_detach_pdev,
   rte_eal_dev_attach_vdev() and rte_eal_dev_detach_vdev() to
   rte_eal_dev_attach() and rte_eal_dev_detach().
 - Add parameter values checking.
 - Refashion a few functions.
   (Thanks to Iremonger, Bernard)

PATCH v1 Changes:
 - Fix error checking code of librte_eth APIs.
 - Fix issue that port from pcap PMD cannot be detached correctly.
 - Fix issue that testpmd could hang after forwarding, if attaching and detaching
   is repeatedly.
 - Fix if-condition of rte_eth_dev_get_port_by_addr().
   (Thanks to Mark Enright)

RFC PATCH v2 Changes:
- remove 'rte_eth_dev_validate_port()', and cleanup codes.


Michael Qiu (2):
  eal_pci: Add flag to hold kernel driver type
  eal_pci: pci memory map work with driver type

Tetsuya Mukawa (12):
  eal: Enable port Hotplug framework in Linux
  eal/pci,ethdev: Remove assumption that port will not be detached
  eal/pci: Consolidate pci address comparison APIs
  ethdev: Add rte_eth_dev_release_port to release specified port
  eal,ethdev: Add a function and function pointers to close ether device
  ethdev: Add functions that will be used by port hotplug functions
  eal/linux/pci: Add functions for unmapping igb_uio resources
  eal/pci: Add a function to remove the entry of devargs list
  eal/pci: Add probe and close functions of pci driver
  ethdev: Add one dev_type parameter to rte_eth_dev_allocate
  eal/pci: Add rte_eal_dev_attach/detach() functions
  doc: Add port hotplug framework section to programmers guide

 app/test/virtual_pmd.c                           |   2 +-
 config/common_bsdapp                             |   6 +
 config/common_linuxapp                           |   5 +
 doc/guides/prog_guide/index.rst                  |   1 +
 doc/guides/prog_guide/port_hotplug_framework.rst | 110 ++++++
 lib/librte_eal/bsdapp/eal/eal_pci.c              |  29 +-
 lib/librte_eal/common/eal_common_dev.c           | 315 +++++++++++++++
 lib/librte_eal/common/eal_common_devargs.c       |  61 +++
 lib/librte_eal/common/eal_common_pci.c           | 100 ++++-
 lib/librte_eal/common/eal_private.h              |  26 ++
 lib/librte_eal/common/include/rte_dev.h          |  33 ++
 lib/librte_eal/common/include/rte_devargs.h      |  21 +
 lib/librte_eal/common/include/rte_pci.h          |  82 ++++
 lib/librte_eal/linuxapp/eal/Makefile             |   1 +
 lib/librte_eal/linuxapp/eal/eal_pci.c            | 230 +++++++++--
 lib/librte_eal/linuxapp/eal/eal_pci_init.h       |   7 +
 lib/librte_eal/linuxapp/eal/eal_pci_uio.c        |  67 +++-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map  |   2 +
 lib/librte_ether/rte_ethdev.c                    | 476 ++++++++++++++++++-----
 lib/librte_ether/rte_ethdev.h                    | 149 ++++++-
 lib/librte_ether/rte_ether_version.map           |   8 +
 lib/librte_pmd_af_packet/rte_eth_af_packet.c     |   2 +-
 lib/librte_pmd_bond/rte_eth_bond_api.c           |   2 +-
 lib/librte_pmd_pcap/rte_eth_pcap.c               |   2 +-
 lib/librte_pmd_ring/rte_eth_ring.c               |   2 +-
 lib/librte_pmd_xenvirt/rte_eth_xenvirt.c         |   2 +-
 26 files changed, 1573 insertions(+), 168 deletions(-)
 create mode 100644 doc/guides/prog_guide/port_hotplug_framework.rst
Tetsuya Mukawa Feb. 23, 2015, 12:45 p.m. UTC | #5
This patch series adds a dynamic port hotplug framework to DPDK.
With the patches, DPDK apps can attach or detach ports at runtime.

The basic concept of the port hotplug is like followings.
- DPDK apps must have responsibility to manage ports.
  DPDK apps only know which ports are attached or detached at the moment.
  The port hotplug framework is implemented to allow DPDK apps to manage ports.
  For example, when DPDK apps call port attach function, attached port number
  will be returned. Also, DPDK apps can detach port by port number.
- Kernel support is needed for attaching or detaching physical device ports.
  To attach a new physical device port, the device will be recognized by
  userspace directly I/O framework in kernel at first. Then DPDK apps can
  call the port hotplug functions to attach ports.
  For detaching, steps are vice versa.
- Before detach ports, ports must be stopped and closed.
  DPDK application must call rte_eth_dev_stop() and rte_eth_dev_close() before
  detaching ports. These function will call finalization codes of PMDs.
  But so far, no PMD frees all resources allocated by initialization.
  It means PMDs are needed to be fixed to support the port hotplug.
  'RTE_PCI_DRV_DETACHABLE' is a new flag indicating a PMD supports detaching.
  Without this flag, detaching will be failed.
- Mustn't affect legacy DPDK apps.
  No DPDK EAL behavior is changed, if the port hotplug functions are't called.
  So all legacy DPDK apps can still work without modifications.

And a few limitations.
- The port hotplug functions are not thread safe.
  DPDK apps should handle it.
- Only support Linux and igb_uio so far.
  BSD and VFIO is not supported. I will send VFIO patches at least, but I don't
  have a plan to submit BSD patch so far.


Here is port hotplug APIs.
-------------------------------------------------------------------------------
/**
 * Attach a new device.
 *
 * @param devargs
 *   A pointer to a strings array describing the new device
 *   to be attached. The strings should be a pci address like
 *   '0000:01:00.0' or virtual device name like 'eth_pcap0'.
 * @param port_id
 *  A pointer to a port identifier actually attached.
 * @return
 *  0 on success and port_id is filled, negative on error
 */
int rte_eal_dev_attach(const char *devargs, uint8_t *port_id);

/**
 * Detach a device.
 *
 * @param port_id
 *   The port identifier of the device to detach.
 * @param addr
 *  A pointer to a device name actually detached.
 * @return
 *  0 on success and devname is filled, negative on error
 */
int rte_eal_dev_detach(uint8_t port_id, char *devname);
-------------------------------------------------------------------------------

This patch series are for DPDK EAL. To use port hotplug function by DPDK apps,
each PMD should be fixed to support 'RTE_PCI_DRV_DETACHABLE' flag. Please check
a patch for pcap PMD.

Also, please check testpmd patch. It will show you how to fix your legacy
applications to support port hotplug feature.

PATCH v12 changes
 - Add missing symbol in version map.
   (Thanks to Iremonger, Bernard)

PATCH v11 changes
 - Remove needless devargs handling codes.
 - Replace get_vdev_name() by rte_eal_parse_devargs_str().
 - Replace rte_eal_vdev_find_and_init by rte_eal_vdev_init()
 - Replace rte_eal_vdev_find_and_uninit by rte_eal_vdev_uninit()
 - Fix rte_eal_dev_init() to use rte_eal_vdev_init().
 - Remove needless patch.
   (Thanks to Maxime Leroy)

PATCH v10 changes
 - Add comments.
 - Chagne order of version.map.
 - Fix comment of "rte_ethdev.h".
   (Thanks to Thomas Monjalon)
 - Add size parameter to rte_eth_dev_create_unique_device_name().
   (Thanks to Iremonger, Bernard)

PATCH v9 changes
 - Fix commit title.
 - Fix commit log.
 - Fix comments.
 - Define CONFIG_RTE_LIBRTE_EAL_HOTPLUG at the top of this patch series.
 - DEV_INVALID/VALID are removed.
 - DEV_DISCONNECTED is replaced by DEV_DETACHED.
 - DEV_CONNECTED is replaced by DEV_ATTACHED.
 - rte_eth_dev_allocate_new_port() is renamed to
   rte_eth_dev_find_free_port().
 - rte_eth_dev_validate_port() is renamed to rte_eth_dev_is_valid_port().
 - rte_eth_dev_is_valid_port() is changed not to handle log toggle.
 - eal_compare_pci_addr() is replaced by rte_eal_compare_pci_addr().
 - rte_eth_dev_free() is replaced by rte_eth_dev_release_port().
 - Add a function to create a unique device name.
 - Change parameter of pci_devuninit_t and rte_eth_dev_uninit.
 - Remove code that initiaize callback of ethdev from
   rte_eth_dev_uninit().
 - Remove pci_unmap_device(). It will be implemented in later patch.
 - rte_eth_dev_check_detachable() is replaced by
   rte_eth_dev_is_detachable().
 - strncpy() is replaced by strcpy().
 - Implement pci_unmap_device() in this patch.
 - Remove "rte_dev_hotplug.h".
 - Remove needless "#ifdef".
 - Remove RTE_EAL_INVOKE_TYPE_PROBE/CLOSE.
 - RTE_ETH_DEV_PHYSICAL is replaced by RTE_ETH_DEV_PCI.
 - Use strcmp() instead of strncmp().
 - Remove RTE_EAL_INVOKE_TYPE_PROBE/CLOSE.
   (Thanks to Thomas Monjalon)
 - Change definition of rte_dev_uninit_t.
   (Thanks to Thomas Monjalon and Maxime Leroy)
 - Add missing symbol in version map.
   (Thanks to Nail Horman)

PATCH v8 changes
 - Fix Makefile and add version map file.
 - Add missing symbol in version map.
 - Fix pci_scan_one() to update sysfs values.
   (Thanks to Qiu, Michael and Iremonger, Bernard)
 - NONE_TRACE is replaced by NO_TRACE.
 - Fix typo.
 - Add size parameter to rte_eth_dev_save().
   (Thanks to Iremonger, Bernard)

PATCH v7 changes
 - Add a new section to programmer's guide.
   (Thanks to Iremonger, Bernard)
 - Fix port checking implementation of star_port().
 - Fix typo of warning messages.
 - Add pt_driver checking to rte_eth_dev_check_detachable().
   (Thanks to Qiu, Michael)

PATCH v6 changes
 - Fix rte_eth_dev_uninit() to handle a return value of uninit
   function of PMD. To do this, below changes also be applied.
   - Fix a parameter of rte_eth_dev_free().
   - Use rte_eth_dev structure as the paramter of rte_eth_dev_free().

PATCH v5 changes
 - Add runtime check passthrough driver type, like vfio-pci, igb_uio
   and uio_pci_generic.
   This was done by Qiu, Michael. Thanks a lot.
 - Change function names like below.
   - rte_eal_dev_find_and_invoke() to rte_eal_vdev_find_and_invoke().
   - rte_eal_dev_invoke() to rte_eal_vdev_invoke().
 - Add code to handle a return value of rte_eal_devargs_remove().
 - Fix pci address format in rte_eal_dev_detach().
 - Remove RTE_EAL_INVOKE_TYPE_UNKNOWN, because it's unused.
 - Change function definition of rte_eal_devargs_remove().
 - Fix pci_unmap_device() to check pt_driver.
 - Fix return value of below functions.
   - rte_eth_dev_get_changed_port().
   - rte_eth_dev_get_port_by_addr().
 - Change paramters of rte_eth_dev_validate_port() to cleanup code.
 - Fix pci_scan_one to handle pt_driver correctly.
   (Thanks to Qiu, Michael for above suggestions)

PATCH v4 changes
 - Merge patches to review easier.
 - Fix indent of 'if' statement.
 - Fix calculation method of eal_compare_pci_addr().
 - Fix header file declaration.
 - Add header file to determine if hotplug can be enabled.
   (Thanks to Qiu, Michael)
 - Use braces with 'for' loop.
 - Add parameter checking.
 - Fix sanity check code
 - Fix comments of rte_eth_dev_type.
 - Change function names.
   (Thanks to Iremonger, Bernard)

PATCH v3 changes:
 - Fix enum definition used in rte_ethdev.c.
   (Thanks to Zhang, Helin)

PATCH v2 changes:
 - Replace rte_eal_dev_attach_pdev(), rte_eal_dev_detach_pdev,
   rte_eal_dev_attach_vdev() and rte_eal_dev_detach_vdev() to
   rte_eal_dev_attach() and rte_eal_dev_detach().
 - Add parameter values checking.
 - Refashion a few functions.
   (Thanks to Iremonger, Bernard)

PATCH v1 Changes:
 - Fix error checking code of librte_eth APIs.
 - Fix issue that port from pcap PMD cannot be detached correctly.
 - Fix issue that testpmd could hang after forwarding, if attaching and detaching
   is repeatedly.
 - Fix if-condition of rte_eth_dev_get_port_by_addr().
   (Thanks to Mark Enright)

RFC PATCH v2 Changes:
- remove 'rte_eth_dev_validate_port()', and cleanup codes.



Michael Qiu (2):
  eal_pci: Add flag to hold kernel driver type
  eal_pci: pci memory map work with driver type

Tetsuya Mukawa (11):
  eal: Enable port Hotplug framework in Linux
  eal/pci,ethdev: Remove assumption that port will not be detached
  eal/pci: Consolidate pci address comparison APIs
  ethdev: Add rte_eth_dev_release_port to release specified port
  eal,ethdev: Add a function and function pointers to close ether device
  ethdev: Add functions that will be used by port hotplug functions
  eal/linux/pci: Add functions for unmapping igb_uio resources
  eal/pci: Add probe and close functions of pci driver
  ethdev: Add one dev_type parameter to rte_eth_dev_allocate
  eal/pci: Add rte_eal_dev_attach/detach() functions
  doc: Add port hotplug framework section to programmers guide

 app/test/virtual_pmd.c                           |   2 +-
 config/common_bsdapp                             |   6 +
 config/common_linuxapp                           |   5 +
 doc/guides/prog_guide/index.rst                  |   1 +
 doc/guides/prog_guide/port_hotplug_framework.rst | 110 ++++++
 lib/librte_eal/bsdapp/eal/eal_pci.c              |  29 +-
 lib/librte_eal/common/eal_common_dev.c           | 285 +++++++++++++-
 lib/librte_eal/common/eal_common_devargs.c       |  46 ++-
 lib/librte_eal/common/eal_common_pci.c           | 100 ++++-
 lib/librte_eal/common/eal_private.h              |  26 ++
 lib/librte_eal/common/include/rte_dev.h          |  33 ++
 lib/librte_eal/common/include/rte_devargs.h      |  28 ++
 lib/librte_eal/common/include/rte_pci.h          |  82 +++++
 lib/librte_eal/linuxapp/eal/Makefile             |   1 +
 lib/librte_eal/linuxapp/eal/eal_pci.c            | 230 ++++++++++--
 lib/librte_eal/linuxapp/eal/eal_pci_init.h       |   7 +
 lib/librte_eal/linuxapp/eal/eal_pci_uio.c        |  67 +++-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map  |   2 +
 lib/librte_ether/rte_ethdev.c                    | 451 ++++++++++++++++++-----
 lib/librte_ether/rte_ethdev.h                    | 149 +++++++-
 lib/librte_ether/rte_ether_version.map           |   8 +
 lib/librte_pmd_af_packet/rte_eth_af_packet.c     |   2 +-
 lib/librte_pmd_bond/rte_eth_bond_api.c           |   2 +-
 lib/librte_pmd_pcap/rte_eth_pcap.c               |   2 +-
 lib/librte_pmd_ring/rte_eth_ring.c               |   2 +-
 lib/librte_pmd_xenvirt/rte_eth_xenvirt.c         |   2 +-
 26 files changed, 1491 insertions(+), 187 deletions(-)
 create mode 100644 doc/guides/prog_guide/port_hotplug_framework.rst
Tetsuya Mukawa Feb. 24, 2015, 4:49 a.m. UTC | #6
This patch series adds a dynamic port hotplug framework to DPDK.
With the patches, DPDK apps can attach or detach ports at runtime.

The basic concept of the port hotplug is like followings.
- DPDK apps must have responsibility to manage ports.
  DPDK apps only know which ports are attached or detached at the moment.
  The port hotplug framework is implemented to allow DPDK apps to manage ports.
  For example, when DPDK apps call port attach function, attached port number
  will be returned. Also, DPDK apps can detach port by port number.
- Kernel support is needed for attaching or detaching physical device ports.
  To attach a new physical device port, the device will be recognized by
  userspace directly I/O framework in kernel at first. Then DPDK apps can
  call the port hotplug functions to attach ports.
  For detaching, steps are vice versa.
- Before detach ports, ports must be stopped and closed.
  DPDK application must call rte_eth_dev_stop() and rte_eth_dev_close() before
  detaching ports. These function will call finalization codes of PMDs.
  But so far, no PMD frees all resources allocated by initialization.
  It means PMDs are needed to be fixed to support the port hotplug.
  'RTE_PCI_DRV_DETACHABLE' is a new flag indicating a PMD supports detaching.
  Without this flag, detaching will be failed.
- Mustn't affect legacy DPDK apps.
  No DPDK EAL behavior is changed, if the port hotplug functions are't called.
  So all legacy DPDK apps can still work without modifications.

And a few limitations.
- The port hotplug functions are not thread safe.
  DPDK apps should handle it.
- Only support Linux and igb_uio so far.
  BSD and VFIO is not supported. I will send VFIO patches at least, but I don't
  have a plan to submit BSD patch so far.


Here is port hotplug APIs.
-------------------------------------------------------------------------------
/**
 * Attach a new device.
 *
 * @param devargs
 *   A pointer to a strings array describing the new device
 *   to be attached. The strings should be a pci address like
 *   '0000:01:00.0' or virtual device name like 'eth_pcap0'.
 * @param port_id
 *  A pointer to a port identifier actually attached.
 * @return
 *  0 on success and port_id is filled, negative on error
 */
int rte_eal_dev_attach(const char *devargs, uint8_t *port_id);

/**
 * Detach a device.
 *
 * @param port_id
 *   The port identifier of the device to detach.
 * @param addr
 *  A pointer to a device name actually detached.
 * @return
 *  0 on success and devname is filled, negative on error
 */
int rte_eal_dev_detach(uint8_t port_id, char *devname);
-------------------------------------------------------------------------------

This patch series are for DPDK EAL. To use port hotplug function by DPDK apps,
each PMD should be fixed to support 'RTE_PCI_DRV_DETACHABLE' flag. Please check
a patch for pcap PMD.

Also, please check testpmd patch. It will show you how to fix your legacy
applications to support port hotplug feature.

PATCH v13 changes
 - Change log level when error occurs in rte_eal_vdev_init() and
   rte_eal_dev_init().
 - Return value of driver init and uninit functions.
 - Replace rte_panic by RTE_LOG in rte_eal_dev_init()
 - Fix return value of rte_eal_vdev_uninit().
 - Fix rte_eal_dev_attach_vdev to set port_id correctly.
   (Thanks to Maxime Leroy)

PATCH v12 changes
 - Add missing symbol in version map.
   (Thanks to Iremonger, Bernard)

PATCH v11 changes
 - Remove needless devargs handling codes.
 - Replace get_vdev_name() by rte_eal_parse_devargs_str().
 - Replace rte_eal_vdev_find_and_init by rte_eal_vdev_init()
 - Replace rte_eal_vdev_find_and_uninit by rte_eal_vdev_uninit()
 - Fix rte_eal_dev_init() to use rte_eal_vdev_init().
 - Remove needless patch.
   (Thanks to Maxime Leroy)

PATCH v10 changes
 - Add comments.
 - Chagne order of version.map.
 - Fix comment of "rte_ethdev.h".
   (Thanks to Thomas Monjalon)
 - Add size parameter to rte_eth_dev_create_unique_device_name().
   (Thanks to Iremonger, Bernard)

PATCH v9 changes
 - Fix commit title.
 - Fix commit log.
 - Fix comments.
 - Define CONFIG_RTE_LIBRTE_EAL_HOTPLUG at the top of this patch series.
 - DEV_INVALID/VALID are removed.
 - DEV_DISCONNECTED is replaced by DEV_DETACHED.
 - DEV_CONNECTED is replaced by DEV_ATTACHED.
 - rte_eth_dev_allocate_new_port() is renamed to
   rte_eth_dev_find_free_port().
 - rte_eth_dev_validate_port() is renamed to rte_eth_dev_is_valid_port().
 - rte_eth_dev_is_valid_port() is changed not to handle log toggle.
 - eal_compare_pci_addr() is replaced by rte_eal_compare_pci_addr().
 - rte_eth_dev_free() is replaced by rte_eth_dev_release_port().
 - Add a function to create a unique device name.
 - Change parameter of pci_devuninit_t and rte_eth_dev_uninit.
 - Remove code that initiaize callback of ethdev from
   rte_eth_dev_uninit().
 - Remove pci_unmap_device(). It will be implemented in later patch.
 - rte_eth_dev_check_detachable() is replaced by
   rte_eth_dev_is_detachable().
 - strncpy() is replaced by strcpy().
 - Implement pci_unmap_device() in this patch.
 - Remove "rte_dev_hotplug.h".
 - Remove needless "#ifdef".
 - Remove RTE_EAL_INVOKE_TYPE_PROBE/CLOSE.
 - RTE_ETH_DEV_PHYSICAL is replaced by RTE_ETH_DEV_PCI.
 - Use strcmp() instead of strncmp().
 - Remove RTE_EAL_INVOKE_TYPE_PROBE/CLOSE.
   (Thanks to Thomas Monjalon)
 - Change definition of rte_dev_uninit_t.
   (Thanks to Thomas Monjalon and Maxime Leroy)
 - Add missing symbol in version map.
   (Thanks to Nail Horman)

PATCH v8 changes
 - Fix Makefile and add version map file.
 - Add missing symbol in version map.
 - Fix pci_scan_one() to update sysfs values.
   (Thanks to Qiu, Michael and Iremonger, Bernard)
 - NONE_TRACE is replaced by NO_TRACE.
 - Fix typo.
 - Add size parameter to rte_eth_dev_save().
   (Thanks to Iremonger, Bernard)

PATCH v7 changes
 - Add a new section to programmer's guide.
   (Thanks to Iremonger, Bernard)
 - Fix port checking implementation of star_port().
 - Fix typo of warning messages.
 - Add pt_driver checking to rte_eth_dev_check_detachable().
   (Thanks to Qiu, Michael)

PATCH v6 changes
 - Fix rte_eth_dev_uninit() to handle a return value of uninit
   function of PMD. To do this, below changes also be applied.
   - Fix a parameter of rte_eth_dev_free().
   - Use rte_eth_dev structure as the paramter of rte_eth_dev_free().

PATCH v5 changes
 - Add runtime check passthrough driver type, like vfio-pci, igb_uio
   and uio_pci_generic.
   This was done by Qiu, Michael. Thanks a lot.
 - Change function names like below.
   - rte_eal_dev_find_and_invoke() to rte_eal_vdev_find_and_invoke().
   - rte_eal_dev_invoke() to rte_eal_vdev_invoke().
 - Add code to handle a return value of rte_eal_devargs_remove().
 - Fix pci address format in rte_eal_dev_detach().
 - Remove RTE_EAL_INVOKE_TYPE_UNKNOWN, because it's unused.
 - Change function definition of rte_eal_devargs_remove().
 - Fix pci_unmap_device() to check pt_driver.
 - Fix return value of below functions.
   - rte_eth_dev_get_changed_port().
   - rte_eth_dev_get_port_by_addr().
 - Change paramters of rte_eth_dev_validate_port() to cleanup code.
 - Fix pci_scan_one to handle pt_driver correctly.
   (Thanks to Qiu, Michael for above suggestions)

PATCH v4 changes
 - Merge patches to review easier.
 - Fix indent of 'if' statement.
 - Fix calculation method of eal_compare_pci_addr().
 - Fix header file declaration.
 - Add header file to determine if hotplug can be enabled.
   (Thanks to Qiu, Michael)
 - Use braces with 'for' loop.
 - Add parameter checking.
 - Fix sanity check code
 - Fix comments of rte_eth_dev_type.
 - Change function names.
   (Thanks to Iremonger, Bernard)

PATCH v3 changes:
 - Fix enum definition used in rte_ethdev.c.
   (Thanks to Zhang, Helin)

PATCH v2 changes:
 - Replace rte_eal_dev_attach_pdev(), rte_eal_dev_detach_pdev,
   rte_eal_dev_attach_vdev() and rte_eal_dev_detach_vdev() to
   rte_eal_dev_attach() and rte_eal_dev_detach().
 - Add parameter values checking.
 - Refashion a few functions.
   (Thanks to Iremonger, Bernard)

PATCH v1 Changes:
 - Fix error checking code of librte_eth APIs.
 - Fix issue that port from pcap PMD cannot be detached correctly.
 - Fix issue that testpmd could hang after forwarding, if attaching and detaching
   is repeatedly.
 - Fix if-condition of rte_eth_dev_get_port_by_addr().
   (Thanks to Mark Enright)

RFC PATCH v2 Changes:
- remove 'rte_eth_dev_validate_port()', and cleanup codes.


Michael Qiu (2):
  eal_pci: Add flag to hold kernel driver type
  eal_pci: pci memory map work with driver type

Tetsuya Mukawa (11):
  eal: Enable port Hotplug framework in Linux
  eal/pci,ethdev: Remove assumption that port will not be detached
  eal/pci: Consolidate pci address comparison APIs
  ethdev: Add rte_eth_dev_release_port to release specified port
  eal,ethdev: Add a function and function pointers to close ether device
  ethdev: Add functions that will be used by port hotplug functions
  eal/linux/pci: Add functions for unmapping igb_uio resources
  eal/pci: Add probe and close functions of pci driver
  ethdev: Add one dev_type parameter to rte_eth_dev_allocate
  eal/pci: Add rte_eal_dev_attach/detach() functions
  doc: Add port hotplug framework section to programmers guide

 app/test/virtual_pmd.c                           |   2 +-
 config/common_bsdapp                             |   6 +
 config/common_linuxapp                           |   5 +
 doc/guides/prog_guide/index.rst                  |   1 +
 doc/guides/prog_guide/port_hotplug_framework.rst | 110 ++++++
 lib/librte_eal/bsdapp/eal/eal_pci.c              |  29 +-
 lib/librte_eal/common/eal_common_dev.c           | 284 +++++++++++++-
 lib/librte_eal/common/eal_common_devargs.c       |  46 ++-
 lib/librte_eal/common/eal_common_pci.c           | 100 ++++-
 lib/librte_eal/common/eal_private.h              |  26 ++
 lib/librte_eal/common/include/rte_dev.h          |  33 ++
 lib/librte_eal/common/include/rte_devargs.h      |  28 ++
 lib/librte_eal/common/include/rte_pci.h          |  82 +++++
 lib/librte_eal/linuxapp/eal/Makefile             |   1 +
 lib/librte_eal/linuxapp/eal/eal_pci.c            | 230 ++++++++++--
 lib/librte_eal/linuxapp/eal/eal_pci_init.h       |   7 +
 lib/librte_eal/linuxapp/eal/eal_pci_uio.c        |  67 +++-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map  |   2 +
 lib/librte_ether/rte_ethdev.c                    | 451 ++++++++++++++++++-----
 lib/librte_ether/rte_ethdev.h                    | 149 +++++++-
 lib/librte_ether/rte_ether_version.map           |   8 +
 lib/librte_pmd_af_packet/rte_eth_af_packet.c     |   2 +-
 lib/librte_pmd_bond/rte_eth_bond_api.c           |   2 +-
 lib/librte_pmd_pcap/rte_eth_pcap.c               |   2 +-
 lib/librte_pmd_ring/rte_eth_ring.c               |   2 +-
 lib/librte_pmd_xenvirt/rte_eth_xenvirt.c         |   2 +-
 26 files changed, 1489 insertions(+), 188 deletions(-)
 create mode 100644 doc/guides/prog_guide/port_hotplug_framework.rst
Tetsuya Mukawa Feb. 25, 2015, 4:04 a.m. UTC | #7
This patch series adds a dynamic port hotplug framework to DPDK.
With the patches, DPDK apps can attach or detach ports at runtime.

The basic concept of the port hotplug is like followings.
- DPDK apps must have responsibility to manage ports.
  DPDK apps only know which ports are attached or detached at the moment.
  The port hotplug framework is implemented to allow DPDK apps to manage ports.
  For example, when DPDK apps call port attach function, attached port number
  will be returned. Also, DPDK apps can detach port by port number.
- Kernel support is needed for attaching or detaching physical device ports.
  To attach a new physical device port, the device will be recognized by
  userspace directly I/O framework in kernel at first. Then DPDK apps can
  call the port hotplug functions to attach ports.
  For detaching, steps are vice versa.
- Before detach ports, ports must be stopped and closed.
  DPDK application must call rte_eth_dev_stop() and rte_eth_dev_close() before
  detaching ports. These function will call finalization codes of PMDs.
  But so far, no PMD frees all resources allocated by initialization.
  It means PMDs are needed to be fixed to support the port hotplug.
  'RTE_PCI_DRV_DETACHABLE' is a new flag indicating a PMD supports detaching.
  Without this flag, detaching will be failed.
- Mustn't affect legacy DPDK apps.
  No DPDK EAL behavior is changed, if the port hotplug functions are't called.
  So all legacy DPDK apps can still work without modifications.

And a few limitations.
- The port hotplug functions are not thread safe.
  DPDK apps should handle it.
- Only support Linux and igb_uio so far.
  BSD and VFIO is not supported. I will send VFIO patches at least, but I don't
  have a plan to submit BSD patch so far.


Here is port hotplug APIs.
-------------------------------------------------------------------------------
/**
 * Attach a new device.
 *
 * @param devargs
 *   A pointer to a strings array describing the new device
 *   to be attached. The strings should be a pci address like
 *   '0000:01:00.0' or virtual device name like 'eth_pcap0'.
 * @param port_id
 *  A pointer to a port identifier actually attached.
 * @return
 *  0 on success and port_id is filled, negative on error
 */
int rte_eal_dev_attach(const char *devargs, uint8_t *port_id);

/**
 * Detach a device.
 *
 * @param port_id
 *   The port identifier of the device to detach.
 * @param addr
 *  A pointer to a device name actually detached.
 * @return
 *  0 on success and devname is filled, negative on error
 */
int rte_eal_dev_detach(uint8_t port_id, char *devname);
-------------------------------------------------------------------------------

This patch series are for DPDK EAL. To use port hotplug function by DPDK apps,
each PMD should be fixed to support 'RTE_PCI_DRV_DETACHABLE' flag. Please check
a patch for pcap PMD.

Also, please check testpmd patch. It will show you how to fix your legacy
applications to support port hotplug feature.

PATCH v14 changes
 - Remove needless if statement.
   (Thanks to Maxime Leroy)

PATCH v13 changes
 - Change log level when error occurs in rte_eal_vdev_init() and
   rte_eal_dev_init().
 - Return value of driver init and uninit functions.
 - Replace rte_panic by RTE_LOG in rte_eal_dev_init()
 - Fix return value of rte_eal_vdev_uninit().
 - Fix rte_eal_dev_attach_vdev to set port_id correctly.
   (Thanks to Maxime Leroy)

PATCH v12 changes
 - Add missing symbol in version map.
   (Thanks to Iremonger, Bernard)

PATCH v11 changes
 - Remove needless devargs handling codes.
 - Replace get_vdev_name() by rte_eal_parse_devargs_str().
 - Replace rte_eal_vdev_find_and_init by rte_eal_vdev_init()
 - Replace rte_eal_vdev_find_and_uninit by rte_eal_vdev_uninit()
 - Fix rte_eal_dev_init() to use rte_eal_vdev_init().
 - Remove needless patch.
   (Thanks to Maxime Leroy)

PATCH v10 changes
 - Add comments.
 - Chagne order of version.map.
 - Fix comment of "rte_ethdev.h".
   (Thanks to Thomas Monjalon)
 - Add size parameter to rte_eth_dev_create_unique_device_name().
   (Thanks to Iremonger, Bernard)

PATCH v9 changes
 - Fix commit title.
 - Fix commit log.
 - Fix comments.
 - Define CONFIG_RTE_LIBRTE_EAL_HOTPLUG at the top of this patch series.
 - DEV_INVALID/VALID are removed.
 - DEV_DISCONNECTED is replaced by DEV_DETACHED.
 - DEV_CONNECTED is replaced by DEV_ATTACHED.
 - rte_eth_dev_allocate_new_port() is renamed to
   rte_eth_dev_find_free_port().
 - rte_eth_dev_validate_port() is renamed to rte_eth_dev_is_valid_port().
 - rte_eth_dev_is_valid_port() is changed not to handle log toggle.
 - eal_compare_pci_addr() is replaced by rte_eal_compare_pci_addr().
 - rte_eth_dev_free() is replaced by rte_eth_dev_release_port().
 - Add a function to create a unique device name.
 - Change parameter of pci_devuninit_t and rte_eth_dev_uninit.
 - Remove code that initiaize callback of ethdev from
   rte_eth_dev_uninit().
 - Remove pci_unmap_device(). It will be implemented in later patch.
 - rte_eth_dev_check_detachable() is replaced by
   rte_eth_dev_is_detachable().
 - strncpy() is replaced by strcpy().
 - Implement pci_unmap_device() in this patch.
 - Remove "rte_dev_hotplug.h".
 - Remove needless "#ifdef".
 - Remove RTE_EAL_INVOKE_TYPE_PROBE/CLOSE.
 - RTE_ETH_DEV_PHYSICAL is replaced by RTE_ETH_DEV_PCI.
 - Use strcmp() instead of strncmp().
 - Remove RTE_EAL_INVOKE_TYPE_PROBE/CLOSE.
   (Thanks to Thomas Monjalon)
 - Change definition of rte_dev_uninit_t.
   (Thanks to Thomas Monjalon and Maxime Leroy)
 - Add missing symbol in version map.
   (Thanks to Nail Horman)

PATCH v8 changes
 - Fix Makefile and add version map file.
 - Add missing symbol in version map.
 - Fix pci_scan_one() to update sysfs values.
   (Thanks to Qiu, Michael and Iremonger, Bernard)
 - NONE_TRACE is replaced by NO_TRACE.
 - Fix typo.
 - Add size parameter to rte_eth_dev_save().
   (Thanks to Iremonger, Bernard)

PATCH v7 changes
 - Add a new section to programmer's guide.
   (Thanks to Iremonger, Bernard)
 - Fix port checking implementation of star_port().
 - Fix typo of warning messages.
 - Add pt_driver checking to rte_eth_dev_check_detachable().
   (Thanks to Qiu, Michael)

PATCH v6 changes
 - Fix rte_eth_dev_uninit() to handle a return value of uninit
   function of PMD. To do this, below changes also be applied.
   - Fix a parameter of rte_eth_dev_free().
   - Use rte_eth_dev structure as the paramter of rte_eth_dev_free().

PATCH v5 changes
 - Add runtime check passthrough driver type, like vfio-pci, igb_uio
   and uio_pci_generic.
   This was done by Qiu, Michael. Thanks a lot.
 - Change function names like below.
   - rte_eal_dev_find_and_invoke() to rte_eal_vdev_find_and_invoke().
   - rte_eal_dev_invoke() to rte_eal_vdev_invoke().
 - Add code to handle a return value of rte_eal_devargs_remove().
 - Fix pci address format in rte_eal_dev_detach().
 - Remove RTE_EAL_INVOKE_TYPE_UNKNOWN, because it's unused.
 - Change function definition of rte_eal_devargs_remove().
 - Fix pci_unmap_device() to check pt_driver.
 - Fix return value of below functions.
   - rte_eth_dev_get_changed_port().
   - rte_eth_dev_get_port_by_addr().
 - Change paramters of rte_eth_dev_validate_port() to cleanup code.
 - Fix pci_scan_one to handle pt_driver correctly.
   (Thanks to Qiu, Michael for above suggestions)

PATCH v4 changes
 - Merge patches to review easier.
 - Fix indent of 'if' statement.
 - Fix calculation method of eal_compare_pci_addr().
 - Fix header file declaration.
 - Add header file to determine if hotplug can be enabled.
   (Thanks to Qiu, Michael)
 - Use braces with 'for' loop.
 - Add parameter checking.
 - Fix sanity check code
 - Fix comments of rte_eth_dev_type.
 - Change function names.
   (Thanks to Iremonger, Bernard)

PATCH v3 changes:
 - Fix enum definition used in rte_ethdev.c.
   (Thanks to Zhang, Helin)

PATCH v2 changes:
 - Replace rte_eal_dev_attach_pdev(), rte_eal_dev_detach_pdev,
   rte_eal_dev_attach_vdev() and rte_eal_dev_detach_vdev() to
   rte_eal_dev_attach() and rte_eal_dev_detach().
 - Add parameter values checking.
 - Refashion a few functions.
   (Thanks to Iremonger, Bernard)

PATCH v1 Changes:
 - Fix error checking code of librte_eth APIs.
 - Fix issue that port from pcap PMD cannot be detached correctly.
 - Fix issue that testpmd could hang after forwarding, if attaching and detaching
   is repeatedly.
 - Fix if-condition of rte_eth_dev_get_port_by_addr().
   (Thanks to Mark Enright)

RFC PATCH v2 Changes:
- remove 'rte_eth_dev_validate_port()', and cleanup codes.

Michael Qiu (2):
  eal_pci: Add flag to hold kernel driver type
  eal_pci: pci memory map work with driver type

Tetsuya Mukawa (11):
  eal: Enable port Hotplug framework in Linux
  eal/pci,ethdev: Remove assumption that port will not be detached
  eal/pci: Consolidate pci address comparison APIs
  ethdev: Add rte_eth_dev_release_port to release specified port
  eal,ethdev: Add a function and function pointers to close ether device
  ethdev: Add functions that will be used by port hotplug functions
  eal/linux/pci: Add functions for unmapping igb_uio resources
  eal/pci: Add probe and close functions of pci driver
  ethdev: Add one dev_type parameter to rte_eth_dev_allocate
  eal/pci: Add rte_eal_dev_attach/detach() functions
  doc: Add port hotplug framework section to programmers guide

 app/test/virtual_pmd.c                           |   2 +-
 config/common_bsdapp                             |   6 +
 config/common_linuxapp                           |   5 +
 doc/guides/prog_guide/index.rst                  |   1 +
 doc/guides/prog_guide/port_hotplug_framework.rst | 110 ++++++
 lib/librte_eal/bsdapp/eal/eal_pci.c              |  29 +-
 lib/librte_eal/common/eal_common_dev.c           | 278 +++++++++++++-
 lib/librte_eal/common/eal_common_devargs.c       |  54 ++-
 lib/librte_eal/common/eal_common_pci.c           | 100 ++++-
 lib/librte_eal/common/eal_private.h              |  26 ++
 lib/librte_eal/common/include/rte_dev.h          |  33 ++
 lib/librte_eal/common/include/rte_devargs.h      |  28 ++
 lib/librte_eal/common/include/rte_pci.h          |  82 +++++
 lib/librte_eal/linuxapp/eal/Makefile             |   1 +
 lib/librte_eal/linuxapp/eal/eal_pci.c            | 230 ++++++++++--
 lib/librte_eal/linuxapp/eal/eal_pci_init.h       |   7 +
 lib/librte_eal/linuxapp/eal/eal_pci_uio.c        |  67 +++-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map  |   2 +
 lib/librte_ether/rte_ethdev.c                    | 451 ++++++++++++++++++-----
 lib/librte_ether/rte_ethdev.h                    | 149 +++++++-
 lib/librte_ether/rte_ether_version.map           |   8 +
 lib/librte_pmd_af_packet/rte_eth_af_packet.c     |   2 +-
 lib/librte_pmd_bond/rte_eth_bond_api.c           |   2 +-
 lib/librte_pmd_pcap/rte_eth_pcap.c               |   2 +-
 lib/librte_pmd_ring/rte_eth_ring.c               |   2 +-
 lib/librte_pmd_xenvirt/rte_eth_xenvirt.c         |   2 +-
 26 files changed, 1488 insertions(+), 191 deletions(-)
 create mode 100644 doc/guides/prog_guide/port_hotplug_framework.rst
diff mbox

Patch

diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h
index 66ed793..7b48b55 100644
--- a/lib/librte_eal/common/include/rte_pci.h
+++ b/lib/librte_eal/common/include/rte_pci.h
@@ -139,6 +139,13 @@  struct rte_pci_addr {
 
 struct rte_devargs;
 
+enum rte_pt_driver {
+	RTE_PT_UNKNOWN		= 0,
+	RTE_PT_IGB_UIO		= 1,
+	RTE_PT_VFIO		= 2,
+	RTE_PT_UIO_GENERIC	= 3,
+};
+
 /**
  * A structure describing a PCI device.
  */
@@ -152,6 +159,7 @@  struct rte_pci_device {
 	uint16_t max_vfs;                       /**< sriov enable if not zero */
 	int numa_node;                          /**< NUMA node connection */
 	struct rte_devargs *devargs;            /**< Device user arguments */
+	enum rte_pt_driver pt_driver;		/**< Driver of passthrough */
 };
 
 /** Any PCI device identifier (vendor, device, ...) */
diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c
index 15db9c4..e760452 100644
--- a/lib/librte_eal/linuxapp/eal/eal_pci.c
+++ b/lib/librte_eal/linuxapp/eal/eal_pci.c
@@ -97,6 +97,35 @@  error:
 	return -1;
 }
 
+static int
+pci_get_kernel_driver_by_path(const char *filename, char *dri_name)
+{
+	int count;
+	char path[PATH_MAX];
+	char *name;
+
+	if (!filename || !dri_name)
+		return -1;
+
+	count = readlink(filename, path, PATH_MAX);
+	if (count >= PATH_MAX)
+		return -1;
+
+	/* For device does not have a driver */
+	if (count < 0)
+		return 1;
+
+	path[count] = '\0';
+
+	name = strrchr(path, '/');
+	if (name) {
+		strncpy(dri_name, name + 1, strlen(name + 1) + 1);
+		return 0;
+	}
+
+	return -1;
+}
+
 void *
 pci_find_max_end_va(void)
 {
@@ -222,11 +251,12 @@  pci_scan_one(const char *dirname, uint16_t domain, uint8_t bus,
 	char filename[PATH_MAX];
 	unsigned long tmp;
 	struct rte_pci_device *dev;
+	char driver[PATH_MAX];
+	int ret;
 
 	dev = malloc(sizeof(*dev));
-	if (dev == NULL) {
+	if (dev == NULL)
 		return -1;
-	}
 
 	memset(dev, 0, sizeof(*dev));
 	dev->addr.domain = domain;
@@ -305,6 +335,25 @@  pci_scan_one(const char *dirname, uint16_t domain, uint8_t bus,
 		return -1;
 	}
 
+	/* parse driver */
+	snprintf(filename, sizeof(filename), "%s/driver", dirname);
+	ret = pci_get_kernel_driver_by_path(filename, driver);
+	if (!ret) {
+		if (!strcmp(driver, "vfio-pci"))
+			dev->pt_driver = RTE_PT_VFIO;
+		else if (!strcmp(driver, "igb_uio"))
+			dev->pt_driver = RTE_PT_IGB_UIO;
+		else if (!strcmp(driver, "uio_pci_generic"))
+			dev->pt_driver = RTE_PT_UIO_GENERIC;
+		else
+			dev->pt_driver = RTE_PT_UNKNOWN;
+	} else if (ret < 0) {
+		RTE_LOG(ERR, EAL, "Fail to get kernel driver\n");
+		free(dev);
+		return -1;
+	} else
+		dev->pt_driver = RTE_PT_UNKNOWN;
+
 	/* device is valid, add in list (sorted) */
 	if (TAILQ_EMPTY(&pci_device_list)) {
 		TAILQ_INSERT_TAIL(&pci_device_list, dev, next);