[v4,04/11] dma/ioat: add configuration functions

Add functions for device configuration. The info_get and close functions
are included here also. info_get can be useful for checking successful
configuration and close is used by the dmadev api when releasing a
configured device.

Signed-off-by: Conor Walsh <conor.walsh@intel.com>
Reviewed-by: Kevin Laatz <kevin.laatz@intel.com>
---
 doc/guides/dmadevs/ioat.rst    | 24 +++++++++
 drivers/dma/ioat/ioat_dmadev.c | 93 ++++++++++++++++++++++++++++++++++
 2 files changed, 117 insertions(+)

On 2021/9/17 23:42, Conor Walsh wrote:
> Add functions for device configuration. The info_get and close functions
> are included here also. info_get can be useful for checking successful
> configuration and close is used by the dmadev api when releasing a
> configured device.
> 
> Signed-off-by: Conor Walsh <conor.walsh@intel.com>
> Reviewed-by: Kevin Laatz <kevin.laatz@intel.com>
> ---
>  doc/guides/dmadevs/ioat.rst    | 24 +++++++++
>  drivers/dma/ioat/ioat_dmadev.c | 93 ++++++++++++++++++++++++++++++++++
>  2 files changed, 117 insertions(+)
> 
> diff --git a/doc/guides/dmadevs/ioat.rst b/doc/guides/dmadevs/ioat.rst
> index 45a2e65d70..6092599fe4 100644
> --- a/doc/guides/dmadevs/ioat.rst
> +++ b/doc/guides/dmadevs/ioat.rst
> @@ -62,3 +62,27 @@ For example::
>  Once probed successfully, the device will appear as a ``dmadev``, that is a
>  "DMA device type" inside DPDK, and can be accessed using APIs from the
>  ``rte_dmadev`` library.
> +
> +Device Configuration
> +~~~~~~~~~~~~~~~~~~~~~
> +
> +Configuring an IOAT dmadev device is done using the ``rte_dma_configure()``
> +and ``rte_dma_vchan_setup()`` APIs. ``rte_dma_configure()`` uses the structure
> +``rte_dma_conf`` to configure an IOAT device, within this struct the number
> +of virtual DMA channels for the device is set and silent mode can be enabled.
> +Each IOAT device can only use 1 virtual DMA channel and silent mode is not
> +supported so these will always be set to ``1`` and ``false``.
> +``rte_dma_vchan_setup()`` uses the structure ``rte_dma_vchan_conf`` to setup
> +a virtual DMA channel within this struct the transfer direction and ring size
> +are set. Generally for an onboard Intel\ |reg| IOAT device the transfer direction
> +will be set to ``RTE_DMA_DIR_MEM_TO_MEM`` to copy from memory to memory (more info
> +available in the dmadev API guide). The ring size must be a power of two, between
> +64 and 4096.
> +
> +The following code shows how the device is configured in ``test_dmadev.c``:
> +
> +.. literalinclude:: ../../../app/test/test_dmadev.c
> +   :language: c
> +   :start-after: Setup of the dmadev device. 8<
> +   :end-before: >8 End of setup of the dmadev device.
> +   :dedent: 1
> diff --git a/drivers/dma/ioat/ioat_dmadev.c b/drivers/dma/ioat/ioat_dmadev.c
> index 370be3ec72..d58022e258 100644
> --- a/drivers/dma/ioat/ioat_dmadev.c
> +++ b/drivers/dma/ioat/ioat_dmadev.c
> @@ -12,9 +12,98 @@ static struct rte_pci_driver ioat_pmd_drv;
>  
>  RTE_LOG_REGISTER_DEFAULT(ioat_pmd_logtype, INFO);
>  
> +#define DESC_SZ sizeof(struct ioat_dma_hw_desc)
> +
>  #define IOAT_PMD_NAME dmadev_ioat
>  #define IOAT_PMD_NAME_STR RTE_STR(IOAT_PMD_NAME)
>  
> +/* Configure a device. */
> +static int
> +ioat_dev_configure(struct rte_dma_dev *dev __rte_unused, const struct rte_dma_conf *dev_conf,
> +		uint32_t conf_sz)
> +{
> +	if (sizeof(struct rte_dma_conf) != conf_sz)
> +		return -EINVAL;
> +
> +	if (dev_conf->nb_vchans != 1)
> +		return -EINVAL;
> +
> +	return 0;
> +}
> +
> +/* Setup a virtual channel for IOAT, only 1 vchan is supported. */
> +static int
> +ioat_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan __rte_unused,
> +		const struct rte_dma_vchan_conf *qconf, uint32_t qconf_sz)
> +{
> +	struct ioat_dmadev *ioat = dev->dev_private;
> +	uint16_t max_desc = qconf->nb_desc;
> +	int i;
> +
> +	if (sizeof(struct rte_dma_vchan_conf) != qconf_sz)
> +		return -EINVAL;
> +
> +	ioat->qcfg = *qconf;
> +
> +	if (!rte_is_power_of_2(max_desc)) {
> +		max_desc = rte_align32pow2(max_desc);
> +		IOAT_PMD_DEBUG("DMA dev %u using %u descriptors", dev->data->dev_id, max_desc);
> +		ioat->qcfg.nb_desc = max_desc;
> +	}
> +
> +	/* In case we are reconfiguring a device, free any existing memory. */
> +	rte_free(ioat->desc_ring);
> +
> +	ioat->desc_ring = rte_zmalloc(NULL, sizeof(*ioat->desc_ring) * max_desc, 0);
> +	if (ioat->desc_ring == NULL)
> +		return -ENOMEM;
> +
> +	ioat->ring_addr = rte_mem_virt2iova(ioat->desc_ring);
> +
> +	ioat->status_addr = rte_mem_virt2iova(ioat) + offsetof(struct ioat_dmadev, status);
> +
> +	/* Ensure all counters are reset, if reconfiguring/restarting device. */
> +	ioat->next_read = 0;
> +	ioat->next_write = 0;
> +	ioat->last_write = 0;
> +	ioat->offset = 0;
> +	ioat->failure = 0;
> +
> +	/* Configure descriptor ring - each one points to next. */
> +	for (i = 0; i < ioat->qcfg.nb_desc; i++) {
> +		ioat->desc_ring[i].next = ioat->ring_addr +
> +				(((i + 1) % ioat->qcfg.nb_desc) * DESC_SZ);
> +	}
> +
> +	return 0;
> +}
> +
> +/* Get device information of a device. */
> +static int
> +ioat_dev_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *info, uint32_t size)
> +{
> +	struct ioat_dmadev *ioat = dev->dev_private;
> +
> +	if (size < sizeof(*info))
> +		return -EINVAL;
> +	info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
> +			RTE_DMA_CAPA_OPS_COPY |
> +			RTE_DMA_CAPA_OPS_FILL;
> +	info->max_vchans = 1;
> +	info->min_desc = 32;
> +	info->max_desc = 4096;
> +	info->nb_vchans = (ioat->desc_ring == NULL);

The nb_vchans will overwrite by library.

> +	return 0;
> +}
> +
> +/* Close a configured device. */
> +static int
> +ioat_dev_close(struct rte_dma_dev *dev)
> +{
> +	RTE_SET_USED(dev);
> +	return 0;
> +}
> +
>  /* Dump DMA device info. */
>  static int
>  ioat_dev_dump(const struct rte_dma_dev *dev, FILE *f)
> @@ -77,7 +166,11 @@ static int
>  ioat_dmadev_create(const char *name, struct rte_pci_device *dev)
>  {
>  	static const struct rte_dma_dev_ops ioat_dmadev_ops = {
> +		.dev_close = ioat_dev_close,
> +		.dev_configure = ioat_dev_configure,
>  		.dev_dump = ioat_dev_dump,
> +		.dev_info_get = ioat_dev_info_get,
> +		.vchan_setup = ioat_vchan_setup,
>  	};
>  
>  	struct rte_dma_dev *dmadev = NULL;
>

On 22/09/2021 09:08, fengchengwen wrote:
> On 2021/9/17 23:42, Conor Walsh wrote:
>> Add functions for device configuration. The info_get and close functions
>> are included here also. info_get can be useful for checking successful
>> configuration and close is used by the dmadev api when releasing a
>> configured device.
>>
>> Signed-off-by: Conor Walsh <conor.walsh@intel.com>
>> Reviewed-by: Kevin Laatz <kevin.laatz@intel.com>
>> ---
>>   doc/guides/dmadevs/ioat.rst    | 24 +++++++++
>>   drivers/dma/ioat/ioat_dmadev.c | 93 ++++++++++++++++++++++++++++++++++
>>   2 files changed, 117 insertions(+)
>>
>> diff --git a/doc/guides/dmadevs/ioat.rst b/doc/guides/dmadevs/ioat.rst
>> index 45a2e65d70..6092599fe4 100644
>> --- a/doc/guides/dmadevs/ioat.rst
>> +++ b/doc/guides/dmadevs/ioat.rst
>> @@ -62,3 +62,27 @@ For example::
>>   Once probed successfully, the device will appear as a ``dmadev``, that is a
>>   "DMA device type" inside DPDK, and can be accessed using APIs from the
>>   ``rte_dmadev`` library.
>> +
>> +Device Configuration
>> +~~~~~~~~~~~~~~~~~~~~~
>> +
>> +Configuring an IOAT dmadev device is done using the ``rte_dma_configure()``
>> +and ``rte_dma_vchan_setup()`` APIs. ``rte_dma_configure()`` uses the structure
>> +``rte_dma_conf`` to configure an IOAT device, within this struct the number
>> +of virtual DMA channels for the device is set and silent mode can be enabled.
>> +Each IOAT device can only use 1 virtual DMA channel and silent mode is not
>> +supported so these will always be set to ``1`` and ``false``.
>> +``rte_dma_vchan_setup()`` uses the structure ``rte_dma_vchan_conf`` to setup
>> +a virtual DMA channel within this struct the transfer direction and ring size
>> +are set. Generally for an onboard Intel\ |reg| IOAT device the transfer direction
>> +will be set to ``RTE_DMA_DIR_MEM_TO_MEM`` to copy from memory to memory (more info
>> +available in the dmadev API guide). The ring size must be a power of two, between
>> +64 and 4096.
>> +
>> +The following code shows how the device is configured in ``test_dmadev.c``:
>> +
>> +.. literalinclude:: ../../../app/test/test_dmadev.c
>> +   :language: c
>> +   :start-after: Setup of the dmadev device. 8<
>> +   :end-before: >8 End of setup of the dmadev device.
>> +   :dedent: 1
>> diff --git a/drivers/dma/ioat/ioat_dmadev.c b/drivers/dma/ioat/ioat_dmadev.c
>> index 370be3ec72..d58022e258 100644
>> --- a/drivers/dma/ioat/ioat_dmadev.c
>> +++ b/drivers/dma/ioat/ioat_dmadev.c
>> @@ -12,9 +12,98 @@ static struct rte_pci_driver ioat_pmd_drv;
>>   
>>   RTE_LOG_REGISTER_DEFAULT(ioat_pmd_logtype, INFO);
>>   
>> +#define DESC_SZ sizeof(struct ioat_dma_hw_desc)
>> +
>>   #define IOAT_PMD_NAME dmadev_ioat
>>   #define IOAT_PMD_NAME_STR RTE_STR(IOAT_PMD_NAME)
>>   
>> +/* Configure a device. */
>> +static int
>> +ioat_dev_configure(struct rte_dma_dev *dev __rte_unused, const struct rte_dma_conf *dev_conf,
>> +		uint32_t conf_sz)
>> +{
>> +	if (sizeof(struct rte_dma_conf) != conf_sz)
>> +		return -EINVAL;
>> +
>> +	if (dev_conf->nb_vchans != 1)
>> +		return -EINVAL;
>> +
>> +	return 0;
>> +}
>> +
>> +/* Setup a virtual channel for IOAT, only 1 vchan is supported. */
>> +static int
>> +ioat_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan __rte_unused,
>> +		const struct rte_dma_vchan_conf *qconf, uint32_t qconf_sz)
>> +{
>> +	struct ioat_dmadev *ioat = dev->dev_private;
>> +	uint16_t max_desc = qconf->nb_desc;
>> +	int i;
>> +
>> +	if (sizeof(struct rte_dma_vchan_conf) != qconf_sz)
>> +		return -EINVAL;
>> +
>> +	ioat->qcfg = *qconf;
>> +
>> +	if (!rte_is_power_of_2(max_desc)) {
>> +		max_desc = rte_align32pow2(max_desc);
>> +		IOAT_PMD_DEBUG("DMA dev %u using %u descriptors", dev->data->dev_id, max_desc);
>> +		ioat->qcfg.nb_desc = max_desc;
>> +	}
>> +
>> +	/* In case we are reconfiguring a device, free any existing memory. */
>> +	rte_free(ioat->desc_ring);
>> +
>> +	ioat->desc_ring = rte_zmalloc(NULL, sizeof(*ioat->desc_ring) * max_desc, 0);
>> +	if (ioat->desc_ring == NULL)
>> +		return -ENOMEM;
>> +
>> +	ioat->ring_addr = rte_mem_virt2iova(ioat->desc_ring);
>> +
>> +	ioat->status_addr = rte_mem_virt2iova(ioat) + offsetof(struct ioat_dmadev, status);
>> +
>> +	/* Ensure all counters are reset, if reconfiguring/restarting device. */
>> +	ioat->next_read = 0;
>> +	ioat->next_write = 0;
>> +	ioat->last_write = 0;
>> +	ioat->offset = 0;
>> +	ioat->failure = 0;
>> +
>> +	/* Configure descriptor ring - each one points to next. */
>> +	for (i = 0; i < ioat->qcfg.nb_desc; i++) {
>> +		ioat->desc_ring[i].next = ioat->ring_addr +
>> +				(((i + 1) % ioat->qcfg.nb_desc) * DESC_SZ);
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>> +/* Get device information of a device. */
>> +static int
>> +ioat_dev_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *info, uint32_t size)
>> +{
>> +	struct ioat_dmadev *ioat = dev->dev_private;
>> +
>> +	if (size < sizeof(*info))
>> +		return -EINVAL;
>> +	info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
>> +			RTE_DMA_CAPA_OPS_COPY |
>> +			RTE_DMA_CAPA_OPS_FILL;
>> +	info->max_vchans = 1;
>> +	info->min_desc = 32;
>> +	info->max_desc = 4096;
>> +	info->nb_vchans = (ioat->desc_ring == NULL);
> The nb_vchans will overwrite by library.

Updated for v5, should be upstreamed in the next few days.

Thanks for the review Chengwen!

/Conor.

>
>> +	return 0;
>> +}
>> +
>> +/* Close a configured device. */
>> +static int
>> +ioat_dev_close(struct rte_dma_dev *dev)
>> +{
>> +	RTE_SET_USED(dev);
>> +	return 0;
>> +}
>> +
>>   /* Dump DMA device info. */
>>   static int
>>   ioat_dev_dump(const struct rte_dma_dev *dev, FILE *f)
>> @@ -77,7 +166,11 @@ static int
>>   ioat_dmadev_create(const char *name, struct rte_pci_device *dev)
>>   {
>>   	static const struct rte_dma_dev_ops ioat_dmadev_ops = {
>> +		.dev_close = ioat_dev_close,
>> +		.dev_configure = ioat_dev_configure,
>>   		.dev_dump = ioat_dev_dump,
>> +		.dev_info_get = ioat_dev_info_get,
>> +		.vchan_setup = ioat_vchan_setup,
>>   	};
>>   
>>   	struct rte_dma_dev *dmadev = NULL;
>>