diff mbox series

[v2,1/3] dma/idxd: fix memory leak in pci close

Message ID 20220703122243.929642-2-kevin.laatz@intel.com (mailing list archive)
State Superseded, archived
Delegated to: Thomas Monjalon
Headers
Series Fix IDXD PCI device close |

Checks

Context Check Description
ci/checkpatch warning coding style issues

Commit Message

Kevin Laatz July 3, 2022, 12:22 p.m. UTC 
  ASAN reports a memory leak for the 'pci' pointer in the 'idxd_dmadev'
struct.

This is fixed by free'ing the struct when the last queue on the PCI
device is being closed.

Fixes: 9449330a8458 ("dma/idxd: create dmadev instances on PCI probe")
Cc: stable@dpdk.org
Cc: bruce.richardson@intel.com

Reported-by: Xingguang He <xingguang.he@intel.com>
Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
---
 drivers/dma/idxd/idxd_common.c   |  2 ++
 drivers/dma/idxd/idxd_internal.h |  2 ++
 drivers/dma/idxd/idxd_pci.c      | 34 +++++++++++++++++++++++++-------
 3 files changed, 31 insertions(+), 7 deletions(-)

Comments

Bruce Richardson July 4, 2022, 1:19 p.m. UTC | #1 
On Sun, Jul 03, 2022 at 01:22:41PM +0100, Kevin Laatz wrote:
> ASAN reports a memory leak for the 'pci' pointer in the 'idxd_dmadev'
> struct.
> 
> This is fixed by free'ing the struct when the last queue on the PCI
> device is being closed.
> 
> Fixes: 9449330a8458 ("dma/idxd: create dmadev instances on PCI probe")
> Cc: stable@dpdk.org
> Cc: bruce.richardson@intel.com
> 
> Reported-by: Xingguang He <xingguang.he@intel.com>
> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> ---
>  drivers/dma/idxd/idxd_common.c   |  2 ++
>  drivers/dma/idxd/idxd_internal.h |  2 ++
>  drivers/dma/idxd/idxd_pci.c      | 34 +++++++++++++++++++++++++-------
>  3 files changed, 31 insertions(+), 7 deletions(-)
> 

Some comments inline below.

/Bruce

> diff --git a/drivers/dma/idxd/idxd_common.c b/drivers/dma/idxd/idxd_common.c
> index c77200a457..d347bbed21 100644
> --- a/drivers/dma/idxd/idxd_common.c
> +++ b/drivers/dma/idxd/idxd_common.c
> @@ -620,6 +620,8 @@ idxd_dmadev_create(const char *name, struct rte_device *dev,
>  	dmadev->fp_obj->dev_private = idxd;
>  
>  	idxd->dmadev->state = RTE_DMA_DEV_READY;
> +	if (idxd->u.pci != NULL)
> +		rte_atomic16_inc(&idxd->u.pci->ref_count);
>  
>  	return 0;
>  

I don't think this belongs in the common code. Can it be put somewhere in
the pci-specific driver code to avoid issues, e.g. after idxd_dmadev_create
returns in probe_pci() function.


> diff --git a/drivers/dma/idxd/idxd_internal.h b/drivers/dma/idxd/idxd_internal.h
> index 3375600217..180a8587c6 100644
> --- a/drivers/dma/idxd/idxd_internal.h
> +++ b/drivers/dma/idxd/idxd_internal.h
> @@ -7,6 +7,7 @@
>  
>  #include <rte_dmadev_pmd.h>
>  #include <rte_spinlock.h>
> +#include <rte_atomic.h>
>  
>  #include "idxd_hw_defs.h"
>  
> @@ -33,6 +34,7 @@ struct idxd_pci_common {
>  	rte_spinlock_t lk;
>  
>  	uint8_t wq_cfg_sz;
> +	rte_atomic16_t ref_count;
>  	volatile struct rte_idxd_bar0 *regs;
>  	volatile uint32_t *wq_regs_base;
>  	volatile struct rte_idxd_grpcfg *grp_regs;
> diff --git a/drivers/dma/idxd/idxd_pci.c b/drivers/dma/idxd/idxd_pci.c
> index 65c6bbf4c1..918981f2ea 100644
> --- a/drivers/dma/idxd/idxd_pci.c
> +++ b/drivers/dma/idxd/idxd_pci.c
> @@ -6,6 +6,7 @@
>  #include <rte_devargs.h>
>  #include <rte_dmadev_pmd.h>
>  #include <rte_malloc.h>
> +#include <rte_atomic.h>
>  
>  #include "idxd_internal.h"
>  
> @@ -115,20 +116,38 @@ idxd_pci_dev_close(struct rte_dma_dev *dev)
>  {
>  	struct idxd_dmadev *idxd = dev->fp_obj->dev_private;
>  	uint8_t err_code;
> +	int is_last_wq;
>  
> -	/* disable the device */
> -	err_code = idxd_pci_dev_command(idxd, idxd_disable_dev);
> -	if (err_code) {
> -		IDXD_PMD_ERR("Error disabling device: code %#x", err_code);
> -		return err_code;
> +	if (idxd_is_wq_enabled(idxd)) {
> +		/* disable the wq */
> +		err_code = idxd_pci_dev_command(idxd, idxd_disable_wq);
> +		if (err_code) {
> +			IDXD_PMD_ERR("Error disabling wq: code %#x", err_code);
> +			return err_code;
> +		}
> +		IDXD_PMD_DEBUG("IDXD WQ disabled OK");
>  	}
> -	IDXD_PMD_DEBUG("IDXD Device disabled OK");
>  
>  	/* free device memory */
>  	IDXD_PMD_DEBUG("Freeing device driver memory");
>  	rte_free(idxd->batch_idx_ring);
>  	rte_free(idxd->desc_ring);
>  
> +	/* if this is the last WQ on the device, disable the device and free
> +	 * the PCI struct
> +	 */
> +	is_last_wq = rte_atomic16_dec_and_test(&idxd->u.pci->ref_count);
> +	if (is_last_wq) {
> +		/* disable the device */
> +		err_code = idxd_pci_dev_command(idxd, idxd_disable_dev);
> +		if (err_code) {
> +			IDXD_PMD_ERR("Error disabling device: code %#x", err_code);
> +			return err_code;
> +		}
> +		IDXD_PMD_DEBUG("IDXD device disabled OK");
> +		rte_free(idxd->u.pci);
> +	}
> +
>  	return 0;
>  }
>  
> @@ -159,12 +178,13 @@ init_pci_device(struct rte_pci_device *dev, struct idxd_dmadev *idxd,
>  	uint8_t lg2_max_batch, lg2_max_copy_size;
>  	unsigned int i, err_code;
>  
> -	pci = malloc(sizeof(*pci));
> +	pci = rte_malloc(NULL, sizeof(*pci), 0);

Any particular reason for the change from regular malloc to rte_malloc?

>  	if (pci == NULL) {
>  		IDXD_PMD_ERR("%s: Can't allocate memory", __func__);
>  		err_code = -1;
>  		goto err;
>  	}
> +	memset(pci, 0, sizeof(*pci));
>  	rte_spinlock_init(&pci->lk);
>  
>  	/* assign the bar registers, and then configure device */
> -- 
> 2.31.1
>
Kevin Laatz July 4, 2022, 1:34 p.m. UTC | #2 
On 04/07/2022 14:19, Bruce Richardson wrote:
> On Sun, Jul 03, 2022 at 01:22:41PM +0100, Kevin Laatz wrote:
>> ASAN reports a memory leak for the 'pci' pointer in the 'idxd_dmadev'
>> struct.
>>
>> This is fixed by free'ing the struct when the last queue on the PCI
>> device is being closed.
>>
>> Fixes: 9449330a8458 ("dma/idxd: create dmadev instances on PCI probe")
>> Cc: stable@dpdk.org
>> Cc: bruce.richardson@intel.com
>>
>> Reported-by: Xingguang He <xingguang.he@intel.com>
>> Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
>> ---
>>   drivers/dma/idxd/idxd_common.c   |  2 ++
>>   drivers/dma/idxd/idxd_internal.h |  2 ++
>>   drivers/dma/idxd/idxd_pci.c      | 34 +++++++++++++++++++++++++-------
>>   3 files changed, 31 insertions(+), 7 deletions(-)
>>
> Some comments inline below.
>
> /Bruce
>
>> diff --git a/drivers/dma/idxd/idxd_common.c b/drivers/dma/idxd/idxd_common.c
>> index c77200a457..d347bbed21 100644
>> --- a/drivers/dma/idxd/idxd_common.c
>> +++ b/drivers/dma/idxd/idxd_common.c
>> @@ -620,6 +620,8 @@ idxd_dmadev_create(const char *name, struct rte_device *dev,
>>   	dmadev->fp_obj->dev_private = idxd;
>>   
>>   	idxd->dmadev->state = RTE_DMA_DEV_READY;
>> +	if (idxd->u.pci != NULL)
>> +		rte_atomic16_inc(&idxd->u.pci->ref_count);
>>   
>>   	return 0;
>>   
> I don't think this belongs in the common code. Can it be put somewhere in
> the pci-specific driver code to avoid issues, e.g. after idxd_dmadev_create
> returns in probe_pci() function.

Sure, I'll look to move it in v3.

[snip]
>   
> @@ -159,12 +178,13 @@ init_pci_device(struct rte_pci_device *dev, struct idxd_dmadev *idxd,
>   	uint8_t lg2_max_batch, lg2_max_copy_size;
>   	unsigned int i, err_code;
>   
> -	pci = malloc(sizeof(*pci));
> +	pci = rte_malloc(NULL, sizeof(*pci), 0);
> Any particular reason for the change from regular malloc to rte_malloc?

Mainly consistency, its the only place in the driver where malloc is 
used instead of rte_malloc.

I have no strong preference here - I can remove the change for v3.

/Kevin
Bruce Richardson July 4, 2022, 1:44 p.m. UTC | #3 
On Mon, Jul 04, 2022 at 02:34:31PM +0100, Kevin Laatz wrote:
> On 04/07/2022 14:19, Bruce Richardson wrote:
> > On Sun, Jul 03, 2022 at 01:22:41PM +0100, Kevin Laatz wrote:
> > > ASAN reports a memory leak for the 'pci' pointer in the 'idxd_dmadev'
> > > struct.
> > > 
> > > This is fixed by free'ing the struct when the last queue on the PCI
> > > device is being closed.
> > > 
> > > Fixes: 9449330a8458 ("dma/idxd: create dmadev instances on PCI probe")
> > > Cc: stable@dpdk.org
> > > Cc: bruce.richardson@intel.com
> > > 
> > > Reported-by: Xingguang He <xingguang.he@intel.com>
> > > Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
> > > ---
> > >   drivers/dma/idxd/idxd_common.c   |  2 ++
> > >   drivers/dma/idxd/idxd_internal.h |  2 ++
> > >   drivers/dma/idxd/idxd_pci.c      | 34 +++++++++++++++++++++++++-------
> > >   3 files changed, 31 insertions(+), 7 deletions(-)
> > > 
> > Some comments inline below.
> > 
> > /Bruce
> > 
> > > diff --git a/drivers/dma/idxd/idxd_common.c b/drivers/dma/idxd/idxd_common.c
> > > index c77200a457..d347bbed21 100644
> > > --- a/drivers/dma/idxd/idxd_common.c
> > > +++ b/drivers/dma/idxd/idxd_common.c
> > > @@ -620,6 +620,8 @@ idxd_dmadev_create(const char *name, struct rte_device *dev,
> > >   	dmadev->fp_obj->dev_private = idxd;
> > >   	idxd->dmadev->state = RTE_DMA_DEV_READY;
> > > +	if (idxd->u.pci != NULL)
> > > +		rte_atomic16_inc(&idxd->u.pci->ref_count);
> > >   	return 0;
> > I don't think this belongs in the common code. Can it be put somewhere in
> > the pci-specific driver code to avoid issues, e.g. after idxd_dmadev_create
> > returns in probe_pci() function.
> 
> Sure, I'll look to move it in v3.
> 
> [snip]
> > @@ -159,12 +178,13 @@ init_pci_device(struct rte_pci_device *dev, struct idxd_dmadev *idxd,
> >   	uint8_t lg2_max_batch, lg2_max_copy_size;
> >   	unsigned int i, err_code;
> > -	pci = malloc(sizeof(*pci));
> > +	pci = rte_malloc(NULL, sizeof(*pci), 0);
> > Any particular reason for the change from regular malloc to rte_malloc?
> 
> Mainly consistency, its the only place in the driver where malloc is used
> instead of rte_malloc.
> 
> I have no strong preference here - I can remove the change for v3.
> 
If it's inconsistent with the rest, please do keep the change in v3.
diff mbox series

Patch

diff --git a/drivers/dma/idxd/idxd_common.c b/drivers/dma/idxd/idxd_common.c
index c77200a457..d347bbed21 100644
--- a/drivers/dma/idxd/idxd_common.c
+++ b/drivers/dma/idxd/idxd_common.c
@@ -620,6 +620,8 @@  idxd_dmadev_create(const char *name, struct rte_device *dev,
 	dmadev->fp_obj->dev_private = idxd;
 
 	idxd->dmadev->state = RTE_DMA_DEV_READY;
+	if (idxd->u.pci != NULL)
+		rte_atomic16_inc(&idxd->u.pci->ref_count);
 
 	return 0;
 
diff --git a/drivers/dma/idxd/idxd_internal.h b/drivers/dma/idxd/idxd_internal.h
index 3375600217..180a8587c6 100644
--- a/drivers/dma/idxd/idxd_internal.h
+++ b/drivers/dma/idxd/idxd_internal.h
@@ -7,6 +7,7 @@ 
 
 #include <rte_dmadev_pmd.h>
 #include <rte_spinlock.h>
+#include <rte_atomic.h>
 
 #include "idxd_hw_defs.h"
 
@@ -33,6 +34,7 @@  struct idxd_pci_common {
 	rte_spinlock_t lk;
 
 	uint8_t wq_cfg_sz;
+	rte_atomic16_t ref_count;
 	volatile struct rte_idxd_bar0 *regs;
 	volatile uint32_t *wq_regs_base;
 	volatile struct rte_idxd_grpcfg *grp_regs;
diff --git a/drivers/dma/idxd/idxd_pci.c b/drivers/dma/idxd/idxd_pci.c
index 65c6bbf4c1..918981f2ea 100644
--- a/drivers/dma/idxd/idxd_pci.c
+++ b/drivers/dma/idxd/idxd_pci.c
@@ -6,6 +6,7 @@ 
 #include <rte_devargs.h>
 #include <rte_dmadev_pmd.h>
 #include <rte_malloc.h>
+#include <rte_atomic.h>
 
 #include "idxd_internal.h"
 
@@ -115,20 +116,38 @@  idxd_pci_dev_close(struct rte_dma_dev *dev)
 {
 	struct idxd_dmadev *idxd = dev->fp_obj->dev_private;
 	uint8_t err_code;
+	int is_last_wq;
 
-	/* disable the device */
-	err_code = idxd_pci_dev_command(idxd, idxd_disable_dev);
-	if (err_code) {
-		IDXD_PMD_ERR("Error disabling device: code %#x", err_code);
-		return err_code;
+	if (idxd_is_wq_enabled(idxd)) {
+		/* disable the wq */
+		err_code = idxd_pci_dev_command(idxd, idxd_disable_wq);
+		if (err_code) {
+			IDXD_PMD_ERR("Error disabling wq: code %#x", err_code);
+			return err_code;
+		}
+		IDXD_PMD_DEBUG("IDXD WQ disabled OK");
 	}
-	IDXD_PMD_DEBUG("IDXD Device disabled OK");
 
 	/* free device memory */
 	IDXD_PMD_DEBUG("Freeing device driver memory");
 	rte_free(idxd->batch_idx_ring);
 	rte_free(idxd->desc_ring);
 
+	/* if this is the last WQ on the device, disable the device and free
+	 * the PCI struct
+	 */
+	is_last_wq = rte_atomic16_dec_and_test(&idxd->u.pci->ref_count);
+	if (is_last_wq) {
+		/* disable the device */
+		err_code = idxd_pci_dev_command(idxd, idxd_disable_dev);
+		if (err_code) {
+			IDXD_PMD_ERR("Error disabling device: code %#x", err_code);
+			return err_code;
+		}
+		IDXD_PMD_DEBUG("IDXD device disabled OK");
+		rte_free(idxd->u.pci);
+	}
+
 	return 0;
 }
 
@@ -159,12 +178,13 @@  init_pci_device(struct rte_pci_device *dev, struct idxd_dmadev *idxd,
 	uint8_t lg2_max_batch, lg2_max_copy_size;
 	unsigned int i, err_code;
 
-	pci = malloc(sizeof(*pci));
+	pci = rte_malloc(NULL, sizeof(*pci), 0);
 	if (pci == NULL) {
 		IDXD_PMD_ERR("%s: Can't allocate memory", __func__);
 		err_code = -1;
 		goto err;
 	}
+	memset(pci, 0, sizeof(*pci));
 	rte_spinlock_init(&pci->lk);
 
 	/* assign the bar registers, and then configure device */