[RFC,1/2] vhost: populate guest memory for DMA-accelerated vhost-user

  DMA engines, like I/OAT, are efficient in moving large data
within memory. Offloading large copies in vhost side to DMA
engines can save precious CPU cycles and improve vhost
performance.

However, using the DMA engine requires to populate guest's
memory. This patch is to enable DMA-accelerated vhost-user
to populate guest's memory.

Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
---
 lib/librte_vhost/rte_vhost.h  |  1 +
 lib/librte_vhost/socket.c     | 10 ++++++++++
 lib/librte_vhost/vhost.h      |  2 ++
 lib/librte_vhost/vhost_user.c |  3 ++-
 4 files changed, 15 insertions(+), 1 deletion(-)
  

On 9/26/19 4:26 PM, Jiayu Hu wrote:
> DMA engines, like I/OAT, are efficient in moving large data
> within memory. Offloading large copies in vhost side to DMA
> engines can save precious CPU cycles and improve vhost
> performance.
> 
> However, using the DMA engine requires to populate guest's
> memory. This patch is to enable DMA-accelerated vhost-user
> to populate guest's memory.
> 
> Signed-off-by: Jiayu Hu <jiayu.hu@intel.com>
> ---
>  lib/librte_vhost/rte_vhost.h  |  1 +
>  lib/librte_vhost/socket.c     | 10 ++++++++++
>  lib/librte_vhost/vhost.h      |  2 ++
>  lib/librte_vhost/vhost_user.c |  3 ++-
>  4 files changed, 15 insertions(+), 1 deletion(-)
> 
> diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
> index 7fb1729..61f8bef 100644
> --- a/lib/librte_vhost/rte_vhost.h
> +++ b/lib/librte_vhost/rte_vhost.h
> @@ -30,6 +30,7 @@ extern "C" {
>  #define RTE_VHOST_USER_DEQUEUE_ZERO_COPY	(1ULL << 2)
>  #define RTE_VHOST_USER_IOMMU_SUPPORT	(1ULL << 3)
>  #define RTE_VHOST_USER_POSTCOPY_SUPPORT		(1ULL << 4)
> +#define RTE_VHOST_USER_DMA_COPY		(1ULL << 5)
>  
>  /** Protocol features. */
>  #ifndef VHOST_USER_PROTOCOL_F_MQ
> diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
> index 274988c..55498c4 100644
> --- a/lib/librte_vhost/socket.c
> +++ b/lib/librte_vhost/socket.c
> @@ -60,6 +60,8 @@ struct vhost_user_socket {
>  	 */
>  	int vdpa_dev_id;
>  
> +	bool dma_enabled;
> +
>  	struct vhost_device_ops const *notify_ops;
>  };
>  
> @@ -232,6 +234,13 @@ vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket)
>  	if (vsocket->dequeue_zero_copy)
>  		vhost_enable_dequeue_zero_copy(vid);
>  
> +	if (vsocket->dma_enabled) {
> +		struct virtio_net *dev;
> +
> +		dev = get_device(vid);
> +		dev->dma_enabled = true;
> +	}
> +
>  	RTE_LOG(INFO, VHOST_CONFIG, "new device, handle is %d\n", vid);
>  
>  	if (vsocket->notify_ops->new_connection) {
> @@ -870,6 +879,7 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
>  		goto out_free;
>  	}
>  	vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
> +	vsocket->dma_enabled = flags & RTE_VHOST_USER_DMA_COPY;
>  
>  	/*
>  	 * Set the supported features correctly for the builtin vhost-user

I think you also need to disable VHOST_USER_PROTOCOL_F_PAGEFAULT from
the supported protocol features if DMA is enabled, since it requires
MAP_POPULATE which is not compatible with postcopy.

> diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
> index 884befa..8f564f1 100644
> --- a/lib/librte_vhost/vhost.h
> +++ b/lib/librte_vhost/vhost.h
> @@ -329,6 +329,8 @@ struct virtio_net {
>  	 */
>  	int			vdpa_dev_id;
>  
> +	bool			dma_enabled;
> +
>  	/* context data for the external message handlers */
>  	void			*extern_data;
>  	/* pre and post vhost user message handlers for the device */
> diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
> index 0b72648..6a4969b 100644
> --- a/lib/librte_vhost/vhost_user.c
> +++ b/lib/librte_vhost/vhost_user.c
> @@ -990,7 +990,8 @@ vhost_user_set_mem_table(struct virtio_net **pdev, struct VhostUserMsg *msg,
>  		}
>  		mmap_size = RTE_ALIGN_CEIL(mmap_size, alignment);
>  
> -		populate = (dev->dequeue_zero_copy) ? MAP_POPULATE : 0;
> +		populate = (dev->dequeue_zero_copy || dev->dma_enabled) ?
> +			MAP_POPULATE : 0;
>  		mmap_addr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
>  				 MAP_SHARED | populate, fd, 0);
>  
>