diff mbox

[dpdk-dev,v5,5/6] vhost: batch update used ring

Message ID 1473392368-84903-6-git-send-email-zhihong.wang@intel.com (mailing list archive)
State Superseded, archived
Delegated to: Yuanhan Liu
Headers

Commit Message

Zhihong Wang Sept. 9, 2016, 3:39 a.m. UTC 
  This patch enables batch update of the used ring for better efficiency.

Signed-off-by: Zhihong Wang <zhihong.wang@intel.com>
---
Changes in v4:

 1. Free shadow used ring in the right place.

 2. Add failure check for shadow used ring malloc.

 lib/librte_vhost/vhost.c      | 20 ++++++++++++--
 lib/librte_vhost/vhost.h      |  4 +++
 lib/librte_vhost/vhost_user.c | 31 +++++++++++++++++----
 lib/librte_vhost/virtio_net.c | 64 +++++++++++++++++++++++++++++++++++--------
 4 files changed, 101 insertions(+), 18 deletions(-)

Comments

Maxime Coquelin Sept. 12, 2016, 3:45 p.m. UTC | #1 
On 09/09/2016 05:39 AM, Zhihong Wang wrote:
> This patch enables batch update of the used ring for better efficiency.
>
> Signed-off-by: Zhihong Wang <zhihong.wang@intel.com>
> ---
> Changes in v4:
>
>  1. Free shadow used ring in the right place.
>
>  2. Add failure check for shadow used ring malloc.
>
>  lib/librte_vhost/vhost.c      | 20 ++++++++++++--
>  lib/librte_vhost/vhost.h      |  4 +++
>  lib/librte_vhost/vhost_user.c | 31 +++++++++++++++++----
>  lib/librte_vhost/virtio_net.c | 64 +++++++++++++++++++++++++++++++++++--------
>  4 files changed, 101 insertions(+), 18 deletions(-)
>
> diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
> index 46095c3..cb31cdd 100644
> --- a/lib/librte_vhost/vhost.c
> +++ b/lib/librte_vhost/vhost.c
> @@ -119,10 +119,26 @@ cleanup_device(struct virtio_net *dev, int destroy)
>  static void
>  free_device(struct virtio_net *dev)
>  {
> +	struct vhost_virtqueue *vq_0;
> +	struct vhost_virtqueue *vq_1;
>  	uint32_t i;
>
> -	for (i = 0; i < dev->virt_qp_nb; i++)
> -		rte_free(dev->virtqueue[i * VIRTIO_QNUM]);
> +	for (i = 0; i < dev->virt_qp_nb; i++) {
> +		vq_0 = dev->virtqueue[i * VIRTIO_QNUM];
> +		if (vq_0->shadow_used_ring) {
> +			rte_free(vq_0->shadow_used_ring);
> +			vq_0->shadow_used_ring = NULL;
> +		}
> +
> +		vq_1 = dev->virtqueue[i * VIRTIO_QNUM + 1];
> +		if (vq_1->shadow_used_ring) {
> +			rte_free(vq_1->shadow_used_ring);
> +			vq_1->shadow_used_ring = NULL;
> +		}
> +
> +		/* malloc together, free together */
> +		rte_free(vq_0);
> +	}
>
>  	rte_free(dev);
>  }
> diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
> index 9707dfc..381dc27 100644
> --- a/lib/librte_vhost/vhost.h
> +++ b/lib/librte_vhost/vhost.h
> @@ -85,6 +85,10 @@ struct vhost_virtqueue {
>
>  	/* Physical address of used ring, for logging */
>  	uint64_t		log_guest_addr;
> +
> +	/* Shadow used ring for performance */
> +	struct vring_used_elem	*shadow_used_ring;
> +	uint32_t		shadow_used_idx;
>  } __rte_cache_aligned;
>
>  /* Old kernels have no such macro defined */
> diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
> index eee99e9..d7cf1ed 100644
> --- a/lib/librte_vhost/vhost_user.c
> +++ b/lib/librte_vhost/vhost_user.c
> @@ -193,7 +193,21 @@ static int
>  vhost_user_set_vring_num(struct virtio_net *dev,
>  			 struct vhost_vring_state *state)
>  {
> -	dev->virtqueue[state->index]->size = state->num;
> +	struct vhost_virtqueue *vq;
> +
> +	vq = dev->virtqueue[state->index];
> +	vq->size = state->num;
> +	if (!vq->shadow_used_ring) {
> +		vq->shadow_used_ring = rte_malloc(NULL,
> +				vq->size * sizeof(struct vring_used_elem),
> +				RTE_CACHE_LINE_SIZE);
> +		if (!vq->shadow_used_ring) {
> +			RTE_LOG(ERR, VHOST_CONFIG,
> +				"Failed to allocate memory"
> +				" for shadow used ring.\n");
> +			return -1;
> +		}
> +	}
>
>  	return 0;
>  }
> @@ -611,14 +625,21 @@ static int
>  vhost_user_get_vring_base(struct virtio_net *dev,
>  			  struct vhost_vring_state *state)
>  {
> +	struct vhost_virtqueue *vq;
> +
>  	/* We have to stop the queue (virtio) if it is running. */
>  	if (dev->flags & VIRTIO_DEV_RUNNING) {
>  		dev->flags &= ~VIRTIO_DEV_RUNNING;
>  		notify_ops->destroy_device(dev->vid);
>  	}
>
> +	vq = dev->virtqueue[state->index];
>  	/* Here we are safe to get the last used index */
> -	state->num = dev->virtqueue[state->index]->last_used_idx;
> +	state->num = vq->last_used_idx;
> +	if (vq->shadow_used_ring) {
> +		rte_free(vq->shadow_used_ring);
> +		vq->shadow_used_ring = NULL;
> +	}
>
>  	RTE_LOG(INFO, VHOST_CONFIG,
>  		"vring base idx:%d file:%d\n", state->index, state->num);
> @@ -627,10 +648,10 @@ vhost_user_get_vring_base(struct virtio_net *dev,
>  	 * sent and only sent in vhost_vring_stop.
>  	 * TODO: cleanup the vring, it isn't usable since here.
>  	 */
> -	if (dev->virtqueue[state->index]->kickfd >= 0)
> -		close(dev->virtqueue[state->index]->kickfd);
> +	if (vq->kickfd >= 0)
> +		close(vq->kickfd);
>
> -	dev->virtqueue[state->index]->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
> +	vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
>
>  	return 0;
>  }
> diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
> index b38f18f..e9f6353 100644
> --- a/lib/librte_vhost/virtio_net.c
> +++ b/lib/librte_vhost/virtio_net.c
> @@ -134,17 +134,52 @@ virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)
>  }
>
>  static inline void __attribute__((always_inline))
> -update_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
> -		uint32_t desc_chain_head, uint32_t desc_chain_len)
> +update_used_ring(struct vhost_virtqueue *vq, uint32_t desc_chain_head,
> +		uint32_t desc_chain_len)
>  {
> -	uint32_t used_idx = vq->last_used_idx & (vq->size - 1);
> -
> -	vq->used->ring[used_idx].id = desc_chain_head;
> -	vq->used->ring[used_idx].len = desc_chain_len;
> +	vq->shadow_used_ring[vq->shadow_used_idx].id  = desc_chain_head;
> +	vq->shadow_used_ring[vq->shadow_used_idx].len = desc_chain_len;
> +	vq->shadow_used_idx++;
>  	vq->last_used_idx++;
> -	vhost_log_used_vring(dev, vq, offsetof(struct vring_used,
> -				ring[used_idx]),
> -			sizeof(vq->used->ring[used_idx]));
> +}
> +
> +static inline void __attribute__((always_inline))
> +flush_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
> +		uint32_t used_idx_start)
> +{
> +	if (used_idx_start + vq->shadow_used_idx < vq->size) {
> +		rte_memcpy(&vq->used->ring[used_idx_start],
> +				&vq->shadow_used_ring[0],
> +				vq->shadow_used_idx *
> +				sizeof(struct vring_used_elem));
> +		vhost_log_used_vring(dev, vq,
> +				offsetof(struct vring_used,
> +					ring[used_idx_start]),
> +				vq->shadow_used_idx *
> +				sizeof(struct vring_used_elem));
> +	} else {
> +		uint32_t part_1 = vq->size - used_idx_start;
> +		uint32_t part_2 = vq->shadow_used_idx - part_1;
> +
> +		rte_memcpy(&vq->used->ring[used_idx_start],
> +				&vq->shadow_used_ring[0],
> +				part_1 *
> +				sizeof(struct vring_used_elem));
> +		vhost_log_used_vring(dev, vq,
> +				offsetof(struct vring_used,
> +					ring[used_idx_start]),
> +				part_1 *
> +				sizeof(struct vring_used_elem));
> +		rte_memcpy(&vq->used->ring[0],
> +				&vq->shadow_used_ring[part_1],
> +				part_2 *
> +				sizeof(struct vring_used_elem));
> +		vhost_log_used_vring(dev, vq,
> +				offsetof(struct vring_used,
> +					ring[0]),
> +				part_2 *
> +				sizeof(struct vring_used_elem));
> +	}
>  }
Is expanding the code done for performance purpose?
Or maybe we could have a loop to do that?
Something like this (not compiled, not tested):

static inline void __attribute__((always_inline))
flush_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
{
	uint32_t to = used_idx_start;
	uint32_t from = 0;
	uint32_t count;

	if (used_idx_start + vq->shadow_used_idx < vq->size)
		count = vq->shadow_used_idx;
	else
		count = vq->size - used_idx_start;

	do {
		rte_memcpy(&vq->used->ring[to],
				&vq->shadow_used_ring[from],
				count * sizeof(struct vring_used_elem));
		vhost_log_used_vring(dev, vq,
				offsetof(struct vring_used, ring[to]),
				count * sizeof(struct vring_used_elem));

		to = (to + count) & (vq->size - 1);
		from += count;
		count = vq->shadow_used_idx - count;
	} while (count);
}

Regards,
Maxime
Zhihong Wang Sept. 14, 2016, 8:43 a.m. UTC | #2 
> -----Original Message-----
> From: Maxime Coquelin [mailto:maxime.coquelin@redhat.com]
> Sent: Monday, September 12, 2016 11:46 PM
> To: Wang, Zhihong <zhihong.wang@intel.com>; dev@dpdk.org
> Cc: yuanhan.liu@linux.intel.com; thomas.monjalon@6wind.com
> Subject: Re: [PATCH v5 5/6] vhost: batch update used ring
> 
> 
> 
> On 09/09/2016 05:39 AM, Zhihong Wang wrote:
> > This patch enables batch update of the used ring for better efficiency.
> >
> > Signed-off-by: Zhihong Wang <zhihong.wang@intel.com>
> > ---
> > Changes in v4:
> >
> >  1. Free shadow used ring in the right place.
> >
> >  2. Add failure check for shadow used ring malloc.
> >
> >  lib/librte_vhost/vhost.c      | 20 ++++++++++++--
> >  lib/librte_vhost/vhost.h      |  4 +++
> >  lib/librte_vhost/vhost_user.c | 31 +++++++++++++++++----
> >  lib/librte_vhost/virtio_net.c | 64
> +++++++++++++++++++++++++++++++++++--------
> >  4 files changed, 101 insertions(+), 18 deletions(-)
> >
> > diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
> > index 46095c3..cb31cdd 100644
> > --- a/lib/librte_vhost/vhost.c
> > +++ b/lib/librte_vhost/vhost.c
> > @@ -119,10 +119,26 @@ cleanup_device(struct virtio_net *dev, int
> destroy)
> >  static void
> >  free_device(struct virtio_net *dev)
> >  {
> > +	struct vhost_virtqueue *vq_0;
> > +	struct vhost_virtqueue *vq_1;
> >  	uint32_t i;
> >
> > -	for (i = 0; i < dev->virt_qp_nb; i++)
> > -		rte_free(dev->virtqueue[i * VIRTIO_QNUM]);
> > +	for (i = 0; i < dev->virt_qp_nb; i++) {
> > +		vq_0 = dev->virtqueue[i * VIRTIO_QNUM];
> > +		if (vq_0->shadow_used_ring) {
> > +			rte_free(vq_0->shadow_used_ring);
> > +			vq_0->shadow_used_ring = NULL;
> > +		}
> > +
> > +		vq_1 = dev->virtqueue[i * VIRTIO_QNUM + 1];
> > +		if (vq_1->shadow_used_ring) {
> > +			rte_free(vq_1->shadow_used_ring);
> > +			vq_1->shadow_used_ring = NULL;
> > +		}
> > +
> > +		/* malloc together, free together */
> > +		rte_free(vq_0);
> > +	}
> >
> >  	rte_free(dev);
> >  }
> > diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
> > index 9707dfc..381dc27 100644
> > --- a/lib/librte_vhost/vhost.h
> > +++ b/lib/librte_vhost/vhost.h
> > @@ -85,6 +85,10 @@ struct vhost_virtqueue {
> >
> >  	/* Physical address of used ring, for logging */
> >  	uint64_t		log_guest_addr;
> > +
> > +	/* Shadow used ring for performance */
> > +	struct vring_used_elem	*shadow_used_ring;
> > +	uint32_t		shadow_used_idx;
> >  } __rte_cache_aligned;
> >
> >  /* Old kernels have no such macro defined */
> > diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
> > index eee99e9..d7cf1ed 100644
> > --- a/lib/librte_vhost/vhost_user.c
> > +++ b/lib/librte_vhost/vhost_user.c
> > @@ -193,7 +193,21 @@ static int
> >  vhost_user_set_vring_num(struct virtio_net *dev,
> >  			 struct vhost_vring_state *state)
> >  {
> > -	dev->virtqueue[state->index]->size = state->num;
> > +	struct vhost_virtqueue *vq;
> > +
> > +	vq = dev->virtqueue[state->index];
> > +	vq->size = state->num;
> > +	if (!vq->shadow_used_ring) {
> > +		vq->shadow_used_ring = rte_malloc(NULL,
> > +				vq->size * sizeof(struct vring_used_elem),
> > +				RTE_CACHE_LINE_SIZE);
> > +		if (!vq->shadow_used_ring) {
> > +			RTE_LOG(ERR, VHOST_CONFIG,
> > +				"Failed to allocate memory"
> > +				" for shadow used ring.\n");
> > +			return -1;
> > +		}
> > +	}
> >
> >  	return 0;
> >  }
> > @@ -611,14 +625,21 @@ static int
> >  vhost_user_get_vring_base(struct virtio_net *dev,
> >  			  struct vhost_vring_state *state)
> >  {
> > +	struct vhost_virtqueue *vq;
> > +
> >  	/* We have to stop the queue (virtio) if it is running. */
> >  	if (dev->flags & VIRTIO_DEV_RUNNING) {
> >  		dev->flags &= ~VIRTIO_DEV_RUNNING;
> >  		notify_ops->destroy_device(dev->vid);
> >  	}
> >
> > +	vq = dev->virtqueue[state->index];
> >  	/* Here we are safe to get the last used index */
> > -	state->num = dev->virtqueue[state->index]->last_used_idx;
> > +	state->num = vq->last_used_idx;
> > +	if (vq->shadow_used_ring) {
> > +		rte_free(vq->shadow_used_ring);
> > +		vq->shadow_used_ring = NULL;
> > +	}
> >
> >  	RTE_LOG(INFO, VHOST_CONFIG,
> >  		"vring base idx:%d file:%d\n", state->index, state->num);
> > @@ -627,10 +648,10 @@ vhost_user_get_vring_base(struct virtio_net
> *dev,
> >  	 * sent and only sent in vhost_vring_stop.
> >  	 * TODO: cleanup the vring, it isn't usable since here.
> >  	 */
> > -	if (dev->virtqueue[state->index]->kickfd >= 0)
> > -		close(dev->virtqueue[state->index]->kickfd);
> > +	if (vq->kickfd >= 0)
> > +		close(vq->kickfd);
> >
> > -	dev->virtqueue[state->index]->kickfd =
> VIRTIO_UNINITIALIZED_EVENTFD;
> > +	vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
> >
> >  	return 0;
> >  }
> > diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
> > index b38f18f..e9f6353 100644
> > --- a/lib/librte_vhost/virtio_net.c
> > +++ b/lib/librte_vhost/virtio_net.c
> > @@ -134,17 +134,52 @@ virtio_enqueue_offload(struct rte_mbuf
> *m_buf, struct virtio_net_hdr *net_hdr)
> >  }
> >
> >  static inline void __attribute__((always_inline))
> > -update_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
> > -		uint32_t desc_chain_head, uint32_t desc_chain_len)
> > +update_used_ring(struct vhost_virtqueue *vq, uint32_t
> desc_chain_head,
> > +		uint32_t desc_chain_len)
> >  {
> > -	uint32_t used_idx = vq->last_used_idx & (vq->size - 1);
> > -
> > -	vq->used->ring[used_idx].id = desc_chain_head;
> > -	vq->used->ring[used_idx].len = desc_chain_len;
> > +	vq->shadow_used_ring[vq->shadow_used_idx].id  =
> desc_chain_head;
> > +	vq->shadow_used_ring[vq->shadow_used_idx].len =
> desc_chain_len;
> > +	vq->shadow_used_idx++;
> >  	vq->last_used_idx++;
> > -	vhost_log_used_vring(dev, vq, offsetof(struct vring_used,
> > -				ring[used_idx]),
> > -			sizeof(vq->used->ring[used_idx]));
> > +}
> > +
> > +static inline void __attribute__((always_inline))
> > +flush_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
> > +		uint32_t used_idx_start)
> > +{
> > +	if (used_idx_start + vq->shadow_used_idx < vq->size) {
> > +		rte_memcpy(&vq->used->ring[used_idx_start],
> > +				&vq->shadow_used_ring[0],
> > +				vq->shadow_used_idx *
> > +				sizeof(struct vring_used_elem));
> > +		vhost_log_used_vring(dev, vq,
> > +				offsetof(struct vring_used,
> > +					ring[used_idx_start]),
> > +				vq->shadow_used_idx *
> > +				sizeof(struct vring_used_elem));
> > +	} else {
> > +		uint32_t part_1 = vq->size - used_idx_start;
> > +		uint32_t part_2 = vq->shadow_used_idx - part_1;
> > +
> > +		rte_memcpy(&vq->used->ring[used_idx_start],
> > +				&vq->shadow_used_ring[0],
> > +				part_1 *
> > +				sizeof(struct vring_used_elem));
> > +		vhost_log_used_vring(dev, vq,
> > +				offsetof(struct vring_used,
> > +					ring[used_idx_start]),
> > +				part_1 *
> > +				sizeof(struct vring_used_elem));
> > +		rte_memcpy(&vq->used->ring[0],
> > +				&vq->shadow_used_ring[part_1],
> > +				part_2 *
> > +				sizeof(struct vring_used_elem));
> > +		vhost_log_used_vring(dev, vq,
> > +				offsetof(struct vring_used,
> > +					ring[0]),
> > +				part_2 *
> > +				sizeof(struct vring_used_elem));
> > +	}
> >  }
> Is expanding the code done for performance purpose?

Hi Maxime,

Yes theoretically this has the least branch number.
And I think the logic is simpler this way.

Thanks
Zhihong

> Or maybe we could have a loop to do that?
> Something like this (not compiled, not tested):
> 
> static inline void __attribute__((always_inline))
> flush_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
> {
> 	uint32_t to = used_idx_start;
> 	uint32_t from = 0;
> 	uint32_t count;
> 
> 	if (used_idx_start + vq->shadow_used_idx < vq->size)
> 		count = vq->shadow_used_idx;
> 	else
> 		count = vq->size - used_idx_start;
> 
> 	do {
> 		rte_memcpy(&vq->used->ring[to],
> 				&vq->shadow_used_ring[from],
> 				count * sizeof(struct vring_used_elem));
> 		vhost_log_used_vring(dev, vq,
> 				offsetof(struct vring_used, ring[to]),
> 				count * sizeof(struct vring_used_elem));
> 
> 		to = (to + count) & (vq->size - 1);
> 		from += count;
> 		count = vq->shadow_used_idx - count;
> 	} while (count);
> }
> 
> Regards,
> Maxime
Maxime Coquelin Sept. 15, 2016, 4:38 p.m. UTC | #3 
On 09/14/2016 10:43 AM, Wang, Zhihong wrote:
>
>
>> -----Original Message-----
>> From: Maxime Coquelin [mailto:maxime.coquelin@redhat.com]
>> Sent: Monday, September 12, 2016 11:46 PM
>> To: Wang, Zhihong <zhihong.wang@intel.com>; dev@dpdk.org
>> Cc: yuanhan.liu@linux.intel.com; thomas.monjalon@6wind.com
>> Subject: Re: [PATCH v5 5/6] vhost: batch update used ring
>>
>>
>>
>> On 09/09/2016 05:39 AM, Zhihong Wang wrote:
>>> This patch enables batch update of the used ring for better efficiency.
>>>
>>> Signed-off-by: Zhihong Wang <zhihong.wang@intel.com>
>>> ---
>>> Changes in v4:
>>>
>>>  1. Free shadow used ring in the right place.
>>>
>>>  2. Add failure check for shadow used ring malloc.
>>>
>>>  lib/librte_vhost/vhost.c      | 20 ++++++++++++--
>>>  lib/librte_vhost/vhost.h      |  4 +++
>>>  lib/librte_vhost/vhost_user.c | 31 +++++++++++++++++----
>>>  lib/librte_vhost/virtio_net.c | 64
>> +++++++++++++++++++++++++++++++++++--------
>>>  4 files changed, 101 insertions(+), 18 deletions(-)
>>>
>>> diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
>>> index 46095c3..cb31cdd 100644
>>> --- a/lib/librte_vhost/vhost.c
>>> +++ b/lib/librte_vhost/vhost.c
>>> @@ -119,10 +119,26 @@ cleanup_device(struct virtio_net *dev, int
>> destroy)
>>>  static void
>>>  free_device(struct virtio_net *dev)
>>>  {
>>> +	struct vhost_virtqueue *vq_0;
>>> +	struct vhost_virtqueue *vq_1;
>>>  	uint32_t i;
>>>
>>> -	for (i = 0; i < dev->virt_qp_nb; i++)
>>> -		rte_free(dev->virtqueue[i * VIRTIO_QNUM]);
>>> +	for (i = 0; i < dev->virt_qp_nb; i++) {
>>> +		vq_0 = dev->virtqueue[i * VIRTIO_QNUM];
>>> +		if (vq_0->shadow_used_ring) {
>>> +			rte_free(vq_0->shadow_used_ring);
>>> +			vq_0->shadow_used_ring = NULL;
>>> +		}
>>> +
>>> +		vq_1 = dev->virtqueue[i * VIRTIO_QNUM + 1];
>>> +		if (vq_1->shadow_used_ring) {
>>> +			rte_free(vq_1->shadow_used_ring);
>>> +			vq_1->shadow_used_ring = NULL;
>>> +		}
>>> +
>>> +		/* malloc together, free together */
>>> +		rte_free(vq_0);
>>> +	}
>>>
>>>  	rte_free(dev);
>>>  }
>>> diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
>>> index 9707dfc..381dc27 100644
>>> --- a/lib/librte_vhost/vhost.h
>>> +++ b/lib/librte_vhost/vhost.h
>>> @@ -85,6 +85,10 @@ struct vhost_virtqueue {
>>>
>>>  	/* Physical address of used ring, for logging */
>>>  	uint64_t		log_guest_addr;
>>> +
>>> +	/* Shadow used ring for performance */
>>> +	struct vring_used_elem	*shadow_used_ring;
>>> +	uint32_t		shadow_used_idx;
>>>  } __rte_cache_aligned;
>>>
>>>  /* Old kernels have no such macro defined */
>>> diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
>>> index eee99e9..d7cf1ed 100644
>>> --- a/lib/librte_vhost/vhost_user.c
>>> +++ b/lib/librte_vhost/vhost_user.c
>>> @@ -193,7 +193,21 @@ static int
>>>  vhost_user_set_vring_num(struct virtio_net *dev,
>>>  			 struct vhost_vring_state *state)
>>>  {
>>> -	dev->virtqueue[state->index]->size = state->num;
>>> +	struct vhost_virtqueue *vq;
>>> +
>>> +	vq = dev->virtqueue[state->index];
>>> +	vq->size = state->num;
>>> +	if (!vq->shadow_used_ring) {
>>> +		vq->shadow_used_ring = rte_malloc(NULL,
>>> +				vq->size * sizeof(struct vring_used_elem),
>>> +				RTE_CACHE_LINE_SIZE);
>>> +		if (!vq->shadow_used_ring) {
>>> +			RTE_LOG(ERR, VHOST_CONFIG,
>>> +				"Failed to allocate memory"
>>> +				" for shadow used ring.\n");
>>> +			return -1;
>>> +		}
>>> +	}
>>>
>>>  	return 0;
>>>  }
>>> @@ -611,14 +625,21 @@ static int
>>>  vhost_user_get_vring_base(struct virtio_net *dev,
>>>  			  struct vhost_vring_state *state)
>>>  {
>>> +	struct vhost_virtqueue *vq;
>>> +
>>>  	/* We have to stop the queue (virtio) if it is running. */
>>>  	if (dev->flags & VIRTIO_DEV_RUNNING) {
>>>  		dev->flags &= ~VIRTIO_DEV_RUNNING;
>>>  		notify_ops->destroy_device(dev->vid);
>>>  	}
>>>
>>> +	vq = dev->virtqueue[state->index];
>>>  	/* Here we are safe to get the last used index */
>>> -	state->num = dev->virtqueue[state->index]->last_used_idx;
>>> +	state->num = vq->last_used_idx;
>>> +	if (vq->shadow_used_ring) {
>>> +		rte_free(vq->shadow_used_ring);
>>> +		vq->shadow_used_ring = NULL;
>>> +	}
>>>
>>>  	RTE_LOG(INFO, VHOST_CONFIG,
>>>  		"vring base idx:%d file:%d\n", state->index, state->num);
>>> @@ -627,10 +648,10 @@ vhost_user_get_vring_base(struct virtio_net
>> *dev,
>>>  	 * sent and only sent in vhost_vring_stop.
>>>  	 * TODO: cleanup the vring, it isn't usable since here.
>>>  	 */
>>> -	if (dev->virtqueue[state->index]->kickfd >= 0)
>>> -		close(dev->virtqueue[state->index]->kickfd);
>>> +	if (vq->kickfd >= 0)
>>> +		close(vq->kickfd);
>>>
>>> -	dev->virtqueue[state->index]->kickfd =
>> VIRTIO_UNINITIALIZED_EVENTFD;
>>> +	vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
>>>
>>>  	return 0;
>>>  }
>>> diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
>>> index b38f18f..e9f6353 100644
>>> --- a/lib/librte_vhost/virtio_net.c
>>> +++ b/lib/librte_vhost/virtio_net.c
>>> @@ -134,17 +134,52 @@ virtio_enqueue_offload(struct rte_mbuf
>> *m_buf, struct virtio_net_hdr *net_hdr)
>>>  }
>>>
>>>  static inline void __attribute__((always_inline))
>>> -update_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>> -		uint32_t desc_chain_head, uint32_t desc_chain_len)
>>> +update_used_ring(struct vhost_virtqueue *vq, uint32_t
>> desc_chain_head,
>>> +		uint32_t desc_chain_len)
>>>  {
>>> -	uint32_t used_idx = vq->last_used_idx & (vq->size - 1);
>>> -
>>> -	vq->used->ring[used_idx].id = desc_chain_head;
>>> -	vq->used->ring[used_idx].len = desc_chain_len;
>>> +	vq->shadow_used_ring[vq->shadow_used_idx].id  =
>> desc_chain_head;
>>> +	vq->shadow_used_ring[vq->shadow_used_idx].len =
>> desc_chain_len;
>>> +	vq->shadow_used_idx++;
>>>  	vq->last_used_idx++;
>>> -	vhost_log_used_vring(dev, vq, offsetof(struct vring_used,
>>> -				ring[used_idx]),
>>> -			sizeof(vq->used->ring[used_idx]));
>>> +}
>>> +
>>> +static inline void __attribute__((always_inline))
>>> +flush_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>> +		uint32_t used_idx_start)
>>> +{
>>> +	if (used_idx_start + vq->shadow_used_idx < vq->size) {
>>> +		rte_memcpy(&vq->used->ring[used_idx_start],
>>> +				&vq->shadow_used_ring[0],
>>> +				vq->shadow_used_idx *
>>> +				sizeof(struct vring_used_elem));
>>> +		vhost_log_used_vring(dev, vq,
>>> +				offsetof(struct vring_used,
>>> +					ring[used_idx_start]),
>>> +				vq->shadow_used_idx *
>>> +				sizeof(struct vring_used_elem));
>>> +	} else {
>>> +		uint32_t part_1 = vq->size - used_idx_start;
>>> +		uint32_t part_2 = vq->shadow_used_idx - part_1;
>>> +
>>> +		rte_memcpy(&vq->used->ring[used_idx_start],
>>> +				&vq->shadow_used_ring[0],
>>> +				part_1 *
>>> +				sizeof(struct vring_used_elem));
>>> +		vhost_log_used_vring(dev, vq,
>>> +				offsetof(struct vring_used,
>>> +					ring[used_idx_start]),
>>> +				part_1 *
>>> +				sizeof(struct vring_used_elem));
>>> +		rte_memcpy(&vq->used->ring[0],
>>> +				&vq->shadow_used_ring[part_1],
>>> +				part_2 *
>>> +				sizeof(struct vring_used_elem));
>>> +		vhost_log_used_vring(dev, vq,
>>> +				offsetof(struct vring_used,
>>> +					ring[0]),
>>> +				part_2 *
>>> +				sizeof(struct vring_used_elem));
>>> +	}
>>>  }
>> Is expanding the code done for performance purpose?
>
> Hi Maxime,
>
> Yes theoretically this has the least branch number.
> And I think the logic is simpler this way.
Ok, in that case, maybe you could create a function to
do the rte_memcpy and the vhost_log_used on a given range.

I don't have a strong opinion on this, if Yuanhan is fine
with current code, that's ok for me.
Thanks,
Maxime
>
> Thanks
> Zhihong
>
>> Or maybe we could have a loop to do that?
>> Something like this (not compiled, not tested):
>>
>> static inline void __attribute__((always_inline))
>> flush_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
>> {
>> 	uint32_t to = used_idx_start;
>> 	uint32_t from = 0;
>> 	uint32_t count;
>>
>> 	if (used_idx_start + vq->shadow_used_idx < vq->size)
>> 		count = vq->shadow_used_idx;
>> 	else
>> 		count = vq->size - used_idx_start;
>>
>> 	do {
>> 		rte_memcpy(&vq->used->ring[to],
>> 				&vq->shadow_used_ring[from],
>> 				count * sizeof(struct vring_used_elem));
>> 		vhost_log_used_vring(dev, vq,
>> 				offsetof(struct vring_used, ring[to]),
>> 				count * sizeof(struct vring_used_elem));
>>
>> 		to = (to + count) & (vq->size - 1);
>> 		from += count;
>> 		count = vq->shadow_used_idx - count;
>> 	} while (count);
>> }
>>
>> Regards,
>> Maxime
Yuanhan Liu Sept. 18, 2016, 2:55 a.m. UTC | #4 
On Thu, Sep 15, 2016 at 06:38:06PM +0200, Maxime Coquelin wrote:
> >>>+static inline void __attribute__((always_inline))
> >>>+flush_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
> >>>+		uint32_t used_idx_start)
> >>>+{
> >>>+	if (used_idx_start + vq->shadow_used_idx < vq->size) {
> >>>+		rte_memcpy(&vq->used->ring[used_idx_start],
> >>>+				&vq->shadow_used_ring[0],
> >>>+				vq->shadow_used_idx *
> >>>+				sizeof(struct vring_used_elem));
> >>>+		vhost_log_used_vring(dev, vq,
> >>>+				offsetof(struct vring_used,
> >>>+					ring[used_idx_start]),
> >>>+				vq->shadow_used_idx *
> >>>+				sizeof(struct vring_used_elem));
> >>>+	} else {
> >>>+		uint32_t part_1 = vq->size - used_idx_start;
> >>>+		uint32_t part_2 = vq->shadow_used_idx - part_1;
> >>>+
> >>>+		rte_memcpy(&vq->used->ring[used_idx_start],
> >>>+				&vq->shadow_used_ring[0],
> >>>+				part_1 *
> >>>+				sizeof(struct vring_used_elem));
> >>>+		vhost_log_used_vring(dev, vq,
> >>>+				offsetof(struct vring_used,
> >>>+					ring[used_idx_start]),
> >>>+				part_1 *
> >>>+				sizeof(struct vring_used_elem));
> >>>+		rte_memcpy(&vq->used->ring[0],
> >>>+				&vq->shadow_used_ring[part_1],
> >>>+				part_2 *
> >>>+				sizeof(struct vring_used_elem));
> >>>+		vhost_log_used_vring(dev, vq,
> >>>+				offsetof(struct vring_used,
> >>>+					ring[0]),
> >>>+				part_2 *
> >>>+				sizeof(struct vring_used_elem));
> >>>+	}
> >>> }
> >>Is expanding the code done for performance purpose?
> >
> >Hi Maxime,
> >
> >Yes theoretically this has the least branch number.
> >And I think the logic is simpler this way.
> Ok, in that case, maybe you could create a function to
> do the rte_memcpy and the vhost_log_used on a given range.

Agreed, that will be better; it could avoid repeating similar code
block 3 times.

> I don't have a strong opinion on this, if Yuanhan is fine
> with current code, that's ok for me.

From what I know, that's kind of DPDK prefered way, to expand code
when necessary. For example, 9ec201f5d6e7 ("mbuf: provide bulk
allocation").

So I'm fine with it.

	--yliu
Zhihong Wang Sept. 18, 2016, 2:57 a.m. UTC | #5 
> -----Original Message-----
> From: Yuanhan Liu [mailto:yuanhan.liu@linux.intel.com]
> Sent: Sunday, September 18, 2016 10:56 AM
> To: Maxime Coquelin <maxime.coquelin@redhat.com>
> Cc: Wang, Zhihong <zhihong.wang@intel.com>; dev@dpdk.org;
> thomas.monjalon@6wind.com
> Subject: Re: [PATCH v5 5/6] vhost: batch update used ring
> 
> On Thu, Sep 15, 2016 at 06:38:06PM +0200, Maxime Coquelin wrote:
> > >>>+static inline void __attribute__((always_inline))
> > >>>+flush_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
> > >>>+		uint32_t used_idx_start)
> > >>>+{
> > >>>+	if (used_idx_start + vq->shadow_used_idx < vq->size) {
> > >>>+		rte_memcpy(&vq->used->ring[used_idx_start],
> > >>>+				&vq->shadow_used_ring[0],
> > >>>+				vq->shadow_used_idx *
> > >>>+				sizeof(struct vring_used_elem));
> > >>>+		vhost_log_used_vring(dev, vq,
> > >>>+				offsetof(struct vring_used,
> > >>>+					ring[used_idx_start]),
> > >>>+				vq->shadow_used_idx *
> > >>>+				sizeof(struct vring_used_elem));
> > >>>+	} else {
> > >>>+		uint32_t part_1 = vq->size - used_idx_start;
> > >>>+		uint32_t part_2 = vq->shadow_used_idx - part_1;
> > >>>+
> > >>>+		rte_memcpy(&vq->used->ring[used_idx_start],
> > >>>+				&vq->shadow_used_ring[0],
> > >>>+				part_1 *
> > >>>+				sizeof(struct vring_used_elem));
> > >>>+		vhost_log_used_vring(dev, vq,
> > >>>+				offsetof(struct vring_used,
> > >>>+					ring[used_idx_start]),
> > >>>+				part_1 *
> > >>>+				sizeof(struct vring_used_elem));
> > >>>+		rte_memcpy(&vq->used->ring[0],
> > >>>+				&vq->shadow_used_ring[part_1],
> > >>>+				part_2 *
> > >>>+				sizeof(struct vring_used_elem));
> > >>>+		vhost_log_used_vring(dev, vq,
> > >>>+				offsetof(struct vring_used,
> > >>>+					ring[0]),
> > >>>+				part_2 *
> > >>>+				sizeof(struct vring_used_elem));
> > >>>+	}
> > >>> }
> > >>Is expanding the code done for performance purpose?
> > >
> > >Hi Maxime,
> > >
> > >Yes theoretically this has the least branch number.
> > >And I think the logic is simpler this way.
> > Ok, in that case, maybe you could create a function to
> > do the rte_memcpy and the vhost_log_used on a given range.
> 
> Agreed, that will be better; it could avoid repeating similar code
> block 3 times.

Okay. Thanks for the suggestion, Maxime and Yuanhan.

> 
> > I don't have a strong opinion on this, if Yuanhan is fine
> > with current code, that's ok for me.
> 
> From what I know, that's kind of DPDK prefered way, to expand code
> when necessary. For example, 9ec201f5d6e7 ("mbuf: provide bulk
> allocation").
> 
> So I'm fine with it.
> 
> 	--yliu
diff mbox

Patch

diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index 46095c3..cb31cdd 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -119,10 +119,26 @@  cleanup_device(struct virtio_net *dev, int destroy)
 static void
 free_device(struct virtio_net *dev)
 {
+	struct vhost_virtqueue *vq_0;
+	struct vhost_virtqueue *vq_1;
 	uint32_t i;
 
-	for (i = 0; i < dev->virt_qp_nb; i++)
-		rte_free(dev->virtqueue[i * VIRTIO_QNUM]);
+	for (i = 0; i < dev->virt_qp_nb; i++) {
+		vq_0 = dev->virtqueue[i * VIRTIO_QNUM];
+		if (vq_0->shadow_used_ring) {
+			rte_free(vq_0->shadow_used_ring);
+			vq_0->shadow_used_ring = NULL;
+		}
+
+		vq_1 = dev->virtqueue[i * VIRTIO_QNUM + 1];
+		if (vq_1->shadow_used_ring) {
+			rte_free(vq_1->shadow_used_ring);
+			vq_1->shadow_used_ring = NULL;
+		}
+
+		/* malloc together, free together */
+		rte_free(vq_0);
+	}
 
 	rte_free(dev);
 }
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 9707dfc..381dc27 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -85,6 +85,10 @@  struct vhost_virtqueue {
 
 	/* Physical address of used ring, for logging */
 	uint64_t		log_guest_addr;
+
+	/* Shadow used ring for performance */
+	struct vring_used_elem	*shadow_used_ring;
+	uint32_t		shadow_used_idx;
 } __rte_cache_aligned;
 
 /* Old kernels have no such macro defined */
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index eee99e9..d7cf1ed 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -193,7 +193,21 @@  static int
 vhost_user_set_vring_num(struct virtio_net *dev,
 			 struct vhost_vring_state *state)
 {
-	dev->virtqueue[state->index]->size = state->num;
+	struct vhost_virtqueue *vq;
+
+	vq = dev->virtqueue[state->index];
+	vq->size = state->num;
+	if (!vq->shadow_used_ring) {
+		vq->shadow_used_ring = rte_malloc(NULL,
+				vq->size * sizeof(struct vring_used_elem),
+				RTE_CACHE_LINE_SIZE);
+		if (!vq->shadow_used_ring) {
+			RTE_LOG(ERR, VHOST_CONFIG,
+				"Failed to allocate memory"
+				" for shadow used ring.\n");
+			return -1;
+		}
+	}
 
 	return 0;
 }
@@ -611,14 +625,21 @@  static int
 vhost_user_get_vring_base(struct virtio_net *dev,
 			  struct vhost_vring_state *state)
 {
+	struct vhost_virtqueue *vq;
+
 	/* We have to stop the queue (virtio) if it is running. */
 	if (dev->flags & VIRTIO_DEV_RUNNING) {
 		dev->flags &= ~VIRTIO_DEV_RUNNING;
 		notify_ops->destroy_device(dev->vid);
 	}
 
+	vq = dev->virtqueue[state->index];
 	/* Here we are safe to get the last used index */
-	state->num = dev->virtqueue[state->index]->last_used_idx;
+	state->num = vq->last_used_idx;
+	if (vq->shadow_used_ring) {
+		rte_free(vq->shadow_used_ring);
+		vq->shadow_used_ring = NULL;
+	}
 
 	RTE_LOG(INFO, VHOST_CONFIG,
 		"vring base idx:%d file:%d\n", state->index, state->num);
@@ -627,10 +648,10 @@  vhost_user_get_vring_base(struct virtio_net *dev,
 	 * sent and only sent in vhost_vring_stop.
 	 * TODO: cleanup the vring, it isn't usable since here.
 	 */
-	if (dev->virtqueue[state->index]->kickfd >= 0)
-		close(dev->virtqueue[state->index]->kickfd);
+	if (vq->kickfd >= 0)
+		close(vq->kickfd);
 
-	dev->virtqueue[state->index]->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
+	vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
 
 	return 0;
 }
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index b38f18f..e9f6353 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -134,17 +134,52 @@  virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)
 }
 
 static inline void __attribute__((always_inline))
-update_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
-		uint32_t desc_chain_head, uint32_t desc_chain_len)
+update_used_ring(struct vhost_virtqueue *vq, uint32_t desc_chain_head,
+		uint32_t desc_chain_len)
 {
-	uint32_t used_idx = vq->last_used_idx & (vq->size - 1);
-
-	vq->used->ring[used_idx].id = desc_chain_head;
-	vq->used->ring[used_idx].len = desc_chain_len;
+	vq->shadow_used_ring[vq->shadow_used_idx].id  = desc_chain_head;
+	vq->shadow_used_ring[vq->shadow_used_idx].len = desc_chain_len;
+	vq->shadow_used_idx++;
 	vq->last_used_idx++;
-	vhost_log_used_vring(dev, vq, offsetof(struct vring_used,
-				ring[used_idx]),
-			sizeof(vq->used->ring[used_idx]));
+}
+
+static inline void __attribute__((always_inline))
+flush_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq,
+		uint32_t used_idx_start)
+{
+	if (used_idx_start + vq->shadow_used_idx < vq->size) {
+		rte_memcpy(&vq->used->ring[used_idx_start],
+				&vq->shadow_used_ring[0],
+				vq->shadow_used_idx *
+				sizeof(struct vring_used_elem));
+		vhost_log_used_vring(dev, vq,
+				offsetof(struct vring_used,
+					ring[used_idx_start]),
+				vq->shadow_used_idx *
+				sizeof(struct vring_used_elem));
+	} else {
+		uint32_t part_1 = vq->size - used_idx_start;
+		uint32_t part_2 = vq->shadow_used_idx - part_1;
+
+		rte_memcpy(&vq->used->ring[used_idx_start],
+				&vq->shadow_used_ring[0],
+				part_1 *
+				sizeof(struct vring_used_elem));
+		vhost_log_used_vring(dev, vq,
+				offsetof(struct vring_used,
+					ring[used_idx_start]),
+				part_1 *
+				sizeof(struct vring_used_elem));
+		rte_memcpy(&vq->used->ring[0],
+				&vq->shadow_used_ring[part_1],
+				part_2 *
+				sizeof(struct vring_used_elem));
+		vhost_log_used_vring(dev, vq,
+				offsetof(struct vring_used,
+					ring[0]),
+				part_2 *
+				sizeof(struct vring_used_elem));
+	}
 }
 
 static inline int __attribute__((always_inline))
@@ -209,7 +244,7 @@  enqueue_packet(struct virtio_net *dev, struct vhost_virtqueue *vq,
 					goto error;
 			} else if (is_mrg_rxbuf) {
 				/* start with the next desc chain */
-				update_used_ring(dev, vq, desc_chain_head,
+				update_used_ring(vq, desc_chain_head,
 						desc_chain_len);
 				num_buffers++;
 				virtio_hdr->num_buffers++;
@@ -245,7 +280,7 @@  enqueue_packet(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		desc_chain_len += cpy_len;
 	}
 
-	update_used_ring(dev, vq, desc_chain_head, desc_chain_len);
+	update_used_ring(vq, desc_chain_head, desc_chain_len);
 
 	return 0;
 
@@ -275,6 +310,7 @@  rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
 {
 	struct vhost_virtqueue *vq;
 	struct virtio_net *dev;
+	uint32_t used_idx_start;
 	uint32_t is_mrg_rxbuf = 0;
 	uint32_t pkt_idx      = 0;
 	uint32_t pkt_left     = count;
@@ -300,6 +336,8 @@  rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
 		is_mrg_rxbuf = 1;
 
 	/* start enqueuing packets 1 by 1 */
+	vq->shadow_used_idx = 0;
+	used_idx_start = vq->last_used_idx & (vq->size - 1);
 	avail_idx = *((volatile uint16_t *)&vq->avail->idx);
 	while (pkt_left && avail_idx != vq->last_used_idx) {
 		/* prefetch the next desc */
@@ -316,6 +354,10 @@  rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
 		pkt_left--;
 	}
 
+	/* batch update used ring for better performance */
+	if (likely(vq->shadow_used_idx > 0))
+		flush_used_ring(dev, vq, used_idx_start);
+
 	/* update used idx and kick the guest if necessary */
 	if (pkt_idx)
 		notify_guest(dev, vq);