diff mbox series

[v8,09/13] vhost: buffer packed ring dequeue updates

Message ID	20191021220813.55236-10-yong.liu@intel.com (mailing list archive)
State	Superseded, archived
Delegated to:	Maxime Coquelin
Headers	From: Marvin Liu <yong.liu@intel.com> To: maxime.coquelin@redhat.com, tiwei.bie@intel.com, zhihong.wang@intel.com, stephen@networkplumber.org, gavin.hu@arm.com Cc: dev@dpdk.org, Marvin Liu <yong.liu@intel.com> Date: Tue, 22 Oct 2019 06:08:09 +0800 Message-Id: <20191021220813.55236-10-yong.liu@intel.com> In-Reply-To: <20191021220813.55236-1-yong.liu@intel.com> References: <20191021154016.16274-1-yong.liu@intel.com> <20191021220813.55236-1-yong.liu@intel.com> Subject: [dpdk-dev] [PATCH v8 09/13] vhost: buffer packed ring dequeue updates Precedence: list Errors-To: dev-bounces@dpdk.org Sender: "dev" <dev-bounces@dpdk.org>
Series	vhost packed ring performance optimization \| [v8,00/13] vhost packed ring performance optimization [v8,01/13] vhost: add packed ring indexes increasing function [v8,02/13] vhost: add packed ring single enqueue [v8,03/13] vhost: try to unroll for each loop [v8,04/13] vhost: add packed ring batch enqueue [v8,05/13] vhost: add packed ring single dequeue [v8,06/13] vhost: add packed ring batch dequeue [v8,07/13] vhost: flush enqueue updates by cacheline [v8,08/13] vhost: flush batched enqueue descs directly [v8,09/13] vhost: buffer packed ring dequeue updates [v8,10/13] vhost: optimize packed ring enqueue [v8,11/13] vhost: add packed ring zcopy batch and single dequeue [v8,12/13] vhost: optimize packed ring dequeue [v8,13/13] vhost: optimize packed ring dequeue when in-order

Checks

Context	Check	Description
ci/checkpatch	success	coding style OK
ci/Intel-compilation	success	Compilation OK

Commit Message

Marvin Liu Oct. 21, 2019, 10:08 p.m. UTC

  Buffer used ring updates as many as possible in vhost dequeue function
for coordinating with virtio driver. For supporting buffer, shadow used
ring element should contain descriptor's flags. First shadowed ring
index was recorded for calculating buffered number.

Signed-off-by: Marvin Liu <yong.liu@intel.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

diff mbox series

Patch

diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index f8dbe841c..9f11b28a3 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -42,6 +42,8 @@ 
 #define PACKED_DESC_ENQUEUE_USED_FLAG(w)	\
 	((w) ? (VRING_DESC_F_AVAIL | VRING_DESC_F_USED | VRING_DESC_F_WRITE) : \
 		VRING_DESC_F_WRITE)
+#define PACKED_DESC_DEQUEUE_USED_FLAG(w)	\
+	((w) ? (VRING_DESC_F_AVAIL | VRING_DESC_F_USED) : 0x0)
 #define PACKED_DESC_SINGLE_DEQUEUE_FLAG (VRING_DESC_F_NEXT | \
 					 VRING_DESC_F_INDIRECT)
 
@@ -114,6 +116,7 @@  struct log_cache_entry {
 
 struct vring_used_elem_packed {
 	uint16_t id;
+	uint16_t flags;
 	uint32_t len;
 	uint32_t count;
 };
@@ -179,6 +182,8 @@  struct vhost_virtqueue {
 	uint16_t                shadow_used_idx;
 	/* Record packed ring enqueue latest desc cache aligned index */
 	uint16_t		shadow_aligned_idx;
+	/* Record packed ring first dequeue desc index */
+	uint16_t		shadow_last_used_idx;
 	struct vhost_vring_addr ring_addrs;
 
 	struct batch_copy_elem	*batch_copy_elems;
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 51ce32064..b09e03fbc 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -154,6 +154,23 @@  vhost_flush_enqueue_shadow_packed(struct virtio_net *dev,
 	vhost_log_cache_sync(dev, vq);
 }
 
+static __rte_always_inline void
+vhost_flush_dequeue_shadow_packed(struct virtio_net *dev,
+				  struct vhost_virtqueue *vq)
+{
+	struct vring_used_elem_packed *used_elem = &vq->shadow_used_packed[0];
+
+	vq->desc_packed[vq->shadow_last_used_idx].id = used_elem->id;
+	rte_smp_wmb();
+	vq->desc_packed[vq->shadow_last_used_idx].flags = used_elem->flags;
+
+	vhost_log_cache_used_vring(dev, vq, vq->shadow_last_used_idx *
+				   sizeof(struct vring_packed_desc),
+				   sizeof(struct vring_packed_desc));
+	vq->shadow_used_idx = 0;
+	vhost_log_cache_sync(dev, vq);
+}
+
 static __rte_always_inline void
 vhost_flush_enqueue_batch_packed(struct virtio_net *dev,
 				 struct vhost_virtqueue *vq,
@@ -246,6 +263,78 @@  flush_shadow_used_ring_packed(struct virtio_net *dev,
 	vhost_log_cache_sync(dev, vq);
 }
 
+static __rte_always_inline void
+vhost_shadow_dequeue_batch_packed(struct virtio_net *dev,
+				  struct vhost_virtqueue *vq,
+				  uint16_t *ids)
+{
+	uint16_t flags;
+	uint16_t i;
+	uint16_t begin;
+
+	flags = PACKED_DESC_DEQUEUE_USED_FLAG(vq->used_wrap_counter);
+
+	if (!vq->shadow_used_idx) {
+		vq->shadow_last_used_idx = vq->last_used_idx;
+		vq->shadow_used_packed[0].id  = ids[0];
+		vq->shadow_used_packed[0].len = 0;
+		vq->shadow_used_packed[0].count = 1;
+		vq->shadow_used_packed[0].flags = flags;
+		vq->shadow_used_idx++;
+		begin = 1;
+	} else
+		begin = 0;
+
+	vhost_for_each_try_unroll(i, begin, PACKED_BATCH_SIZE) {
+		vq->desc_packed[vq->last_used_idx + i].id = ids[i];
+		vq->desc_packed[vq->last_used_idx + i].len = 0;
+	}
+
+	rte_smp_wmb();
+	vhost_for_each_try_unroll(i, begin, PACKED_BATCH_SIZE)
+		vq->desc_packed[vq->last_used_idx + i].flags = flags;
+
+	vhost_log_cache_used_vring(dev, vq, vq->last_used_idx *
+				   sizeof(struct vring_packed_desc),
+				   sizeof(struct vring_packed_desc) *
+				   PACKED_BATCH_SIZE);
+	vhost_log_cache_sync(dev, vq);
+
+	vq_inc_last_used_packed(vq, PACKED_BATCH_SIZE);
+}
+
+static __rte_always_inline void
+vhost_shadow_dequeue_single_packed(struct vhost_virtqueue *vq,
+				   uint16_t buf_id,
+				   uint16_t count)
+{
+	uint16_t flags;
+
+	flags = vq->desc_packed[vq->last_used_idx].flags;
+	if (vq->used_wrap_counter) {
+		flags |= VRING_DESC_F_USED;
+		flags |= VRING_DESC_F_AVAIL;
+	} else {
+		flags &= ~VRING_DESC_F_USED;
+		flags &= ~VRING_DESC_F_AVAIL;
+	}
+
+	if (!vq->shadow_used_idx) {
+		vq->shadow_last_used_idx = vq->last_used_idx;
+
+		vq->shadow_used_packed[0].id  = buf_id;
+		vq->shadow_used_packed[0].len = 0;
+		vq->shadow_used_packed[0].flags = flags;
+		vq->shadow_used_idx++;
+	} else {
+		vq->desc_packed[vq->last_used_idx].id = buf_id;
+		vq->desc_packed[vq->last_used_idx].len = 0;
+		vq->desc_packed[vq->last_used_idx].flags = flags;
+	}
+
+	vq_inc_last_used_packed(vq, count);
+}
+
 static __rte_always_inline void
 update_shadow_used_ring_packed(struct vhost_virtqueue *vq,
 			 uint16_t desc_idx, uint32_t len, uint16_t count)
@@ -314,6 +403,25 @@  vhost_shadow_enqueue_single_packed(struct virtio_net *dev,
 	}
 }
 
+static __rte_unused void
+vhost_flush_dequeue_packed(struct virtio_net *dev,
+			   struct vhost_virtqueue *vq)
+{
+	int shadow_count;
+	if (!vq->shadow_used_idx)
+		return;
+
+	shadow_count = vq->last_used_idx - vq->shadow_last_used_idx;
+	if (shadow_count <= 0)
+		shadow_count += vq->size;
+
+	if ((uint32_t)shadow_count >= (vq->size - MAX_PKT_BURST)) {
+		do_data_copy_dequeue(vq);
+		vhost_flush_dequeue_shadow_packed(dev, vq);
+		vhost_vring_call_packed(dev, vq);
+	}
+}
+
 /* avoid write operation when necessary, to lessen cache issues */
 #define ASSIGN_UNLESS_EQUAL(var, val) do {	\
 	if ((var) != (val))			\
@@ -1876,6 +1984,8 @@  virtio_dev_tx_batch_packed(struct virtio_net *dev,
 			   (void *)(uintptr_t)(desc_addrs[i] + buf_offset),
 			   pkts[i]->pkt_len);
 
+	vhost_shadow_dequeue_batch_packed(dev, vq, ids);
+
 	vq_inc_last_avail_packed(vq, PACKED_BATCH_SIZE);
 
 	return 0;
@@ -1931,6 +2041,8 @@  virtio_dev_tx_single_packed(struct virtio_net *dev,
 					&desc_count))
 		return -1;
 
+	vhost_shadow_dequeue_single_packed(vq, buf_id, desc_count);
+
 	vq_inc_last_avail_packed(vq, desc_count);
 
 	return 0;