| Message ID | 20180702081629.29258-14-maxime.coquelin@redhat.com (mailing list archive) |
|---|---|
| State | Superseded, archived |
| Delegated to: | Maxime Coquelin |
| Headers | show |
| Series | Vhost: add support to packed ring layout | expand |
| Context | Check | Description |
|---|---|---|
| ci/Intel-compilation | fail | Compilation issues |
On Mon, Jul 02, 2018 at 10:16:27AM +0200, Maxime Coquelin wrote: > Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com> > --- > lib/librte_vhost/vhost.h | 1 + > lib/librte_vhost/virtio_net.c | 121 +++++++++++++++++++++++++++++++++++++++++- > 2 files changed, 121 insertions(+), 1 deletion(-) > > diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h > index a7320469a..6ea8fb896 100644 > --- a/lib/librte_vhost/vhost.h > +++ b/lib/librte_vhost/vhost.h > @@ -56,6 +56,7 @@ struct buf_vector { > struct zcopy_mbuf { > struct rte_mbuf *mbuf; > uint32_t desc_idx; > + uint16_t desc_count; > uint16_t in_use; > > TAILQ_ENTRY(zcopy_mbuf) next; > diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c > index 2e286e228..03dd38235 100644 > --- a/lib/librte_vhost/virtio_net.c > +++ b/lib/librte_vhost/virtio_net.c > @@ -1445,6 +1445,122 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, > return i; > } > > +static __rte_always_inline uint16_t > +virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, > + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) > +{ > + uint16_t i; > + > + rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]); > + > + if (unlikely(dev->dequeue_zero_copy)) { > + struct zcopy_mbuf *zmbuf, *next; > + int nr_updated = 0; > + > + for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list); > + zmbuf != NULL; zmbuf = next) { > + next = TAILQ_NEXT(zmbuf, next); > + > + if (mbuf_is_consumed(zmbuf->mbuf)) { > + update_shadow_used_ring_packed(vq, > + zmbuf->desc_idx, > + 0, > + zmbuf->desc_count); > + nr_updated += 1; nr_updated isn't really used. > + > + TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next); > + restore_mbuf(zmbuf->mbuf); > + rte_pktmbuf_free(zmbuf->mbuf); > + put_zmbuf(zmbuf); > + vq->nr_zmbuf -= 1; > + } > + } > + > + flush_shadow_used_ring_packed(dev, vq); > + vhost_vring_call(dev, vq); > + } > + [...]
On 07/04/2018 07:45 AM, Tiwei Bie wrote: > On Mon, Jul 02, 2018 at 10:16:27AM +0200, Maxime Coquelin wrote: >> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com> >> --- >> lib/librte_vhost/vhost.h | 1 + >> lib/librte_vhost/virtio_net.c | 121 +++++++++++++++++++++++++++++++++++++++++- >> 2 files changed, 121 insertions(+), 1 deletion(-) >> >> diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h >> index a7320469a..6ea8fb896 100644 >> --- a/lib/librte_vhost/vhost.h >> +++ b/lib/librte_vhost/vhost.h >> @@ -56,6 +56,7 @@ struct buf_vector { >> struct zcopy_mbuf { >> struct rte_mbuf *mbuf; >> uint32_t desc_idx; >> + uint16_t desc_count; >> uint16_t in_use; >> >> TAILQ_ENTRY(zcopy_mbuf) next; >> diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c >> index 2e286e228..03dd38235 100644 >> --- a/lib/librte_vhost/virtio_net.c >> +++ b/lib/librte_vhost/virtio_net.c >> @@ -1445,6 +1445,122 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, >> return i; >> } >> >> +static __rte_always_inline uint16_t >> +virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, >> + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) >> +{ >> + uint16_t i; >> + >> + rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]); >> + >> + if (unlikely(dev->dequeue_zero_copy)) { >> + struct zcopy_mbuf *zmbuf, *next; >> + int nr_updated = 0; >> + >> + for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list); >> + zmbuf != NULL; zmbuf = next) { >> + next = TAILQ_NEXT(zmbuf, next); >> + >> + if (mbuf_is_consumed(zmbuf->mbuf)) { >> + update_shadow_used_ring_packed(vq, >> + zmbuf->desc_idx, >> + 0, >> + zmbuf->desc_count); >> + nr_updated += 1; > > nr_updated isn't really used. Right, it seems it should be removed in the _split version too, but I'll do that in a separate patch. Thanks, Maxime >> + >> + TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next); >> + restore_mbuf(zmbuf->mbuf); >> + rte_pktmbuf_free(zmbuf->mbuf); >> + put_zmbuf(zmbuf); >> + vq->nr_zmbuf -= 1; >> + } >> + } >> + >> + flush_shadow_used_ring_packed(dev, vq); >> + vhost_vring_call(dev, vq); >> + } >> + > [...] >
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index a7320469a..6ea8fb896 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -56,6 +56,7 @@ struct buf_vector { struct zcopy_mbuf { struct rte_mbuf *mbuf; uint32_t desc_idx; + uint16_t desc_count; uint16_t in_use; TAILQ_ENTRY(zcopy_mbuf) next; diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c index 2e286e228..03dd38235 100644 --- a/lib/librte_vhost/virtio_net.c +++ b/lib/librte_vhost/virtio_net.c @@ -1445,6 +1445,122 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, return i; } +static __rte_always_inline uint16_t +virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) +{ + uint16_t i; + + rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]); + + if (unlikely(dev->dequeue_zero_copy)) { + struct zcopy_mbuf *zmbuf, *next; + int nr_updated = 0; + + for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list); + zmbuf != NULL; zmbuf = next) { + next = TAILQ_NEXT(zmbuf, next); + + if (mbuf_is_consumed(zmbuf->mbuf)) { + update_shadow_used_ring_packed(vq, + zmbuf->desc_idx, + 0, + zmbuf->desc_count); + nr_updated += 1; + + TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next); + restore_mbuf(zmbuf->mbuf); + rte_pktmbuf_free(zmbuf->mbuf); + put_zmbuf(zmbuf); + vq->nr_zmbuf -= 1; + } + } + + flush_shadow_used_ring_packed(dev, vq); + vhost_vring_call(dev, vq); + } + + VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__); + + count = RTE_MIN(count, MAX_PKT_BURST); + VHOST_LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n", + dev->vid, count); + + for (i = 0; i < count; i++) { + struct buf_vector buf_vec[BUF_VECTOR_MAX]; + uint16_t buf_id, dummy_len; + uint16_t desc_count, nr_vec = 0; + int err; + + if (unlikely(fill_vec_buf_packed(dev, vq, + vq->last_avail_idx, &desc_count, + buf_vec, &nr_vec, + &buf_id, &dummy_len, + VHOST_ACCESS_RW) < 0)) + break; + + if (likely(dev->dequeue_zero_copy == 0)) + update_shadow_used_ring_packed(vq, buf_id, 0, + desc_count); + + rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr); + + pkts[i] = rte_pktmbuf_alloc(mbuf_pool); + if (unlikely(pkts[i] == NULL)) { + RTE_LOG(ERR, VHOST_DATA, + "Failed to allocate memory for mbuf.\n"); + break; + } + + err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i], + mbuf_pool); + if (unlikely(err)) { + rte_pktmbuf_free(pkts[i]); + break; + } + + if (unlikely(dev->dequeue_zero_copy)) { + struct zcopy_mbuf *zmbuf; + + zmbuf = get_zmbuf(vq); + if (!zmbuf) { + rte_pktmbuf_free(pkts[i]); + break; + } + zmbuf->mbuf = pkts[i]; + zmbuf->desc_idx = buf_id; + zmbuf->desc_count = desc_count; + + /* + * Pin lock the mbuf; we will check later to see + * whether the mbuf is freed (when we are the last + * user) or not. If that's the case, we then could + * update the used ring safely. + */ + rte_mbuf_refcnt_update(pkts[i], 1); + + vq->nr_zmbuf += 1; + TAILQ_INSERT_TAIL(&vq->zmbuf_list, zmbuf, next); + } + + vq->last_avail_idx += desc_count; + if (vq->last_avail_idx >= vq->size) { + vq->last_avail_idx -= vq->size; + vq->avail_wrap_counter ^= 1; + } + } + + if (likely(dev->dequeue_zero_copy == 0)) { + do_data_copy_dequeue(vq); + if (unlikely(i < count)) + vq->shadow_used_idx = i; + flush_shadow_used_ring_packed(dev, vq); + vhost_vring_call(dev, vq); + } + + return i; +} + uint16_t rte_vhost_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) @@ -1515,7 +1631,10 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id, count -= 1; } - count = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count); + if (vq_is_packed(dev)) + count = virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count); + else + count = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count); out: if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com> --- lib/librte_vhost/vhost.h | 1 + lib/librte_vhost/virtio_net.c | 121 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 121 insertions(+), 1 deletion(-)