diff mbox series

[v4,05/26] vhost: change to single IOTLB cache per device

Message ID 20230601200812.672233-6-maxime.coquelin@redhat.com (mailing list archive)
State Superseded, archived
Delegated to: Maxime Coquelin
Headers
Series Add VDUSE support to Vhost library |

Checks

Context Check Description
ci/checkpatch success coding style OK

Commit Message

Maxime Coquelin June 1, 2023, 8:07 p.m. UTC 
  This patch simplifies IOTLB implementation and improves
IOTLB memory consumption by having a single IOTLB cache
per device, instead of having one per queue.

In order to not impact performance, it keeps an IOTLB lock
per virtqueue, so that there is no contention between
multiple queue trying to acquire it.

Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 lib/vhost/iotlb.c      | 212 +++++++++++++++++++----------------------
 lib/vhost/iotlb.h      |  43 ++++++---
 lib/vhost/vhost.c      |  18 ++--
 lib/vhost/vhost.h      |  16 ++--
 lib/vhost/vhost_user.c |  23 +++--
 5 files changed, 159 insertions(+), 153 deletions(-)
diff mbox series

Patch

diff --git a/lib/vhost/iotlb.c b/lib/vhost/iotlb.c
index 95d67ac832..6d49bf6b30 100644
--- a/lib/vhost/iotlb.c
+++ b/lib/vhost/iotlb.c
@@ -74,86 +74,81 @@  vhost_user_iotlb_clear_dump(struct virtio_net *dev, struct vhost_iotlb_entry *no
 }
 
 static struct vhost_iotlb_entry *
-vhost_user_iotlb_pool_get(struct vhost_virtqueue *vq)
+vhost_user_iotlb_pool_get(struct virtio_net *dev)
 {
 	struct vhost_iotlb_entry *node;
 
-	rte_spinlock_lock(&vq->iotlb_free_lock);
-	node = SLIST_FIRST(&vq->iotlb_free_list);
+	rte_spinlock_lock(&dev->iotlb_free_lock);
+	node = SLIST_FIRST(&dev->iotlb_free_list);
 	if (node != NULL)
-		SLIST_REMOVE_HEAD(&vq->iotlb_free_list, next_free);
-	rte_spinlock_unlock(&vq->iotlb_free_lock);
+		SLIST_REMOVE_HEAD(&dev->iotlb_free_list, next_free);
+	rte_spinlock_unlock(&dev->iotlb_free_lock);
 	return node;
 }
 
 static void
-vhost_user_iotlb_pool_put(struct vhost_virtqueue *vq,
-	struct vhost_iotlb_entry *node)
+vhost_user_iotlb_pool_put(struct virtio_net *dev, struct vhost_iotlb_entry *node)
 {
-	rte_spinlock_lock(&vq->iotlb_free_lock);
-	SLIST_INSERT_HEAD(&vq->iotlb_free_list, node, next_free);
-	rte_spinlock_unlock(&vq->iotlb_free_lock);
+	rte_spinlock_lock(&dev->iotlb_free_lock);
+	SLIST_INSERT_HEAD(&dev->iotlb_free_list, node, next_free);
+	rte_spinlock_unlock(&dev->iotlb_free_lock);
 }
 
 static void
-vhost_user_iotlb_cache_random_evict(struct virtio_net *dev, struct vhost_virtqueue *vq);
+vhost_user_iotlb_cache_random_evict(struct virtio_net *dev);
 
 static void
-vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq)
+vhost_user_iotlb_pending_remove_all(struct virtio_net *dev)
 {
 	struct vhost_iotlb_entry *node, *temp_node;
 
-	rte_rwlock_write_lock(&vq->iotlb_pending_lock);
+	rte_rwlock_write_lock(&dev->iotlb_pending_lock);
 
-	RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
-		TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
-		vhost_user_iotlb_pool_put(vq, node);
+	RTE_TAILQ_FOREACH_SAFE(node, &dev->iotlb_pending_list, next, temp_node) {
+		TAILQ_REMOVE(&dev->iotlb_pending_list, node, next);
+		vhost_user_iotlb_pool_put(dev, node);
 	}
 
-	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
+	rte_rwlock_write_unlock(&dev->iotlb_pending_lock);
 }
 
 bool
-vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,
-				uint8_t perm)
+vhost_user_iotlb_pending_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm)
 {
 	struct vhost_iotlb_entry *node;
 	bool found = false;
 
-	rte_rwlock_read_lock(&vq->iotlb_pending_lock);
+	rte_rwlock_read_lock(&dev->iotlb_pending_lock);
 
-	TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) {
+	TAILQ_FOREACH(node, &dev->iotlb_pending_list, next) {
 		if ((node->iova == iova) && (node->perm == perm)) {
 			found = true;
 			break;
 		}
 	}
 
-	rte_rwlock_read_unlock(&vq->iotlb_pending_lock);
+	rte_rwlock_read_unlock(&dev->iotlb_pending_lock);
 
 	return found;
 }
 
 void
-vhost_user_iotlb_pending_insert(struct virtio_net *dev, struct vhost_virtqueue *vq,
-				uint64_t iova, uint8_t perm)
+vhost_user_iotlb_pending_insert(struct virtio_net *dev, uint64_t iova, uint8_t perm)
 {
 	struct vhost_iotlb_entry *node;
 
-	node = vhost_user_iotlb_pool_get(vq);
+	node = vhost_user_iotlb_pool_get(dev);
 	if (node == NULL) {
 		VHOST_LOG_CONFIG(dev->ifname, DEBUG,
-			"IOTLB pool for vq %"PRIu32" empty, clear entries for pending insertion\n",
-			vq->index);
-		if (!TAILQ_EMPTY(&vq->iotlb_pending_list))
-			vhost_user_iotlb_pending_remove_all(vq);
+			"IOTLB pool empty, clear entries for pending insertion\n");
+		if (!TAILQ_EMPTY(&dev->iotlb_pending_list))
+			vhost_user_iotlb_pending_remove_all(dev);
 		else
-			vhost_user_iotlb_cache_random_evict(dev, vq);
-		node = vhost_user_iotlb_pool_get(vq);
+			vhost_user_iotlb_cache_random_evict(dev);
+		node = vhost_user_iotlb_pool_get(dev);
 		if (node == NULL) {
 			VHOST_LOG_CONFIG(dev->ifname, ERR,
-				"IOTLB pool vq %"PRIu32" still empty, pending insertion failure\n",
-				vq->index);
+				"IOTLB pool still empty, pending insertion failure\n");
 			return;
 		}
 	}
@@ -161,22 +156,21 @@  vhost_user_iotlb_pending_insert(struct virtio_net *dev, struct vhost_virtqueue *
 	node->iova = iova;
 	node->perm = perm;
 
-	rte_rwlock_write_lock(&vq->iotlb_pending_lock);
+	rte_rwlock_write_lock(&dev->iotlb_pending_lock);
 
-	TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next);
+	TAILQ_INSERT_TAIL(&dev->iotlb_pending_list, node, next);
 
-	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
+	rte_rwlock_write_unlock(&dev->iotlb_pending_lock);
 }
 
 void
-vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,
-				uint64_t iova, uint64_t size, uint8_t perm)
+vhost_user_iotlb_pending_remove(struct virtio_net *dev, uint64_t iova, uint64_t size, uint8_t perm)
 {
 	struct vhost_iotlb_entry *node, *temp_node;
 
-	rte_rwlock_write_lock(&vq->iotlb_pending_lock);
+	rte_rwlock_write_lock(&dev->iotlb_pending_lock);
 
-	RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next,
+	RTE_TAILQ_FOREACH_SAFE(node, &dev->iotlb_pending_list, next,
 				temp_node) {
 		if (node->iova < iova)
 			continue;
@@ -184,81 +178,78 @@  vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,
 			continue;
 		if ((node->perm & perm) != node->perm)
 			continue;
-		TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
-		vhost_user_iotlb_pool_put(vq, node);
+		TAILQ_REMOVE(&dev->iotlb_pending_list, node, next);
+		vhost_user_iotlb_pool_put(dev, node);
 	}
 
-	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
+	rte_rwlock_write_unlock(&dev->iotlb_pending_lock);
 }
 
 static void
-vhost_user_iotlb_cache_remove_all(struct virtio_net *dev, struct vhost_virtqueue *vq)
+vhost_user_iotlb_cache_remove_all(struct virtio_net *dev)
 {
 	struct vhost_iotlb_entry *node, *temp_node;
 
-	rte_rwlock_write_lock(&vq->iotlb_lock);
+	vhost_user_iotlb_wr_lock_all(dev);
 
-	RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
+	RTE_TAILQ_FOREACH_SAFE(node, &dev->iotlb_list, next, temp_node) {
 		vhost_user_iotlb_clear_dump(dev, node, NULL, NULL);
 
-		TAILQ_REMOVE(&vq->iotlb_list, node, next);
-		vhost_user_iotlb_pool_put(vq, node);
+		TAILQ_REMOVE(&dev->iotlb_list, node, next);
+		vhost_user_iotlb_pool_put(dev, node);
 	}
 
-	vq->iotlb_cache_nr = 0;
+	dev->iotlb_cache_nr = 0;
 
-	rte_rwlock_write_unlock(&vq->iotlb_lock);
+	vhost_user_iotlb_wr_unlock_all(dev);
 }
 
 static void
-vhost_user_iotlb_cache_random_evict(struct virtio_net *dev, struct vhost_virtqueue *vq)
+vhost_user_iotlb_cache_random_evict(struct virtio_net *dev)
 {
 	struct vhost_iotlb_entry *node, *temp_node, *prev_node = NULL;
 	int entry_idx;
 
-	rte_rwlock_write_lock(&vq->iotlb_lock);
+	vhost_user_iotlb_wr_lock_all(dev);
 
-	entry_idx = rte_rand() % vq->iotlb_cache_nr;
+	entry_idx = rte_rand() % dev->iotlb_cache_nr;
 
-	RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
+	RTE_TAILQ_FOREACH_SAFE(node, &dev->iotlb_list, next, temp_node) {
 		if (!entry_idx) {
 			struct vhost_iotlb_entry *next_node = RTE_TAILQ_NEXT(node, next);
 
 			vhost_user_iotlb_clear_dump(dev, node, prev_node, next_node);
 
-			TAILQ_REMOVE(&vq->iotlb_list, node, next);
-			vhost_user_iotlb_pool_put(vq, node);
-			vq->iotlb_cache_nr--;
+			TAILQ_REMOVE(&dev->iotlb_list, node, next);
+			vhost_user_iotlb_pool_put(dev, node);
+			dev->iotlb_cache_nr--;
 			break;
 		}
 		prev_node = node;
 		entry_idx--;
 	}
 
-	rte_rwlock_write_unlock(&vq->iotlb_lock);
+	vhost_user_iotlb_wr_unlock_all(dev);
 }
 
 void
-vhost_user_iotlb_cache_insert(struct virtio_net *dev, struct vhost_virtqueue *vq,
-				uint64_t iova, uint64_t uaddr,
+vhost_user_iotlb_cache_insert(struct virtio_net *dev, uint64_t iova, uint64_t uaddr,
 				uint64_t size, uint8_t perm)
 {
 	struct vhost_iotlb_entry *node, *new_node;
 
-	new_node = vhost_user_iotlb_pool_get(vq);
+	new_node = vhost_user_iotlb_pool_get(dev);
 	if (new_node == NULL) {
 		VHOST_LOG_CONFIG(dev->ifname, DEBUG,
-			"IOTLB pool vq %"PRIu32" empty, clear entries for cache insertion\n",
-			vq->index);
-		if (!TAILQ_EMPTY(&vq->iotlb_list))
-			vhost_user_iotlb_cache_random_evict(dev, vq);
+			"IOTLB pool empty, clear entries for cache insertion\n");
+		if (!TAILQ_EMPTY(&dev->iotlb_list))
+			vhost_user_iotlb_cache_random_evict(dev);
 		else
-			vhost_user_iotlb_pending_remove_all(vq);
-		new_node = vhost_user_iotlb_pool_get(vq);
+			vhost_user_iotlb_pending_remove_all(dev);
+		new_node = vhost_user_iotlb_pool_get(dev);
 		if (new_node == NULL) {
 			VHOST_LOG_CONFIG(dev->ifname, ERR,
-				"IOTLB pool vq %"PRIu32" still empty, cache insertion failed\n",
-				vq->index);
+				"IOTLB pool still empty, cache insertion failed\n");
 			return;
 		}
 	}
@@ -268,49 +259,47 @@  vhost_user_iotlb_cache_insert(struct virtio_net *dev, struct vhost_virtqueue *vq
 	new_node->size = size;
 	new_node->perm = perm;
 
-	rte_rwlock_write_lock(&vq->iotlb_lock);
+	vhost_user_iotlb_wr_lock_all(dev);
 
-	TAILQ_FOREACH(node, &vq->iotlb_list, next) {
+	TAILQ_FOREACH(node, &dev->iotlb_list, next) {
 		/*
 		 * Entries must be invalidated before being updated.
 		 * So if iova already in list, assume identical.
 		 */
 		if (node->iova == new_node->iova) {
-			vhost_user_iotlb_pool_put(vq, new_node);
+			vhost_user_iotlb_pool_put(dev, new_node);
 			goto unlock;
 		} else if (node->iova > new_node->iova) {
 			vhost_user_iotlb_set_dump(dev, new_node);
 
 			TAILQ_INSERT_BEFORE(node, new_node, next);
-			vq->iotlb_cache_nr++;
+			dev->iotlb_cache_nr++;
 			goto unlock;
 		}
 	}
 
 	vhost_user_iotlb_set_dump(dev, new_node);
 
-	TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next);
-	vq->iotlb_cache_nr++;
+	TAILQ_INSERT_TAIL(&dev->iotlb_list, new_node, next);
+	dev->iotlb_cache_nr++;
 
 unlock:
-	vhost_user_iotlb_pending_remove(vq, iova, size, perm);
-
-	rte_rwlock_write_unlock(&vq->iotlb_lock);
+	vhost_user_iotlb_pending_remove(dev, iova, size, perm);
 
+	vhost_user_iotlb_wr_unlock_all(dev);
 }
 
 void
-vhost_user_iotlb_cache_remove(struct virtio_net *dev, struct vhost_virtqueue *vq,
-					uint64_t iova, uint64_t size)
+vhost_user_iotlb_cache_remove(struct virtio_net *dev, uint64_t iova, uint64_t size)
 {
 	struct vhost_iotlb_entry *node, *temp_node, *prev_node = NULL;
 
 	if (unlikely(!size))
 		return;
 
-	rte_rwlock_write_lock(&vq->iotlb_lock);
+	vhost_user_iotlb_wr_lock_all(dev);
 
-	RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
+	RTE_TAILQ_FOREACH_SAFE(node, &dev->iotlb_list, next, temp_node) {
 		/* Sorted list */
 		if (unlikely(iova + size < node->iova))
 			break;
@@ -320,19 +309,19 @@  vhost_user_iotlb_cache_remove(struct virtio_net *dev, struct vhost_virtqueue *vq
 
 			vhost_user_iotlb_clear_dump(dev, node, prev_node, next_node);
 
-			TAILQ_REMOVE(&vq->iotlb_list, node, next);
-			vhost_user_iotlb_pool_put(vq, node);
-			vq->iotlb_cache_nr--;
-		} else
+			TAILQ_REMOVE(&dev->iotlb_list, node, next);
+			vhost_user_iotlb_pool_put(dev, node);
+			dev->iotlb_cache_nr--;
+		} else {
 			prev_node = node;
+		}
 	}
 
-	rte_rwlock_write_unlock(&vq->iotlb_lock);
+	vhost_user_iotlb_wr_unlock_all(dev);
 }
 
 uint64_t
-vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
-						uint64_t *size, uint8_t perm)
+vhost_user_iotlb_cache_find(struct virtio_net *dev, uint64_t iova, uint64_t *size, uint8_t perm)
 {
 	struct vhost_iotlb_entry *node;
 	uint64_t offset, vva = 0, mapped = 0;
@@ -340,7 +329,7 @@  vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
 	if (unlikely(!*size))
 		goto out;
 
-	TAILQ_FOREACH(node, &vq->iotlb_list, next) {
+	TAILQ_FOREACH(node, &dev->iotlb_list, next) {
 		/* List sorted by iova */
 		if (unlikely(iova < node->iova))
 			break;
@@ -373,60 +362,57 @@  vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
 }
 
 void
-vhost_user_iotlb_flush_all(struct virtio_net *dev, struct vhost_virtqueue *vq)
+vhost_user_iotlb_flush_all(struct virtio_net *dev)
 {
-	vhost_user_iotlb_cache_remove_all(dev, vq);
-	vhost_user_iotlb_pending_remove_all(vq);
+	vhost_user_iotlb_cache_remove_all(dev);
+	vhost_user_iotlb_pending_remove_all(dev);
 }
 
 int
-vhost_user_iotlb_init(struct virtio_net *dev, struct vhost_virtqueue *vq)
+vhost_user_iotlb_init(struct virtio_net *dev)
 {
 	unsigned int i;
 	int socket = 0;
 
-	if (vq->iotlb_pool) {
+	if (dev->iotlb_pool) {
 		/*
 		 * The cache has already been initialized,
 		 * just drop all cached and pending entries.
 		 */
-		vhost_user_iotlb_flush_all(dev, vq);
-		rte_free(vq->iotlb_pool);
+		vhost_user_iotlb_flush_all(dev);
+		rte_free(dev->iotlb_pool);
 	}
 
 #ifdef RTE_LIBRTE_VHOST_NUMA
-	if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0)
+	if (get_mempolicy(&socket, NULL, 0, dev, MPOL_F_NODE | MPOL_F_ADDR) != 0)
 		socket = 0;
 #endif
 
-	rte_spinlock_init(&vq->iotlb_free_lock);
-	rte_rwlock_init(&vq->iotlb_lock);
-	rte_rwlock_init(&vq->iotlb_pending_lock);
+	rte_spinlock_init(&dev->iotlb_free_lock);
+	rte_rwlock_init(&dev->iotlb_pending_lock);
 
-	SLIST_INIT(&vq->iotlb_free_list);
-	TAILQ_INIT(&vq->iotlb_list);
-	TAILQ_INIT(&vq->iotlb_pending_list);
+	SLIST_INIT(&dev->iotlb_free_list);
+	TAILQ_INIT(&dev->iotlb_list);
+	TAILQ_INIT(&dev->iotlb_pending_list);
 
 	if (dev->flags & VIRTIO_DEV_SUPPORT_IOMMU) {
-		vq->iotlb_pool = rte_calloc_socket("iotlb", IOTLB_CACHE_SIZE,
+		dev->iotlb_pool = rte_calloc_socket("iotlb", IOTLB_CACHE_SIZE,
 			sizeof(struct vhost_iotlb_entry), 0, socket);
-		if (!vq->iotlb_pool) {
-			VHOST_LOG_CONFIG(dev->ifname, ERR,
-				"Failed to create IOTLB cache pool for vq %"PRIu32"\n",
-				vq->index);
+		if (!dev->iotlb_pool) {
+			VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to create IOTLB cache pool\n");
 			return -1;
 		}
 		for (i = 0; i < IOTLB_CACHE_SIZE; i++)
-			vhost_user_iotlb_pool_put(vq, &vq->iotlb_pool[i]);
+			vhost_user_iotlb_pool_put(dev, &dev->iotlb_pool[i]);
 	}
 
-	vq->iotlb_cache_nr = 0;
+	dev->iotlb_cache_nr = 0;
 
 	return 0;
 }
 
 void
-vhost_user_iotlb_destroy(struct vhost_virtqueue *vq)
+vhost_user_iotlb_destroy(struct virtio_net *dev)
 {
-	rte_free(vq->iotlb_pool);
+	rte_free(dev->iotlb_pool);
 }
diff --git a/lib/vhost/iotlb.h b/lib/vhost/iotlb.h
index 73b5465b41..3490b9e6be 100644
--- a/lib/vhost/iotlb.h
+++ b/lib/vhost/iotlb.h
@@ -37,20 +37,37 @@  vhost_user_iotlb_wr_unlock(struct vhost_virtqueue *vq)
 	rte_rwlock_write_unlock(&vq->iotlb_lock);
 }
 
-void vhost_user_iotlb_cache_insert(struct virtio_net *dev, struct vhost_virtqueue *vq,
-					uint64_t iova, uint64_t uaddr,
+static __rte_always_inline void
+vhost_user_iotlb_wr_lock_all(struct virtio_net *dev)
+	__rte_no_thread_safety_analysis
+{
+	uint32_t i;
+
+	for (i = 0; i < dev->nr_vring; i++)
+		rte_rwlock_write_lock(&dev->virtqueue[i]->iotlb_lock);
+}
+
+static __rte_always_inline void
+vhost_user_iotlb_wr_unlock_all(struct virtio_net *dev)
+	__rte_no_thread_safety_analysis
+{
+	uint32_t i;
+
+	for (i = 0; i < dev->nr_vring; i++)
+		rte_rwlock_write_unlock(&dev->virtqueue[i]->iotlb_lock);
+}
+
+void vhost_user_iotlb_cache_insert(struct virtio_net *dev, uint64_t iova, uint64_t uaddr,
 					uint64_t size, uint8_t perm);
-void vhost_user_iotlb_cache_remove(struct virtio_net *dev, struct vhost_virtqueue *vq,
-					uint64_t iova, uint64_t size);
-uint64_t vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
+void vhost_user_iotlb_cache_remove(struct virtio_net *dev, uint64_t iova, uint64_t size);
+uint64_t vhost_user_iotlb_cache_find(struct virtio_net *dev, uint64_t iova,
 					uint64_t *size, uint8_t perm);
-bool vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,
-						uint8_t perm);
-void vhost_user_iotlb_pending_insert(struct virtio_net *dev, struct vhost_virtqueue *vq,
-						uint64_t iova, uint8_t perm);
-void vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq, uint64_t iova,
+bool vhost_user_iotlb_pending_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm);
+void vhost_user_iotlb_pending_insert(struct virtio_net *dev, uint64_t iova, uint8_t perm);
+void vhost_user_iotlb_pending_remove(struct virtio_net *dev, uint64_t iova,
 						uint64_t size, uint8_t perm);
-void vhost_user_iotlb_flush_all(struct virtio_net *dev, struct vhost_virtqueue *vq);
-int vhost_user_iotlb_init(struct virtio_net *dev, struct vhost_virtqueue *vq);
-void vhost_user_iotlb_destroy(struct vhost_virtqueue *vq);
+void vhost_user_iotlb_flush_all(struct virtio_net *dev);
+int vhost_user_iotlb_init(struct virtio_net *dev);
+void vhost_user_iotlb_destroy(struct virtio_net *dev);
+
 #endif /* _VHOST_IOTLB_H_ */
diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c
index 79e88f986e..3ddd2a963f 100644
--- a/lib/vhost/vhost.c
+++ b/lib/vhost/vhost.c
@@ -67,7 +67,7 @@  __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
 
 	tmp_size = *size;
 
-	vva = vhost_user_iotlb_cache_find(vq, iova, &tmp_size, perm);
+	vva = vhost_user_iotlb_cache_find(dev, iova, &tmp_size, perm);
 	if (tmp_size == *size) {
 		if (dev->flags & VIRTIO_DEV_STATS_ENABLED)
 			vq->stats.iotlb_hits++;
@@ -79,7 +79,7 @@  __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
 
 	iova += tmp_size;
 
-	if (!vhost_user_iotlb_pending_miss(vq, iova, perm)) {
+	if (!vhost_user_iotlb_pending_miss(dev, iova, perm)) {
 		/*
 		 * iotlb_lock is read-locked for a full burst,
 		 * but it only protects the iotlb cache.
@@ -89,12 +89,12 @@  __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		 */
 		vhost_user_iotlb_rd_unlock(vq);
 
-		vhost_user_iotlb_pending_insert(dev, vq, iova, perm);
+		vhost_user_iotlb_pending_insert(dev, iova, perm);
 		if (vhost_user_iotlb_miss(dev, iova, perm)) {
 			VHOST_LOG_DATA(dev->ifname, ERR,
 				"IOTLB miss req failed for IOVA 0x%" PRIx64 "\n",
 				iova);
-			vhost_user_iotlb_pending_remove(vq, iova, 1, perm);
+			vhost_user_iotlb_pending_remove(dev, iova, 1, perm);
 		}
 
 		vhost_user_iotlb_rd_lock(vq);
@@ -401,7 +401,6 @@  free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq)
 	vhost_free_async_mem(vq);
 	rte_rwlock_write_unlock(&vq->access_lock);
 	rte_free(vq->batch_copy_elems);
-	vhost_user_iotlb_destroy(vq);
 	rte_free(vq->log_cache);
 	rte_free(vq);
 }
@@ -579,7 +578,7 @@  vring_invalidate(struct virtio_net *dev __rte_unused, struct vhost_virtqueue *vq
 }
 
 static void
-init_vring_queue(struct virtio_net *dev, struct vhost_virtqueue *vq,
+init_vring_queue(struct virtio_net *dev __rte_unused, struct vhost_virtqueue *vq,
 	uint32_t vring_idx)
 {
 	int numa_node = SOCKET_ID_ANY;
@@ -599,8 +598,6 @@  init_vring_queue(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	}
 #endif
 	vq->numa_node = numa_node;
-
-	vhost_user_iotlb_init(dev, vq);
 }
 
 static void
@@ -635,6 +632,7 @@  alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
 		dev->virtqueue[i] = vq;
 		init_vring_queue(dev, vq, i);
 		rte_rwlock_init(&vq->access_lock);
+		rte_rwlock_init(&vq->iotlb_lock);
 		vq->avail_wrap_counter = 1;
 		vq->used_wrap_counter = 1;
 		vq->signalled_used_valid = false;
@@ -799,6 +797,10 @@  vhost_setup_virtio_net(int vid, bool enable, bool compliant_ol_flags, bool stats
 		dev->flags |= VIRTIO_DEV_SUPPORT_IOMMU;
 	else
 		dev->flags &= ~VIRTIO_DEV_SUPPORT_IOMMU;
+
+	if (vhost_user_iotlb_init(dev) < 0)
+		VHOST_LOG_CONFIG("device", ERR, "failed to init IOTLB\n");
+
 }
 
 void
diff --git a/lib/vhost/vhost.h b/lib/vhost/vhost.h
index eaf3b0d392..ee952de175 100644
--- a/lib/vhost/vhost.h
+++ b/lib/vhost/vhost.h
@@ -305,13 +305,6 @@  struct vhost_virtqueue {
 	struct log_cache_entry	*log_cache;
 
 	rte_rwlock_t	iotlb_lock;
-	rte_rwlock_t	iotlb_pending_lock;
-	struct vhost_iotlb_entry *iotlb_pool;
-	TAILQ_HEAD(, vhost_iotlb_entry) iotlb_list;
-	TAILQ_HEAD(, vhost_iotlb_entry) iotlb_pending_list;
-	int				iotlb_cache_nr;
-	rte_spinlock_t	iotlb_free_lock;
-	SLIST_HEAD(, vhost_iotlb_entry) iotlb_free_list;
 
 	/* Used to notify the guest (trigger interrupt) */
 	int			callfd;
@@ -486,6 +479,15 @@  struct virtio_net {
 	int			extbuf;
 	int			linearbuf;
 	struct vhost_virtqueue	*virtqueue[VHOST_MAX_QUEUE_PAIRS * 2];
+
+	rte_rwlock_t	iotlb_pending_lock;
+	struct vhost_iotlb_entry *iotlb_pool;
+	TAILQ_HEAD(, vhost_iotlb_entry) iotlb_list;
+	TAILQ_HEAD(, vhost_iotlb_entry) iotlb_pending_list;
+	int				iotlb_cache_nr;
+	rte_spinlock_t	iotlb_free_lock;
+	SLIST_HEAD(, vhost_iotlb_entry) iotlb_free_list;
+
 	struct inflight_mem_info *inflight_info;
 #define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ)
 	char			ifname[IF_NAME_SZ];
diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c
index c9454ce3d9..f2fe7ebc93 100644
--- a/lib/vhost/vhost_user.c
+++ b/lib/vhost/vhost_user.c
@@ -237,6 +237,8 @@  vhost_backend_cleanup(struct virtio_net *dev)
 	}
 
 	dev->postcopy_listening = 0;
+
+	vhost_user_iotlb_destroy(dev);
 }
 
 static void
@@ -539,7 +541,6 @@  numa_realloc(struct virtio_net **pdev, struct vhost_virtqueue **pvq)
 	if (vq != dev->virtqueue[vq->index]) {
 		VHOST_LOG_CONFIG(dev->ifname, INFO, "reallocated virtqueue on node %d\n", node);
 		dev->virtqueue[vq->index] = vq;
-		vhost_user_iotlb_init(dev, vq);
 	}
 
 	if (vq_is_packed(dev)) {
@@ -664,6 +665,8 @@  numa_realloc(struct virtio_net **pdev, struct vhost_virtqueue **pvq)
 		return;
 	}
 	dev->guest_pages = gp;
+
+	vhost_user_iotlb_init(dev);
 }
 #else
 static void
@@ -1360,8 +1363,7 @@  vhost_user_set_mem_table(struct virtio_net **pdev,
 
 		/* Flush IOTLB cache as previous HVAs are now invalid */
 		if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
-			for (i = 0; i < dev->nr_vring; i++)
-				vhost_user_iotlb_flush_all(dev, dev->virtqueue[i]);
+			vhost_user_iotlb_flush_all(dev);
 
 		free_mem_region(dev);
 		rte_free(dev->mem);
@@ -2194,7 +2196,7 @@  vhost_user_get_vring_base(struct virtio_net **pdev,
 	ctx->msg.size = sizeof(ctx->msg.payload.state);
 	ctx->fd_num = 0;
 
-	vhost_user_iotlb_flush_all(dev, vq);
+	vhost_user_iotlb_flush_all(dev);
 
 	vring_invalidate(dev, vq);
 
@@ -2639,15 +2641,14 @@  vhost_user_iotlb_msg(struct virtio_net **pdev,
 		if (!vva)
 			return RTE_VHOST_MSG_RESULT_ERR;
 
+		vhost_user_iotlb_cache_insert(dev, imsg->iova, vva, len, imsg->perm);
+
 		for (i = 0; i < dev->nr_vring; i++) {
 			struct vhost_virtqueue *vq = dev->virtqueue[i];
 
 			if (!vq)
 				continue;
 
-			vhost_user_iotlb_cache_insert(dev, vq, imsg->iova, vva,
-					len, imsg->perm);
-
 			if (is_vring_iotlb(dev, vq, imsg)) {
 				rte_rwlock_write_lock(&vq->access_lock);
 				translate_ring_addresses(&dev, &vq);
@@ -2657,15 +2658,14 @@  vhost_user_iotlb_msg(struct virtio_net **pdev,
 		}
 		break;
 	case VHOST_IOTLB_INVALIDATE:
+		vhost_user_iotlb_cache_remove(dev, imsg->iova, imsg->size);
+
 		for (i = 0; i < dev->nr_vring; i++) {
 			struct vhost_virtqueue *vq = dev->virtqueue[i];
 
 			if (!vq)
 				continue;
 
-			vhost_user_iotlb_cache_remove(dev, vq, imsg->iova,
-					imsg->size);
-
 			if (is_vring_iotlb(dev, vq, imsg)) {
 				rte_rwlock_write_lock(&vq->access_lock);
 				vring_invalidate(dev, vq);
@@ -2674,8 +2674,7 @@  vhost_user_iotlb_msg(struct virtio_net **pdev,
 		}
 		break;
 	default:
-		VHOST_LOG_CONFIG(dev->ifname, ERR,
-			"invalid IOTLB message type (%d)\n",
+		VHOST_LOG_CONFIG(dev->ifname, ERR, "invalid IOTLB message type (%d)\n",
 			imsg->type);
 		return RTE_VHOST_MSG_RESULT_ERR;
 	}