==================|   |==================|

In RX side, it firstly polls each queue of the pool and gets the
packets from it and enqueue them into its corresponding queue in
virtio device/port.  In TX side, it dequeue packets from each queue
of virtio device/port and send them to either physical port or
another virtio device according to its destination MAC address.

We bind the virtq to a specific core by rte_vhost_core_id_set(),
and later we can retrieve it by rte_vhost_core_id_get().

Signed-off-by: Changchun Ouyang <changchun.ouyang@intel.com>
Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
---
 examples/vhost/main.c | 325 ++++++++++++++++++++++++++++++++++----------------
 examples/vhost/main.h |   3 +-
 2 files changed, 225 insertions(+), 103 deletions(-)

diff --git a/examples/vhost/main.c b/examples/vhost/main.c
index 9eac2d0..23b7aa7 100644
--- a/examples/vhost/main.c
+++ b/examples/vhost/main.c
@@ -163,6 +163,9 @@ static int mergeable;
 /* Do vlan strip on host, enabled on default */
 static uint32_t vlan_strip = 1;
 
+/* Rx queue number per virtio device */
+static uint32_t rxq = 1;
+
 /* number of descriptors to apply*/
 static uint32_t num_rx_descriptor = RTE_TEST_RX_DESC_DEFAULT_ZCP;
 static uint32_t num_tx_descriptor = RTE_TEST_TX_DESC_DEFAULT_ZCP;
@@ -365,6 +368,37 @@ validate_num_devices(uint32_t max_nb_devices)
 	return 0;
 }
 
+static int
+get_dev_nb_for_82599(struct rte_eth_dev_info dev_info)
+{
+	int dev_nb = -1;
+	switch (rxq) {
+	case 1:
+	case 2:
+		/*
+		 * for 82599, dev_info.max_vmdq_pools always 64 dispite rx mode.
+		 */
+		dev_nb = (int)dev_info.max_vmdq_pools;
+		break;
+	case 4:
+		dev_nb = (int)dev_info.max_vmdq_pools / 2;
+		break;
+	default:
+		RTE_LOG(ERR, VHOST_CONFIG, "invalid rxq for VMDq.\n");
+	}
+	return dev_nb;
+}
+
+static int
+get_dev_nb_for_fvl(struct rte_eth_dev_info dev_info)
+{
+	/*
+	 * for FVL, dev_info.max_vmdq_pools is calculated according to
+	 * the configured value: CONFIG_RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM.
+	 */
+	return (int)dev_info.max_vmdq_pools;
+}
+
 /*
  * Initialises a given port using global settings and with the rx buffers
  * coming from the mbuf_pool passed as parameter
@@ -380,6 +414,7 @@ port_init(uint8_t port)
 	uint16_t rx_ring_size, tx_ring_size;
 	int retval;
 	uint16_t q;
+	struct rte_eth_dev *eth_dev;
 
 	/* The max pool number from dev_info will be used to validate the pool number specified in cmd line */
 	rte_eth_dev_info_get (port, &dev_info);
@@ -408,8 +443,16 @@ port_init(uint8_t port)
 		txconf->tx_deferred_start = 1;
 	}
 
-	/*configure the number of supported virtio devices based on VMDQ limits */
-	num_devices = dev_info.max_vmdq_pools;
+	/* Configure the virtio devices num based on VMDQ limits */
+	if (dev_info.max_vmdq_pools == ETH_64_POOLS) {
+		num_devices = (uint32_t)get_dev_nb_for_82599(dev_info);
+		if (num_devices == (uint32_t)-1)
+			return -1;
+	} else {
+		num_devices = (uint32_t)get_dev_nb_for_fvl(dev_info);
+		if (num_devices == (uint32_t)-1)
+			return -1;
+	}
 
 	if (zero_copy) {
 		rx_ring_size = num_rx_descriptor;
@@ -431,7 +474,7 @@ port_init(uint8_t port)
 		return retval;
 	/* NIC queues are divided into pf queues and vmdq queues.  */
 	num_pf_queues = dev_info.max_rx_queues - dev_info.vmdq_queue_num;
-	queues_per_pool = dev_info.vmdq_queue_num / dev_info.max_vmdq_pools;
+	queues_per_pool = dev_info.vmdq_queue_num / num_devices;
 	num_vmdq_queues = num_devices * queues_per_pool;
 	num_queues = num_pf_queues + num_vmdq_queues;
 	vmdq_queue_base = dev_info.vmdq_queue_base;
@@ -447,6 +490,14 @@ port_init(uint8_t port)
 	if (retval != 0)
 		return retval;
 
+	eth_dev = &rte_eth_devices[port];
+	if (RTE_ETH_DEV_SRIOV(eth_dev).nb_q_per_pool > 4) {
+		RTE_LOG(ERR, VHOST_CONFIG, "ethdev port_id=%d SRIOV active, "
+			"invalid queue number for VMDQ RSS, allowed value "
+			"are 1, 2 or 4\n", port);
+		return -EINVAL;
+	}
+
 	/* Setup the queues. */
 	for (q = 0; q < rx_rings; q ++) {
 		retval = rte_eth_rx_queue_setup(port, q, rx_ring_size,
@@ -576,7 +627,8 @@ us_vhost_usage(const char *prgname)
 	"		--rx-desc-num [0-N]: the number of descriptors on rx, "
 			"used only when zero copy is enabled.\n"
 	"		--tx-desc-num [0-N]: the number of descriptors on tx, "
-			"used only when zero copy is enabled.\n",
+			"used only when zero copy is enabled.\n"
+	"		--rxq [1,2,4]: rx queue number for each vhost device\n",
 	       prgname);
 }
 
@@ -602,6 +654,7 @@ us_vhost_parse_args(int argc, char **argv)
 		{"zero-copy", required_argument, NULL, 0},
 		{"rx-desc-num", required_argument, NULL, 0},
 		{"tx-desc-num", required_argument, NULL, 0},
+		{"rxq", required_argument, NULL, 0},
 		{NULL, 0, 0, 0},
 	};
 
@@ -778,6 +831,18 @@ us_vhost_parse_args(int argc, char **argv)
 				}
 			}
 
+			/* Specify the Rx queue number for each vhost dev. */
+			if (!strncmp(long_option[option_index].name,
+				"rxq", MAX_LONG_OPT_SZ)) {
+				ret = parse_num_opt(optarg, 4);
+				if ((ret == -1) || (ret == 0) || (!POWEROF2(ret))) {
+					RTE_LOG(INFO, VHOST_CONFIG,
+					"Valid value for rxq is [1,2,4]\n");
+					us_vhost_usage(prgname);
+					return -1;
+				} else
+					rxq = ret;
+			}
 			break;
 
 			/* Invalid option - print options. */
@@ -813,6 +878,19 @@ us_vhost_parse_args(int argc, char **argv)
 		return -1;
 	}
 
+	if (rxq > 1) {
+		vmdq_conf_default.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS;
+		vmdq_conf_default.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_IP |
+				ETH_RSS_UDP | ETH_RSS_TCP | ETH_RSS_SCTP;
+	}
+
+	if ((zero_copy == 1) && (rxq > 1)) {
+		RTE_LOG(INFO, VHOST_PORT,
+			"Vhost zero copy doesn't support mq mode,"
+			"please specify '--rxq 1' to disable it.\n");
+		return -1;
+	}
+
 	return 0;
 }
 
@@ -959,9 +1037,11 @@ link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m)
 					dev->device_fh);
 
 	/* Enable stripping of the vlan tag as we handle routing. */
-	if (vlan_strip)
-		rte_eth_dev_set_vlan_strip_on_queue(ports[0],
-			(uint16_t)vdev->vmdq_rx_q, 1);
+	if (vlan_strip) {
+		for (i = 0; i < (int)rxq; i++)
+			rte_eth_dev_set_vlan_strip_on_queue(ports[0],
+				(uint16_t)(vdev->vmdq_rx_q + i), 1);
+	}
 
 	/* Set device as ready for RX. */
 	vdev->ready = DEVICE_RX;
@@ -976,7 +1056,7 @@ link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m)
 static inline void
 unlink_vmdq(struct vhost_dev *vdev)
 {
-	unsigned i = 0;
+	unsigned i = 0, j = 0;
 	unsigned rx_count;
 	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 
@@ -989,15 +1069,19 @@ unlink_vmdq(struct vhost_dev *vdev)
 		vdev->vlan_tag = 0;
 
 		/*Clear out the receive buffers*/
-		rx_count = rte_eth_rx_burst(ports[0],
-					(uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
+		for (i = 0; i < rxq; i++) {
+			rx_count = rte_eth_rx_burst(ports[0],
+					(uint16_t)vdev->vmdq_rx_q + i,
+					pkts_burst, MAX_PKT_BURST);
 
-		while (rx_count) {
-			for (i = 0; i < rx_count; i++)
-				rte_pktmbuf_free(pkts_burst[i]);
+			while (rx_count) {
+				for (j = 0; j < rx_count; j++)
+					rte_pktmbuf_free(pkts_burst[j]);
 
-			rx_count = rte_eth_rx_burst(ports[0],
-					(uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
+				rx_count = rte_eth_rx_burst(ports[0],
+					(uint16_t)vdev->vmdq_rx_q + i,
+					pkts_burst, MAX_PKT_BURST);
+			}
 		}
 
 		vdev->ready = DEVICE_MAC_LEARNING;
@@ -1009,7 +1093,7 @@ unlink_vmdq(struct vhost_dev *vdev)
  * the packet on that devices RX queue. If not then return.
  */
 static inline int __attribute__((always_inline))
-virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)
+virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m, uint32_t qp_idx)
 {
 	struct virtio_net_data_ll *dev_ll;
 	struct ether_hdr *pkt_hdr;
@@ -1024,7 +1108,7 @@ virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)
 
 	while (dev_ll != NULL) {
 		if ((dev_ll->vdev->ready == DEVICE_RX) && ether_addr_cmp(&(pkt_hdr->d_addr),
-				          &dev_ll->vdev->mac_address)) {
+					&dev_ll->vdev->mac_address)) {
 
 			/* Drop the packet if the TX packet is destined for the TX device. */
 			if (dev_ll->vdev->dev->device_fh == dev->device_fh) {
@@ -1042,7 +1126,9 @@ virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)
 				LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Device is marked for removal\n", tdev->device_fh);
 			} else {
 				/*send the packet to the local virtio device*/
-				ret = rte_vhost_enqueue_burst(tdev, VIRTIO_RXQ, &m, 1);
+				ret = rte_vhost_enqueue_burst(tdev,
+					VIRTIO_RXQ + qp_idx * VIRTIO_QNUM,
+					&m, 1);
 				if (enable_stats) {
 					rte_atomic64_add(
 					&dev_statistics[tdev->device_fh].rx_total_atomic,
@@ -1119,7 +1205,8 @@ find_local_dest(struct virtio_net *dev, struct rte_mbuf *m,
  * or the physical port.
  */
 static inline void __attribute__((always_inline))
-virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag)
+virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m,
+		uint16_t vlan_tag, uint32_t qp_idx)
 {
 	struct mbuf_table *tx_q;
 	struct rte_mbuf **m_table;
@@ -1129,7 +1216,8 @@ virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag)
 	struct ether_hdr *nh;
 
 	/*check if destination is local VM*/
-	if ((vm2vm_mode == VM2VM_SOFTWARE) && (virtio_tx_local(vdev, m) == 0)) {
+	if ((vm2vm_mode == VM2VM_SOFTWARE) &&
+	    (virtio_tx_local(vdev, m, qp_idx) == 0)) {
 		rte_pktmbuf_free(m);
 		return;
 	}
@@ -1293,22 +1381,26 @@ switch_worker(__attribute__((unused)) void *arg)
 			}
 			if (likely(vdev->ready == DEVICE_RX)) {
 				/*Handle guest RX*/
+				uint16_t qp_idx = dev_ll->work_qp_idx;
 				rx_count = rte_eth_rx_burst(ports[0],
-					vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
+					vdev->vmdq_rx_q + qp_idx, pkts_burst, MAX_PKT_BURST);
 
 				if (rx_count) {
 					/*
 					* Retry is enabled and the queue is full then we wait and retry to avoid packet loss
 					* Here MAX_PKT_BURST must be less than virtio queue size
 					*/
-					if (enable_retry && unlikely(rx_count > rte_vring_available_entries(dev, VIRTIO_RXQ))) {
+					if (enable_retry && unlikely(rx_count > rte_vring_available_entries(dev,
+										VIRTIO_RXQ + qp_idx * VIRTIO_QNUM))) {
 						for (retry = 0; retry < burst_rx_retry_num; retry++) {
 							rte_delay_us(burst_rx_delay_time);
-							if (rx_count <= rte_vring_available_entries(dev, VIRTIO_RXQ))
+							if (rx_count <= rte_vring_available_entries(dev,
+										VIRTIO_RXQ + qp_idx * VIRTIO_QNUM))
 								break;
 						}
 					}
-					ret_count = rte_vhost_enqueue_burst(dev, VIRTIO_RXQ, pkts_burst, rx_count);
+					ret_count = rte_vhost_enqueue_burst(dev, VIRTIO_RXQ + qp_idx * VIRTIO_QNUM,
+										pkts_burst, rx_count);
 					if (enable_stats) {
 						rte_atomic64_add(
 						&dev_statistics[dev_ll->vdev->dev->device_fh].rx_total_atomic,
@@ -1320,14 +1412,18 @@ switch_worker(__attribute__((unused)) void *arg)
 						rx_count--;
 						rte_pktmbuf_free(pkts_burst[rx_count]);
 					}
-
 				}
 			}
 
 			if (likely(!vdev->remove)) {
 				/* Handle guest TX*/
-				tx_count = rte_vhost_dequeue_burst(dev, VIRTIO_TXQ, mbuf_pool, pkts_burst, MAX_PKT_BURST);
-				/* If this is the first received packet we need to learn the MAC and setup VMDQ */
+				uint16_t qp_idx = dev_ll->work_qp_idx;
+				tx_count = rte_vhost_dequeue_burst(dev, VIRTIO_TXQ + qp_idx * VIRTIO_QNUM,
+						mbuf_pool, pkts_burst, MAX_PKT_BURST);
+				/*
+				 * If this is the first received packet we need to learn
+				 * the MAC and setup VMDQ
+				 */
 				if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && tx_count) {
 					if (vdev->remove || (link_vmdq(vdev, pkts_burst[0]) == -1)) {
 						while (tx_count)
@@ -1335,7 +1431,8 @@ switch_worker(__attribute__((unused)) void *arg)
 					}
 				}
 				while (tx_count)
-					virtio_tx_route(vdev, pkts_burst[--tx_count], (uint16_t)dev->device_fh);
+					virtio_tx_route(vdev, pkts_burst[--tx_count],
+						(uint16_t)dev->device_fh, qp_idx);
 			}
 
 			/*move to the next device in the list*/
@@ -2323,6 +2420,7 @@ destroy_device (volatile struct virtio_net *dev)
 	struct virtio_net_data_ll *ll_main_dev_last = NULL;
 	struct vhost_dev *vdev;
 	int lcore;
+	uint32_t i;
 
 	dev->flags &= ~VIRTIO_DEV_RUNNING;
 
@@ -2334,61 +2432,73 @@ destroy_device (volatile struct virtio_net *dev)
 	}
 
 	/* Search for entry to be removed from lcore ll */
-	ll_lcore_dev_cur = lcore_info[vdev->coreid].lcore_ll->ll_root_used;
-	while (ll_lcore_dev_cur != NULL) {
-		if (ll_lcore_dev_cur->vdev == vdev) {
-			break;
-		} else {
-			ll_lcore_dev_last = ll_lcore_dev_cur;
-			ll_lcore_dev_cur = ll_lcore_dev_cur->next;
+	for (i = 0; i < rxq; i++) {
+		uint16_t core_id = rte_vhost_core_id_get(dev, i);
+
+		ll_lcore_dev_cur = lcore_info[core_id].lcore_ll->ll_root_used;
+
+		while (ll_lcore_dev_cur != NULL) {
+			if (ll_lcore_dev_cur->vdev == vdev) {
+				break;
+			} else {
+				ll_lcore_dev_last = ll_lcore_dev_cur;
+				ll_lcore_dev_cur = ll_lcore_dev_cur->next;
+			}
 		}
-	}
 
-	if (ll_lcore_dev_cur == NULL) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"(%"PRIu64") Failed to find the dev to be destroy.\n",
-			dev->device_fh);
-		return;
-	}
+		if (ll_lcore_dev_cur == NULL) {
+			RTE_LOG(ERR, VHOST_CONFIG,
+				"(%"PRIu64") Failed to find the dev to be destroy.\n",
+				dev->device_fh);
+			if (i == 0)
+				return;
+			else
+				break;
+		}
 
-	/* Search for entry to be removed from main ll */
-	ll_main_dev_cur = ll_root_used;
-	ll_main_dev_last = NULL;
-	while (ll_main_dev_cur != NULL) {
-		if (ll_main_dev_cur->vdev == vdev) {
-			break;
-		} else {
-			ll_main_dev_last = ll_main_dev_cur;
-			ll_main_dev_cur = ll_main_dev_cur->next;
+		/* Search for entry to be removed from main ll */
+		if (i == 0) {
+			ll_main_dev_cur = ll_root_used;
+			ll_main_dev_last = NULL;
+			while (ll_main_dev_cur != NULL) {
+				if (ll_main_dev_cur->vdev == vdev) {
+					break;
+				} else {
+					ll_main_dev_last = ll_main_dev_cur;
+					ll_main_dev_cur = ll_main_dev_cur->next;
+				}
+			}
 		}
-	}
 
-	/* Remove entries from the lcore and main ll. */
-	rm_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used, ll_lcore_dev_cur, ll_lcore_dev_last);
-	rm_data_ll_entry(&ll_root_used, ll_main_dev_cur, ll_main_dev_last);
+		/* Remove entries from the lcore and main ll. */
+		rm_data_ll_entry(&lcore_info[core_id].lcore_ll->ll_root_used, ll_lcore_dev_cur, ll_lcore_dev_last);
+		if (i == 0)
+			rm_data_ll_entry(&ll_root_used, ll_main_dev_cur, ll_main_dev_last);
 
-	/* Set the dev_removal_flag on each lcore. */
-	RTE_LCORE_FOREACH_SLAVE(lcore) {
-		lcore_info[lcore].lcore_ll->dev_removal_flag = REQUEST_DEV_REMOVAL;
-	}
+		/* Set the dev_removal_flag on each lcore. */
+		RTE_LCORE_FOREACH_SLAVE(lcore) {
+			lcore_info[lcore].lcore_ll->dev_removal_flag = REQUEST_DEV_REMOVAL;
+		}
 
-	/*
-	 * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL we can be sure that
-	 * they can no longer access the device removed from the linked lists and that the devices
-	 * are no longer in use.
-	 */
-	RTE_LCORE_FOREACH_SLAVE(lcore) {
-		while (lcore_info[lcore].lcore_ll->dev_removal_flag != ACK_DEV_REMOVAL) {
-			rte_pause();
+		/*
+		 * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL we can be sure that
+		 * they can no longer access the device removed from the linked lists and that the devices
+		 * are no longer in use.
+		 */
+		RTE_LCORE_FOREACH_SLAVE(lcore) {
+			while (lcore_info[lcore].lcore_ll->dev_removal_flag != ACK_DEV_REMOVAL)
+				rte_pause();
 		}
-	}
 
-	/* Add the entries back to the lcore and main free ll.*/
-	put_data_ll_free_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_free, ll_lcore_dev_cur);
-	put_data_ll_free_entry(&ll_root_free, ll_main_dev_cur);
+		/* Add the entries back to the lcore and main free ll.*/
+		put_data_ll_free_entry(&lcore_info[core_id].lcore_ll->ll_root_free, ll_lcore_dev_cur);
+
+		if (i == 0)
+			put_data_ll_free_entry(&ll_root_free, ll_main_dev_cur);
 
-	/* Decrement number of device on the lcore. */
-	lcore_info[vdev->coreid].lcore_ll->device_num--;
+		/* Decrement number of device on the lcore. */
+		lcore_info[core_id].lcore_ll->device_num--;
+	}
 
 	RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been removed from data core\n", dev->device_fh);
 
@@ -2593,6 +2703,14 @@ new_device (struct virtio_net *dev)
 	uint32_t device_num_min = num_devices;
 	struct vhost_dev *vdev;
 	uint32_t regionidx;
+	uint32_t i;
+
+	if ((rxq > 1) && (dev->virt_qp_nb != rxq)) {
+		RTE_LOG(ERR, VHOST_DATA, "(%"PRIu64") queue num in VMDq pool:"
+			"%d != queue pair num in vhost dev:%d\n",
+			dev->device_fh, rxq, dev->virt_qp_nb);
+		return -1;
+	}
 
 	vdev = rte_zmalloc("vhost device", sizeof(*vdev), RTE_CACHE_LINE_SIZE);
 	if (vdev == NULL) {
@@ -2638,12 +2756,12 @@ new_device (struct virtio_net *dev)
 		}
 	}
 
-
 	/* Add device to main ll */
 	ll_dev = get_data_ll_free_entry(&ll_root_free);
 	if (ll_dev == NULL) {
-		RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") No free entry found in linked list. Device limit "
-			"of %d devices per core has been reached\n",
+		RTE_LOG(INFO, VHOST_DATA,
+			"(%"PRIu64") No free entry found in linked list."
+			"Device limit of %d devices per core has been reached\n",
 			dev->device_fh, num_devices);
 		if (vdev->regions_hpa)
 			rte_free(vdev->regions_hpa);
@@ -2652,8 +2770,7 @@ new_device (struct virtio_net *dev)
 	}
 	ll_dev->vdev = vdev;
 	add_data_ll_entry(&ll_root_used, ll_dev);
-	vdev->vmdq_rx_q
-		= dev->device_fh * queues_per_pool + vmdq_queue_base;
+	vdev->vmdq_rx_q	= dev->device_fh * rxq + vmdq_queue_base;
 
 	if (zero_copy) {
 		uint32_t index = vdev->vmdq_rx_q;
@@ -2734,37 +2851,42 @@ new_device (struct virtio_net *dev)
 	vdev->remove = 0;
 
 	/* Find a suitable lcore to add the device. */
-	RTE_LCORE_FOREACH_SLAVE(lcore) {
-		if (lcore_info[lcore].lcore_ll->device_num < device_num_min) {
-			device_num_min = lcore_info[lcore].lcore_ll->device_num;
-			core_add = lcore;
+	for (i = 0; i < rxq; i++) {
+		device_num_min = num_devices;
+		RTE_LCORE_FOREACH_SLAVE(lcore) {
+			if (lcore_info[lcore].lcore_ll->device_num < device_num_min) {
+				device_num_min = lcore_info[lcore].lcore_ll->device_num;
+				core_add = lcore;
+			}
 		}
-	}
-	/* Add device to lcore ll */
-	ll_dev = get_data_ll_free_entry(&lcore_info[core_add].lcore_ll->ll_root_free);
-	if (ll_dev == NULL) {
-		RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Failed to add device to data core\n", dev->device_fh);
-		vdev->ready = DEVICE_SAFE_REMOVE;
-		destroy_device(dev);
-		rte_free(vdev->regions_hpa);
-		rte_free(vdev);
-		return -1;
-	}
-	ll_dev->vdev = vdev;
-	vdev->coreid = core_add;
+		/* Add device to lcore ll */
+		ll_dev = get_data_ll_free_entry(&lcore_info[core_add].lcore_ll->ll_root_free);
+		if (ll_dev == NULL) {
+			RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Failed to add device to data core\n", dev->device_fh);
+			vdev->ready = DEVICE_SAFE_REMOVE;
+			destroy_device(dev);
+			rte_free(vdev->regions_hpa);
+			rte_free(vdev);
+			return -1;
+		}
+		ll_dev->vdev = vdev;
+		ll_dev->work_qp_idx = i;
+		rte_vhost_core_id_set(dev, i, core_add);
+		add_data_ll_entry(&lcore_info[core_add].lcore_ll->ll_root_used, ll_dev);
 
-	add_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used, ll_dev);
+		/* Disable notifications. */
+		rte_vhost_enable_guest_notification(dev, i * VIRTIO_QNUM + VIRTIO_RXQ, 0);
+		rte_vhost_enable_guest_notification(dev, i * VIRTIO_QNUM + VIRTIO_TXQ, 0);
+		lcore_info[core_add].lcore_ll->device_num++;
+		RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been added to data core %d for vq: %d\n",
+			dev->device_fh, core_add, i);
+	}
 
 	/* Initialize device stats */
 	memset(&dev_statistics[dev->device_fh], 0, sizeof(struct device_statistics));
 
-	/* Disable notifications. */
-	rte_vhost_enable_guest_notification(dev, VIRTIO_RXQ, 0);
-	rte_vhost_enable_guest_notification(dev, VIRTIO_TXQ, 0);
-	lcore_info[vdev->coreid].lcore_ll->device_num++;
 	dev->flags |= VIRTIO_DEV_RUNNING;
 
-	RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been added to data core %d\n", dev->device_fh, vdev->coreid);
 
 	return 0;
 }
@@ -2833,6 +2955,7 @@ print_stats(void)
 					rx_dropped,
 					rx);
 
+
 			dev_ll = dev_ll->next;
 		}
 		printf("\n======================================================\n");
diff --git a/examples/vhost/main.h b/examples/vhost/main.h
index d04e2be..5561c82 100644
--- a/examples/vhost/main.h
+++ b/examples/vhost/main.h
@@ -82,8 +82,6 @@ struct vhost_dev {
 	uint16_t vmdq_rx_q;
 	/**< Vlan tag assigned to the pool */
 	uint32_t vlan_tag;
-	/**< Data core that the device is added to. */
-	uint16_t coreid;
 	/**< A device is set as ready if the MAC address has been set. */
 	volatile uint8_t ready;
 	/**< Device is marked for removal from the data core. */
@@ -94,6 +92,7 @@ struct virtio_net_data_ll
 {
 	struct vhost_dev		*vdev;	/* Pointer to device created by configuration core. */
 	struct virtio_net_data_ll	*next;  /* Pointer to next device in linked list. */
+	uint32_t work_qp_idx;
 };
 
 /*
