Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/7090/?format=api
{ "id": 7090, "url": "https://patches.dpdk.org/api/patches/7090/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/1442589061-19225-12-git-send-email-yuanhan.liu@linux.intel.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1442589061-19225-12-git-send-email-yuanhan.liu@linux.intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1442589061-19225-12-git-send-email-yuanhan.liu@linux.intel.com", "date": "2015-09-18T15:11:00", "name": "[dpdk-dev,v5,resend,11/12] examples/vhost: demonstrate the usage of vhost mq feature", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "9268b7ecf01d6cf4f9845770c6fdcdc9f1463021", "submitter": { "id": 307, "url": "https://patches.dpdk.org/api/people/307/?format=api", "name": "Yuanhan Liu", "email": "yuanhan.liu@linux.intel.com" }, "delegate": null, "mbox": "https://patches.dpdk.org/project/dpdk/patch/1442589061-19225-12-git-send-email-yuanhan.liu@linux.intel.com/mbox/", "series": [], "comments": "https://patches.dpdk.org/api/patches/7090/comments/", "check": "pending", "checks": "https://patches.dpdk.org/api/patches/7090/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [IPv6:::1])\n\tby dpdk.org (Postfix) with ESMTP id 21EB08E94;\n\tFri, 18 Sep 2015 17:10:00 +0200 (CEST)", "from mga11.intel.com (mga11.intel.com [192.55.52.93])\n\tby dpdk.org (Postfix) with ESMTP id 4E9898E94\n\tfor <dev@dpdk.org>; Fri, 18 Sep 2015 17:09:57 +0200 (CEST)", "from fmsmga001.fm.intel.com ([10.253.24.23])\n\tby fmsmga102.fm.intel.com with ESMTP; 18 Sep 2015 08:09:56 -0700", "from yliu-dev.sh.intel.com ([10.239.66.60])\n\tby fmsmga001.fm.intel.com with ESMTP; 18 Sep 2015 08:09:54 -0700" ], "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.17,553,1437462000\"; d=\"scan'208\";a=\"792565989\"", "From": "Yuanhan Liu <yuanhan.liu@linux.intel.com>", "To": "dev@dpdk.org", "Date": "Fri, 18 Sep 2015 23:11:00 +0800", "Message-Id": "<1442589061-19225-12-git-send-email-yuanhan.liu@linux.intel.com>", "X-Mailer": "git-send-email 1.9.0", "In-Reply-To": "<1442589061-19225-1-git-send-email-yuanhan.liu@linux.intel.com>", "References": "<1442589061-19225-1-git-send-email-yuanhan.liu@linux.intel.com>", "Cc": "\"Michael S. Tsirkin\" <mst@redhat.com>", "Subject": "[dpdk-dev] [PATCH v5 resend 11/12] examples/vhost: demonstrate the\n\tusage of vhost mq feature", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "patches and discussions about DPDK <dev.dpdk.org>", "List-Unsubscribe": "<http://dpdk.org/ml/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://dpdk.org/ml/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<http://dpdk.org/ml/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "From: Changchun Ouyang <changchun.ouyang@intel.com>\n\nThis patch demonstrates the usage of vhost mq feature, by leveraging\nthe VMDq+RSS HW feature to receive packets and distribute them into\ndifferent queue in the pool according to 5 tuples.\n\nQueue number is specified by the --rxq option.\n\nHW queue numbers in pool is exactly same with the queue number in virtio\ndevice, e.g. rxq = 4, the queue number is 4, it means 4 HW queues in\neach VMDq pool, and 4 queues in each virtio device/port, one maps to\neach.\n\n=========================================\n==================| |==================|\n vport0 | | vport1 |\n--- --- --- ---| |--- --- --- ---|\nq0 | q1 | q2 | q3 | |q0 | q1 | q2 | q3 |\n/\\= =/\\= =/\\= =/\\=| |/\\= =/\\= =/\\= =/\\=|\n|| || || || || || || ||\n|| || || || || || || ||\n||= =||= =||= =||=| =||== ||== ||== ||=|\nq0 | q1 | q2 | q3 | |q0 | q1 | q2 | q3 |\n\n------------------| |------------------|\n VMDq pool0 | | VMDq pool1 |", "diff": "==================| |==================|\n\nIn RX side, it firstly polls each queue of the pool and gets the\npackets from it and enqueue them into its corresponding queue in\nvirtio device/port. In TX side, it dequeue packets from each queue\nof virtio device/port and send them to either physical port or\nanother virtio device according to its destination MAC address.\n\nWe bind the virtq to a specific core by rte_vhost_core_id_set(),\nand later we can retrieve it by rte_vhost_core_id_get().\n\nSigned-off-by: Changchun Ouyang <changchun.ouyang@intel.com>\nSigned-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>\n---\n examples/vhost/main.c | 325 ++++++++++++++++++++++++++++++++++----------------\n examples/vhost/main.h | 3 +-\n 2 files changed, 225 insertions(+), 103 deletions(-)\n\ndiff --git a/examples/vhost/main.c b/examples/vhost/main.c\nindex 9eac2d0..23b7aa7 100644\n--- a/examples/vhost/main.c\n+++ b/examples/vhost/main.c\n@@ -163,6 +163,9 @@ static int mergeable;\n /* Do vlan strip on host, enabled on default */\n static uint32_t vlan_strip = 1;\n \n+/* Rx queue number per virtio device */\n+static uint32_t rxq = 1;\n+\n /* number of descriptors to apply*/\n static uint32_t num_rx_descriptor = RTE_TEST_RX_DESC_DEFAULT_ZCP;\n static uint32_t num_tx_descriptor = RTE_TEST_TX_DESC_DEFAULT_ZCP;\n@@ -365,6 +368,37 @@ validate_num_devices(uint32_t max_nb_devices)\n \treturn 0;\n }\n \n+static int\n+get_dev_nb_for_82599(struct rte_eth_dev_info dev_info)\n+{\n+\tint dev_nb = -1;\n+\tswitch (rxq) {\n+\tcase 1:\n+\tcase 2:\n+\t\t/*\n+\t\t * for 82599, dev_info.max_vmdq_pools always 64 dispite rx mode.\n+\t\t */\n+\t\tdev_nb = (int)dev_info.max_vmdq_pools;\n+\t\tbreak;\n+\tcase 4:\n+\t\tdev_nb = (int)dev_info.max_vmdq_pools / 2;\n+\t\tbreak;\n+\tdefault:\n+\t\tRTE_LOG(ERR, VHOST_CONFIG, \"invalid rxq for VMDq.\\n\");\n+\t}\n+\treturn dev_nb;\n+}\n+\n+static int\n+get_dev_nb_for_fvl(struct rte_eth_dev_info dev_info)\n+{\n+\t/*\n+\t * for FVL, dev_info.max_vmdq_pools is calculated according to\n+\t * the configured value: CONFIG_RTE_LIBRTE_I40E_QUEUE_NUM_PER_VM.\n+\t */\n+\treturn (int)dev_info.max_vmdq_pools;\n+}\n+\n /*\n * Initialises a given port using global settings and with the rx buffers\n * coming from the mbuf_pool passed as parameter\n@@ -380,6 +414,7 @@ port_init(uint8_t port)\n \tuint16_t rx_ring_size, tx_ring_size;\n \tint retval;\n \tuint16_t q;\n+\tstruct rte_eth_dev *eth_dev;\n \n \t/* The max pool number from dev_info will be used to validate the pool number specified in cmd line */\n \trte_eth_dev_info_get (port, &dev_info);\n@@ -408,8 +443,16 @@ port_init(uint8_t port)\n \t\ttxconf->tx_deferred_start = 1;\n \t}\n \n-\t/*configure the number of supported virtio devices based on VMDQ limits */\n-\tnum_devices = dev_info.max_vmdq_pools;\n+\t/* Configure the virtio devices num based on VMDQ limits */\n+\tif (dev_info.max_vmdq_pools == ETH_64_POOLS) {\n+\t\tnum_devices = (uint32_t)get_dev_nb_for_82599(dev_info);\n+\t\tif (num_devices == (uint32_t)-1)\n+\t\t\treturn -1;\n+\t} else {\n+\t\tnum_devices = (uint32_t)get_dev_nb_for_fvl(dev_info);\n+\t\tif (num_devices == (uint32_t)-1)\n+\t\t\treturn -1;\n+\t}\n \n \tif (zero_copy) {\n \t\trx_ring_size = num_rx_descriptor;\n@@ -431,7 +474,7 @@ port_init(uint8_t port)\n \t\treturn retval;\n \t/* NIC queues are divided into pf queues and vmdq queues. */\n \tnum_pf_queues = dev_info.max_rx_queues - dev_info.vmdq_queue_num;\n-\tqueues_per_pool = dev_info.vmdq_queue_num / dev_info.max_vmdq_pools;\n+\tqueues_per_pool = dev_info.vmdq_queue_num / num_devices;\n \tnum_vmdq_queues = num_devices * queues_per_pool;\n \tnum_queues = num_pf_queues + num_vmdq_queues;\n \tvmdq_queue_base = dev_info.vmdq_queue_base;\n@@ -447,6 +490,14 @@ port_init(uint8_t port)\n \tif (retval != 0)\n \t\treturn retval;\n \n+\teth_dev = &rte_eth_devices[port];\n+\tif (RTE_ETH_DEV_SRIOV(eth_dev).nb_q_per_pool > 4) {\n+\t\tRTE_LOG(ERR, VHOST_CONFIG, \"ethdev port_id=%d SRIOV active, \"\n+\t\t\t\"invalid queue number for VMDQ RSS, allowed value \"\n+\t\t\t\"are 1, 2 or 4\\n\", port);\n+\t\treturn -EINVAL;\n+\t}\n+\n \t/* Setup the queues. */\n \tfor (q = 0; q < rx_rings; q ++) {\n \t\tretval = rte_eth_rx_queue_setup(port, q, rx_ring_size,\n@@ -576,7 +627,8 @@ us_vhost_usage(const char *prgname)\n \t\"\t\t--rx-desc-num [0-N]: the number of descriptors on rx, \"\n \t\t\t\"used only when zero copy is enabled.\\n\"\n \t\"\t\t--tx-desc-num [0-N]: the number of descriptors on tx, \"\n-\t\t\t\"used only when zero copy is enabled.\\n\",\n+\t\t\t\"used only when zero copy is enabled.\\n\"\n+\t\"\t\t--rxq [1,2,4]: rx queue number for each vhost device\\n\",\n \t prgname);\n }\n \n@@ -602,6 +654,7 @@ us_vhost_parse_args(int argc, char **argv)\n \t\t{\"zero-copy\", required_argument, NULL, 0},\n \t\t{\"rx-desc-num\", required_argument, NULL, 0},\n \t\t{\"tx-desc-num\", required_argument, NULL, 0},\n+\t\t{\"rxq\", required_argument, NULL, 0},\n \t\t{NULL, 0, 0, 0},\n \t};\n \n@@ -778,6 +831,18 @@ us_vhost_parse_args(int argc, char **argv)\n \t\t\t\t}\n \t\t\t}\n \n+\t\t\t/* Specify the Rx queue number for each vhost dev. */\n+\t\t\tif (!strncmp(long_option[option_index].name,\n+\t\t\t\t\"rxq\", MAX_LONG_OPT_SZ)) {\n+\t\t\t\tret = parse_num_opt(optarg, 4);\n+\t\t\t\tif ((ret == -1) || (ret == 0) || (!POWEROF2(ret))) {\n+\t\t\t\t\tRTE_LOG(INFO, VHOST_CONFIG,\n+\t\t\t\t\t\"Valid value for rxq is [1,2,4]\\n\");\n+\t\t\t\t\tus_vhost_usage(prgname);\n+\t\t\t\t\treturn -1;\n+\t\t\t\t} else\n+\t\t\t\t\trxq = ret;\n+\t\t\t}\n \t\t\tbreak;\n \n \t\t\t/* Invalid option - print options. */\n@@ -813,6 +878,19 @@ us_vhost_parse_args(int argc, char **argv)\n \t\treturn -1;\n \t}\n \n+\tif (rxq > 1) {\n+\t\tvmdq_conf_default.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS;\n+\t\tvmdq_conf_default.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_IP |\n+\t\t\t\tETH_RSS_UDP | ETH_RSS_TCP | ETH_RSS_SCTP;\n+\t}\n+\n+\tif ((zero_copy == 1) && (rxq > 1)) {\n+\t\tRTE_LOG(INFO, VHOST_PORT,\n+\t\t\t\"Vhost zero copy doesn't support mq mode,\"\n+\t\t\t\"please specify '--rxq 1' to disable it.\\n\");\n+\t\treturn -1;\n+\t}\n+\n \treturn 0;\n }\n \n@@ -959,9 +1037,11 @@ link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m)\n \t\t\t\t\tdev->device_fh);\n \n \t/* Enable stripping of the vlan tag as we handle routing. */\n-\tif (vlan_strip)\n-\t\trte_eth_dev_set_vlan_strip_on_queue(ports[0],\n-\t\t\t(uint16_t)vdev->vmdq_rx_q, 1);\n+\tif (vlan_strip) {\n+\t\tfor (i = 0; i < (int)rxq; i++)\n+\t\t\trte_eth_dev_set_vlan_strip_on_queue(ports[0],\n+\t\t\t\t(uint16_t)(vdev->vmdq_rx_q + i), 1);\n+\t}\n \n \t/* Set device as ready for RX. */\n \tvdev->ready = DEVICE_RX;\n@@ -976,7 +1056,7 @@ link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m)\n static inline void\n unlink_vmdq(struct vhost_dev *vdev)\n {\n-\tunsigned i = 0;\n+\tunsigned i = 0, j = 0;\n \tunsigned rx_count;\n \tstruct rte_mbuf *pkts_burst[MAX_PKT_BURST];\n \n@@ -989,15 +1069,19 @@ unlink_vmdq(struct vhost_dev *vdev)\n \t\tvdev->vlan_tag = 0;\n \n \t\t/*Clear out the receive buffers*/\n-\t\trx_count = rte_eth_rx_burst(ports[0],\n-\t\t\t\t\t(uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);\n+\t\tfor (i = 0; i < rxq; i++) {\n+\t\t\trx_count = rte_eth_rx_burst(ports[0],\n+\t\t\t\t\t(uint16_t)vdev->vmdq_rx_q + i,\n+\t\t\t\t\tpkts_burst, MAX_PKT_BURST);\n \n-\t\twhile (rx_count) {\n-\t\t\tfor (i = 0; i < rx_count; i++)\n-\t\t\t\trte_pktmbuf_free(pkts_burst[i]);\n+\t\t\twhile (rx_count) {\n+\t\t\t\tfor (j = 0; j < rx_count; j++)\n+\t\t\t\t\trte_pktmbuf_free(pkts_burst[j]);\n \n-\t\t\trx_count = rte_eth_rx_burst(ports[0],\n-\t\t\t\t\t(uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);\n+\t\t\t\trx_count = rte_eth_rx_burst(ports[0],\n+\t\t\t\t\t(uint16_t)vdev->vmdq_rx_q + i,\n+\t\t\t\t\tpkts_burst, MAX_PKT_BURST);\n+\t\t\t}\n \t\t}\n \n \t\tvdev->ready = DEVICE_MAC_LEARNING;\n@@ -1009,7 +1093,7 @@ unlink_vmdq(struct vhost_dev *vdev)\n * the packet on that devices RX queue. If not then return.\n */\n static inline int __attribute__((always_inline))\n-virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)\n+virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m, uint32_t qp_idx)\n {\n \tstruct virtio_net_data_ll *dev_ll;\n \tstruct ether_hdr *pkt_hdr;\n@@ -1024,7 +1108,7 @@ virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)\n \n \twhile (dev_ll != NULL) {\n \t\tif ((dev_ll->vdev->ready == DEVICE_RX) && ether_addr_cmp(&(pkt_hdr->d_addr),\n-\t\t\t\t &dev_ll->vdev->mac_address)) {\n+\t\t\t\t\t&dev_ll->vdev->mac_address)) {\n \n \t\t\t/* Drop the packet if the TX packet is destined for the TX device. */\n \t\t\tif (dev_ll->vdev->dev->device_fh == dev->device_fh) {\n@@ -1042,7 +1126,9 @@ virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)\n \t\t\t\tLOG_DEBUG(VHOST_DATA, \"(%\"PRIu64\") Device is marked for removal\\n\", tdev->device_fh);\n \t\t\t} else {\n \t\t\t\t/*send the packet to the local virtio device*/\n-\t\t\t\tret = rte_vhost_enqueue_burst(tdev, VIRTIO_RXQ, &m, 1);\n+\t\t\t\tret = rte_vhost_enqueue_burst(tdev,\n+\t\t\t\t\tVIRTIO_RXQ + qp_idx * VIRTIO_QNUM,\n+\t\t\t\t\t&m, 1);\n \t\t\t\tif (enable_stats) {\n \t\t\t\t\trte_atomic64_add(\n \t\t\t\t\t&dev_statistics[tdev->device_fh].rx_total_atomic,\n@@ -1119,7 +1205,8 @@ find_local_dest(struct virtio_net *dev, struct rte_mbuf *m,\n * or the physical port.\n */\n static inline void __attribute__((always_inline))\n-virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag)\n+virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m,\n+\t\tuint16_t vlan_tag, uint32_t qp_idx)\n {\n \tstruct mbuf_table *tx_q;\n \tstruct rte_mbuf **m_table;\n@@ -1129,7 +1216,8 @@ virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag)\n \tstruct ether_hdr *nh;\n \n \t/*check if destination is local VM*/\n-\tif ((vm2vm_mode == VM2VM_SOFTWARE) && (virtio_tx_local(vdev, m) == 0)) {\n+\tif ((vm2vm_mode == VM2VM_SOFTWARE) &&\n+\t (virtio_tx_local(vdev, m, qp_idx) == 0)) {\n \t\trte_pktmbuf_free(m);\n \t\treturn;\n \t}\n@@ -1293,22 +1381,26 @@ switch_worker(__attribute__((unused)) void *arg)\n \t\t\t}\n \t\t\tif (likely(vdev->ready == DEVICE_RX)) {\n \t\t\t\t/*Handle guest RX*/\n+\t\t\t\tuint16_t qp_idx = dev_ll->work_qp_idx;\n \t\t\t\trx_count = rte_eth_rx_burst(ports[0],\n-\t\t\t\t\tvdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);\n+\t\t\t\t\tvdev->vmdq_rx_q + qp_idx, pkts_burst, MAX_PKT_BURST);\n \n \t\t\t\tif (rx_count) {\n \t\t\t\t\t/*\n \t\t\t\t\t* Retry is enabled and the queue is full then we wait and retry to avoid packet loss\n \t\t\t\t\t* Here MAX_PKT_BURST must be less than virtio queue size\n \t\t\t\t\t*/\n-\t\t\t\t\tif (enable_retry && unlikely(rx_count > rte_vring_available_entries(dev, VIRTIO_RXQ))) {\n+\t\t\t\t\tif (enable_retry && unlikely(rx_count > rte_vring_available_entries(dev,\n+\t\t\t\t\t\t\t\t\t\tVIRTIO_RXQ + qp_idx * VIRTIO_QNUM))) {\n \t\t\t\t\t\tfor (retry = 0; retry < burst_rx_retry_num; retry++) {\n \t\t\t\t\t\t\trte_delay_us(burst_rx_delay_time);\n-\t\t\t\t\t\t\tif (rx_count <= rte_vring_available_entries(dev, VIRTIO_RXQ))\n+\t\t\t\t\t\t\tif (rx_count <= rte_vring_available_entries(dev,\n+\t\t\t\t\t\t\t\t\t\tVIRTIO_RXQ + qp_idx * VIRTIO_QNUM))\n \t\t\t\t\t\t\t\tbreak;\n \t\t\t\t\t\t}\n \t\t\t\t\t}\n-\t\t\t\t\tret_count = rte_vhost_enqueue_burst(dev, VIRTIO_RXQ, pkts_burst, rx_count);\n+\t\t\t\t\tret_count = rte_vhost_enqueue_burst(dev, VIRTIO_RXQ + qp_idx * VIRTIO_QNUM,\n+\t\t\t\t\t\t\t\t\t\tpkts_burst, rx_count);\n \t\t\t\t\tif (enable_stats) {\n \t\t\t\t\t\trte_atomic64_add(\n \t\t\t\t\t\t&dev_statistics[dev_ll->vdev->dev->device_fh].rx_total_atomic,\n@@ -1320,14 +1412,18 @@ switch_worker(__attribute__((unused)) void *arg)\n \t\t\t\t\t\trx_count--;\n \t\t\t\t\t\trte_pktmbuf_free(pkts_burst[rx_count]);\n \t\t\t\t\t}\n-\n \t\t\t\t}\n \t\t\t}\n \n \t\t\tif (likely(!vdev->remove)) {\n \t\t\t\t/* Handle guest TX*/\n-\t\t\t\ttx_count = rte_vhost_dequeue_burst(dev, VIRTIO_TXQ, mbuf_pool, pkts_burst, MAX_PKT_BURST);\n-\t\t\t\t/* If this is the first received packet we need to learn the MAC and setup VMDQ */\n+\t\t\t\tuint16_t qp_idx = dev_ll->work_qp_idx;\n+\t\t\t\ttx_count = rte_vhost_dequeue_burst(dev, VIRTIO_TXQ + qp_idx * VIRTIO_QNUM,\n+\t\t\t\t\t\tmbuf_pool, pkts_burst, MAX_PKT_BURST);\n+\t\t\t\t/*\n+\t\t\t\t * If this is the first received packet we need to learn\n+\t\t\t\t * the MAC and setup VMDQ\n+\t\t\t\t */\n \t\t\t\tif (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && tx_count) {\n \t\t\t\t\tif (vdev->remove || (link_vmdq(vdev, pkts_burst[0]) == -1)) {\n \t\t\t\t\t\twhile (tx_count)\n@@ -1335,7 +1431,8 @@ switch_worker(__attribute__((unused)) void *arg)\n \t\t\t\t\t}\n \t\t\t\t}\n \t\t\t\twhile (tx_count)\n-\t\t\t\t\tvirtio_tx_route(vdev, pkts_burst[--tx_count], (uint16_t)dev->device_fh);\n+\t\t\t\t\tvirtio_tx_route(vdev, pkts_burst[--tx_count],\n+\t\t\t\t\t\t(uint16_t)dev->device_fh, qp_idx);\n \t\t\t}\n \n \t\t\t/*move to the next device in the list*/\n@@ -2323,6 +2420,7 @@ destroy_device (volatile struct virtio_net *dev)\n \tstruct virtio_net_data_ll *ll_main_dev_last = NULL;\n \tstruct vhost_dev *vdev;\n \tint lcore;\n+\tuint32_t i;\n \n \tdev->flags &= ~VIRTIO_DEV_RUNNING;\n \n@@ -2334,61 +2432,73 @@ destroy_device (volatile struct virtio_net *dev)\n \t}\n \n \t/* Search for entry to be removed from lcore ll */\n-\tll_lcore_dev_cur = lcore_info[vdev->coreid].lcore_ll->ll_root_used;\n-\twhile (ll_lcore_dev_cur != NULL) {\n-\t\tif (ll_lcore_dev_cur->vdev == vdev) {\n-\t\t\tbreak;\n-\t\t} else {\n-\t\t\tll_lcore_dev_last = ll_lcore_dev_cur;\n-\t\t\tll_lcore_dev_cur = ll_lcore_dev_cur->next;\n+\tfor (i = 0; i < rxq; i++) {\n+\t\tuint16_t core_id = rte_vhost_core_id_get(dev, i);\n+\n+\t\tll_lcore_dev_cur = lcore_info[core_id].lcore_ll->ll_root_used;\n+\n+\t\twhile (ll_lcore_dev_cur != NULL) {\n+\t\t\tif (ll_lcore_dev_cur->vdev == vdev) {\n+\t\t\t\tbreak;\n+\t\t\t} else {\n+\t\t\t\tll_lcore_dev_last = ll_lcore_dev_cur;\n+\t\t\t\tll_lcore_dev_cur = ll_lcore_dev_cur->next;\n+\t\t\t}\n \t\t}\n-\t}\n \n-\tif (ll_lcore_dev_cur == NULL) {\n-\t\tRTE_LOG(ERR, VHOST_CONFIG,\n-\t\t\t\"(%\"PRIu64\") Failed to find the dev to be destroy.\\n\",\n-\t\t\tdev->device_fh);\n-\t\treturn;\n-\t}\n+\t\tif (ll_lcore_dev_cur == NULL) {\n+\t\t\tRTE_LOG(ERR, VHOST_CONFIG,\n+\t\t\t\t\"(%\"PRIu64\") Failed to find the dev to be destroy.\\n\",\n+\t\t\t\tdev->device_fh);\n+\t\t\tif (i == 0)\n+\t\t\t\treturn;\n+\t\t\telse\n+\t\t\t\tbreak;\n+\t\t}\n \n-\t/* Search for entry to be removed from main ll */\n-\tll_main_dev_cur = ll_root_used;\n-\tll_main_dev_last = NULL;\n-\twhile (ll_main_dev_cur != NULL) {\n-\t\tif (ll_main_dev_cur->vdev == vdev) {\n-\t\t\tbreak;\n-\t\t} else {\n-\t\t\tll_main_dev_last = ll_main_dev_cur;\n-\t\t\tll_main_dev_cur = ll_main_dev_cur->next;\n+\t\t/* Search for entry to be removed from main ll */\n+\t\tif (i == 0) {\n+\t\t\tll_main_dev_cur = ll_root_used;\n+\t\t\tll_main_dev_last = NULL;\n+\t\t\twhile (ll_main_dev_cur != NULL) {\n+\t\t\t\tif (ll_main_dev_cur->vdev == vdev) {\n+\t\t\t\t\tbreak;\n+\t\t\t\t} else {\n+\t\t\t\t\tll_main_dev_last = ll_main_dev_cur;\n+\t\t\t\t\tll_main_dev_cur = ll_main_dev_cur->next;\n+\t\t\t\t}\n+\t\t\t}\n \t\t}\n-\t}\n \n-\t/* Remove entries from the lcore and main ll. */\n-\trm_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used, ll_lcore_dev_cur, ll_lcore_dev_last);\n-\trm_data_ll_entry(&ll_root_used, ll_main_dev_cur, ll_main_dev_last);\n+\t\t/* Remove entries from the lcore and main ll. */\n+\t\trm_data_ll_entry(&lcore_info[core_id].lcore_ll->ll_root_used, ll_lcore_dev_cur, ll_lcore_dev_last);\n+\t\tif (i == 0)\n+\t\t\trm_data_ll_entry(&ll_root_used, ll_main_dev_cur, ll_main_dev_last);\n \n-\t/* Set the dev_removal_flag on each lcore. */\n-\tRTE_LCORE_FOREACH_SLAVE(lcore) {\n-\t\tlcore_info[lcore].lcore_ll->dev_removal_flag = REQUEST_DEV_REMOVAL;\n-\t}\n+\t\t/* Set the dev_removal_flag on each lcore. */\n+\t\tRTE_LCORE_FOREACH_SLAVE(lcore) {\n+\t\t\tlcore_info[lcore].lcore_ll->dev_removal_flag = REQUEST_DEV_REMOVAL;\n+\t\t}\n \n-\t/*\n-\t * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL we can be sure that\n-\t * they can no longer access the device removed from the linked lists and that the devices\n-\t * are no longer in use.\n-\t */\n-\tRTE_LCORE_FOREACH_SLAVE(lcore) {\n-\t\twhile (lcore_info[lcore].lcore_ll->dev_removal_flag != ACK_DEV_REMOVAL) {\n-\t\t\trte_pause();\n+\t\t/*\n+\t\t * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL we can be sure that\n+\t\t * they can no longer access the device removed from the linked lists and that the devices\n+\t\t * are no longer in use.\n+\t\t */\n+\t\tRTE_LCORE_FOREACH_SLAVE(lcore) {\n+\t\t\twhile (lcore_info[lcore].lcore_ll->dev_removal_flag != ACK_DEV_REMOVAL)\n+\t\t\t\trte_pause();\n \t\t}\n-\t}\n \n-\t/* Add the entries back to the lcore and main free ll.*/\n-\tput_data_ll_free_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_free, ll_lcore_dev_cur);\n-\tput_data_ll_free_entry(&ll_root_free, ll_main_dev_cur);\n+\t\t/* Add the entries back to the lcore and main free ll.*/\n+\t\tput_data_ll_free_entry(&lcore_info[core_id].lcore_ll->ll_root_free, ll_lcore_dev_cur);\n+\n+\t\tif (i == 0)\n+\t\t\tput_data_ll_free_entry(&ll_root_free, ll_main_dev_cur);\n \n-\t/* Decrement number of device on the lcore. */\n-\tlcore_info[vdev->coreid].lcore_ll->device_num--;\n+\t\t/* Decrement number of device on the lcore. */\n+\t\tlcore_info[core_id].lcore_ll->device_num--;\n+\t}\n \n \tRTE_LOG(INFO, VHOST_DATA, \"(%\"PRIu64\") Device has been removed from data core\\n\", dev->device_fh);\n \n@@ -2593,6 +2703,14 @@ new_device (struct virtio_net *dev)\n \tuint32_t device_num_min = num_devices;\n \tstruct vhost_dev *vdev;\n \tuint32_t regionidx;\n+\tuint32_t i;\n+\n+\tif ((rxq > 1) && (dev->virt_qp_nb != rxq)) {\n+\t\tRTE_LOG(ERR, VHOST_DATA, \"(%\"PRIu64\") queue num in VMDq pool:\"\n+\t\t\t\"%d != queue pair num in vhost dev:%d\\n\",\n+\t\t\tdev->device_fh, rxq, dev->virt_qp_nb);\n+\t\treturn -1;\n+\t}\n \n \tvdev = rte_zmalloc(\"vhost device\", sizeof(*vdev), RTE_CACHE_LINE_SIZE);\n \tif (vdev == NULL) {\n@@ -2638,12 +2756,12 @@ new_device (struct virtio_net *dev)\n \t\t}\n \t}\n \n-\n \t/* Add device to main ll */\n \tll_dev = get_data_ll_free_entry(&ll_root_free);\n \tif (ll_dev == NULL) {\n-\t\tRTE_LOG(INFO, VHOST_DATA, \"(%\"PRIu64\") No free entry found in linked list. Device limit \"\n-\t\t\t\"of %d devices per core has been reached\\n\",\n+\t\tRTE_LOG(INFO, VHOST_DATA,\n+\t\t\t\"(%\"PRIu64\") No free entry found in linked list.\"\n+\t\t\t\"Device limit of %d devices per core has been reached\\n\",\n \t\t\tdev->device_fh, num_devices);\n \t\tif (vdev->regions_hpa)\n \t\t\trte_free(vdev->regions_hpa);\n@@ -2652,8 +2770,7 @@ new_device (struct virtio_net *dev)\n \t}\n \tll_dev->vdev = vdev;\n \tadd_data_ll_entry(&ll_root_used, ll_dev);\n-\tvdev->vmdq_rx_q\n-\t\t= dev->device_fh * queues_per_pool + vmdq_queue_base;\n+\tvdev->vmdq_rx_q\t= dev->device_fh * rxq + vmdq_queue_base;\n \n \tif (zero_copy) {\n \t\tuint32_t index = vdev->vmdq_rx_q;\n@@ -2734,37 +2851,42 @@ new_device (struct virtio_net *dev)\n \tvdev->remove = 0;\n \n \t/* Find a suitable lcore to add the device. */\n-\tRTE_LCORE_FOREACH_SLAVE(lcore) {\n-\t\tif (lcore_info[lcore].lcore_ll->device_num < device_num_min) {\n-\t\t\tdevice_num_min = lcore_info[lcore].lcore_ll->device_num;\n-\t\t\tcore_add = lcore;\n+\tfor (i = 0; i < rxq; i++) {\n+\t\tdevice_num_min = num_devices;\n+\t\tRTE_LCORE_FOREACH_SLAVE(lcore) {\n+\t\t\tif (lcore_info[lcore].lcore_ll->device_num < device_num_min) {\n+\t\t\t\tdevice_num_min = lcore_info[lcore].lcore_ll->device_num;\n+\t\t\t\tcore_add = lcore;\n+\t\t\t}\n \t\t}\n-\t}\n-\t/* Add device to lcore ll */\n-\tll_dev = get_data_ll_free_entry(&lcore_info[core_add].lcore_ll->ll_root_free);\n-\tif (ll_dev == NULL) {\n-\t\tRTE_LOG(INFO, VHOST_DATA, \"(%\"PRIu64\") Failed to add device to data core\\n\", dev->device_fh);\n-\t\tvdev->ready = DEVICE_SAFE_REMOVE;\n-\t\tdestroy_device(dev);\n-\t\trte_free(vdev->regions_hpa);\n-\t\trte_free(vdev);\n-\t\treturn -1;\n-\t}\n-\tll_dev->vdev = vdev;\n-\tvdev->coreid = core_add;\n+\t\t/* Add device to lcore ll */\n+\t\tll_dev = get_data_ll_free_entry(&lcore_info[core_add].lcore_ll->ll_root_free);\n+\t\tif (ll_dev == NULL) {\n+\t\t\tRTE_LOG(INFO, VHOST_DATA, \"(%\"PRIu64\") Failed to add device to data core\\n\", dev->device_fh);\n+\t\t\tvdev->ready = DEVICE_SAFE_REMOVE;\n+\t\t\tdestroy_device(dev);\n+\t\t\trte_free(vdev->regions_hpa);\n+\t\t\trte_free(vdev);\n+\t\t\treturn -1;\n+\t\t}\n+\t\tll_dev->vdev = vdev;\n+\t\tll_dev->work_qp_idx = i;\n+\t\trte_vhost_core_id_set(dev, i, core_add);\n+\t\tadd_data_ll_entry(&lcore_info[core_add].lcore_ll->ll_root_used, ll_dev);\n \n-\tadd_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used, ll_dev);\n+\t\t/* Disable notifications. */\n+\t\trte_vhost_enable_guest_notification(dev, i * VIRTIO_QNUM + VIRTIO_RXQ, 0);\n+\t\trte_vhost_enable_guest_notification(dev, i * VIRTIO_QNUM + VIRTIO_TXQ, 0);\n+\t\tlcore_info[core_add].lcore_ll->device_num++;\n+\t\tRTE_LOG(INFO, VHOST_DATA, \"(%\"PRIu64\") Device has been added to data core %d for vq: %d\\n\",\n+\t\t\tdev->device_fh, core_add, i);\n+\t}\n \n \t/* Initialize device stats */\n \tmemset(&dev_statistics[dev->device_fh], 0, sizeof(struct device_statistics));\n \n-\t/* Disable notifications. */\n-\trte_vhost_enable_guest_notification(dev, VIRTIO_RXQ, 0);\n-\trte_vhost_enable_guest_notification(dev, VIRTIO_TXQ, 0);\n-\tlcore_info[vdev->coreid].lcore_ll->device_num++;\n \tdev->flags |= VIRTIO_DEV_RUNNING;\n \n-\tRTE_LOG(INFO, VHOST_DATA, \"(%\"PRIu64\") Device has been added to data core %d\\n\", dev->device_fh, vdev->coreid);\n \n \treturn 0;\n }\n@@ -2833,6 +2955,7 @@ print_stats(void)\n \t\t\t\t\trx_dropped,\n \t\t\t\t\trx);\n \n+\n \t\t\tdev_ll = dev_ll->next;\n \t\t}\n \t\tprintf(\"\\n======================================================\\n\");\ndiff --git a/examples/vhost/main.h b/examples/vhost/main.h\nindex d04e2be..5561c82 100644\n--- a/examples/vhost/main.h\n+++ b/examples/vhost/main.h\n@@ -82,8 +82,6 @@ struct vhost_dev {\n \tuint16_t vmdq_rx_q;\n \t/**< Vlan tag assigned to the pool */\n \tuint32_t vlan_tag;\n-\t/**< Data core that the device is added to. */\n-\tuint16_t coreid;\n \t/**< A device is set as ready if the MAC address has been set. */\n \tvolatile uint8_t ready;\n \t/**< Device is marked for removal from the data core. */\n@@ -94,6 +92,7 @@ struct virtio_net_data_ll\n {\n \tstruct vhost_dev\t\t*vdev;\t/* Pointer to device created by configuration core. */\n \tstruct virtio_net_data_ll\t*next; /* Pointer to next device in linked list. */\n+\tuint32_t work_qp_idx;\n };\n \n /*\n", "prefixes": [ "dpdk-dev", "v5", "resend", "11/12" ] }