From patchwork Fri Jan 22 09:47:55 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jiawen Wu X-Patchwork-Id: 87073 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 45773A0A0A; Fri, 22 Jan 2021 10:50:16 +0100 (CET) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 29304140EC1; Fri, 22 Jan 2021 10:48:22 +0100 (CET) Received: from smtpbgbr2.qq.com (smtpbgbr2.qq.com [54.207.22.56]) by mails.dpdk.org (Postfix) with ESMTP id 254D9140EA9 for ; Fri, 22 Jan 2021 10:48:17 +0100 (CET) X-QQ-mid: bizesmtp9t1611308894t8r1gbsxr Received: from wxdbg.localdomain.com (unknown [183.129.236.74]) by esmtp6.qq.com (ESMTP) with id ; Fri, 22 Jan 2021 17:48:13 +0800 (CST) X-QQ-SSF: 01400000002000C0D000B00A0000000 X-QQ-FEAT: Ude1zx7N41SF8VgcIPKrFSDyZZFhoJ2LS6yC6LECQ2cJDVlDLQyUZdBE9lzAU 9lAQ9/a31Vcu5zLZ50at6dlC8Tcq5HIBHn5AIIMbXWlXJOb5bE7Oa3mjN5wsxnvWhnH1tDY 8jaeegl9Y3ib/mFo2aSlTITzmvlvPNyb1XboTX6aEU0zIkarIjgum9hecKrDFcbHyaW6u2K axpcNUKGPd1HMIJt/AKsabGO3Cw0CYgOCIEng9SN43uwWDFtPPucusNpMWFrcQ8zedW+RHK bhUcDEmw9cTfuufI3womXp/Ph8qhJjjAt1shoEOv2YplptpmiXtl6FUUjNU8k/oMc3N755U 2kbxgvcpbOxMTGvhxKobOh955qrJQ== X-QQ-GoodBg: 2 From: Jiawen Wu To: dev@dpdk.org Cc: Jiawen Wu Date: Fri, 22 Jan 2021 17:47:55 +0800 Message-Id: <20210122094800.197748-16-jiawenwu@trustnetic.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20210122094800.197748-1-jiawenwu@trustnetic.com> References: <20210122094800.197748-1-jiawenwu@trustnetic.com> MIME-Version: 1.0 X-QQ-SENDSIZE: 520 Feedback-ID: bizesmtp:trustnetic.com:qybgforeign:qybgforeign7 X-QQ-Bgrelay: 1 Subject: [dpdk-dev] [PATCH v1 15/20] net/txgbe: start and stop VF device X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add support to start, stop and reset VF device. Signed-off-by: Jiawen Wu --- drivers/net/txgbe/txgbe_ethdev.h | 2 + drivers/net/txgbe/txgbe_ethdev_vf.c | 176 +++++++++++++++++++++++++++- drivers/net/txgbe/txgbe_rxtx.c | 57 +++++++++ 3 files changed, 234 insertions(+), 1 deletion(-) diff --git a/drivers/net/txgbe/txgbe_ethdev.h b/drivers/net/txgbe/txgbe_ethdev.h index 52ce9c31e..5d4d9434a 100644 --- a/drivers/net/txgbe/txgbe_ethdev.h +++ b/drivers/net/txgbe/txgbe_ethdev.h @@ -479,6 +479,8 @@ int txgbevf_dev_rx_init(struct rte_eth_dev *dev); void txgbevf_dev_tx_init(struct rte_eth_dev *dev); +void txgbevf_dev_rxtx_start(struct rte_eth_dev *dev); + uint16_t txgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); diff --git a/drivers/net/txgbe/txgbe_ethdev_vf.c b/drivers/net/txgbe/txgbe_ethdev_vf.c index 05e666d57..1ae221167 100644 --- a/drivers/net/txgbe/txgbe_ethdev_vf.c +++ b/drivers/net/txgbe/txgbe_ethdev_vf.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "txgbe_logs.h" #include "base/txgbe.h" @@ -51,8 +52,10 @@ static int txgbevf_dev_xstats_get(struct rte_eth_dev *dev, static int txgbevf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static int txgbevf_dev_configure(struct rte_eth_dev *dev); +static int txgbevf_dev_start(struct rte_eth_dev *dev); static int txgbevf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete); +static int txgbevf_dev_stop(struct rte_eth_dev *dev); static int txgbevf_dev_close(struct rte_eth_dev *dev); static void txgbevf_intr_disable(struct rte_eth_dev *dev); static void txgbevf_intr_enable(struct rte_eth_dev *dev); @@ -605,18 +608,168 @@ txgbevf_dev_configure(struct rte_eth_dev *dev) return 0; } +static int +txgbevf_dev_start(struct rte_eth_dev *dev) +{ + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + uint32_t intr_vector = 0; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + + int err, mask = 0; + + PMD_INIT_FUNC_TRACE(); + + /* Stop the link setup handler before resetting the HW. */ + rte_eal_alarm_cancel(txgbe_dev_setup_link_alarm_handler, dev); + + err = hw->mac.reset_hw(hw); + if (err) { + PMD_INIT_LOG(ERR, "Unable to reset vf hardware (%d)", err); + return err; + } + hw->mac.get_link_status = true; + + /* negotiate mailbox API version to use with the PF. */ + txgbevf_negotiate_api(hw); + + txgbevf_dev_tx_init(dev); + + /* This can fail when allocating mbufs for descriptor rings */ + err = txgbevf_dev_rx_init(dev); + + /** + * In this case, reuses the MAC address assigned by VF + * initialization. + */ + if (err != 0 && err != TXGBE_ERR_INVALID_MAC_ADDR) { + PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)", err); + txgbe_dev_clear_queues(dev); + return err; + } + + /* Set vfta */ + txgbevf_set_vfta_all(dev, 1); + + /* Set HW strip */ + mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | + ETH_VLAN_EXTEND_MASK; + err = txgbevf_vlan_offload_config(dev, mask); + if (err) { + PMD_INIT_LOG(ERR, "Unable to set VLAN offload (%d)", err); + txgbe_dev_clear_queues(dev); + return err; + } + + txgbevf_dev_rxtx_start(dev); + + /* check and configure queue intr-vector mapping */ + if (rte_intr_cap_multiple(intr_handle) && + dev->data->dev_conf.intr_conf.rxq) { + /* According to datasheet, only vector 0/1/2 can be used, + * now only one vector is used for Rx queue + */ + intr_vector = 1; + if (rte_intr_efd_enable(intr_handle, intr_vector)) + return -1; + } + + if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { + intr_handle->intr_vec = + rte_zmalloc("intr_vec", + dev->data->nb_rx_queues * sizeof(int), 0); + if (intr_handle->intr_vec == NULL) { + PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" + " intr_vec", dev->data->nb_rx_queues); + return -ENOMEM; + } + } + txgbevf_configure_msix(dev); + + /* When a VF port is bound to VFIO-PCI, only miscellaneous interrupt + * is mapped to VFIO vector 0 in eth_txgbevf_dev_init( ). + * If previous VFIO interrupt mapping setting in eth_txgbevf_dev_init( ) + * is not cleared, it will fail when following rte_intr_enable( ) tries + * to map Rx queue interrupt to other VFIO vectors. + * So clear uio/vfio intr/evevnfd first to avoid failure. + */ + rte_intr_disable(intr_handle); + + rte_intr_enable(intr_handle); + + /* Re-enable interrupt for VF */ + txgbevf_intr_enable(dev); + + /* + * Update link status right before return, because it may + * start link configuration process in a separate thread. + */ + txgbevf_dev_link_update(dev, 0); + + hw->adapter_stopped = false; + + return 0; +} + +static int +txgbevf_dev_stop(struct rte_eth_dev *dev) +{ + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + + if (hw->adapter_stopped) + return 0; + + PMD_INIT_FUNC_TRACE(); + + rte_eal_alarm_cancel(txgbe_dev_setup_link_alarm_handler, dev); + + txgbevf_intr_disable(dev); + + hw->adapter_stopped = 1; + hw->mac.stop_hw(hw); + + /* + * Clear what we set, but we still keep shadow_vfta to + * restore after device starts + */ + txgbevf_set_vfta_all(dev, 0); + + /* Clear stored conf */ + dev->data->scattered_rx = 0; + + txgbe_dev_clear_queues(dev); + + /* Clean datapath event and queue/vec mapping */ + rte_intr_efd_disable(intr_handle); + if (intr_handle->intr_vec != NULL) { + rte_free(intr_handle->intr_vec); + intr_handle->intr_vec = NULL; + } + + adapter->rss_reta_updated = 0; + + return 0; +} + static int txgbevf_dev_close(struct rte_eth_dev *dev) { struct txgbe_hw *hw = TXGBE_DEV_HW(dev); struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + int ret; + PMD_INIT_FUNC_TRACE(); if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; hw->mac.reset_hw(hw); + ret = txgbevf_dev_stop(dev); + txgbe_dev_free_queues(dev); /** @@ -640,7 +793,24 @@ txgbevf_dev_close(struct rte_eth_dev *dev) rte_intr_callback_unregister(intr_handle, txgbevf_dev_interrupt_handler, dev); - return 0; + return ret; +} + +/* + * Reset VF device + */ +static int +txgbevf_dev_reset(struct rte_eth_dev *dev) +{ + int ret; + + ret = eth_txgbevf_dev_uninit(dev); + if (ret) + return ret; + + ret = eth_txgbevf_dev_init(dev); + + return ret; } static void txgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on) @@ -1173,12 +1343,16 @@ txgbevf_dev_interrupt_handler(void *param) */ static const struct eth_dev_ops txgbevf_eth_dev_ops = { .dev_configure = txgbevf_dev_configure, + .dev_start = txgbevf_dev_start, + .dev_stop = txgbevf_dev_stop, .link_update = txgbevf_dev_link_update, .stats_get = txgbevf_dev_stats_get, .xstats_get = txgbevf_dev_xstats_get, .stats_reset = txgbevf_dev_stats_reset, .xstats_reset = txgbevf_dev_stats_reset, .xstats_get_names = txgbevf_dev_xstats_get_names, + .dev_close = txgbevf_dev_close, + .dev_reset = txgbevf_dev_reset, .promiscuous_enable = txgbevf_dev_promiscuous_enable, .promiscuous_disable = txgbevf_dev_promiscuous_disable, .allmulticast_enable = txgbevf_dev_allmulticast_enable, diff --git a/drivers/net/txgbe/txgbe_rxtx.c b/drivers/net/txgbe/txgbe_rxtx.c index 4a6ab16ca..6d596d9e0 100644 --- a/drivers/net/txgbe/txgbe_rxtx.c +++ b/drivers/net/txgbe/txgbe_rxtx.c @@ -4938,6 +4938,63 @@ txgbevf_dev_tx_init(struct rte_eth_dev *dev) } } +/* + * [VF] Start Transmit and Receive Units. + */ +void __rte_cold +txgbevf_dev_rxtx_start(struct rte_eth_dev *dev) +{ + struct txgbe_hw *hw; + struct txgbe_tx_queue *txq; + struct txgbe_rx_queue *rxq; + uint32_t txdctl; + uint32_t rxdctl; + uint16_t i; + int poll_ms; + + PMD_INIT_FUNC_TRACE(); + hw = TXGBE_DEV_HW(dev); + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + /* Setup Transmit Threshold Registers */ + wr32m(hw, TXGBE_TXCFG(txq->reg_idx), + TXGBE_TXCFG_HTHRESH_MASK | + TXGBE_TXCFG_WTHRESH_MASK, + TXGBE_TXCFG_HTHRESH(txq->hthresh) | + TXGBE_TXCFG_WTHRESH(txq->wthresh)); + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + wr32m(hw, TXGBE_TXCFG(i), TXGBE_TXCFG_ENA, TXGBE_TXCFG_ENA); + + poll_ms = 10; + /* Wait until TX Enable ready */ + do { + rte_delay_ms(1); + txdctl = rd32(hw, TXGBE_TXCFG(i)); + } while (--poll_ms && !(txdctl & TXGBE_TXCFG_ENA)); + if (!poll_ms) + PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i); + } + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + + wr32m(hw, TXGBE_RXCFG(i), TXGBE_RXCFG_ENA, TXGBE_RXCFG_ENA); + + /* Wait until RX Enable ready */ + poll_ms = 10; + do { + rte_delay_ms(1); + rxdctl = rd32(hw, TXGBE_RXCFG(i)); + } while (--poll_ms && !(rxdctl & TXGBE_RXCFG_ENA)); + if (!poll_ms) + PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i); + rte_wmb(); + wr32(hw, TXGBE_RXWP(i), rxq->nb_rx_desc - 1); + } +} + int txgbe_rss_conf_init(struct txgbe_rte_flow_rss_conf *out, const struct rte_flow_action_rss *in)