From patchwork Fri Oct 25 12:37:05 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Wei Hu (Xavier)" X-Patchwork-Id: 61979 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id E1FBA1C1C6; Fri, 25 Oct 2019 14:36:47 +0200 (CEST) Received: from huawei.com (szxga06-in.huawei.com [45.249.212.32]) by dpdk.org (Postfix) with ESMTP id 5F0251C197 for ; Fri, 25 Oct 2019 14:36:37 +0200 (CEST) Received: from DGGEMS407-HUB.china.huawei.com (unknown [172.30.72.58]) by Forcepoint Email with ESMTP id 17DEDDF488ED045168DE; Fri, 25 Oct 2019 20:36:36 +0800 (CST) Received: from localhost.localdomain (10.69.192.56) by DGGEMS407-HUB.china.huawei.com (10.3.19.207) with Microsoft SMTP Server id 14.3.439.0; Fri, 25 Oct 2019 20:36:27 +0800 From: "Wei Hu (Xavier)" To: CC: , , , , Date: Fri, 25 Oct 2019 20:37:05 +0800 Message-ID: <1572007025-43391-5-git-send-email-xavier.huwei@huawei.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1572007025-43391-1-git-send-email-xavier.huwei@huawei.com> References: <1572007025-43391-1-git-send-email-xavier.huwei@huawei.com> MIME-Version: 1.0 X-Originating-IP: [10.69.192.56] X-CFilter-Loop: Reflected Subject: [dpdk-dev] [PATCH v2 4/4] net/hns3: fix FLR reset failure X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Chunsong Feng PF FLR resets the PCIe ECAM space of all VFs under the PF and does not automatically recover. Therefore, the VF driver needs to restore the ECAM configuration, including bus_master_en, msix_enable to avoid FLR reset failure. Fixes: 2790c6464725 ("net/hns3: support device reset") Signed-off-by: Chunsong Feng Signed-off-by: Wei Hu (Xavier) --- drivers/net/hns3/hns3_cmd.c | 15 ----- drivers/net/hns3/hns3_ethdev.c | 5 ++ drivers/net/hns3/hns3_ethdev_vf.c | 135 ++++++++++++++++++++++++++++++++++++-- drivers/net/hns3/hns3_intr.c | 5 +- 4 files changed, 139 insertions(+), 21 deletions(-) diff --git a/drivers/net/hns3/hns3_cmd.c b/drivers/net/hns3/hns3_cmd.c index 58776c2..65a5af8 100644 --- a/drivers/net/hns3/hns3_cmd.c +++ b/drivers/net/hns3/hns3_cmd.c @@ -216,24 +216,9 @@ hns3_cmd_csq_clean(struct hns3_hw *hw) if (!is_valid_csq_clean_head(csq, head)) { struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); - uint32_t global; - uint32_t fun_rst; hns3_err(hw, "wrong cmd head (%u, %u-%u)", head, csq->next_to_use, csq->next_to_clean); rte_atomic16_set(&hw->reset.disable_cmd, 1); - if (hns->is_vf) { - global = hns3_read_dev(hw, HNS3_VF_RST_ING); - fun_rst = hns3_read_dev(hw, HNS3_FUN_RST_ING); - hns3_err(hw, "Delayed VF reset global: %x fun_rst: %x", - global, fun_rst); - hns3_atomic_set_bit(HNS3_VF_RESET, &hw->reset.pending); - } else { - global = hns3_read_dev(hw, HNS3_GLOBAL_RESET_REG); - fun_rst = hns3_read_dev(hw, HNS3_FUN_RST_ING); - hns3_err(hw, "Delayed IMP reset global: %x fun_rst: %x", - global, fun_rst); - hns3_atomic_set_bit(HNS3_IMP_RESET, &hw->reset.pending); - } hns3_schedule_delayed_reset(hns); diff --git a/drivers/net/hns3/hns3_ethdev.c b/drivers/net/hns3/hns3_ethdev.c index 862a717..3435bce 100644 --- a/drivers/net/hns3/hns3_ethdev.c +++ b/drivers/net/hns3/hns3_ethdev.c @@ -4680,6 +4680,11 @@ hns3_reset_service(void *param) rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_REQUESTED); hns3_err(hw, "Handling interrupts in delayed tasks"); hns3_interrupt_handler(&rte_eth_devices[hw->data->port_id]); + reset_level = hns3_get_reset_level(hns, &hw->reset.pending); + if (reset_level == HNS3_NONE_RESET) { + hns3_err(hw, "No reset level is set, try IMP reset"); + hns3_atomic_set_bit(HNS3_IMP_RESET, &hw->reset.pending); + } } rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_NONE); diff --git a/drivers/net/hns3/hns3_ethdev_vf.c b/drivers/net/hns3/hns3_ethdev_vf.c index 121beb5..4036749 100644 --- a/drivers/net/hns3/hns3_ethdev_vf.c +++ b/drivers/net/hns3/hns3_ethdev_vf.c @@ -9,6 +9,8 @@ #include #include #include +#include + #include #include #include @@ -24,6 +26,7 @@ #include #include #include +#include #include "hns3_ethdev.h" #include "hns3_logs.h" @@ -56,6 +59,81 @@ static enum hns3_reset_level hns3vf_get_reset_level(struct hns3_hw *hw, static int hns3vf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); static int hns3vf_dev_configure_vlan(struct rte_eth_dev *dev); +/* set PCI bus mastering */ +static void +hns3vf_set_bus_master(const struct rte_pci_device *device, bool op) +{ + uint16_t reg; + + rte_pci_read_config(device, ®, sizeof(reg), PCI_COMMAND); + + if (op) + /* set the master bit */ + reg |= PCI_COMMAND_MASTER; + else + reg &= ~(PCI_COMMAND_MASTER); + + rte_pci_write_config(device, ®, sizeof(reg), PCI_COMMAND); +} + +/** + * hns3vf_find_pci_capability - lookup a capability in the PCI capability list + * @cap: the capability + * + * Return the address of the given capability within the PCI capability list. + */ +static int +hns3vf_find_pci_capability(const struct rte_pci_device *device, int cap) +{ +#define MAX_PCIE_CAPABILITY 48 + uint16_t status; + uint8_t pos; + uint8_t id; + int ttl; + + rte_pci_read_config(device, &status, sizeof(status), PCI_STATUS); + if (!(status & PCI_STATUS_CAP_LIST)) + return 0; + + ttl = MAX_PCIE_CAPABILITY; + rte_pci_read_config(device, &pos, sizeof(pos), PCI_CAPABILITY_LIST); + while (ttl-- && pos >= PCI_STD_HEADER_SIZEOF) { + rte_pci_read_config(device, &id, sizeof(id), + (pos + PCI_CAP_LIST_ID)); + + if (id == 0xFF) + break; + + if (id == cap) + return (int)pos; + + rte_pci_read_config(device, &pos, sizeof(pos), + (pos + PCI_CAP_LIST_NEXT)); + } + return 0; +} + +static int +hns3vf_enable_msix(const struct rte_pci_device *device, bool op) +{ + uint16_t control; + int pos; + + pos = hns3vf_find_pci_capability(device, PCI_CAP_ID_MSIX); + if (pos) { + rte_pci_read_config(device, &control, sizeof(control), + (pos + PCI_MSIX_FLAGS)); + if (op) + control |= PCI_MSIX_FLAGS_ENABLE; + else + control &= ~PCI_MSIX_FLAGS_ENABLE; + rte_pci_write_config(device, &control, sizeof(control), + (pos + PCI_MSIX_FLAGS)); + return 0; + } + return -1; +} + static int hns3vf_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, __attribute__ ((unused)) uint32_t idx, @@ -1308,9 +1386,30 @@ hns3vf_wait_hardware_ready(struct hns3_adapter *hns) struct hns3_wait_data *wait_data = hw->reset.wait_data; struct timeval tv; - if (wait_data->result == HNS3_WAIT_SUCCESS) - return 0; - else if (wait_data->result == HNS3_WAIT_TIMEOUT) { + if (wait_data->result == HNS3_WAIT_SUCCESS) { + /* + * After vf reset is ready, the PF may not have completed + * the reset processing. The vf sending mbox to PF may fail + * during the pf reset, so it is better to add extra delay. + */ + if (hw->reset.level == HNS3_VF_FUNC_RESET || + hw->reset.level == HNS3_FLR_RESET) + return 0; + /* Reset retry process, no need to add extra delay. */ + if (hw->reset.attempts) + return 0; + if (wait_data->check_completion == NULL) + return 0; + + wait_data->check_completion = NULL; + wait_data->interval = 1 * MSEC_PER_SEC * USEC_PER_MSEC; + wait_data->count = 1; + wait_data->result = HNS3_WAIT_REQUEST; + rte_eal_alarm_set(wait_data->interval, hns3_wait_callback, + wait_data); + hns3_warn(hw, "hardware is ready, delay 1 sec for PF reset complete"); + return -EAGAIN; + } else if (wait_data->result == HNS3_WAIT_TIMEOUT) { gettimeofday(&tv, NULL); hns3_warn(hw, "Reset step4 hardware not ready after reset time=%ld.%.6ld", tv.tv_sec, tv.tv_usec); @@ -1473,6 +1572,11 @@ hns3vf_reset_service(void *param) rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_REQUESTED); hns3_err(hw, "Handling interrupts in delayed tasks"); hns3vf_interrupt_handler(&rte_eth_devices[hw->data->port_id]); + reset_level = hns3vf_get_reset_level(hw, &hw->reset.pending); + if (reset_level == HNS3_NONE_RESET) { + hns3_err(hw, "No reset level is set, try global reset"); + hns3_atomic_set_bit(HNS3_VF_RESET, &hw->reset.pending); + } } rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_NONE); @@ -1498,14 +1602,35 @@ hns3vf_reset_service(void *param) static int hns3vf_reinit_dev(struct hns3_adapter *hns) { + struct rte_eth_dev *eth_dev = &rte_eth_devices[hns->hw.data->port_id]; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct hns3_hw *hw = &hns->hw; int ret; + if (hw->reset.level == HNS3_VF_FULL_RESET) { + rte_intr_disable(&pci_dev->intr_handle); + hns3vf_set_bus_master(pci_dev, true); + } + /* Firmware command initialize */ ret = hns3_cmd_init(hw); if (ret) { hns3_err(hw, "Failed to init cmd: %d", ret); - return ret; + goto err_cmd_init; + } + + if (hw->reset.level == HNS3_VF_FULL_RESET) { + /* + * UIO enables msix by writing the pcie configuration space + * vfio_pci enables msix in rte_intr_enable. + */ + if (pci_dev->kdrv == RTE_KDRV_IGB_UIO || + pci_dev->kdrv == RTE_KDRV_UIO_GENERIC) { + if (hns3vf_enable_msix(pci_dev, true)) + hns3_err(hw, "Failed to enable msix"); + } + + rte_intr_enable(&pci_dev->intr_handle); } ret = hns3_reset_all_queues(hns); @@ -1522,6 +1647,8 @@ hns3vf_reinit_dev(struct hns3_adapter *hns) return 0; +err_cmd_init: + hns3vf_set_bus_master(pci_dev, false); err_init: hns3_cmd_uninit(hw); return ret; diff --git a/drivers/net/hns3/hns3_intr.c b/drivers/net/hns3/hns3_intr.c index 9e2d811..6c3ebd3 100644 --- a/drivers/net/hns3/hns3_intr.c +++ b/drivers/net/hns3/hns3_intr.c @@ -890,11 +890,12 @@ hns3_reset_err_handle(struct hns3_adapter *hns) hns3_warn(hw, "%s reset fail fail_cnt:%" PRIx64 " success_cnt:%" PRIx64 " global_cnt:%" PRIx64 " imp_cnt:%" PRIx64 " request_cnt:%" PRIx64 " exec_cnt:%" PRIx64 - " merge_cnt:%" PRIx64, + " merge_cnt:%" PRIx64 "adapter_state:%d", reset_string[hw->reset.level], hw->reset.stats.fail_cnt, hw->reset.stats.success_cnt, hw->reset.stats.global_cnt, hw->reset.stats.imp_cnt, hw->reset.stats.request_cnt, - hw->reset.stats.exec_cnt, hw->reset.stats.merge_cnt); + hw->reset.stats.exec_cnt, hw->reset.stats.merge_cnt, + hw->adapter_state); /* IMP no longer waiting the ready flag */ hns3_notify_reset_ready(hw, true);