From patchwork Thu Sep 26 14:02:08 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Wei Hu (Xavier)" X-Patchwork-Id: 59901 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id B48FC1BF2C; Thu, 26 Sep 2019 16:06:03 +0200 (CEST) Received: from huawei.com (szxga04-in.huawei.com [45.249.212.190]) by dpdk.org (Postfix) with ESMTP id 8675F1BEAD for ; Thu, 26 Sep 2019 16:05:00 +0200 (CEST) Received: from DGGEMS405-HUB.china.huawei.com (unknown [172.30.72.59]) by Forcepoint Email with ESMTP id 6FD9C956B856E89CBD1A; Thu, 26 Sep 2019 22:04:58 +0800 (CST) Received: from localhost.localdomain (10.67.212.132) by DGGEMS405-HUB.china.huawei.com (10.3.19.205) with Microsoft SMTP Server id 14.3.439.0; Thu, 26 Sep 2019 22:04:48 +0800 From: "Wei Hu (Xavier)" To: CC: , , , Date: Thu, 26 Sep 2019 22:02:08 +0800 Message-ID: <1569506528-60464-23-git-send-email-xavier.huwei@huawei.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1569506528-60464-1-git-send-email-xavier.huwei@huawei.com> References: <1569506528-60464-1-git-send-email-xavier.huwei@huawei.com> MIME-Version: 1.0 X-Originating-IP: [10.67.212.132] X-CFilter-Loop: Reflected Subject: [dpdk-dev] [PATCH v3 22/22] net/hns3: add multiple process support for hns3 PMD driver X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This patch adds multiple process support for hns3 PMD driver. Multi-process support selection queue by configuring RSS or flow director. The primary process supports various management ops, and the secondary process only supports queries ops. The primary process notifies the secondary processes to start or stop tranceiver. Signed-off-by: Chunsong Feng Signed-off-by: Min Wang (Jushui) Signed-off-by: Wei Hu (Xavier) Signed-off-by: Min Hu (Connor) Signed-off-by: Hao Chen Signed-off-by: Huisong Li --- v1 -> v2: Address Ferruh Yigit's comments as follows: https://inbox.dpdk.org/dev/a8e60175-3b08-c3ca-6766-8b98e661eb68@intel.com --- drivers/net/hns3/Makefile | 7 ++ drivers/net/hns3/hns3_ethdev.c | 25 ++++- drivers/net/hns3/hns3_ethdev_vf.c | 23 +++- drivers/net/hns3/hns3_mp.c | 214 ++++++++++++++++++++++++++++++++++++++ drivers/net/hns3/hns3_mp.h | 14 +++ drivers/net/hns3/meson.build | 10 +- 6 files changed, 290 insertions(+), 3 deletions(-) create mode 100644 drivers/net/hns3/hns3_mp.c create mode 100644 drivers/net/hns3/hns3_mp.h diff --git a/drivers/net/hns3/Makefile b/drivers/net/hns3/Makefile index fede05d..cbbbe04 100644 --- a/drivers/net/hns3/Makefile +++ b/drivers/net/hns3/Makefile @@ -10,6 +10,12 @@ LIB = librte_pmd_hns3.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +CFLAGS += -DALLOW_EXPERIMENTAL_API +# Experimantal APIs: +# - rte_mp_action_register +# - rte_mp_action_unregister +# - rte_mp_reply +# - rte_mp_request_sync LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring LDLIBS += -lrte_ethdev -lrte_net -lrte_hash @@ -34,5 +40,6 @@ SRCS-$(CONFIG_RTE_LIBRTE_HNS3_PMD) += hns3_intr.c SRCS-$(CONFIG_RTE_LIBRTE_HNS3_PMD) += hns3_stats.c SRCS-$(CONFIG_RTE_LIBRTE_HNS3_PMD) += hns3_regs.c SRCS-$(CONFIG_RTE_LIBRTE_HNS3_PMD) += hns3_dcb.c +SRCS-$(CONFIG_RTE_LIBRTE_HNS3_PMD) += hns3_mp.c include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/net/hns3/hns3_ethdev.c b/drivers/net/hns3/hns3_ethdev.c index 361060d..574eb92 100644 --- a/drivers/net/hns3/hns3_ethdev.c +++ b/drivers/net/hns3/hns3_ethdev.c @@ -29,6 +29,7 @@ #include "hns3_intr.h" #include "hns3_regs.h" #include "hns3_dcb.h" +#include "hns3_mp.h" #define HNS3_DEFAULT_PORT_CONF_BURST_SIZE 32 #define HNS3_DEFAULT_PORT_CONF_QUEUES_NUM 1 @@ -4036,6 +4037,7 @@ hns3_dev_start(struct rte_eth_dev *eth_dev) hw->adapter_state = HNS3_NIC_STARTED; rte_spinlock_unlock(&hw->lock); hns3_set_rxtx_function(eth_dev); + hns3_mp_req_start_rxtx(eth_dev); hns3_info(hw, "hns3 dev start successful!"); return 0; @@ -4072,6 +4074,11 @@ hns3_dev_stop(struct rte_eth_dev *eth_dev) hw->adapter_state = HNS3_NIC_STOPPING; hns3_set_rxtx_function(eth_dev); + rte_wmb(); + /* Disable datapath on secondary process. */ + hns3_mp_req_stop_rxtx(eth_dev); + /* Prevent crashes when queues are still in use. */ + rte_delay_ms(hw->tqps_num); rte_spinlock_lock(&hw->lock); if (rte_atomic16_read(&hw->reset.resetting) == 0) { @@ -4088,6 +4095,12 @@ hns3_dev_close(struct rte_eth_dev *eth_dev) struct hns3_adapter *hns = eth_dev->data->dev_private; struct hns3_hw *hw = &hns->hw; + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + rte_free(eth_dev->process_private); + eth_dev->process_private = NULL; + return; + } + if (hw->adapter_state == HNS3_NIC_STARTED) hns3_dev_stop(eth_dev); @@ -4104,6 +4117,7 @@ hns3_dev_close(struct rte_eth_dev *eth_dev) rte_free(hw->reset.wait_data); rte_free(eth_dev->process_private); eth_dev->process_private = NULL; + hns3_mp_uninit_primary(); hns3_warn(hw, "Close port %d finished", hw->data->port_id); } @@ -4558,6 +4572,10 @@ hns3_stop_service(struct hns3_adapter *hns) hw->mac.link_status = ETH_LINK_DOWN; hns3_set_rxtx_function(eth_dev); + rte_wmb(); + /* Disable datapath on secondary process. */ + hns3_mp_req_stop_rxtx(eth_dev); + rte_delay_ms(hw->tqps_num); rte_spinlock_lock(&hw->lock); if (hns->hw.adapter_state == HNS3_NIC_STARTED || @@ -4590,6 +4608,7 @@ hns3_start_service(struct hns3_adapter *hns) hns3_set_rst_done(hw); eth_dev = &rte_eth_devices[hw->data->port_id]; hns3_set_rxtx_function(eth_dev); + hns3_mp_req_start_rxtx(eth_dev); hns3_service_handler(eth_dev); return 0; } @@ -4776,9 +4795,13 @@ hns3_dev_init(struct rte_eth_dev *eth_dev) hns3_set_rxtx_function(eth_dev); eth_dev->dev_ops = &hns3_eth_dev_ops; - if (rte_eal_process_type() != RTE_PROC_PRIMARY) + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + hns3_mp_init_secondary(); + hw->secondary_cnt++; return 0; + } + hns3_mp_init_primary(); hw->adapter_state = HNS3_NIC_UNINITIALIZED; if (device_id == HNS3_DEV_ID_25GE_RDMA || diff --git a/drivers/net/hns3/hns3_ethdev_vf.c b/drivers/net/hns3/hns3_ethdev_vf.c index fc98ebb..07a4b01 100644 --- a/drivers/net/hns3/hns3_ethdev_vf.c +++ b/drivers/net/hns3/hns3_ethdev_vf.c @@ -31,6 +31,7 @@ #include "hns3_regs.h" #include "hns3_intr.h" #include "hns3_dcb.h" +#include "hns3_mp.h" #define HNS3VF_KEEP_ALIVE_INTERVAL 2000000 /* us */ #define HNS3VF_SERVICE_INTERVAL 1000000 /* us */ @@ -1141,6 +1142,11 @@ hns3vf_dev_stop(struct rte_eth_dev *eth_dev) hw->adapter_state = HNS3_NIC_STOPPING; hns3_set_rxtx_function(eth_dev); + rte_wmb(); + /* Disable datapath on secondary process. */ + hns3_mp_req_stop_rxtx(eth_dev); + /* Prevent crashes when queues are still in use. */ + rte_delay_ms(hw->tqps_num); rte_spinlock_lock(&hw->lock); if (rte_atomic16_read(&hw->reset.resetting) == 0) { @@ -1157,6 +1163,9 @@ hns3vf_dev_close(struct rte_eth_dev *eth_dev) struct hns3_adapter *hns = eth_dev->data->dev_private; struct hns3_hw *hw = &hns->hw; + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return; + if (hw->adapter_state == HNS3_NIC_STARTED) hns3vf_dev_stop(eth_dev); @@ -1172,6 +1181,7 @@ hns3vf_dev_close(struct rte_eth_dev *eth_dev) rte_free(hw->reset.wait_data); rte_free(eth_dev->process_private); eth_dev->process_private = NULL; + hns3_mp_uninit_primary(); hns3_warn(hw, "Close port %d finished", hw->data->port_id); } @@ -1249,6 +1259,7 @@ hns3vf_dev_start(struct rte_eth_dev *eth_dev) hw->adapter_state = HNS3_NIC_STARTED; rte_spinlock_unlock(&hw->lock); hns3_set_rxtx_function(eth_dev); + hns3_mp_req_start_rxtx(eth_dev); return 0; } @@ -1344,6 +1355,10 @@ hns3vf_stop_service(struct hns3_adapter *hns) hw->mac.link_status = ETH_LINK_DOWN; hns3_set_rxtx_function(eth_dev); + rte_wmb(); + /* Disable datapath on secondary process. */ + hns3_mp_req_stop_rxtx(eth_dev); + rte_delay_ms(hw->tqps_num); rte_spinlock_lock(&hw->lock); if (hw->adapter_state == HNS3_NIC_STARTED || @@ -1373,6 +1388,7 @@ hns3vf_start_service(struct hns3_adapter *hns) eth_dev = &rte_eth_devices[hw->data->port_id]; hns3_set_rxtx_function(eth_dev); + hns3_mp_req_start_rxtx(eth_dev); hns3vf_service_handler(eth_dev); return 0; @@ -1578,8 +1594,13 @@ hns3vf_dev_init(struct rte_eth_dev *eth_dev) hns3_set_rxtx_function(eth_dev); eth_dev->dev_ops = &hns3vf_eth_dev_ops; - if (rte_eal_process_type() != RTE_PROC_PRIMARY) + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + hns3_mp_init_secondary(); + hw->secondary_cnt++; return 0; + } + + hns3_mp_init_primary(); hw->adapter_state = HNS3_NIC_UNINITIALIZED; hns->is_vf = true; diff --git a/drivers/net/hns3/hns3_mp.c b/drivers/net/hns3/hns3_mp.c new file mode 100644 index 0000000..596c310 --- /dev/null +++ b/drivers/net/hns3/hns3_mp.c @@ -0,0 +1,214 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018-2019 Hisilicon Limited. + */ + +#include + +#include +#include +#include +#include + +#include "hns3_ethdev.h" +#include "hns3_logs.h" +#include "hns3_rxtx.h" +#include "hns3_mp.h" + +/* + * Initialize IPC message. + * + * @param[in] dev + * Pointer to Ethernet structure. + * @param[out] msg + * Pointer to message to fill in. + * @param[in] type + * Message type. + */ +static inline void +mp_init_msg(struct rte_eth_dev *dev, struct rte_mp_msg *msg, + enum hns3_mp_req_type type) +{ + struct hns3_mp_param *param = (struct hns3_mp_param *)msg->param; + + memset(msg, 0, sizeof(*msg)); + strlcpy(msg->name, HNS3_MP_NAME, sizeof(msg->name)); + msg->len_param = sizeof(*param); + param->type = type; + param->port_id = dev->data->port_id; +} + +/* + * IPC message handler of primary process. + * + * @param[in] dev + * Pointer to Ethernet structure. + * @param[in] peer + * Pointer to the peer socket path. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mp_primary_handle(const struct rte_mp_msg *mp_msg __rte_unused, + const void *peer __rte_unused) +{ + return 0; +} + +/* + * IPC message handler of a secondary process. + * + * @param[in] dev + * Pointer to Ethernet structure. + * @param[in] peer + * Pointer to the peer socket path. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mp_secondary_handle(const struct rte_mp_msg *mp_msg, const void *peer) +{ + struct rte_mp_msg mp_res; + struct hns3_mp_param *res = (struct hns3_mp_param *)mp_res.param; + const struct hns3_mp_param *param = + (const struct hns3_mp_param *)mp_msg->param; + struct rte_eth_dev *dev; + int ret; + + if (!rte_eth_dev_is_valid_port(param->port_id)) { + rte_errno = ENODEV; + PMD_INIT_LOG(ERR, "port %u invalid port ID", param->port_id); + return -rte_errno; + } + dev = &rte_eth_devices[param->port_id]; + switch (param->type) { + case HNS3_MP_REQ_START_RXTX: + PMD_INIT_LOG(INFO, "port %u starting datapath", + dev->data->port_id); + rte_mb(); + hns3_set_rxtx_function(dev); + mp_init_msg(dev, &mp_res, param->type); + res->result = 0; + ret = rte_mp_reply(&mp_res, peer); + break; + case HNS3_MP_REQ_STOP_RXTX: + PMD_INIT_LOG(INFO, "port %u stopping datapath", + dev->data->port_id); + hns3_set_rxtx_function(dev); + rte_mb(); + mp_init_msg(dev, &mp_res, param->type); + res->result = 0; + ret = rte_mp_reply(&mp_res, peer); + break; + default: + rte_errno = EINVAL; + PMD_INIT_LOG(ERR, "port %u invalid mp request type", + dev->data->port_id); + return -rte_errno; + } + return ret; +} + +/* + * Broadcast request of stopping/starting data-path to secondary processes. + * + * @param[in] dev + * Pointer to Ethernet structure. + * @param[in] type + * Request type. + */ +static void +mp_req_on_rxtx(struct rte_eth_dev *dev, enum hns3_mp_req_type type) +{ + struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_mp_msg mp_req; + struct rte_mp_msg *mp_res; + struct rte_mp_reply mp_rep; + struct hns3_mp_param *res; + struct timespec ts; + int ret; + int i; + + if (!hw->secondary_cnt) + return; + if (type != HNS3_MP_REQ_START_RXTX && type != HNS3_MP_REQ_STOP_RXTX) { + hns3_err(hw, "port %u unknown request (req_type %d)", + dev->data->port_id, type); + return; + } + mp_init_msg(dev, &mp_req, type); + ts.tv_sec = HNS3_MP_REQ_TIMEOUT_SEC; + ts.tv_nsec = 0; + ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts); + if (ret) { + hns3_err(hw, "port %u failed to request stop/start Rx/Tx (%d)", + dev->data->port_id, type); + goto exit; + } + if (mp_rep.nb_sent != mp_rep.nb_received) { + PMD_INIT_LOG(ERR, + "port %u not all secondaries responded (req_type %d)", + dev->data->port_id, type); + goto exit; + } + for (i = 0; i < mp_rep.nb_received; i++) { + mp_res = &mp_rep.msgs[i]; + res = (struct hns3_mp_param *)mp_res->param; + if (res->result) { + hns3_err(hw, "port %u request failed on secondary #%d", + dev->data->port_id, i); + goto exit; + } + } +exit: + free(mp_rep.msgs); +} + +/* + * Broadcast request of starting data-path to secondary processes. The request + * is synchronous. + * + * @param[in] dev + * Pointer to Ethernet structure. + */ +void hns3_mp_req_start_rxtx(struct rte_eth_dev *dev) +{ + mp_req_on_rxtx(dev, HNS3_MP_REQ_START_RXTX); +} + +/* + * Broadcast request of stopping data-path to secondary processes. The request + * is synchronous. + * + * @param[in] dev + * Pointer to Ethernet structure. + */ +void hns3_mp_req_stop_rxtx(struct rte_eth_dev *dev) +{ + mp_req_on_rxtx(dev, HNS3_MP_REQ_STOP_RXTX); +} + +/* + * Initialize by primary process. + */ +void hns3_mp_init_primary(void) +{ + rte_mp_action_register(HNS3_MP_NAME, mp_primary_handle); +} + +/* + * Un-initialize by primary process. + */ +void hns3_mp_uninit_primary(void) +{ + rte_mp_action_unregister(HNS3_MP_NAME); +} + +/* + * Initialize by secondary process. + */ +void hns3_mp_init_secondary(void) +{ + rte_mp_action_register(HNS3_MP_NAME, mp_secondary_handle); +} diff --git a/drivers/net/hns3/hns3_mp.h b/drivers/net/hns3/hns3_mp.h new file mode 100644 index 0000000..aefbeb1 --- /dev/null +++ b/drivers/net/hns3/hns3_mp.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018-2019 Hisilicon Limited. + */ + +#ifndef _HNS3_MP_H_ +#define _HNS3_MP_H_ + +void hns3_mp_req_start_rxtx(struct rte_eth_dev *dev); +void hns3_mp_req_stop_rxtx(struct rte_eth_dev *dev); +void hns3_mp_init_primary(void); +void hns3_mp_uninit_primary(void); +void hns3_mp_init_secondary(void); + +#endif /* _HNS3_MP_H_ */ diff --git a/drivers/net/hns3/meson.build b/drivers/net/hns3/meson.build index 87e1a0c..1a350fc 100644 --- a/drivers/net/hns3/meson.build +++ b/drivers/net/hns3/meson.build @@ -25,5 +25,13 @@ sources = files('hns3_cmd.c', 'hns3_rss.c', 'hns3_rxtx.c', 'hns3_stats.c', - ) + 'hns3_mp.c') + +allow_experimental_apis = true +# Experimantal APIs: +# - rte_mp_action_register +# - rte_mp_action_unregister +# - rte_mp_reply +# - rte_mp_request_sync + deps += ['hash']