From patchwork Mon Oct 24 13:01:21 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 118991 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 0E285A034C; Mon, 24 Oct 2022 15:03:44 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id EFFD042B87; Mon, 24 Oct 2022 15:03:43 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id AC72942B85 for ; Mon, 24 Oct 2022 15:03:41 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616621; x=1698152621; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=lx01nGox/vbB8wZQJW0pZa8WagijTmYNpImz8oAn1Ag=; b=FYrpKC4k9OVF4+mILuhQLg1+C6lRnDggOqYISjlMaWsi5LpuR/kyU2Aa ZX8p622YVv4Ah0RA79q4vyRt+caZDxPjzkQGOdHXfJiG91ZuWMfdpP7cP ErB5CoBHoaOTLrwhj7XE1xdTa3pozc3LGinZgb5FdBT7f6KaMzBvbZdjC xQnDU6gbwTU2tXOqt2Ha3xkVMFKpbJi2sL1FSk9ujbrBZKWUVkcT5xbju 3ckrn1v98dZD6V6gR1vnHI09DhGum4HtLKkM5JyDngJ2Yk3rpIj6T3Svm yhgpzyLDxF1dqiZyfJpdxS66F0xH8ZgPD8QLRFZSAIhTX7jowq1SZg27B g==; X-IronPort-AV: E=McAfee;i="6500,9779,10509"; a="305018621" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018621" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:12 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248198" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248198" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:09 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo , Xiaoyun Li Subject: [PATCH v10 01/14] net/idpf: add support for device start and stop Date: Mon, 24 Oct 2022 21:01:21 +0800 Message-Id: <20221024130134.1046536-2-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add dev ops dev_start, dev_stop and link_update. Signed-off-by: Beilei Xing Signed-off-by: Xiaoyun Li Signed-off-by: Junfeng Guo --- drivers/net/idpf/idpf_ethdev.c | 89 ++++++++++++++++++++++++++++++++++ drivers/net/idpf/idpf_ethdev.h | 5 ++ 2 files changed, 94 insertions(+) diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index 1d2075f466..4c7a2d0748 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -29,17 +29,42 @@ static const char * const idpf_valid_args[] = { }; static int idpf_dev_configure(struct rte_eth_dev *dev); +static int idpf_dev_start(struct rte_eth_dev *dev); +static int idpf_dev_stop(struct rte_eth_dev *dev); static int idpf_dev_close(struct rte_eth_dev *dev); static int idpf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); static void idpf_adapter_rel(struct idpf_adapter *adapter); +int +idpf_dev_link_update(struct rte_eth_dev *dev, + __rte_unused int wait_to_complete) +{ + struct idpf_vport *vport = dev->data->dev_private; + struct rte_eth_link new_link; + + memset(&new_link, 0, sizeof(new_link)); + + new_link.link_speed = RTE_ETH_SPEED_NUM_NONE; + + new_link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; + new_link.link_status = vport->link_up ? RTE_ETH_LINK_UP : + RTE_ETH_LINK_DOWN; + new_link.link_autoneg = !(dev->data->dev_conf.link_speeds & + RTE_ETH_LINK_SPEED_FIXED); + + return rte_eth_linkstatus_set(dev, &new_link); +} + static const struct eth_dev_ops idpf_eth_dev_ops = { .dev_configure = idpf_dev_configure, + .dev_start = idpf_dev_start, + .dev_stop = idpf_dev_stop, .dev_close = idpf_dev_close, .rx_queue_setup = idpf_rx_queue_setup, .tx_queue_setup = idpf_tx_queue_setup, .dev_infos_get = idpf_dev_info_get, + .link_update = idpf_dev_link_update, }; static int @@ -233,6 +258,70 @@ idpf_dev_configure(struct rte_eth_dev *dev) return 0; } +static int +idpf_start_queues(struct rte_eth_dev *dev) +{ + struct idpf_rx_queue *rxq; + struct idpf_tx_queue *txq; + int err = 0; + int i; + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + if (txq == NULL || txq->tx_deferred_start) + continue; + + PMD_DRV_LOG(ERR, "Start Tx queues not supported yet"); + return -ENOTSUP; + } + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + if (rxq == NULL || rxq->rx_deferred_start) + continue; + + PMD_DRV_LOG(ERR, "Start Rx queues not supported yet"); + return -ENOTSUP; + } + + return err; +} + +static int +idpf_dev_start(struct rte_eth_dev *dev) +{ + struct idpf_vport *vport = dev->data->dev_private; + + if (dev->data->mtu > vport->max_mtu) { + PMD_DRV_LOG(ERR, "MTU should be less than %d", vport->max_mtu); + return -1; + } + + vport->max_pkt_len = dev->data->mtu + IDPF_ETH_OVERHEAD; + + if (idpf_start_queues(dev) != 0) { + PMD_DRV_LOG(ERR, "Failed to start queues"); + return -1; + } + + if (idpf_vc_ena_dis_vport(vport, true) != 0) { + PMD_DRV_LOG(ERR, "Failed to enable vport"); + return -1; + } + + return 0; +} + +static int +idpf_dev_stop(struct rte_eth_dev *dev) +{ + struct idpf_vport *vport = dev->data->dev_private; + + idpf_vc_ena_dis_vport(vport, false); + + return 0; +} + static int idpf_dev_close(struct rte_eth_dev *dev) { diff --git a/drivers/net/idpf/idpf_ethdev.h b/drivers/net/idpf/idpf_ethdev.h index c0ae801fd5..070531cc48 100644 --- a/drivers/net/idpf/idpf_ethdev.h +++ b/drivers/net/idpf/idpf_ethdev.h @@ -105,6 +105,9 @@ struct idpf_vport { /* Chunk info */ struct idpf_chunks_info chunks_info; + /* Event from ipf */ + bool link_up; + uint16_t devarg_id; }; @@ -195,6 +198,8 @@ atomic_set_cmd(struct idpf_adapter *adapter, enum virtchnl_ops ops) } struct idpf_adapter *idpf_find_adapter(struct rte_pci_device *pci_dev); +int idpf_dev_link_update(struct rte_eth_dev *dev, + __rte_unused int wait_to_complete); void idpf_handle_virtchnl_msg(struct rte_eth_dev *dev); int idpf_vc_check_api_version(struct idpf_adapter *adapter); int idpf_vc_get_caps(struct idpf_adapter *adapter); From patchwork Mon Oct 24 13:01:22 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 118992 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 82573A034C; Mon, 24 Oct 2022 15:03:48 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 0034E42B8B; Mon, 24 Oct 2022 15:03:46 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id 30D9D40695 for ; Mon, 24 Oct 2022 15:03:43 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616623; x=1698152623; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=2ePiLxOHhCfZkqPKcyAxtLmMLUU0rh9oAKlkXqb1/Dk=; b=bOiRFXtOjEYRd/AmZMkuWzCBzEnou9RmZHFKzTEi2Cnur3Bw4C9NdP2Q EISqY3zLBDlz7+7FvG0fjxmDDiERWDTvBOU88TIQPxS4w6/b74CKjVS2v s/QsIxLzUGfQ6J0lyOQK7Kw04rP+6Ku/eYB3mdN8cUPy0xx6o2v6+q4w4 RxHpE2ccYux3hHPZEe6JlZuUx4x6DFUu2Xp1nfyOQpAxxZuAZ8gXLyMdR wyQ/7BcSpaOCfozG5uMtSV0kAQzN9ujKHTe5vTMDgknlhBQk18IqzD1bo MXAS25JQbbUQxLb5TKPYaT3I9jYjMonyEK7BeZZIYDsc7ML/YOQk1H1lL w==; X-IronPort-AV: E=McAfee;i="6500,9779,10509"; a="305018645" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018645" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:14 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248229" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248229" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:11 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo , Xiaoyun Li Subject: [PATCH v10 02/14] net/idpf: add support for queue start Date: Mon, 24 Oct 2022 21:01:22 +0800 Message-Id: <20221024130134.1046536-3-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add support for these device ops: - rx_queue_start - tx_queue_start Signed-off-by: Beilei Xing Signed-off-by: Xiaoyun Li Signed-off-by: Junfeng Guo --- drivers/net/idpf/idpf_ethdev.c | 18 +- drivers/net/idpf/idpf_ethdev.h | 7 + drivers/net/idpf/idpf_rxtx.c | 214 ++++++++++++++++ drivers/net/idpf/idpf_rxtx.h | 4 + drivers/net/idpf/idpf_vchnl.c | 447 +++++++++++++++++++++++++++++++++ 5 files changed, 684 insertions(+), 6 deletions(-) diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index 4c7a2d0748..954bd0bf4b 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -61,6 +61,8 @@ static const struct eth_dev_ops idpf_eth_dev_ops = { .dev_start = idpf_dev_start, .dev_stop = idpf_dev_stop, .dev_close = idpf_dev_close, + .rx_queue_start = idpf_rx_queue_start, + .tx_queue_start = idpf_tx_queue_start, .rx_queue_setup = idpf_rx_queue_setup, .tx_queue_setup = idpf_tx_queue_setup, .dev_infos_get = idpf_dev_info_get, @@ -270,18 +272,22 @@ idpf_start_queues(struct rte_eth_dev *dev) txq = dev->data->tx_queues[i]; if (txq == NULL || txq->tx_deferred_start) continue; - - PMD_DRV_LOG(ERR, "Start Tx queues not supported yet"); - return -ENOTSUP; + err = idpf_tx_queue_start(dev, i); + if (err != 0) { + PMD_DRV_LOG(ERR, "Fail to start Tx queue %u", i); + return err; + } } for (i = 0; i < dev->data->nb_rx_queues; i++) { rxq = dev->data->rx_queues[i]; if (rxq == NULL || rxq->rx_deferred_start) continue; - - PMD_DRV_LOG(ERR, "Start Rx queues not supported yet"); - return -ENOTSUP; + err = idpf_rx_queue_start(dev, i); + if (err != 0) { + PMD_DRV_LOG(ERR, "Fail to start Rx queue %u", i); + return err; + } } return err; diff --git a/drivers/net/idpf/idpf_ethdev.h b/drivers/net/idpf/idpf_ethdev.h index 070531cc48..094f123be1 100644 --- a/drivers/net/idpf/idpf_ethdev.h +++ b/drivers/net/idpf/idpf_ethdev.h @@ -205,6 +205,13 @@ int idpf_vc_check_api_version(struct idpf_adapter *adapter); int idpf_vc_get_caps(struct idpf_adapter *adapter); int idpf_vc_create_vport(struct idpf_adapter *adapter); int idpf_vc_destroy_vport(struct idpf_vport *vport); +int idpf_vc_config_rxqs(struct idpf_vport *vport); +int idpf_vc_config_rxq(struct idpf_vport *vport, uint16_t rxq_id); +int idpf_vc_config_txqs(struct idpf_vport *vport); +int idpf_vc_config_txq(struct idpf_vport *vport, uint16_t txq_id); +int idpf_switch_queue(struct idpf_vport *vport, uint16_t qid, + bool rx, bool on); +int idpf_vc_ena_dis_queues(struct idpf_vport *vport, bool enable); int idpf_vc_ena_dis_vport(struct idpf_vport *vport, bool enable); int idpf_read_one_msg(struct idpf_adapter *adapter, uint32_t ops, uint16_t buf_len, uint8_t *buf); diff --git a/drivers/net/idpf/idpf_rxtx.c b/drivers/net/idpf/idpf_rxtx.c index 76669504c3..b71c1ac6db 100644 --- a/drivers/net/idpf/idpf_rxtx.c +++ b/drivers/net/idpf/idpf_rxtx.c @@ -796,3 +796,217 @@ idpf_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, return idpf_tx_split_queue_setup(dev, queue_idx, nb_desc, socket_id, tx_conf); } +static int +idpf_alloc_single_rxq_mbufs(struct idpf_rx_queue *rxq) +{ + volatile struct virtchnl2_singleq_rx_buf_desc *rxd; + struct rte_mbuf *mbuf = NULL; + uint64_t dma_addr; + uint16_t i; + + for (i = 0; i < rxq->nb_rx_desc; i++) { + mbuf = rte_mbuf_raw_alloc(rxq->mp); + if (unlikely(mbuf == NULL)) { + PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX"); + return -ENOMEM; + } + + rte_mbuf_refcnt_set(mbuf, 1); + mbuf->next = NULL; + mbuf->data_off = RTE_PKTMBUF_HEADROOM; + mbuf->nb_segs = 1; + mbuf->port = rxq->port_id; + + dma_addr = + rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); + + rxd = &((volatile struct virtchnl2_singleq_rx_buf_desc *)(rxq->rx_ring))[i]; + rxd->pkt_addr = dma_addr; + rxd->hdr_addr = 0; + rxd->rsvd1 = 0; + rxd->rsvd2 = 0; + rxq->sw_ring[i] = mbuf; + } + + return 0; +} + +static int +idpf_alloc_split_rxq_mbufs(struct idpf_rx_queue *rxq) +{ + volatile struct virtchnl2_splitq_rx_buf_desc *rxd; + struct rte_mbuf *mbuf = NULL; + uint64_t dma_addr; + uint16_t i; + + for (i = 0; i < rxq->nb_rx_desc; i++) { + mbuf = rte_mbuf_raw_alloc(rxq->mp); + if (unlikely(mbuf == NULL)) { + PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX"); + return -ENOMEM; + } + + rte_mbuf_refcnt_set(mbuf, 1); + mbuf->next = NULL; + mbuf->data_off = RTE_PKTMBUF_HEADROOM; + mbuf->nb_segs = 1; + mbuf->port = rxq->port_id; + + dma_addr = + rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); + + rxd = &((volatile struct virtchnl2_splitq_rx_buf_desc *)(rxq->rx_ring))[i]; + rxd->qword0.buf_id = i; + rxd->qword0.rsvd0 = 0; + rxd->qword0.rsvd1 = 0; + rxd->pkt_addr = dma_addr; + rxd->hdr_addr = 0; + rxd->rsvd2 = 0; + + rxq->sw_ring[i] = mbuf; + } + + rxq->nb_rx_hold = 0; + rxq->rx_tail = rxq->nb_rx_desc - 1; + + return 0; +} + +int +idpf_rx_queue_init(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct idpf_rx_queue *rxq; + int err; + + if (rx_queue_id >= dev->data->nb_rx_queues) + return -EINVAL; + + rxq = dev->data->rx_queues[rx_queue_id]; + + if (rxq == NULL || !rxq->q_set) { + PMD_DRV_LOG(ERR, "RX queue %u not available or setup", + rx_queue_id); + return -EINVAL; + } + + if (rxq->bufq1 == NULL) { + /* Single queue */ + err = idpf_alloc_single_rxq_mbufs(rxq); + if (err != 0) { + PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf"); + return err; + } + + rte_wmb(); + + /* Init the RX tail register. */ + IDPF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); + } else { + /* Split queue */ + err = idpf_alloc_split_rxq_mbufs(rxq->bufq1); + if (err != 0) { + PMD_DRV_LOG(ERR, "Failed to allocate RX buffer queue mbuf"); + return err; + } + err = idpf_alloc_split_rxq_mbufs(rxq->bufq2); + if (err != 0) { + PMD_DRV_LOG(ERR, "Failed to allocate RX buffer queue mbuf"); + return err; + } + + rte_wmb(); + + /* Init the RX tail register. */ + IDPF_PCI_REG_WRITE(rxq->bufq1->qrx_tail, rxq->bufq1->rx_tail); + IDPF_PCI_REG_WRITE(rxq->bufq2->qrx_tail, rxq->bufq2->rx_tail); + } + + return err; +} + +int +idpf_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct idpf_vport *vport = dev->data->dev_private; + struct idpf_rx_queue *rxq = + (struct idpf_rx_queue *)dev->data->rx_queues[rx_queue_id]; + int err = 0; + + err = idpf_vc_config_rxq(vport, rx_queue_id); + if (err != 0) { + PMD_DRV_LOG(ERR, "Fail to configure Rx queue %u", rx_queue_id); + return err; + } + + err = idpf_rx_queue_init(dev, rx_queue_id); + if (err != 0) { + PMD_DRV_LOG(ERR, "Failed to init RX queue %u", + rx_queue_id); + return err; + } + + /* Ready to switch the queue on */ + err = idpf_switch_queue(vport, rx_queue_id, true, true); + if (err != 0) { + PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on", + rx_queue_id); + } else { + rxq->q_started = true; + dev->data->rx_queue_state[rx_queue_id] = + RTE_ETH_QUEUE_STATE_STARTED; + } + + return err; +} + +int +idpf_tx_queue_init(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + struct idpf_tx_queue *txq; + + if (tx_queue_id >= dev->data->nb_tx_queues) + return -EINVAL; + + txq = dev->data->tx_queues[tx_queue_id]; + + /* Init the RX tail register. */ + IDPF_PCI_REG_WRITE(txq->qtx_tail, 0); + + return 0; +} + +int +idpf_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + struct idpf_vport *vport = dev->data->dev_private; + struct idpf_tx_queue *txq = + (struct idpf_tx_queue *)dev->data->tx_queues[tx_queue_id]; + int err = 0; + + err = idpf_vc_config_txq(vport, tx_queue_id); + if (err != 0) { + PMD_DRV_LOG(ERR, "Fail to configure Tx queue %u", tx_queue_id); + return err; + } + + err = idpf_tx_queue_init(dev, tx_queue_id); + if (err != 0) { + PMD_DRV_LOG(ERR, "Failed to init TX queue %u", + tx_queue_id); + return err; + } + + /* Ready to switch the queue on */ + err = idpf_switch_queue(vport, tx_queue_id, false, true); + if (err != 0) { + PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on", + tx_queue_id); + } else { + txq->q_started = true; + dev->data->tx_queue_state[tx_queue_id] = + RTE_ETH_QUEUE_STATE_STARTED; + } + + return err; +} + diff --git a/drivers/net/idpf/idpf_rxtx.h b/drivers/net/idpf/idpf_rxtx.h index a79e9a0b26..d076903e3c 100644 --- a/drivers/net/idpf/idpf_rxtx.h +++ b/drivers/net/idpf/idpf_rxtx.h @@ -115,8 +115,12 @@ int idpf_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp); +int idpf_rx_queue_init(struct rte_eth_dev *dev, uint16_t rx_queue_id); +int idpf_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id); int idpf_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf); +int idpf_tx_queue_init(struct rte_eth_dev *dev, uint16_t tx_queue_id); +int idpf_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id); #endif /* _IDPF_RXTX_H_ */ diff --git a/drivers/net/idpf/idpf_vchnl.c b/drivers/net/idpf/idpf_vchnl.c index ec72a8bba7..5e5508e999 100644 --- a/drivers/net/idpf/idpf_vchnl.c +++ b/drivers/net/idpf/idpf_vchnl.c @@ -21,6 +21,7 @@ #include #include "idpf_ethdev.h" +#include "idpf_rxtx.h" #define IDPF_CTLQ_LEN 64 @@ -227,6 +228,10 @@ idpf_execute_vc_cmd(struct idpf_adapter *adapter, struct idpf_cmd_info *args) case VIRTCHNL2_OP_GET_CAPS: case VIRTCHNL2_OP_CREATE_VPORT: case VIRTCHNL2_OP_DESTROY_VPORT: + case VIRTCHNL2_OP_CONFIG_RX_QUEUES: + case VIRTCHNL2_OP_CONFIG_TX_QUEUES: + case VIRTCHNL2_OP_ENABLE_QUEUES: + case VIRTCHNL2_OP_DISABLE_QUEUES: case VIRTCHNL2_OP_ENABLE_VPORT: case VIRTCHNL2_OP_DISABLE_VPORT: /* for init virtchnl ops, need to poll the response */ @@ -444,6 +449,448 @@ idpf_vc_destroy_vport(struct idpf_vport *vport) return err; } +#define IDPF_RX_BUF_STRIDE 64 +int +idpf_vc_config_rxqs(struct idpf_vport *vport) +{ + struct idpf_adapter *adapter = vport->adapter; + struct idpf_rx_queue **rxq = + (struct idpf_rx_queue **)vport->dev_data->rx_queues; + struct virtchnl2_config_rx_queues *vc_rxqs = NULL; + struct virtchnl2_rxq_info *rxq_info; + struct idpf_cmd_info args; + uint16_t total_qs, num_qs; + int size, i, j; + int err = 0; + int k = 0; + + total_qs = vport->num_rx_q + vport->num_rx_bufq; + while (total_qs) { + if (total_qs > adapter->max_rxq_per_msg) { + num_qs = adapter->max_rxq_per_msg; + total_qs -= adapter->max_rxq_per_msg; + } else { + num_qs = total_qs; + total_qs = 0; + } + + size = sizeof(*vc_rxqs) + (num_qs - 1) * + sizeof(struct virtchnl2_rxq_info); + vc_rxqs = rte_zmalloc("cfg_rxqs", size, 0); + if (vc_rxqs == NULL) { + PMD_DRV_LOG(ERR, "Failed to allocate virtchnl2_config_rx_queues"); + err = -ENOMEM; + break; + } + vc_rxqs->vport_id = vport->vport_id; + vc_rxqs->num_qinfo = num_qs; + if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) { + for (i = 0; i < num_qs; i++, k++) { + rxq_info = &vc_rxqs->qinfo[i]; + rxq_info->dma_ring_addr = rxq[k]->rx_ring_phys_addr; + rxq_info->type = VIRTCHNL2_QUEUE_TYPE_RX; + rxq_info->queue_id = rxq[k]->queue_id; + rxq_info->model = VIRTCHNL2_QUEUE_MODEL_SINGLE; + rxq_info->data_buffer_size = rxq[k]->rx_buf_len; + rxq_info->max_pkt_size = vport->max_pkt_len; + + rxq_info->desc_ids = VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M; + rxq_info->qflags |= VIRTCHNL2_RX_DESC_SIZE_32BYTE; + + rxq_info->ring_len = rxq[k]->nb_rx_desc; + } + } else { + for (i = 0; i < num_qs / 3; i++, k++) { + /* Rx queue */ + rxq_info = &vc_rxqs->qinfo[i * 3]; + rxq_info->dma_ring_addr = + rxq[k]->rx_ring_phys_addr; + rxq_info->type = VIRTCHNL2_QUEUE_TYPE_RX; + rxq_info->queue_id = rxq[k]->queue_id; + rxq_info->model = VIRTCHNL2_QUEUE_MODEL_SPLIT; + rxq_info->data_buffer_size = rxq[k]->rx_buf_len; + rxq_info->max_pkt_size = vport->max_pkt_len; + + rxq_info->desc_ids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M; + rxq_info->qflags |= VIRTCHNL2_RX_DESC_SIZE_32BYTE; + + rxq_info->ring_len = rxq[k]->nb_rx_desc; + rxq_info->rx_bufq1_id = rxq[k]->bufq1->queue_id; + rxq_info->rx_bufq2_id = rxq[k]->bufq2->queue_id; + rxq_info->rx_buffer_low_watermark = 64; + + /* Buffer queue */ + for (j = 1; j <= IDPF_RX_BUFQ_PER_GRP; j++) { + struct idpf_rx_queue *bufq = j == 1 ? + rxq[k]->bufq1 : rxq[k]->bufq2; + rxq_info = &vc_rxqs->qinfo[i * 3 + j]; + rxq_info->dma_ring_addr = + bufq->rx_ring_phys_addr; + rxq_info->type = + VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; + rxq_info->queue_id = bufq->queue_id; + rxq_info->model = VIRTCHNL2_QUEUE_MODEL_SPLIT; + rxq_info->data_buffer_size = bufq->rx_buf_len; + rxq_info->desc_ids = + VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M; + rxq_info->ring_len = bufq->nb_rx_desc; + + rxq_info->buffer_notif_stride = + IDPF_RX_BUF_STRIDE; + rxq_info->rx_buffer_low_watermark = 64; + } + } + } + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL2_OP_CONFIG_RX_QUEUES; + args.in_args = (uint8_t *)vc_rxqs; + args.in_args_size = size; + args.out_buffer = adapter->mbx_resp; + args.out_size = IDPF_DFLT_MBX_BUF_SIZE; + + err = idpf_execute_vc_cmd(adapter, &args); + rte_free(vc_rxqs); + if (err != 0) { + PMD_DRV_LOG(ERR, "Failed to execute command of VIRTCHNL2_OP_CONFIG_RX_QUEUES"); + break; + } + } + + return err; +} + +int +idpf_vc_config_rxq(struct idpf_vport *vport, uint16_t rxq_id) +{ + struct idpf_adapter *adapter = vport->adapter; + struct idpf_rx_queue **rxq = + (struct idpf_rx_queue **)vport->dev_data->rx_queues; + struct virtchnl2_config_rx_queues *vc_rxqs = NULL; + struct virtchnl2_rxq_info *rxq_info; + struct idpf_cmd_info args; + uint16_t num_qs; + int size, err, i; + + if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) + num_qs = IDPF_RXQ_PER_GRP; + else + num_qs = IDPF_RXQ_PER_GRP + IDPF_RX_BUFQ_PER_GRP; + + size = sizeof(*vc_rxqs) + (num_qs - 1) * + sizeof(struct virtchnl2_rxq_info); + vc_rxqs = rte_zmalloc("cfg_rxqs", size, 0); + if (vc_rxqs == NULL) { + PMD_DRV_LOG(ERR, "Failed to allocate virtchnl2_config_rx_queues"); + err = -ENOMEM; + return err; + } + vc_rxqs->vport_id = vport->vport_id; + vc_rxqs->num_qinfo = num_qs; + if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) { + rxq_info = &vc_rxqs->qinfo[0]; + rxq_info->dma_ring_addr = rxq[rxq_id]->rx_ring_phys_addr; + rxq_info->type = VIRTCHNL2_QUEUE_TYPE_RX; + rxq_info->queue_id = rxq[rxq_id]->queue_id; + rxq_info->model = VIRTCHNL2_QUEUE_MODEL_SINGLE; + rxq_info->data_buffer_size = rxq[rxq_id]->rx_buf_len; + rxq_info->max_pkt_size = vport->max_pkt_len; + + rxq_info->desc_ids = VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M; + rxq_info->qflags |= VIRTCHNL2_RX_DESC_SIZE_32BYTE; + + rxq_info->ring_len = rxq[rxq_id]->nb_rx_desc; + } else { + /* Rx queue */ + rxq_info = &vc_rxqs->qinfo[0]; + rxq_info->dma_ring_addr = rxq[rxq_id]->rx_ring_phys_addr; + rxq_info->type = VIRTCHNL2_QUEUE_TYPE_RX; + rxq_info->queue_id = rxq[rxq_id]->queue_id; + rxq_info->model = VIRTCHNL2_QUEUE_MODEL_SPLIT; + rxq_info->data_buffer_size = rxq[rxq_id]->rx_buf_len; + rxq_info->max_pkt_size = vport->max_pkt_len; + + rxq_info->desc_ids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M; + rxq_info->qflags |= VIRTCHNL2_RX_DESC_SIZE_32BYTE; + + rxq_info->ring_len = rxq[rxq_id]->nb_rx_desc; + rxq_info->rx_bufq1_id = rxq[rxq_id]->bufq1->queue_id; + rxq_info->rx_bufq2_id = rxq[rxq_id]->bufq2->queue_id; + rxq_info->rx_buffer_low_watermark = 64; + + /* Buffer queue */ + for (i = 1; i <= IDPF_RX_BUFQ_PER_GRP; i++) { + struct idpf_rx_queue *bufq = + i == 1 ? rxq[rxq_id]->bufq1 : rxq[rxq_id]->bufq2; + rxq_info = &vc_rxqs->qinfo[i]; + rxq_info->dma_ring_addr = bufq->rx_ring_phys_addr; + rxq_info->type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; + rxq_info->queue_id = bufq->queue_id; + rxq_info->model = VIRTCHNL2_QUEUE_MODEL_SPLIT; + rxq_info->data_buffer_size = bufq->rx_buf_len; + rxq_info->desc_ids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M; + rxq_info->ring_len = bufq->nb_rx_desc; + + rxq_info->buffer_notif_stride = IDPF_RX_BUF_STRIDE; + rxq_info->rx_buffer_low_watermark = 64; + } + } + + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL2_OP_CONFIG_RX_QUEUES; + args.in_args = (uint8_t *)vc_rxqs; + args.in_args_size = size; + args.out_buffer = adapter->mbx_resp; + args.out_size = IDPF_DFLT_MBX_BUF_SIZE; + + err = idpf_execute_vc_cmd(adapter, &args); + rte_free(vc_rxqs); + if (err != 0) + PMD_DRV_LOG(ERR, "Failed to execute command of VIRTCHNL2_OP_CONFIG_RX_QUEUES"); + + return err; +} + +int +idpf_vc_config_txqs(struct idpf_vport *vport) +{ + struct idpf_adapter *adapter = vport->adapter; + struct idpf_tx_queue **txq = + (struct idpf_tx_queue **)vport->dev_data->tx_queues; + struct virtchnl2_config_tx_queues *vc_txqs = NULL; + struct virtchnl2_txq_info *txq_info; + struct idpf_cmd_info args; + uint16_t total_qs, num_qs; + int size, i; + int err = 0; + int k = 0; + + total_qs = vport->num_tx_q + vport->num_tx_complq; + while (total_qs) { + if (total_qs > adapter->max_txq_per_msg) { + num_qs = adapter->max_txq_per_msg; + total_qs -= adapter->max_txq_per_msg; + } else { + num_qs = total_qs; + total_qs = 0; + } + size = sizeof(*vc_txqs) + (num_qs - 1) * + sizeof(struct virtchnl2_txq_info); + vc_txqs = rte_zmalloc("cfg_txqs", size, 0); + if (vc_txqs == NULL) { + PMD_DRV_LOG(ERR, "Failed to allocate virtchnl2_config_tx_queues"); + err = -ENOMEM; + break; + } + vc_txqs->vport_id = vport->vport_id; + vc_txqs->num_qinfo = num_qs; + if (vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) { + for (i = 0; i < num_qs; i++, k++) { + txq_info = &vc_txqs->qinfo[i]; + txq_info->dma_ring_addr = txq[k]->tx_ring_phys_addr; + txq_info->type = VIRTCHNL2_QUEUE_TYPE_TX; + txq_info->queue_id = txq[k]->queue_id; + txq_info->model = VIRTCHNL2_QUEUE_MODEL_SINGLE; + txq_info->sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; + txq_info->ring_len = txq[k]->nb_tx_desc; + } + } else { + for (i = 0; i < num_qs / 2; i++, k++) { + /* txq info */ + txq_info = &vc_txqs->qinfo[2 * i]; + txq_info->dma_ring_addr = txq[k]->tx_ring_phys_addr; + txq_info->type = VIRTCHNL2_QUEUE_TYPE_TX; + txq_info->queue_id = txq[k]->queue_id; + txq_info->model = VIRTCHNL2_QUEUE_MODEL_SPLIT; + txq_info->sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; + txq_info->ring_len = txq[k]->nb_tx_desc; + txq_info->tx_compl_queue_id = + txq[k]->complq->queue_id; + txq_info->relative_queue_id = txq_info->queue_id; + + /* tx completion queue info */ + txq_info = &vc_txqs->qinfo[2 * i + 1]; + txq_info->dma_ring_addr = + txq[k]->complq->tx_ring_phys_addr; + txq_info->type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + txq_info->queue_id = txq[k]->complq->queue_id; + txq_info->model = VIRTCHNL2_QUEUE_MODEL_SPLIT; + txq_info->sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; + txq_info->ring_len = txq[k]->complq->nb_tx_desc; + } + } + + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL2_OP_CONFIG_TX_QUEUES; + args.in_args = (uint8_t *)vc_txqs; + args.in_args_size = size; + args.out_buffer = adapter->mbx_resp; + args.out_size = IDPF_DFLT_MBX_BUF_SIZE; + + err = idpf_execute_vc_cmd(adapter, &args); + rte_free(vc_txqs); + if (err != 0) { + PMD_DRV_LOG(ERR, "Failed to execute command of VIRTCHNL2_OP_CONFIG_TX_QUEUES"); + break; + } + } + + return err; +} + +int +idpf_vc_config_txq(struct idpf_vport *vport, uint16_t txq_id) +{ + struct idpf_adapter *adapter = vport->adapter; + struct idpf_tx_queue **txq = + (struct idpf_tx_queue **)vport->dev_data->tx_queues; + struct virtchnl2_config_tx_queues *vc_txqs = NULL; + struct virtchnl2_txq_info *txq_info; + struct idpf_cmd_info args; + uint16_t num_qs; + int size, err; + + if (vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) + num_qs = IDPF_TXQ_PER_GRP; + else + num_qs = IDPF_TXQ_PER_GRP + IDPF_TX_COMPLQ_PER_GRP; + + size = sizeof(*vc_txqs) + (num_qs - 1) * + sizeof(struct virtchnl2_txq_info); + vc_txqs = rte_zmalloc("cfg_txqs", size, 0); + if (vc_txqs == NULL) { + PMD_DRV_LOG(ERR, "Failed to allocate virtchnl2_config_tx_queues"); + err = -ENOMEM; + return err; + } + vc_txqs->vport_id = vport->vport_id; + vc_txqs->num_qinfo = num_qs; + + if (vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) { + txq_info = &vc_txqs->qinfo[0]; + txq_info->dma_ring_addr = txq[txq_id]->tx_ring_phys_addr; + txq_info->type = VIRTCHNL2_QUEUE_TYPE_TX; + txq_info->queue_id = txq[txq_id]->queue_id; + txq_info->model = VIRTCHNL2_QUEUE_MODEL_SINGLE; + txq_info->sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; + txq_info->ring_len = txq[txq_id]->nb_tx_desc; + } else { + /* txq info */ + txq_info = &vc_txqs->qinfo[0]; + txq_info->dma_ring_addr = txq[txq_id]->tx_ring_phys_addr; + txq_info->type = VIRTCHNL2_QUEUE_TYPE_TX; + txq_info->queue_id = txq[txq_id]->queue_id; + txq_info->model = VIRTCHNL2_QUEUE_MODEL_SPLIT; + txq_info->sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; + txq_info->ring_len = txq[txq_id]->nb_tx_desc; + txq_info->tx_compl_queue_id = txq[txq_id]->complq->queue_id; + txq_info->relative_queue_id = txq_info->queue_id; + + /* tx completion queue info */ + txq_info = &vc_txqs->qinfo[1]; + txq_info->dma_ring_addr = txq[txq_id]->complq->tx_ring_phys_addr; + txq_info->type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + txq_info->queue_id = txq[txq_id]->complq->queue_id; + txq_info->model = VIRTCHNL2_QUEUE_MODEL_SPLIT; + txq_info->sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; + txq_info->ring_len = txq[txq_id]->complq->nb_tx_desc; + } + + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL2_OP_CONFIG_TX_QUEUES; + args.in_args = (uint8_t *)vc_txqs; + args.in_args_size = size; + args.out_buffer = adapter->mbx_resp; + args.out_size = IDPF_DFLT_MBX_BUF_SIZE; + + err = idpf_execute_vc_cmd(adapter, &args); + rte_free(vc_txqs); + if (err != 0) + PMD_DRV_LOG(ERR, "Failed to execute command of VIRTCHNL2_OP_CONFIG_TX_QUEUES"); + + return err; +} + +static int +idpf_vc_ena_dis_one_queue(struct idpf_vport *vport, uint16_t qid, + uint32_t type, bool on) +{ + struct idpf_adapter *adapter = vport->adapter; + struct virtchnl2_del_ena_dis_queues *queue_select; + struct virtchnl2_queue_chunk *queue_chunk; + struct idpf_cmd_info args; + int err, len; + + len = sizeof(struct virtchnl2_del_ena_dis_queues); + queue_select = rte_zmalloc("queue_select", len, 0); + if (queue_select == NULL) + return -ENOMEM; + + queue_chunk = queue_select->chunks.chunks; + queue_select->chunks.num_chunks = 1; + queue_select->vport_id = vport->vport_id; + + queue_chunk->type = type; + queue_chunk->start_queue_id = qid; + queue_chunk->num_queues = 1; + + args.ops = on ? VIRTCHNL2_OP_ENABLE_QUEUES : + VIRTCHNL2_OP_DISABLE_QUEUES; + args.in_args = (u8 *)queue_select; + args.in_args_size = len; + args.out_buffer = adapter->mbx_resp; + args.out_size = IDPF_DFLT_MBX_BUF_SIZE; + err = idpf_execute_vc_cmd(adapter, &args); + if (err != 0) + PMD_DRV_LOG(ERR, "Failed to execute command of VIRTCHNL2_OP_%s_QUEUES", + on ? "ENABLE" : "DISABLE"); + + rte_free(queue_select); + return err; +} + +int +idpf_switch_queue(struct idpf_vport *vport, uint16_t qid, + bool rx, bool on) +{ + uint32_t type; + int err, queue_id; + + /* switch txq/rxq */ + type = rx ? VIRTCHNL2_QUEUE_TYPE_RX : VIRTCHNL2_QUEUE_TYPE_TX; + + if (type == VIRTCHNL2_QUEUE_TYPE_RX) + queue_id = vport->chunks_info.rx_start_qid + qid; + else + queue_id = vport->chunks_info.tx_start_qid + qid; + err = idpf_vc_ena_dis_one_queue(vport, queue_id, type, on); + if (err != 0) + return err; + + /* switch tx completion queue */ + if (!rx && vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) { + type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + queue_id = vport->chunks_info.tx_compl_start_qid + qid; + err = idpf_vc_ena_dis_one_queue(vport, queue_id, type, on); + if (err != 0) + return err; + } + + /* switch rx buffer queue */ + if (rx && vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) { + type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; + queue_id = vport->chunks_info.rx_buf_start_qid + 2 * qid; + err = idpf_vc_ena_dis_one_queue(vport, queue_id, type, on); + if (err != 0) + return err; + queue_id++; + err = idpf_vc_ena_dis_one_queue(vport, queue_id, type, on); + if (err != 0) + return err; + } + + return err; +} + int idpf_vc_ena_dis_vport(struct idpf_vport *vport, bool enable) { From patchwork Mon Oct 24 13:01:23 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 118993 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id D91ABA034C; Mon, 24 Oct 2022 15:04:09 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id CEEF242B85; Mon, 24 Oct 2022 15:04:09 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id 109264069C for ; Mon, 24 Oct 2022 15:04:07 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616648; x=1698152648; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=+flkplBQ5gntaK8qjKFg6NStq403UHbnt6k+kB0Jnls=; b=Ff+dEZRSdV5qdl5zSqmEAf+URDuT1DQucmMmnOA/BxIETwjppu9WdBKv bZmqssF8wnXR6NTFgfzAzZvo/r3ygamh9KKmFhW+bSBORKaKgag12g43A 57oGBXwEhFKfNgzrbIgFhcW9H5q6lHXptDwqFK5uJ8EfIc4GOkVFCVd5J FeWd5B871slhlg4aKUuSbpgBWpIRf2hb0ekq80Y/Zk8022zt9SHTlcruH wlunffO1r9HvNZKF4YC/IP2gtW6iA6DHtQto9mdLnhh3tuYVgljmxvCXP VbIHPxwnExUDecq8QKdIbLzIjdRW8Gif3qMSaibHiuaTmao5x3mqFqdQF g==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305018683" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018683" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:16 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248245" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248245" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:14 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo , Xiaoyun Li Subject: [PATCH v10 03/14] net/idpf: add support for queue stop Date: Mon, 24 Oct 2022 21:01:23 +0800 Message-Id: <20221024130134.1046536-4-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add support for these device ops: - rx_queue_stop - tx_queue_stop Signed-off-by: Beilei Xing Signed-off-by: Xiaoyun Li Signed-off-by: Junfeng Guo --- doc/guides/nics/features/idpf.ini | 1 + drivers/net/idpf/idpf_ethdev.c | 14 ++- drivers/net/idpf/idpf_rxtx.c | 140 ++++++++++++++++++++++++++++++ drivers/net/idpf/idpf_rxtx.h | 11 +++ drivers/net/idpf/idpf_vchnl.c | 69 +++++++++++++++ 5 files changed, 232 insertions(+), 3 deletions(-) diff --git a/doc/guides/nics/features/idpf.ini b/doc/guides/nics/features/idpf.ini index 7a44b8b5e4..30e1c0831e 100644 --- a/doc/guides/nics/features/idpf.ini +++ b/doc/guides/nics/features/idpf.ini @@ -7,4 +7,5 @@ ; is selected. ; [Features] +Queue start/stop = Y Linux = Y diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index 954bd0bf4b..6040050dd9 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -62,7 +62,9 @@ static const struct eth_dev_ops idpf_eth_dev_ops = { .dev_stop = idpf_dev_stop, .dev_close = idpf_dev_close, .rx_queue_start = idpf_rx_queue_start, + .rx_queue_stop = idpf_rx_queue_stop, .tx_queue_start = idpf_tx_queue_start, + .tx_queue_stop = idpf_tx_queue_stop, .rx_queue_setup = idpf_rx_queue_setup, .tx_queue_setup = idpf_tx_queue_setup, .dev_infos_get = idpf_dev_info_get, @@ -300,22 +302,26 @@ idpf_dev_start(struct rte_eth_dev *dev) if (dev->data->mtu > vport->max_mtu) { PMD_DRV_LOG(ERR, "MTU should be less than %d", vport->max_mtu); - return -1; + goto err_mtu; } vport->max_pkt_len = dev->data->mtu + IDPF_ETH_OVERHEAD; if (idpf_start_queues(dev) != 0) { PMD_DRV_LOG(ERR, "Failed to start queues"); - return -1; + goto err_mtu; } if (idpf_vc_ena_dis_vport(vport, true) != 0) { PMD_DRV_LOG(ERR, "Failed to enable vport"); - return -1; + goto err_vport; } return 0; +err_vport: + idpf_stop_queues(dev); +err_mtu: + return -1; } static int @@ -325,6 +331,8 @@ idpf_dev_stop(struct rte_eth_dev *dev) idpf_vc_ena_dis_vport(vport, false); + idpf_stop_queues(dev); + return 0; } diff --git a/drivers/net/idpf/idpf_rxtx.c b/drivers/net/idpf/idpf_rxtx.c index b71c1ac6db..c14fd4324e 100644 --- a/drivers/net/idpf/idpf_rxtx.c +++ b/drivers/net/idpf/idpf_rxtx.c @@ -71,6 +71,55 @@ check_tx_thresh(uint16_t nb_desc, uint16_t tx_rs_thresh, return 0; } +static inline void +release_rxq_mbufs(struct idpf_rx_queue *rxq) +{ + uint16_t i; + + if (rxq->sw_ring == NULL) + return; + + for (i = 0; i < rxq->nb_rx_desc; i++) { + if (rxq->sw_ring[i] != NULL) { + rte_pktmbuf_free_seg(rxq->sw_ring[i]); + rxq->sw_ring[i] = NULL; + } + } +} + +static inline void +release_txq_mbufs(struct idpf_tx_queue *txq) +{ + uint16_t nb_desc, i; + + if (txq == NULL || txq->sw_ring == NULL) { + PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL"); + return; + } + + if (txq->sw_nb_desc != 0) { + /* For split queue model, descriptor ring */ + nb_desc = txq->sw_nb_desc; + } else { + /* For single queue model */ + nb_desc = txq->nb_tx_desc; + } + for (i = 0; i < nb_desc; i++) { + if (txq->sw_ring[i].mbuf != NULL) { + rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); + txq->sw_ring[i].mbuf = NULL; + } + } +} + +static const struct idpf_rxq_ops def_rxq_ops = { + .release_mbufs = release_rxq_mbufs, +}; + +static const struct idpf_txq_ops def_txq_ops = { + .release_mbufs = release_txq_mbufs, +}; + static inline void reset_split_rx_descq(struct idpf_rx_queue *rxq) { @@ -311,6 +360,7 @@ idpf_rx_split_bufq_setup(struct rte_eth_dev *dev, struct idpf_rx_queue *bufq, bufq->q_set = true; bufq->qrx_tail = hw->hw_addr + (vport->chunks_info.rx_buf_qtail_start + queue_idx * vport->chunks_info.rx_buf_qtail_spacing); + bufq->ops = &def_rxq_ops; /* TODO: allow bulk or vec */ @@ -552,6 +602,7 @@ idpf_rx_single_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, dev->data->rx_queues[queue_idx] = rxq; rxq->qrx_tail = hw->hw_addr + (vport->chunks_info.rx_qtail_start + queue_idx * vport->chunks_info.rx_qtail_spacing); + rxq->ops = &def_rxq_ops; return 0; } @@ -654,6 +705,7 @@ idpf_tx_split_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, reset_split_tx_descq(txq); txq->qtx_tail = hw->hw_addr + (vport->chunks_info.tx_qtail_start + queue_idx * vport->chunks_info.tx_qtail_spacing); + txq->ops = &def_txq_ops; /* Allocate the TX completion queue data structure. */ txq->complq = rte_zmalloc_socket("idpf splitq cq", @@ -778,6 +830,7 @@ idpf_tx_single_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, dev->data->tx_queues[queue_idx] = txq; txq->qtx_tail = hw->hw_addr + (vport->chunks_info.tx_qtail_start + queue_idx * vport->chunks_info.tx_qtail_spacing); + txq->ops = &def_txq_ops; return 0; } @@ -1010,3 +1063,90 @@ idpf_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) return err; } +int +idpf_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct idpf_vport *vport = dev->data->dev_private; + struct idpf_rx_queue *rxq; + int err; + + if (rx_queue_id >= dev->data->nb_rx_queues) + return -EINVAL; + + err = idpf_switch_queue(vport, rx_queue_id, true, false); + if (err != 0) { + PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off", + rx_queue_id); + return err; + } + + rxq = dev->data->rx_queues[rx_queue_id]; + if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) { + rxq->ops->release_mbufs(rxq); + reset_single_rx_queue(rxq); + } else { + rxq->bufq1->ops->release_mbufs(rxq->bufq1); + rxq->bufq2->ops->release_mbufs(rxq->bufq2); + reset_split_rx_queue(rxq); + } + dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; + + return 0; +} + +int +idpf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + struct idpf_vport *vport = dev->data->dev_private; + struct idpf_tx_queue *txq; + int err; + + if (tx_queue_id >= dev->data->nb_tx_queues) + return -EINVAL; + + err = idpf_switch_queue(vport, tx_queue_id, false, false); + if (err != 0) { + PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off", + tx_queue_id); + return err; + } + + txq = dev->data->tx_queues[tx_queue_id]; + txq->ops->release_mbufs(txq); + if (vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SINGLE) { + reset_single_tx_queue(txq); + } else { + reset_split_tx_descq(txq); + reset_split_tx_complq(txq->complq); + } + dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; + + return 0; +} + +void +idpf_stop_queues(struct rte_eth_dev *dev) +{ + struct idpf_rx_queue *rxq; + struct idpf_tx_queue *txq; + int i; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + if (rxq == NULL) + continue; + + if (idpf_rx_queue_stop(dev, i) != 0) + PMD_DRV_LOG(WARNING, "Fail to stop Rx queue %d", i); + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + if (txq == NULL) + continue; + + if (idpf_tx_queue_stop(dev, i) != 0) + PMD_DRV_LOG(WARNING, "Fail to stop Tx queue %d", i); + } +} + diff --git a/drivers/net/idpf/idpf_rxtx.h b/drivers/net/idpf/idpf_rxtx.h index d076903e3c..e1de436a4a 100644 --- a/drivers/net/idpf/idpf_rxtx.h +++ b/drivers/net/idpf/idpf_rxtx.h @@ -111,16 +111,27 @@ struct idpf_tx_queue { struct idpf_tx_queue *complq; }; +struct idpf_rxq_ops { + void (*release_mbufs)(struct idpf_rx_queue *rxq); +}; + +struct idpf_txq_ops { + void (*release_mbufs)(struct idpf_tx_queue *txq); +}; + int idpf_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp); int idpf_rx_queue_init(struct rte_eth_dev *dev, uint16_t rx_queue_id); int idpf_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id); +int idpf_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id); int idpf_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf); int idpf_tx_queue_init(struct rte_eth_dev *dev, uint16_t tx_queue_id); int idpf_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id); +int idpf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id); +void idpf_stop_queues(struct rte_eth_dev *dev); #endif /* _IDPF_RXTX_H_ */ diff --git a/drivers/net/idpf/idpf_vchnl.c b/drivers/net/idpf/idpf_vchnl.c index 5e5508e999..9138799989 100644 --- a/drivers/net/idpf/idpf_vchnl.c +++ b/drivers/net/idpf/idpf_vchnl.c @@ -891,6 +891,75 @@ idpf_switch_queue(struct idpf_vport *vport, uint16_t qid, return err; } +#define IDPF_RXTX_QUEUE_CHUNKS_NUM 2 +int +idpf_vc_ena_dis_queues(struct idpf_vport *vport, bool enable) +{ + struct idpf_adapter *adapter = vport->adapter; + struct virtchnl2_del_ena_dis_queues *queue_select; + struct virtchnl2_queue_chunk *queue_chunk; + uint32_t type; + struct idpf_cmd_info args; + uint16_t num_chunks; + int err, len; + + num_chunks = IDPF_RXTX_QUEUE_CHUNKS_NUM; + if (vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) + num_chunks++; + if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) + num_chunks++; + + len = sizeof(struct virtchnl2_del_ena_dis_queues) + + sizeof(struct virtchnl2_queue_chunk) * (num_chunks - 1); + queue_select = rte_zmalloc("queue_select", len, 0); + if (queue_select == NULL) + return -ENOMEM; + + queue_chunk = queue_select->chunks.chunks; + queue_select->chunks.num_chunks = num_chunks; + queue_select->vport_id = vport->vport_id; + + type = VIRTCHNL_QUEUE_TYPE_RX; + queue_chunk[type].type = type; + queue_chunk[type].start_queue_id = vport->chunks_info.rx_start_qid; + queue_chunk[type].num_queues = vport->num_rx_q; + + type = VIRTCHNL2_QUEUE_TYPE_TX; + queue_chunk[type].type = type; + queue_chunk[type].start_queue_id = vport->chunks_info.tx_start_qid; + queue_chunk[type].num_queues = vport->num_tx_q; + + if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) { + type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; + queue_chunk[type].type = type; + queue_chunk[type].start_queue_id = + vport->chunks_info.rx_buf_start_qid; + queue_chunk[type].num_queues = vport->num_rx_bufq; + } + + if (vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) { + type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + queue_chunk[type].type = type; + queue_chunk[type].start_queue_id = + vport->chunks_info.tx_compl_start_qid; + queue_chunk[type].num_queues = vport->num_tx_complq; + } + + args.ops = enable ? VIRTCHNL2_OP_ENABLE_QUEUES : + VIRTCHNL2_OP_DISABLE_QUEUES; + args.in_args = (u8 *)queue_select; + args.in_args_size = len; + args.out_buffer = adapter->mbx_resp; + args.out_size = IDPF_DFLT_MBX_BUF_SIZE; + err = idpf_execute_vc_cmd(adapter, &args); + if (err != 0) + PMD_DRV_LOG(ERR, "Failed to execute command of VIRTCHNL2_OP_%s_QUEUES", + enable ? "ENABLE" : "DISABLE"); + + rte_free(queue_select); + return err; +} + int idpf_vc_ena_dis_vport(struct idpf_vport *vport, bool enable) { From patchwork Mon Oct 24 13:01:24 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 118994 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id BC0FBA034C; Mon, 24 Oct 2022 15:04:14 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 04C8B42B96; Mon, 24 Oct 2022 15:04:14 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id 8AC8A42B99 for ; Mon, 24 Oct 2022 15:04:11 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616651; x=1698152651; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=pIqeztutWyesrXh+qyNoD84MKo4ZqxZH63mJYd7wXI4=; b=P/il+5fkMjkMQPjciC6Znt/f1n0cA1bdQI83XuLhkL0iq9fvJZmQ9DP6 SKYw+lPGJcLjzj+6BC2sABESPlVkIhA6d1MOPBn+33evzgGgRNTe8Obpo BCfbtabaoHkneSP3EtvwVOtOAShxyYokUnrzpqna4slQaNX6BxIcUyOq2 XGuFJksE63jteA4QIuHsqMAAVzsVBZqH4/WRXZTZ9IPa1KSQJHk5EIcRU tg6OBzmOKP5weCJFcYtKIJdpomGs1M6ecPrV877zXu/LrYWyehL7ncIJk IxUNL1b06mesfGLDmi0L3rihWUx2+d+sQkcoz0ZiAF4OZQz+ab5066w1+ w==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305018712" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018712" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:19 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248275" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248275" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:16 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo , Xiaoyun Li Subject: [PATCH v10 04/14] net/idpf: add queue release Date: Mon, 24 Oct 2022 21:01:24 +0800 Message-Id: <20221024130134.1046536-5-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add support for queue operations: - rx_queue_release - tx_queue_release Signed-off-by: Beilei Xing Signed-off-by: Xiaoyun Li Signed-off-by: Junfeng Guo --- drivers/net/idpf/idpf_ethdev.c | 2 + drivers/net/idpf/idpf_rxtx.c | 81 ++++++++++++++++++++++++++++++++++ drivers/net/idpf/idpf_rxtx.h | 3 ++ 3 files changed, 86 insertions(+) diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index 6040050dd9..4c058660b4 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -66,7 +66,9 @@ static const struct eth_dev_ops idpf_eth_dev_ops = { .tx_queue_start = idpf_tx_queue_start, .tx_queue_stop = idpf_tx_queue_stop, .rx_queue_setup = idpf_rx_queue_setup, + .rx_queue_release = idpf_dev_rx_queue_release, .tx_queue_setup = idpf_tx_queue_setup, + .tx_queue_release = idpf_dev_tx_queue_release, .dev_infos_get = idpf_dev_info_get, .link_update = idpf_dev_link_update, }; diff --git a/drivers/net/idpf/idpf_rxtx.c b/drivers/net/idpf/idpf_rxtx.c index c14fd4324e..298eaf0a1a 100644 --- a/drivers/net/idpf/idpf_rxtx.c +++ b/drivers/net/idpf/idpf_rxtx.c @@ -120,6 +120,51 @@ static const struct idpf_txq_ops def_txq_ops = { .release_mbufs = release_txq_mbufs, }; +static void +idpf_rx_queue_release(void *rxq) +{ + struct idpf_rx_queue *q = (struct idpf_rx_queue *)rxq; + + if (q == NULL) + return; + + /* Split queue */ + if (q->bufq1 != NULL && q->bufq2 != NULL) { + q->bufq1->ops->release_mbufs(q->bufq1); + rte_free(q->bufq1->sw_ring); + rte_memzone_free(q->bufq1->mz); + rte_free(q->bufq1); + q->bufq2->ops->release_mbufs(q->bufq2); + rte_free(q->bufq2->sw_ring); + rte_memzone_free(q->bufq2->mz); + rte_free(q->bufq2); + rte_memzone_free(q->mz); + rte_free(q); + return; + } + + /* Single queue */ + q->ops->release_mbufs(q); + rte_free(q->sw_ring); + rte_memzone_free(q->mz); + rte_free(q); +} + +static void +idpf_tx_queue_release(void *txq) +{ + struct idpf_tx_queue *q = (struct idpf_tx_queue *)txq; + + if (q == NULL) + return; + + rte_free(q->complq); + q->ops->release_mbufs(q); + rte_free(q->sw_ring); + rte_memzone_free(q->mz); + rte_free(q); +} + static inline void reset_split_rx_descq(struct idpf_rx_queue *rxq) { @@ -401,6 +446,12 @@ idpf_rx_split_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, if (check_rx_thresh(nb_desc, rx_free_thresh) != 0) return -EINVAL; + /* Free memory if needed */ + if (dev->data->rx_queues[queue_idx] != NULL) { + idpf_rx_queue_release(dev->data->rx_queues[queue_idx]); + dev->data->rx_queues[queue_idx] = NULL; + } + /* Setup Rx description queue */ rxq = rte_zmalloc_socket("idpf rxq", sizeof(struct idpf_rx_queue), @@ -541,6 +592,12 @@ idpf_rx_single_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, if (check_rx_thresh(nb_desc, rx_free_thresh) != 0) return -EINVAL; + /* Free memory if needed */ + if (dev->data->rx_queues[queue_idx] != NULL) { + idpf_rx_queue_release(dev->data->rx_queues[queue_idx]); + dev->data->rx_queues[queue_idx] = NULL; + } + /* Setup Rx description queue */ rxq = rte_zmalloc_socket("idpf rxq", sizeof(struct idpf_rx_queue), @@ -654,6 +711,12 @@ idpf_tx_split_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, if (check_tx_thresh(nb_desc, tx_rs_thresh, tx_free_thresh) != 0) return -EINVAL; + /* Free memory if needed. */ + if (dev->data->tx_queues[queue_idx] != NULL) { + idpf_tx_queue_release(dev->data->tx_queues[queue_idx]); + dev->data->tx_queues[queue_idx] = NULL; + } + /* Allocate the TX queue data structure. */ txq = rte_zmalloc_socket("idpf split txq", sizeof(struct idpf_tx_queue), @@ -776,6 +839,12 @@ idpf_tx_single_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, if (check_tx_thresh(nb_desc, tx_rs_thresh, tx_free_thresh) != 0) return -EINVAL; + /* Free memory if needed. */ + if (dev->data->tx_queues[queue_idx] != NULL) { + idpf_tx_queue_release(dev->data->tx_queues[queue_idx]); + dev->data->tx_queues[queue_idx] = NULL; + } + /* Allocate the TX queue data structure. */ txq = rte_zmalloc_socket("idpf txq", sizeof(struct idpf_tx_queue), @@ -1124,6 +1193,18 @@ idpf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) return 0; } +void +idpf_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) +{ + idpf_rx_queue_release(dev->data->rx_queues[qid]); +} + +void +idpf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) +{ + idpf_tx_queue_release(dev->data->tx_queues[qid]); +} + void idpf_stop_queues(struct rte_eth_dev *dev) { diff --git a/drivers/net/idpf/idpf_rxtx.h b/drivers/net/idpf/idpf_rxtx.h index e1de436a4a..9f767079b2 100644 --- a/drivers/net/idpf/idpf_rxtx.h +++ b/drivers/net/idpf/idpf_rxtx.h @@ -126,12 +126,15 @@ int idpf_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, int idpf_rx_queue_init(struct rte_eth_dev *dev, uint16_t rx_queue_id); int idpf_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id); int idpf_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id); +void idpf_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid); + int idpf_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf); int idpf_tx_queue_init(struct rte_eth_dev *dev, uint16_t tx_queue_id); int idpf_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id); int idpf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id); +void idpf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid); void idpf_stop_queues(struct rte_eth_dev *dev); #endif /* _IDPF_RXTX_H_ */ From patchwork Mon Oct 24 13:01:25 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 118995 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id E5D02A034C; Mon, 24 Oct 2022 15:04:20 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 1084F42B9B; Mon, 24 Oct 2022 15:04:18 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id 1D39342B9C for ; Mon, 24 Oct 2022 15:04:15 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616656; x=1698152656; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=nErDuAnoETk/YxshWjjVxV6nk6wPC3cBHjQYX4/7YXk=; b=hGcQUQqOCMH1vyJ4ji/4w1LS7VDZnn7HIDIhEDFytJXcxJwDmfrm/P/O ZYK+aj9P0FasDjSqk/L4fcCMgur/SLhvPHG5W1Ws3XFKywZyK5ad6qyDy 6q54j5S8XBQX8+lP1EdUceAluhE/M/ThHVAN6PeQZbzMS40aeJ5htr1dK gO5nCecTcH2emY6p6U8bSUwTzo+mhpoMYH9BTbGMKuAQHGaoIlk1QgZMB 48fEfOjw7CZjufvxtY0jOVqI12Ht9NFdIJ4H2x8al/x1r/xw6k1KIDWtJ hCkMPqx/MjAbOMwnbM7YV2mBZTX+vbgi2u75ZFigHuHu0sDAGlXEU4Z4O w==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305018745" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018745" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:21 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248296" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248296" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:19 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo , Wenjun Wu Subject: [PATCH v10 05/14] net/idpf: add support for packet type get Date: Mon, 24 Oct 2022 21:01:25 +0800 Message-Id: <20221024130134.1046536-6-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add dev ops dev_supported_ptypes_get. Signed-off-by: Wenjun Wu Signed-off-by: Junfeng Guo --- doc/guides/nics/features/idpf.ini | 1 + drivers/net/idpf/idpf_ethdev.c | 7 + drivers/net/idpf/idpf_ethdev.h | 6 + drivers/net/idpf/idpf_rxtx.c | 19 +++ drivers/net/idpf/idpf_rxtx.h | 7 + drivers/net/idpf/idpf_vchnl.c | 240 ++++++++++++++++++++++++++++++ 6 files changed, 280 insertions(+) diff --git a/doc/guides/nics/features/idpf.ini b/doc/guides/nics/features/idpf.ini index 30e1c0831e..a03068df85 100644 --- a/doc/guides/nics/features/idpf.ini +++ b/doc/guides/nics/features/idpf.ini @@ -8,4 +8,5 @@ ; [Features] Queue start/stop = Y +Packet type parsing = Y Linux = Y diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index 4c058660b4..80826569c8 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -57,6 +57,7 @@ idpf_dev_link_update(struct rte_eth_dev *dev, } static const struct eth_dev_ops idpf_eth_dev_ops = { + .dev_supported_ptypes_get = idpf_dev_supported_ptypes_get, .dev_configure = idpf_dev_configure, .dev_start = idpf_dev_start, .dev_stop = idpf_dev_stop, @@ -641,6 +642,12 @@ idpf_adapter_init(struct rte_pci_device *pci_dev, struct idpf_adapter *adapter) goto err_api; } + ret = idpf_get_pkt_type(adapter); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Failed to set ptype table"); + goto err_api; + } + adapter->caps = rte_zmalloc("idpf_caps", sizeof(struct virtchnl2_get_capabilities), 0); if (adapter->caps == NULL) { diff --git a/drivers/net/idpf/idpf_ethdev.h b/drivers/net/idpf/idpf_ethdev.h index 094f123be1..58f070c976 100644 --- a/drivers/net/idpf/idpf_ethdev.h +++ b/drivers/net/idpf/idpf_ethdev.h @@ -47,6 +47,8 @@ #define IDPF_NUM_MACADDR_MAX 64 +#define IDPF_MAX_PKT_TYPE 1024 + #define IDPF_VLAN_TAG_SIZE 4 #define IDPF_ETH_OVERHEAD \ (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + IDPF_VLAN_TAG_SIZE * 2) @@ -141,6 +143,8 @@ struct idpf_adapter { uint32_t max_rxq_per_msg; uint32_t max_txq_per_msg; + uint32_t ptype_tbl[IDPF_MAX_PKT_TYPE] __rte_cache_min_aligned; + bool stopped; }; @@ -202,6 +206,7 @@ int idpf_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete); void idpf_handle_virtchnl_msg(struct rte_eth_dev *dev); int idpf_vc_check_api_version(struct idpf_adapter *adapter); +int idpf_get_pkt_type(struct idpf_adapter *adapter); int idpf_vc_get_caps(struct idpf_adapter *adapter); int idpf_vc_create_vport(struct idpf_adapter *adapter); int idpf_vc_destroy_vport(struct idpf_vport *vport); @@ -213,6 +218,7 @@ int idpf_switch_queue(struct idpf_vport *vport, uint16_t qid, bool rx, bool on); int idpf_vc_ena_dis_queues(struct idpf_vport *vport, bool enable); int idpf_vc_ena_dis_vport(struct idpf_vport *vport, bool enable); +int idpf_vc_query_ptype_info(struct idpf_adapter *adapter); int idpf_read_one_msg(struct idpf_adapter *adapter, uint32_t ops, uint16_t buf_len, uint8_t *buf); diff --git a/drivers/net/idpf/idpf_rxtx.c b/drivers/net/idpf/idpf_rxtx.c index 298eaf0a1a..c3c4acb69f 100644 --- a/drivers/net/idpf/idpf_rxtx.c +++ b/drivers/net/idpf/idpf_rxtx.c @@ -8,6 +8,25 @@ #include "idpf_ethdev.h" #include "idpf_rxtx.h" +const uint32_t * +idpf_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) +{ + static const uint32_t ptypes[] = { + RTE_PTYPE_L2_ETHER, + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_L4_FRAG, + RTE_PTYPE_L4_NONFRAG, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_SCTP, + RTE_PTYPE_L4_ICMP, + RTE_PTYPE_UNKNOWN + }; + + return ptypes; +} + static inline int check_rx_thresh(uint16_t nb_desc, uint16_t thresh) { diff --git a/drivers/net/idpf/idpf_rxtx.h b/drivers/net/idpf/idpf_rxtx.h index 9f767079b2..11f1fde5d4 100644 --- a/drivers/net/idpf/idpf_rxtx.h +++ b/drivers/net/idpf/idpf_rxtx.h @@ -21,6 +21,10 @@ #define IDPF_DEFAULT_TX_RS_THRESH 32 #define IDPF_DEFAULT_TX_FREE_THRESH 32 +#define IDPF_GET_PTYPE_SIZE(p) \ + (sizeof(struct virtchnl2_ptype) + \ + (((p)->proto_id_count ? ((p)->proto_id_count - 1) : 0) * sizeof((p)->proto_id[0]))) + struct idpf_rx_queue { struct idpf_adapter *adapter; /* the adapter this queue belongs to */ struct rte_mempool *mp; /* mbuf pool to populate Rx ring */ @@ -137,4 +141,7 @@ int idpf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id); void idpf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid); void idpf_stop_queues(struct rte_eth_dev *dev); + +const uint32_t *idpf_dev_supported_ptypes_get(struct rte_eth_dev *dev); + #endif /* _IDPF_RXTX_H_ */ diff --git a/drivers/net/idpf/idpf_vchnl.c b/drivers/net/idpf/idpf_vchnl.c index 9138799989..86a55d2ff1 100644 --- a/drivers/net/idpf/idpf_vchnl.c +++ b/drivers/net/idpf/idpf_vchnl.c @@ -238,6 +238,11 @@ idpf_execute_vc_cmd(struct idpf_adapter *adapter, struct idpf_cmd_info *args) err = idpf_read_one_msg(adapter, args->ops, args->out_size, args->out_buffer); clear_cmd(adapter); break; + case VIRTCHNL2_OP_GET_PTYPE_INFO: + /* for multuple response message, + * do not handle the response here. + */ + break; default: /* For other virtchnl ops in running time, * wait for the cmd done flag. @@ -286,6 +291,215 @@ idpf_vc_check_api_version(struct idpf_adapter *adapter) return err; } +int __rte_cold +idpf_get_pkt_type(struct idpf_adapter *adapter) +{ + struct virtchnl2_get_ptype_info *ptype_info; + uint16_t ptype_recvd = 0, ptype_offset, i, j; + int ret; + + ret = idpf_vc_query_ptype_info(adapter); + if (ret != 0) { + PMD_DRV_LOG(ERR, "Fail to query packet type information"); + return ret; + } + + ptype_info = rte_zmalloc("ptype_info", IDPF_DFLT_MBX_BUF_SIZE, 0); + if (ptype_info == NULL) + return -ENOMEM; + + while (ptype_recvd < IDPF_MAX_PKT_TYPE) { + ret = idpf_read_one_msg(adapter, VIRTCHNL2_OP_GET_PTYPE_INFO, + IDPF_DFLT_MBX_BUF_SIZE, (u8 *)ptype_info); + if (ret != 0) { + PMD_DRV_LOG(ERR, "Fail to get packet type information"); + goto free_ptype_info; + } + + ptype_recvd += ptype_info->num_ptypes; + ptype_offset = sizeof(struct virtchnl2_get_ptype_info) - + sizeof(struct virtchnl2_ptype); + + for (i = 0; i < rte_cpu_to_le_16(ptype_info->num_ptypes); i++) { + bool is_inner = false, is_ip = false; + struct virtchnl2_ptype *ptype; + uint32_t proto_hdr = 0; + + ptype = (struct virtchnl2_ptype *) + ((u8 *)ptype_info + ptype_offset); + ptype_offset += IDPF_GET_PTYPE_SIZE(ptype); + if (ptype_offset > IDPF_DFLT_MBX_BUF_SIZE) { + ret = -EINVAL; + goto free_ptype_info; + } + + if (rte_cpu_to_le_16(ptype->ptype_id_10) == 0xFFFF) + goto free_ptype_info; + + for (j = 0; j < ptype->proto_id_count; j++) { + switch (rte_cpu_to_le_16(ptype->proto_id[j])) { + case VIRTCHNL2_PROTO_HDR_GRE: + case VIRTCHNL2_PROTO_HDR_VXLAN: + proto_hdr &= ~RTE_PTYPE_L4_MASK; + proto_hdr |= RTE_PTYPE_TUNNEL_GRENAT; + is_inner = true; + break; + case VIRTCHNL2_PROTO_HDR_MAC: + if (is_inner) { + proto_hdr &= ~RTE_PTYPE_INNER_L2_MASK; + proto_hdr |= RTE_PTYPE_INNER_L2_ETHER; + } else { + proto_hdr &= ~RTE_PTYPE_L2_MASK; + proto_hdr |= RTE_PTYPE_L2_ETHER; + } + break; + case VIRTCHNL2_PROTO_HDR_VLAN: + if (is_inner) { + proto_hdr &= ~RTE_PTYPE_INNER_L2_MASK; + proto_hdr |= RTE_PTYPE_INNER_L2_ETHER_VLAN; + } + break; + case VIRTCHNL2_PROTO_HDR_PTP: + proto_hdr &= ~RTE_PTYPE_L2_MASK; + proto_hdr |= RTE_PTYPE_L2_ETHER_TIMESYNC; + break; + case VIRTCHNL2_PROTO_HDR_LLDP: + proto_hdr &= ~RTE_PTYPE_L2_MASK; + proto_hdr |= RTE_PTYPE_L2_ETHER_LLDP; + break; + case VIRTCHNL2_PROTO_HDR_ARP: + proto_hdr &= ~RTE_PTYPE_L2_MASK; + proto_hdr |= RTE_PTYPE_L2_ETHER_ARP; + break; + case VIRTCHNL2_PROTO_HDR_PPPOE: + proto_hdr &= ~RTE_PTYPE_L2_MASK; + proto_hdr |= RTE_PTYPE_L2_ETHER_PPPOE; + break; + case VIRTCHNL2_PROTO_HDR_IPV4: + if (!is_ip) { + proto_hdr |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; + is_ip = true; + } else { + proto_hdr |= RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP; + is_inner = true; + } + break; + case VIRTCHNL2_PROTO_HDR_IPV6: + if (!is_ip) { + proto_hdr |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; + is_ip = true; + } else { + proto_hdr |= RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_TUNNEL_IP; + is_inner = true; + } + break; + case VIRTCHNL2_PROTO_HDR_IPV4_FRAG: + case VIRTCHNL2_PROTO_HDR_IPV6_FRAG: + if (is_inner) + proto_hdr |= RTE_PTYPE_INNER_L4_FRAG; + else + proto_hdr |= RTE_PTYPE_L4_FRAG; + break; + case VIRTCHNL2_PROTO_HDR_UDP: + if (is_inner) + proto_hdr |= RTE_PTYPE_INNER_L4_UDP; + else + proto_hdr |= RTE_PTYPE_L4_UDP; + break; + case VIRTCHNL2_PROTO_HDR_TCP: + if (is_inner) + proto_hdr |= RTE_PTYPE_INNER_L4_TCP; + else + proto_hdr |= RTE_PTYPE_L4_TCP; + break; + case VIRTCHNL2_PROTO_HDR_SCTP: + if (is_inner) + proto_hdr |= RTE_PTYPE_INNER_L4_SCTP; + else + proto_hdr |= RTE_PTYPE_L4_SCTP; + break; + case VIRTCHNL2_PROTO_HDR_ICMP: + if (is_inner) + proto_hdr |= RTE_PTYPE_INNER_L4_ICMP; + else + proto_hdr |= RTE_PTYPE_L4_ICMP; + break; + case VIRTCHNL2_PROTO_HDR_ICMPV6: + if (is_inner) + proto_hdr |= RTE_PTYPE_INNER_L4_ICMP; + else + proto_hdr |= RTE_PTYPE_L4_ICMP; + break; + case VIRTCHNL2_PROTO_HDR_L2TPV2: + case VIRTCHNL2_PROTO_HDR_L2TPV2_CONTROL: + case VIRTCHNL2_PROTO_HDR_L2TPV3: + is_inner = true; + proto_hdr |= RTE_PTYPE_TUNNEL_L2TP; + break; + case VIRTCHNL2_PROTO_HDR_NVGRE: + is_inner = true; + proto_hdr |= RTE_PTYPE_TUNNEL_NVGRE; + break; + case VIRTCHNL2_PROTO_HDR_GTPC_TEID: + is_inner = true; + proto_hdr |= RTE_PTYPE_TUNNEL_GTPC; + break; + case VIRTCHNL2_PROTO_HDR_GTPU: + case VIRTCHNL2_PROTO_HDR_GTPU_UL: + case VIRTCHNL2_PROTO_HDR_GTPU_DL: + is_inner = true; + proto_hdr |= RTE_PTYPE_TUNNEL_GTPU; + break; + case VIRTCHNL2_PROTO_HDR_PAY: + case VIRTCHNL2_PROTO_HDR_IPV6_EH: + case VIRTCHNL2_PROTO_HDR_PRE_MAC: + case VIRTCHNL2_PROTO_HDR_POST_MAC: + case VIRTCHNL2_PROTO_HDR_ETHERTYPE: + case VIRTCHNL2_PROTO_HDR_SVLAN: + case VIRTCHNL2_PROTO_HDR_CVLAN: + case VIRTCHNL2_PROTO_HDR_MPLS: + case VIRTCHNL2_PROTO_HDR_MMPLS: + case VIRTCHNL2_PROTO_HDR_CTRL: + case VIRTCHNL2_PROTO_HDR_ECP: + case VIRTCHNL2_PROTO_HDR_EAPOL: + case VIRTCHNL2_PROTO_HDR_PPPOD: + case VIRTCHNL2_PROTO_HDR_IGMP: + case VIRTCHNL2_PROTO_HDR_AH: + case VIRTCHNL2_PROTO_HDR_ESP: + case VIRTCHNL2_PROTO_HDR_IKE: + case VIRTCHNL2_PROTO_HDR_NATT_KEEP: + case VIRTCHNL2_PROTO_HDR_GTP: + case VIRTCHNL2_PROTO_HDR_GTP_EH: + case VIRTCHNL2_PROTO_HDR_GTPCV2: + case VIRTCHNL2_PROTO_HDR_ECPRI: + case VIRTCHNL2_PROTO_HDR_VRRP: + case VIRTCHNL2_PROTO_HDR_OSPF: + case VIRTCHNL2_PROTO_HDR_TUN: + case VIRTCHNL2_PROTO_HDR_VXLAN_GPE: + case VIRTCHNL2_PROTO_HDR_GENEVE: + case VIRTCHNL2_PROTO_HDR_NSH: + case VIRTCHNL2_PROTO_HDR_QUIC: + case VIRTCHNL2_PROTO_HDR_PFCP: + case VIRTCHNL2_PROTO_HDR_PFCP_NODE: + case VIRTCHNL2_PROTO_HDR_PFCP_SESSION: + case VIRTCHNL2_PROTO_HDR_RTP: + case VIRTCHNL2_PROTO_HDR_NO_PROTO: + default: + continue; + } + adapter->ptype_tbl[ptype->ptype_id_10] = proto_hdr; + } + } + } + +free_ptype_info: + rte_free(ptype_info); + clear_cmd(adapter); + return ret; +} + int idpf_vc_get_caps(struct idpf_adapter *adapter) { @@ -984,3 +1198,29 @@ idpf_vc_ena_dis_vport(struct idpf_vport *vport, bool enable) return err; } + +int +idpf_vc_query_ptype_info(struct idpf_adapter *adapter) +{ + struct virtchnl2_get_ptype_info *ptype_info; + struct idpf_cmd_info args; + int len, err; + + len = sizeof(struct virtchnl2_get_ptype_info); + ptype_info = rte_zmalloc("ptype_info", len, 0); + if (ptype_info == NULL) + return -ENOMEM; + + ptype_info->start_ptype_id = 0; + ptype_info->num_ptypes = IDPF_MAX_PKT_TYPE; + args.ops = VIRTCHNL2_OP_GET_PTYPE_INFO; + args.in_args = (u8 *)ptype_info; + args.in_args_size = len; + + err = idpf_execute_vc_cmd(adapter, &args); + if (err != 0) + PMD_DRV_LOG(ERR, "Failed to execute command of VIRTCHNL2_OP_GET_PTYPE_INFO"); + + rte_free(ptype_info); + return err; +} From patchwork Mon Oct 24 13:01:26 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 118996 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 46E26A034C; Mon, 24 Oct 2022 15:04:29 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 333AD42B9C; Mon, 24 Oct 2022 15:04:21 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id 7C43A42B9C for ; Mon, 24 Oct 2022 15:04:18 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616658; x=1698152658; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=3G4Zjh0wjq8MW6Fi+hX2Mos/kj+JH/PS/CGq2od2HQI=; b=N2u9TA9aQ9CYJX3XbMmYamuQ10zhSeCpROeKM/U2yWWuVPWgxhi/ipa4 A0m6HMrjYQcYL13Zbum1uWnFOciaZnVhvB8n9EhOswBK84naWZdt5n4EH N21OZDBLZ8k4TX3QX2G5lM5bUHrxv/yQrFAFLefyZ0XNT3Ws0EPuonrln 3krxJ3g2xM8rU0ZWUreTfDlL4MtLVYMGudSeX07SjgE2Gz8AAFR1OtB0i ngDmrap2DaasQnC9q2IhNoOG/niqONpY2kAomMwigPTephj2vR+UXR892 nA+WfrGF4ER7xbvk+8JK53o/RzKoGhAvIlOlQVaE7IiHTnUUaEStveQOf w==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305018770" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018770" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:23 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248313" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248313" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:21 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo Subject: [PATCH v10 06/14] net/idpf: add support for MTU configuration Date: Mon, 24 Oct 2022 21:01:26 +0800 Message-Id: <20221024130134.1046536-7-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add dev ops mtu_set. Signed-off-by: Beilei Xing Signed-off-by: Junfeng Guo --- doc/guides/nics/features/idpf.ini | 1 + drivers/net/idpf/idpf_ethdev.c | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/doc/guides/nics/features/idpf.ini b/doc/guides/nics/features/idpf.ini index a03068df85..d4eb9b374c 100644 --- a/doc/guides/nics/features/idpf.ini +++ b/doc/guides/nics/features/idpf.ini @@ -8,5 +8,6 @@ ; [Features] Queue start/stop = Y +MTU update = Y Packet type parsing = Y Linux = Y diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index 80826569c8..1654af0053 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -34,6 +34,7 @@ static int idpf_dev_stop(struct rte_eth_dev *dev); static int idpf_dev_close(struct rte_eth_dev *dev); static int idpf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); +static int idpf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); static void idpf_adapter_rel(struct idpf_adapter *adapter); int @@ -72,6 +73,7 @@ static const struct eth_dev_ops idpf_eth_dev_ops = { .tx_queue_release = idpf_dev_tx_queue_release, .dev_infos_get = idpf_dev_info_get, .link_update = idpf_dev_link_update, + .mtu_set = idpf_dev_mtu_set, }; static int @@ -122,6 +124,18 @@ idpf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) return 0; } +static int +idpf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu __rte_unused) +{ + /* mtu setting is forbidden if port is start */ + if (dev->data->dev_started) { + PMD_DRV_LOG(ERR, "port must be stopped before configuration"); + return -EBUSY; + } + + return 0; +} + static int idpf_init_vport_req_info(struct rte_eth_dev *dev) { @@ -209,6 +223,7 @@ idpf_init_vport(struct rte_eth_dev *dev) vport->num_tx_complq = vport_info->num_tx_complq; vport->num_rx_q = vport_info->num_rx_q; vport->num_rx_bufq = vport_info->num_rx_bufq; + vport->max_mtu = vport_info->max_mtu; rte_memcpy(vport->default_mac_addr, vport_info->default_mac_addr, ETH_ALEN); vport->sw_idx = idx; From patchwork Mon Oct 24 13:01:27 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 118997 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id EB595A034C; Mon, 24 Oct 2022 15:04:39 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 1847342BB3; Mon, 24 Oct 2022 15:04:27 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id 8F73A42BA3 for ; Mon, 24 Oct 2022 15:04:22 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616662; x=1698152662; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=h/WtSTaxvHGF7foysl+La+sQyeOdS7iWvk9pao/W+Ps=; b=EoGpPdXf843WME6G7ED8yR8gTnyHv2qkf/kEuzflDe5rzDU4Y33VUwmn akmDlTk3KjGtDRx4yxj39NYuQwMws67NpC/PTwzorcPawy10sDaoG+aVl VwYX2ACPL0bkteFhAfknCrUPQgqnnTxGBSrzz1z3HbdneCwHV64DRvMQ0 +kNu5nDn8XhWe+J8znVLYz84o6hyXxZ9BmWbIM5w7iUMLq/5mvAmHzDZt +93ouXF130O0rZvVjtU/ZKw9I560XKSq5mR+Tl7fwSMNIH3e7htEA0JLr 6pTWzFotV4IB8zFJeVZfKSw1ufSyr0KV6Na0Ho5du+W1xwz/LeXVps8tU g==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305018806" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018806" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:25 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248345" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248345" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:23 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo , Xiaoyun Li Subject: [PATCH v10 07/14] net/idpf: add support for basic Rx datapath Date: Mon, 24 Oct 2022 21:01:27 +0800 Message-Id: <20221024130134.1046536-8-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add basic Rx support in split queue mode and single queue mode. Signed-off-by: Beilei Xing Signed-off-by: Xiaoyun Li Signed-off-by: Junfeng Guo --- drivers/net/idpf/idpf_ethdev.c | 2 + drivers/net/idpf/idpf_rxtx.c | 286 +++++++++++++++++++++++++++++++++ drivers/net/idpf/idpf_rxtx.h | 5 + 3 files changed, 293 insertions(+) diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index 1654af0053..f03df0a3ef 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -330,6 +330,8 @@ idpf_dev_start(struct rte_eth_dev *dev) goto err_mtu; } + idpf_set_rx_function(dev); + if (idpf_vc_ena_dis_vport(vport, true) != 0) { PMD_DRV_LOG(ERR, "Failed to enable vport"); goto err_vport; diff --git a/drivers/net/idpf/idpf_rxtx.c b/drivers/net/idpf/idpf_rxtx.c index c3c4acb69f..4037cf5a74 100644 --- a/drivers/net/idpf/idpf_rxtx.c +++ b/drivers/net/idpf/idpf_rxtx.c @@ -1250,3 +1250,289 @@ idpf_stop_queues(struct rte_eth_dev *dev) } } + +static void +idpf_split_rx_bufq_refill(struct idpf_rx_queue *rx_bufq) +{ + volatile struct virtchnl2_splitq_rx_buf_desc *rx_buf_ring; + volatile struct virtchnl2_splitq_rx_buf_desc *rx_buf_desc; + uint16_t nb_refill = rx_bufq->rx_free_thresh; + uint16_t nb_desc = rx_bufq->nb_rx_desc; + uint16_t next_avail = rx_bufq->rx_tail; + struct rte_mbuf *nmb[rx_bufq->rx_free_thresh]; + struct rte_eth_dev *dev; + uint64_t dma_addr; + uint16_t delta; + int i; + + if (rx_bufq->nb_rx_hold < rx_bufq->rx_free_thresh) + return; + + rx_buf_ring = + (volatile struct virtchnl2_splitq_rx_buf_desc *)rx_bufq->rx_ring; + delta = nb_desc - next_avail; + if (unlikely(delta < nb_refill)) { + if (likely(rte_pktmbuf_alloc_bulk(rx_bufq->mp, nmb, delta) == 0)) { + for (i = 0; i < delta; i++) { + rx_buf_desc = &rx_buf_ring[next_avail + i]; + rx_bufq->sw_ring[next_avail + i] = nmb[i]; + dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb[i])); + rx_buf_desc->hdr_addr = 0; + rx_buf_desc->pkt_addr = dma_addr; + } + nb_refill -= delta; + next_avail = 0; + rx_bufq->nb_rx_hold -= delta; + } else { + dev = &rte_eth_devices[rx_bufq->port_id]; + dev->data->rx_mbuf_alloc_failed += nb_desc - next_avail; + PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u queue_id=%u", + rx_bufq->port_id, rx_bufq->queue_id); + return; + } + } + + if (nb_desc - next_avail >= nb_refill) { + if (likely(rte_pktmbuf_alloc_bulk(rx_bufq->mp, nmb, nb_refill) == 0)) { + for (i = 0; i < nb_refill; i++) { + rx_buf_desc = &rx_buf_ring[next_avail + i]; + rx_bufq->sw_ring[next_avail + i] = nmb[i]; + dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb[i])); + rx_buf_desc->hdr_addr = 0; + rx_buf_desc->pkt_addr = dma_addr; + } + next_avail += nb_refill; + rx_bufq->nb_rx_hold -= nb_refill; + } else { + dev = &rte_eth_devices[rx_bufq->port_id]; + dev->data->rx_mbuf_alloc_failed += nb_desc - next_avail; + PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u queue_id=%u", + rx_bufq->port_id, rx_bufq->queue_id); + } + } + + IDPF_PCI_REG_WRITE(rx_bufq->qrx_tail, next_avail); + + rx_bufq->rx_tail = next_avail; +} + +uint16_t +idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + volatile struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc_ring; + volatile struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc; + uint16_t pktlen_gen_bufq_id; + struct idpf_rx_queue *rxq; + const uint32_t *ptype_tbl; + struct rte_mbuf *rxm; + uint16_t rx_id_bufq1; + uint16_t rx_id_bufq2; + uint16_t pkt_len; + uint16_t bufq_id; + uint16_t gen_id; + uint16_t rx_id; + uint16_t nb_rx; + + nb_rx = 0; + rxq = (struct idpf_rx_queue *)rx_queue; + + if (unlikely(rxq == NULL) || unlikely(!rxq->q_started)) + return nb_rx; + + rx_id = rxq->rx_tail; + rx_id_bufq1 = rxq->bufq1->rx_next_avail; + rx_id_bufq2 = rxq->bufq2->rx_next_avail; + rx_desc_ring = + (volatile struct virtchnl2_rx_flex_desc_adv_nic_3 *)rxq->rx_ring; + ptype_tbl = rxq->adapter->ptype_tbl; + + while (nb_rx < nb_pkts) { + rx_desc = &rx_desc_ring[rx_id]; + + pktlen_gen_bufq_id = + rte_le_to_cpu_16(rx_desc->pktlen_gen_bufq_id); + gen_id = (pktlen_gen_bufq_id & + VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_M) >> + VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_S; + if (gen_id != rxq->expected_gen_id) + break; + + pkt_len = (pktlen_gen_bufq_id & + VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_PBUF_M) >> + VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_PBUF_S; + if (pkt_len == 0) + PMD_RX_LOG(ERR, "Packet length is 0"); + + rx_id++; + if (unlikely(rx_id == rxq->nb_rx_desc)) { + rx_id = 0; + rxq->expected_gen_id ^= 1; + } + + bufq_id = (pktlen_gen_bufq_id & + VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_M) >> + VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_S; + if (bufq_id == 0) { + rxm = rxq->bufq1->sw_ring[rx_id_bufq1]; + rx_id_bufq1++; + if (unlikely(rx_id_bufq1 == rxq->bufq1->nb_rx_desc)) + rx_id_bufq1 = 0; + rxq->bufq1->nb_rx_hold++; + } else { + rxm = rxq->bufq2->sw_ring[rx_id_bufq2]; + rx_id_bufq2++; + if (unlikely(rx_id_bufq2 == rxq->bufq2->nb_rx_desc)) + rx_id_bufq2 = 0; + rxq->bufq2->nb_rx_hold++; + } + + rxm->pkt_len = pkt_len; + rxm->data_len = pkt_len; + rxm->data_off = RTE_PKTMBUF_HEADROOM; + rxm->next = NULL; + rxm->nb_segs = 1; + rxm->port = rxq->port_id; + rxm->packet_type = + ptype_tbl[(rte_le_to_cpu_16(rx_desc->ptype_err_fflags0) & + VIRTCHNL2_RX_FLEX_DESC_ADV_PTYPE_M) >> + VIRTCHNL2_RX_FLEX_DESC_ADV_PTYPE_S]; + + rx_pkts[nb_rx++] = rxm; + } + + if (nb_rx > 0) { + rxq->rx_tail = rx_id; + if (rx_id_bufq1 != rxq->bufq1->rx_next_avail) + rxq->bufq1->rx_next_avail = rx_id_bufq1; + if (rx_id_bufq2 != rxq->bufq2->rx_next_avail) + rxq->bufq2->rx_next_avail = rx_id_bufq2; + + idpf_split_rx_bufq_refill(rxq->bufq1); + idpf_split_rx_bufq_refill(rxq->bufq2); + } + + return nb_rx; +} + +static inline void +idpf_update_rx_tail(struct idpf_rx_queue *rxq, uint16_t nb_hold, + uint16_t rx_id) +{ + nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); + + if (nb_hold > rxq->rx_free_thresh) { + PMD_RX_LOG(DEBUG, + "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u", + rxq->port_id, rxq->queue_id, rx_id, nb_hold); + rx_id = (uint16_t)((rx_id == 0) ? + (rxq->nb_rx_desc - 1) : (rx_id - 1)); + IDPF_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + nb_hold = 0; + } + rxq->nb_rx_hold = nb_hold; +} + +uint16_t +idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + volatile union virtchnl2_rx_desc *rx_ring; + volatile union virtchnl2_rx_desc *rxdp; + union virtchnl2_rx_desc rxd; + struct idpf_rx_queue *rxq; + const uint32_t *ptype_tbl; + uint16_t rx_id, nb_hold; + struct rte_eth_dev *dev; + uint16_t rx_packet_len; + struct rte_mbuf *rxm; + struct rte_mbuf *nmb; + uint16_t rx_status0; + uint64_t dma_addr; + uint16_t nb_rx; + + nb_rx = 0; + nb_hold = 0; + rxq = rx_queue; + + if (unlikely(rxq == NULL) || unlikely(!rxq->q_started)) + return nb_rx; + + rx_id = rxq->rx_tail; + rx_ring = rxq->rx_ring; + ptype_tbl = rxq->adapter->ptype_tbl; + + while (nb_rx < nb_pkts) { + rxdp = &rx_ring[rx_id]; + rx_status0 = rte_le_to_cpu_16(rxdp->flex_nic_wb.status_error0); + + /* Check the DD bit first */ + if ((rx_status0 & (1 << VIRTCHNL2_RX_FLEX_DESC_STATUS0_DD_S)) == 0) + break; + + nmb = rte_mbuf_raw_alloc(rxq->mp); + if (unlikely(nmb == NULL)) { + dev = &rte_eth_devices[rxq->port_id]; + dev->data->rx_mbuf_alloc_failed++; + PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " + "queue_id=%u", rxq->port_id, rxq->queue_id); + break; + } + rxd = *rxdp; /* copy descriptor in ring to temp variable*/ + + nb_hold++; + rxm = rxq->sw_ring[rx_id]; + rxq->sw_ring[rx_id] = nmb; + rx_id++; + if (unlikely(rx_id == rxq->nb_rx_desc)) + rx_id = 0; + + /* Prefetch next mbuf */ + rte_prefetch0(rxq->sw_ring[rx_id]); + + /* When next RX descriptor is on a cache line boundary, + * prefetch the next 4 RX descriptors and next 8 pointers + * to mbufs. + */ + if ((rx_id & 0x3) == 0) { + rte_prefetch0(&rx_ring[rx_id]); + rte_prefetch0(rxq->sw_ring[rx_id]); + } + dma_addr = + rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); + rxdp->read.hdr_addr = 0; + rxdp->read.pkt_addr = dma_addr; + + rx_packet_len = (rte_cpu_to_le_16(rxd.flex_nic_wb.pkt_len) & + VIRTCHNL2_RX_FLEX_DESC_PKT_LEN_M); + + rxm->data_off = RTE_PKTMBUF_HEADROOM; + rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM)); + rxm->nb_segs = 1; + rxm->next = NULL; + rxm->pkt_len = rx_packet_len; + rxm->data_len = rx_packet_len; + rxm->port = rxq->port_id; + rxm->packet_type = + ptype_tbl[(uint8_t)(rte_cpu_to_le_16(rxd.flex_nic_wb.ptype_flex_flags0) & + VIRTCHNL2_RX_FLEX_DESC_PTYPE_M)]; + + rx_pkts[nb_rx++] = rxm; + } + rxq->rx_tail = rx_id; + + idpf_update_rx_tail(rxq, nb_hold, rx_id); + + return nb_rx; +} + +void +idpf_set_rx_function(struct rte_eth_dev *dev) +{ + struct idpf_vport *vport = dev->data->dev_private; + + if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) + dev->rx_pkt_burst = idpf_splitq_recv_pkts; + else + dev->rx_pkt_burst = idpf_singleq_recv_pkts; +} diff --git a/drivers/net/idpf/idpf_rxtx.h b/drivers/net/idpf/idpf_rxtx.h index 11f1fde5d4..056596ce94 100644 --- a/drivers/net/idpf/idpf_rxtx.h +++ b/drivers/net/idpf/idpf_rxtx.h @@ -140,8 +140,13 @@ int idpf_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id); int idpf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id); void idpf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid); +uint16_t idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); void idpf_stop_queues(struct rte_eth_dev *dev); +void idpf_set_rx_function(struct rte_eth_dev *dev); const uint32_t *idpf_dev_supported_ptypes_get(struct rte_eth_dev *dev); #endif /* _IDPF_RXTX_H_ */ From patchwork Mon Oct 24 13:01:28 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 118998 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id DE48BA034C; Mon, 24 Oct 2022 15:04:47 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 2270A42BB7; Mon, 24 Oct 2022 15:04:28 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id C277E42BAF for ; Mon, 24 Oct 2022 15:04:24 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616665; x=1698152665; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=yN43E7KYlzBDsR8cBxbA8pIPUy91p9rPAtnVZps04rI=; b=R3MmP2E02ju/t89t1QeE0nNTNOi/xWIwtxggv0MPoTsrmcENVALsrrOE Hxvs9xrULmY5OqJfNzoUocgQhByDFOyUSiuaDFrXKyDniJXuEU9jVOVwZ OQc8URglprlJcXhxcuOeHwDXZJthhJYUguT9BkqRDhAe4zoHuZq5pivzs GqF+O2WcRuMA+h+WQtTi1Pf+oGl+2Hf4kI/wHz8Ptuc18+hGwgiuRsVHj eB13IoiWwgN8jso8h3KGxAJB+lPtqSIc2aIXB5VhtSSjCifU6UWjhI3fx oHiwz7HWWb1R5m4S4ppyRuXcsiBtQScRhcEhNM1okvm0OJW9b01tp0SNn A==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305018826" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018826" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:27 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248365" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248365" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:25 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo , Xiaoyun Li Subject: [PATCH v10 08/14] net/idpf: add support for basic Tx datapath Date: Mon, 24 Oct 2022 21:01:28 +0800 Message-Id: <20221024130134.1046536-9-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add basic Tx support in split queue mode and single queue mode. Signed-off-by: Beilei Xing Signed-off-by: Xiaoyun Li Signed-off-by: Junfeng Guo --- drivers/net/idpf/idpf_ethdev.c | 3 +- drivers/net/idpf/idpf_ethdev.h | 1 + drivers/net/idpf/idpf_rxtx.c | 353 +++++++++++++++++++++++++++++++++ drivers/net/idpf/idpf_rxtx.h | 8 + 4 files changed, 364 insertions(+), 1 deletion(-) diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index f03df0a3ef..d3ac269419 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -94,7 +94,7 @@ idpf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP | RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP; dev_info->rx_offload_capa = 0; - dev_info->tx_offload_capa = 0; + dev_info->tx_offload_capa = RTE_ETH_TX_OFFLOAD_MULTI_SEGS; dev_info->default_rxconf = (struct rte_eth_rxconf) { .rx_free_thresh = IDPF_DEFAULT_RX_FREE_THRESH, @@ -331,6 +331,7 @@ idpf_dev_start(struct rte_eth_dev *dev) } idpf_set_rx_function(dev); + idpf_set_tx_function(dev); if (idpf_vc_ena_dis_vport(vport, true) != 0) { PMD_DRV_LOG(ERR, "Failed to enable vport"); diff --git a/drivers/net/idpf/idpf_ethdev.h b/drivers/net/idpf/idpf_ethdev.h index 58f070c976..238c641c3f 100644 --- a/drivers/net/idpf/idpf_ethdev.h +++ b/drivers/net/idpf/idpf_ethdev.h @@ -44,6 +44,7 @@ #define IDPF_MIN_BUF_SIZE 1024 #define IDPF_MAX_FRAME_SIZE 9728 +#define IDPF_MIN_FRAME_SIZE 14 #define IDPF_NUM_MACADDR_MAX 64 diff --git a/drivers/net/idpf/idpf_rxtx.c b/drivers/net/idpf/idpf_rxtx.c index 4037cf5a74..fd203c985d 100644 --- a/drivers/net/idpf/idpf_rxtx.c +++ b/drivers/net/idpf/idpf_rxtx.c @@ -1415,6 +1415,148 @@ idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, return nb_rx; } +static inline void +idpf_split_tx_free(struct idpf_tx_queue *cq) +{ + volatile struct idpf_splitq_tx_compl_desc *compl_ring = cq->compl_ring; + volatile struct idpf_splitq_tx_compl_desc *txd; + uint16_t next = cq->tx_tail; + struct idpf_tx_entry *txe; + struct idpf_tx_queue *txq; + uint16_t gen, qid, q_head; + uint8_t ctype; + + txd = &compl_ring[next]; + gen = (rte_le_to_cpu_16(txd->qid_comptype_gen) & + IDPF_TXD_COMPLQ_GEN_M) >> IDPF_TXD_COMPLQ_GEN_S; + if (gen != cq->expected_gen_id) + return; + + ctype = (rte_le_to_cpu_16(txd->qid_comptype_gen) & + IDPF_TXD_COMPLQ_COMPL_TYPE_M) >> IDPF_TXD_COMPLQ_COMPL_TYPE_S; + qid = (rte_le_to_cpu_16(txd->qid_comptype_gen) & + IDPF_TXD_COMPLQ_QID_M) >> IDPF_TXD_COMPLQ_QID_S; + q_head = rte_le_to_cpu_16(txd->q_head_compl_tag.compl_tag); + txq = cq->txqs[qid - cq->tx_start_qid]; + + switch (ctype) { + case IDPF_TXD_COMPLT_RE: + if (q_head == 0) + txq->last_desc_cleaned = txq->nb_tx_desc - 1; + else + txq->last_desc_cleaned = q_head - 1; + if (unlikely((txq->last_desc_cleaned % 32) == 0)) { + PMD_DRV_LOG(ERR, "unexpected desc (head = %u) completion.", + q_head); + return; + } + + break; + case IDPF_TXD_COMPLT_RS: + txq->nb_free++; + txq->nb_used--; + txe = &txq->sw_ring[q_head]; + if (txe->mbuf != NULL) { + rte_pktmbuf_free_seg(txe->mbuf); + txe->mbuf = NULL; + } + break; + default: + PMD_DRV_LOG(ERR, "unknown completion type."); + return; + } + + if (++next == cq->nb_tx_desc) { + next = 0; + cq->expected_gen_id ^= 1; + } + + cq->tx_tail = next; +} + +uint16_t +idpf_splitq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct idpf_tx_queue *txq = (struct idpf_tx_queue *)tx_queue; + volatile struct idpf_flex_tx_sched_desc *txr; + volatile struct idpf_flex_tx_sched_desc *txd; + struct idpf_tx_entry *sw_ring; + struct idpf_tx_entry *txe, *txn; + uint16_t nb_used, tx_id, sw_id; + struct rte_mbuf *tx_pkt; + uint16_t nb_to_clean; + uint16_t nb_tx = 0; + + if (unlikely(txq == NULL) || unlikely(!txq->q_started)) + return nb_tx; + + txr = txq->desc_ring; + sw_ring = txq->sw_ring; + tx_id = txq->tx_tail; + sw_id = txq->sw_tail; + txe = &sw_ring[sw_id]; + + for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { + tx_pkt = tx_pkts[nb_tx]; + + if (txq->nb_free <= txq->free_thresh) { + /* TODO: Need to refine + * 1. free and clean: Better to decide a clean destination instead of + * loop times. And don't free mbuf when RS got immediately, free when + * transmit or according to the clean destination. + * Now, just ignore the RE write back, free mbuf when get RS + * 2. out-of-order rewrite back haven't be supported, SW head and HW head + * need to be separated. + **/ + nb_to_clean = 2 * txq->rs_thresh; + while (nb_to_clean--) + idpf_split_tx_free(txq->complq); + } + + if (txq->nb_free < tx_pkt->nb_segs) + break; + nb_used = tx_pkt->nb_segs; + + do { + txd = &txr[tx_id]; + txn = &sw_ring[txe->next_id]; + txe->mbuf = tx_pkt; + + /* Setup TX descriptor */ + txd->buf_addr = + rte_cpu_to_le_64(rte_mbuf_data_iova(tx_pkt)); + txd->qw1.cmd_dtype = + rte_cpu_to_le_16(IDPF_TX_DESC_DTYPE_FLEX_FLOW_SCHE); + txd->qw1.rxr_bufsize = tx_pkt->data_len; + txd->qw1.compl_tag = sw_id; + tx_id++; + if (tx_id == txq->nb_tx_desc) + tx_id = 0; + sw_id = txe->next_id; + txe = txn; + tx_pkt = tx_pkt->next; + } while (tx_pkt); + + /* fill the last descriptor with End of Packet (EOP) bit */ + txd->qw1.cmd_dtype |= IDPF_TXD_FLEX_FLOW_CMD_EOP; + + if (unlikely((tx_id % 32) == 0)) + txd->qw1.cmd_dtype |= IDPF_TXD_FLEX_FLOW_CMD_RE; + txq->nb_free = (uint16_t)(txq->nb_free - nb_used); + txq->nb_used = (uint16_t)(txq->nb_used + nb_used); + } + + /* update the tail pointer if any packets were processed */ + if (likely(nb_tx > 0)) { + IDPF_PCI_REG_WRITE(txq->qtx_tail, tx_id); + txq->tx_tail = tx_id; + txq->sw_tail = sw_id; + } + + return nb_tx; +} + static inline void idpf_update_rx_tail(struct idpf_rx_queue *rxq, uint16_t nb_hold, uint16_t rx_id) @@ -1526,6 +1668,204 @@ idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, return nb_rx; } +static inline int +idpf_xmit_cleanup(struct idpf_tx_queue *txq) +{ + uint16_t last_desc_cleaned = txq->last_desc_cleaned; + struct idpf_tx_entry *sw_ring = txq->sw_ring; + uint16_t nb_tx_desc = txq->nb_tx_desc; + uint16_t desc_to_clean_to; + uint16_t nb_tx_to_clean; + uint16_t i; + + volatile struct idpf_flex_tx_desc *txd = txq->tx_ring; + + desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->rs_thresh); + if (desc_to_clean_to >= nb_tx_desc) + desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); + + desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; + /* In the writeback Tx desccriptor, the only significant fields are the 4-bit DTYPE */ + if ((txd[desc_to_clean_to].qw1.cmd_dtype & + rte_cpu_to_le_16(IDPF_TXD_QW1_DTYPE_M)) != + rte_cpu_to_le_16(IDPF_TX_DESC_DTYPE_DESC_DONE)) { + PMD_TX_LOG(DEBUG, "TX descriptor %4u is not done " + "(port=%d queue=%d)", desc_to_clean_to, + txq->port_id, txq->queue_id); + return -1; + } + + if (last_desc_cleaned > desc_to_clean_to) + nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + + desc_to_clean_to); + else + nb_tx_to_clean = (uint16_t)(desc_to_clean_to - + last_desc_cleaned); + + txd[desc_to_clean_to].qw1.cmd_dtype = 0; + txd[desc_to_clean_to].qw1.buf_size = 0; + for (i = 0; i < RTE_DIM(txd[desc_to_clean_to].qw1.flex.raw); i++) + txd[desc_to_clean_to].qw1.flex.raw[i] = 0; + + txq->last_desc_cleaned = desc_to_clean_to; + txq->nb_free = (uint16_t)(txq->nb_free + nb_tx_to_clean); + + return 0; +} + +/* TX function */ +uint16_t +idpf_singleq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + volatile struct idpf_flex_tx_desc *txd; + volatile struct idpf_flex_tx_desc *txr; + struct idpf_tx_entry *txe, *txn; + struct idpf_tx_entry *sw_ring; + struct idpf_tx_queue *txq; + struct rte_mbuf *tx_pkt; + struct rte_mbuf *m_seg; + uint64_t buf_dma_addr; + uint16_t tx_last; + uint16_t nb_used; + uint16_t td_cmd; + uint16_t tx_id; + uint16_t nb_tx; + uint16_t slen; + + nb_tx = 0; + txq = tx_queue; + + if (unlikely(txq == NULL) || unlikely(!txq->q_started)) + return nb_tx; + + sw_ring = txq->sw_ring; + txr = txq->tx_ring; + tx_id = txq->tx_tail; + txe = &sw_ring[tx_id]; + + /* Check if the descriptor ring needs to be cleaned. */ + if (txq->nb_free < txq->free_thresh) + (void)idpf_xmit_cleanup(txq); + + for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { + td_cmd = 0; + + tx_pkt = *tx_pkts++; + RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf); + + /* The number of descriptors that must be allocated for + * a packet equals to the number of the segments of that + * packet plus 1 context descriptor if needed. + */ + nb_used = (uint16_t)(tx_pkt->nb_segs); + tx_last = (uint16_t)(tx_id + nb_used - 1); + + /* Circular ring */ + if (tx_last >= txq->nb_tx_desc) + tx_last = (uint16_t)(tx_last - txq->nb_tx_desc); + + PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u" + " tx_first=%u tx_last=%u", + txq->port_id, txq->queue_id, tx_id, tx_last); + + if (nb_used > txq->nb_free) { + if (idpf_xmit_cleanup(txq) != 0) { + if (nb_tx == 0) + return 0; + goto end_of_tx; + } + if (unlikely(nb_used > txq->rs_thresh)) { + while (nb_used > txq->nb_free) { + if (idpf_xmit_cleanup(txq) != 0) { + if (nb_tx == 0) + return 0; + goto end_of_tx; + } + } + } + } + + m_seg = tx_pkt; + do { + txd = &txr[tx_id]; + txn = &sw_ring[txe->next_id]; + + if (txe->mbuf != NULL) + rte_pktmbuf_free_seg(txe->mbuf); + txe->mbuf = m_seg; + + /* Setup TX Descriptor */ + slen = m_seg->data_len; + buf_dma_addr = rte_mbuf_data_iova(m_seg); + txd->buf_addr = rte_cpu_to_le_64(buf_dma_addr); + txd->qw1.buf_size = slen; + txd->qw1.cmd_dtype = rte_cpu_to_le_16(IDPF_TX_DESC_DTYPE_FLEX_DATA << + IDPF_FLEX_TXD_QW1_DTYPE_S); + + txe->last_id = tx_last; + tx_id = txe->next_id; + txe = txn; + m_seg = m_seg->next; + } while (m_seg); + + /* The last packet data descriptor needs End Of Packet (EOP) */ + td_cmd |= IDPF_TX_FLEX_DESC_CMD_EOP; + txq->nb_used = (uint16_t)(txq->nb_used + nb_used); + txq->nb_free = (uint16_t)(txq->nb_free - nb_used); + + if (txq->nb_used >= txq->rs_thresh) { + PMD_TX_LOG(DEBUG, "Setting RS bit on TXD id=" + "%4u (port=%d queue=%d)", + tx_last, txq->port_id, txq->queue_id); + + td_cmd |= IDPF_TX_FLEX_DESC_CMD_RS; + + /* Update txq RS bit counters */ + txq->nb_used = 0; + } + + txd->qw1.cmd_dtype |= rte_cpu_to_le_16(td_cmd << IDPF_FLEX_TXD_QW1_CMD_S); + } + +end_of_tx: + rte_wmb(); + + PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", + txq->port_id, txq->queue_id, tx_id, nb_tx); + + IDPF_PCI_REG_WRITE(txq->qtx_tail, tx_id); + txq->tx_tail = tx_id; + + return nb_tx; +} + +/* TX prep functions */ +uint16_t +idpf_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + int i, ret; + struct rte_mbuf *m; + + for (i = 0; i < nb_pkts; i++) { + m = tx_pkts[i]; + + if (m->pkt_len < IDPF_MIN_FRAME_SIZE) { + rte_errno = EINVAL; + return i; + } + + ret = rte_net_intel_cksum_prepare(m); + if (ret != 0) { + rte_errno = -ret; + return i; + } + } + + return i; +} + void idpf_set_rx_function(struct rte_eth_dev *dev) { @@ -1536,3 +1876,16 @@ idpf_set_rx_function(struct rte_eth_dev *dev) else dev->rx_pkt_burst = idpf_singleq_recv_pkts; } + +void +idpf_set_tx_function(struct rte_eth_dev *dev) +{ + struct idpf_vport *vport = dev->data->dev_private; + if (vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) { + dev->tx_pkt_burst = idpf_splitq_xmit_pkts; + dev->tx_pkt_prepare = idpf_prep_pkts; + } else { + dev->tx_pkt_burst = idpf_singleq_xmit_pkts; + dev->tx_pkt_prepare = idpf_prep_pkts; + } +} diff --git a/drivers/net/idpf/idpf_rxtx.h b/drivers/net/idpf/idpf_rxtx.h index 056596ce94..eb0b230d3a 100644 --- a/drivers/net/idpf/idpf_rxtx.h +++ b/drivers/net/idpf/idpf_rxtx.h @@ -144,9 +144,17 @@ uint16_t idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); uint16_t idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +uint16_t idpf_singleq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +uint16_t idpf_splitq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +uint16_t idpf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); void idpf_stop_queues(struct rte_eth_dev *dev); void idpf_set_rx_function(struct rte_eth_dev *dev); +void idpf_set_tx_function(struct rte_eth_dev *dev); + const uint32_t *idpf_dev_supported_ptypes_get(struct rte_eth_dev *dev); #endif /* _IDPF_RXTX_H_ */ From patchwork Mon Oct 24 13:01:29 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 118999 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 16BCCA034C; Mon, 24 Oct 2022 15:04:58 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 5AFDE42BA5; Mon, 24 Oct 2022 15:04:39 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id 2DFE842BBF for ; Mon, 24 Oct 2022 15:04:29 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616669; x=1698152669; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=vG3DmFug8s+uhQ8uVcgc48/m70hm3zp4ngjpeTPNAjc=; b=SqZQgnqY2VO9NifbOs0DuRYcnSe+BR083sjlEceqQ84QOW76AaJFrI5U Yw8r8cJpi8rFXcBIWkEXjcLO/XEA9LbS4sesTu/qEXfJ46f/o+mSztamN rDVr0oG966pQLzhrHfvX8/lA/1oSbTvzz8DrOchIvR/4cE+PS+YbhXDML UXD2kiWAsX7WT1K8eeI6uJi7gglhnNa4voHeoMzq/mrTCo5RKzRGjSzyp lHsXXXlvHakh9c1nn58XhCAvZih3Gr5QlVVzOIVDzDATxiZvFLejNUwHL /Qs1jYVDndp1qJ35HPhsoIruVVZ6OOGYtE8W9A3kaIPa78mEPDvaj/mR/ A==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305018853" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018853" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:30 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248387" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248387" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:27 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo Subject: [PATCH v10 09/14] net/idpf: add support for write back based on ITR expire Date: Mon, 24 Oct 2022 21:01:29 +0800 Message-Id: <20221024130134.1046536-10-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Enable write back on ITR expire, then packets can be received one by one. Signed-off-by: Beilei Xing Signed-off-by: Junfeng Guo --- drivers/net/idpf/idpf_ethdev.c | 116 +++++++++++++++++++++++++++++++++ drivers/net/idpf/idpf_ethdev.h | 13 ++++ drivers/net/idpf/idpf_vchnl.c | 111 +++++++++++++++++++++++++++++++ 3 files changed, 240 insertions(+) diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index d3ac269419..57e4bf0d67 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -280,6 +280,90 @@ idpf_dev_configure(struct rte_eth_dev *dev) return 0; } +static int +idpf_config_rx_queues_irqs(struct rte_eth_dev *dev) +{ + struct idpf_vport *vport = dev->data->dev_private; + struct idpf_adapter *adapter = vport->adapter; + struct virtchnl2_queue_vector *qv_map; + struct idpf_hw *hw = &adapter->hw; + uint32_t dynctl_reg_start; + uint32_t itrn_reg_start; + uint32_t dynctl_val, itrn_val; + uint16_t i; + + qv_map = rte_zmalloc("qv_map", + dev->data->nb_rx_queues * + sizeof(struct virtchnl2_queue_vector), 0); + if (qv_map == NULL) { + PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map", + dev->data->nb_rx_queues); + goto qv_map_alloc_err; + } + + /* Rx interrupt disabled, Map interrupt only for writeback */ + + /* The capability flags adapter->caps->other_caps here should be + * compared with bit VIRTCHNL2_CAP_WB_ON_ITR. The if condition should + * be updated when the FW can return correct flag bits. + */ + if (adapter->caps->other_caps != 0) { + dynctl_reg_start = + vport->recv_vectors->vchunks.vchunks->dynctl_reg_start; + itrn_reg_start = + vport->recv_vectors->vchunks.vchunks->itrn_reg_start; + dynctl_val = IDPF_READ_REG(hw, dynctl_reg_start); + PMD_DRV_LOG(DEBUG, "Value of dynctl_reg_start is 0x%x", + dynctl_val); + itrn_val = IDPF_READ_REG(hw, itrn_reg_start); + PMD_DRV_LOG(DEBUG, "Value of itrn_reg_start is 0x%x", itrn_val); + /* Force write-backs by setting WB_ON_ITR bit in DYN_CTL + * register. WB_ON_ITR and INTENA are mutually exclusive + * bits. Setting WB_ON_ITR bits means TX and RX Descs + * are written back based on ITR expiration irrespective + * of INTENA setting. + */ + /* TBD: need to tune INTERVAL value for better performance. */ + if (itrn_val != 0) + IDPF_WRITE_REG(hw, + dynctl_reg_start, + VIRTCHNL2_ITR_IDX_0 << + PF_GLINT_DYN_CTL_ITR_INDX_S | + PF_GLINT_DYN_CTL_WB_ON_ITR_M | + itrn_val << + PF_GLINT_DYN_CTL_INTERVAL_S); + else + IDPF_WRITE_REG(hw, + dynctl_reg_start, + VIRTCHNL2_ITR_IDX_0 << + PF_GLINT_DYN_CTL_ITR_INDX_S | + PF_GLINT_DYN_CTL_WB_ON_ITR_M | + IDPF_DFLT_INTERVAL << + PF_GLINT_DYN_CTL_INTERVAL_S); + } + for (i = 0; i < dev->data->nb_rx_queues; i++) { + /* map all queues to the same vector */ + qv_map[i].queue_id = vport->chunks_info.rx_start_qid + i; + qv_map[i].vector_id = + vport->recv_vectors->vchunks.vchunks->start_vector_id; + } + vport->qv_map = qv_map; + + if (idpf_vc_config_irq_map_unmap(vport, true) != 0) { + PMD_DRV_LOG(ERR, "config interrupt mapping failed"); + goto config_irq_map_err; + } + + return 0; + +config_irq_map_err: + rte_free(vport->qv_map); + vport->qv_map = NULL; + +qv_map_alloc_err: + return -1; +} + static int idpf_start_queues(struct rte_eth_dev *dev) { @@ -317,6 +401,10 @@ static int idpf_dev_start(struct rte_eth_dev *dev) { struct idpf_vport *vport = dev->data->dev_private; + struct idpf_adapter *adapter = vport->adapter; + uint16_t num_allocated_vectors = + adapter->caps->num_allocated_vectors; + uint16_t req_vecs_num; if (dev->data->mtu > vport->max_mtu) { PMD_DRV_LOG(ERR, "MTU should be less than %d", vport->max_mtu); @@ -325,6 +413,23 @@ idpf_dev_start(struct rte_eth_dev *dev) vport->max_pkt_len = dev->data->mtu + IDPF_ETH_OVERHEAD; + req_vecs_num = IDPF_DFLT_Q_VEC_NUM; + if (req_vecs_num + adapter->used_vecs_num > num_allocated_vectors) { + PMD_DRV_LOG(ERR, "The accumulated request vectors' number should be less than %d", + num_allocated_vectors); + goto err_mtu; + } + if (idpf_vc_alloc_vectors(vport, req_vecs_num) != 0) { + PMD_DRV_LOG(ERR, "Failed to allocate interrupt vectors"); + goto err_mtu; + } + adapter->used_vecs_num += req_vecs_num; + + if (idpf_config_rx_queues_irqs(dev) != 0) { + PMD_DRV_LOG(ERR, "Failed to configure irqs"); + goto err_mtu; + } + if (idpf_start_queues(dev) != 0) { PMD_DRV_LOG(ERR, "Failed to start queues"); goto err_mtu; @@ -354,6 +459,10 @@ idpf_dev_stop(struct rte_eth_dev *dev) idpf_stop_queues(dev); + idpf_vc_config_irq_map_unmap(vport, false); + + idpf_vc_dealloc_vectors(vport); + return 0; } @@ -365,6 +474,11 @@ idpf_dev_close(struct rte_eth_dev *dev) idpf_vc_destroy_vport(vport); + rte_free(vport->recv_vectors); + vport->recv_vectors = NULL; + + rte_free(vport->qv_map); + vport->qv_map = NULL; adapter->cur_vports &= ~RTE_BIT32(vport->devarg_id); @@ -717,6 +831,8 @@ idpf_adapter_init(struct rte_pci_device *pci_dev, struct idpf_adapter *adapter) adapter->cur_vports = 0; adapter->cur_vport_nb = 0; + adapter->used_vecs_num = 0; + return ret; err_vports: diff --git a/drivers/net/idpf/idpf_ethdev.h b/drivers/net/idpf/idpf_ethdev.h index 238c641c3f..d89a06d239 100644 --- a/drivers/net/idpf/idpf_ethdev.h +++ b/drivers/net/idpf/idpf_ethdev.h @@ -42,6 +42,9 @@ #define IDPF_CTLQ_LEN 64 #define IDPF_DFLT_MBX_BUF_SIZE 4096 +#define IDPF_DFLT_Q_VEC_NUM 1 +#define IDPF_DFLT_INTERVAL 16 + #define IDPF_MIN_BUF_SIZE 1024 #define IDPF_MAX_FRAME_SIZE 9728 #define IDPF_MIN_FRAME_SIZE 14 @@ -105,6 +108,11 @@ struct idpf_vport { struct rte_eth_dev_data *dev_data; /* Pointer to the device data */ uint16_t max_pkt_len; /* Maximum packet length */ + /* MSIX info*/ + struct virtchnl2_queue_vector *qv_map; /* queue vector mapping */ + uint16_t max_vectors; + struct virtchnl2_alloc_vectors *recv_vectors; + /* Chunk info */ struct idpf_chunks_info chunks_info; @@ -140,6 +148,8 @@ struct idpf_adapter { uint16_t cur_vport_nb; uint16_t cur_vport_idx; + uint16_t used_vecs_num; + /* Max config queue number per VC message */ uint32_t max_rxq_per_msg; uint32_t max_txq_per_msg; @@ -219,6 +229,9 @@ int idpf_switch_queue(struct idpf_vport *vport, uint16_t qid, bool rx, bool on); int idpf_vc_ena_dis_queues(struct idpf_vport *vport, bool enable); int idpf_vc_ena_dis_vport(struct idpf_vport *vport, bool enable); +int idpf_vc_config_irq_map_unmap(struct idpf_vport *vport, bool map); +int idpf_vc_alloc_vectors(struct idpf_vport *vport, uint16_t num_vectors); +int idpf_vc_dealloc_vectors(struct idpf_vport *vport); int idpf_vc_query_ptype_info(struct idpf_adapter *adapter); int idpf_read_one_msg(struct idpf_adapter *adapter, uint32_t ops, uint16_t buf_len, uint8_t *buf); diff --git a/drivers/net/idpf/idpf_vchnl.c b/drivers/net/idpf/idpf_vchnl.c index 86a55d2ff1..34369cfc96 100644 --- a/drivers/net/idpf/idpf_vchnl.c +++ b/drivers/net/idpf/idpf_vchnl.c @@ -234,6 +234,10 @@ idpf_execute_vc_cmd(struct idpf_adapter *adapter, struct idpf_cmd_info *args) case VIRTCHNL2_OP_DISABLE_QUEUES: case VIRTCHNL2_OP_ENABLE_VPORT: case VIRTCHNL2_OP_DISABLE_VPORT: + case VIRTCHNL2_OP_MAP_QUEUE_VECTOR: + case VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR: + case VIRTCHNL2_OP_ALLOC_VECTORS: + case VIRTCHNL2_OP_DEALLOC_VECTORS: /* for init virtchnl ops, need to poll the response */ err = idpf_read_one_msg(adapter, args->ops, args->out_size, args->out_buffer); clear_cmd(adapter); @@ -1024,6 +1028,113 @@ idpf_vc_config_txq(struct idpf_vport *vport, uint16_t txq_id) return err; } +int +idpf_vc_config_irq_map_unmap(struct idpf_vport *vport, bool map) +{ + struct idpf_adapter *adapter = vport->adapter; + struct virtchnl2_queue_vector_maps *map_info; + struct virtchnl2_queue_vector *vecmap; + uint16_t nb_rxq = vport->dev_data->nb_rx_queues; + struct idpf_cmd_info args; + int len, i, err = 0; + + len = sizeof(struct virtchnl2_queue_vector_maps) + + (nb_rxq - 1) * sizeof(struct virtchnl2_queue_vector); + + map_info = rte_zmalloc("map_info", len, 0); + if (map_info == NULL) + return -ENOMEM; + + map_info->vport_id = vport->vport_id; + map_info->num_qv_maps = nb_rxq; + for (i = 0; i < nb_rxq; i++) { + vecmap = &map_info->qv_maps[i]; + vecmap->queue_id = vport->qv_map[i].queue_id; + vecmap->vector_id = vport->qv_map[i].vector_id; + vecmap->itr_idx = VIRTCHNL2_ITR_IDX_0; + vecmap->queue_type = VIRTCHNL2_QUEUE_TYPE_RX; + } + + args.ops = map ? VIRTCHNL2_OP_MAP_QUEUE_VECTOR : + VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR; + args.in_args = (u8 *)map_info; + args.in_args_size = len; + args.out_buffer = adapter->mbx_resp; + args.out_size = IDPF_DFLT_MBX_BUF_SIZE; + err = idpf_execute_vc_cmd(adapter, &args); + if (err != 0) + PMD_DRV_LOG(ERR, "Failed to execute command of VIRTCHNL2_OP_%s_QUEUE_VECTOR", + map ? "MAP" : "UNMAP"); + + rte_free(map_info); + return err; +} + +int +idpf_vc_alloc_vectors(struct idpf_vport *vport, uint16_t num_vectors) +{ + struct idpf_adapter *adapter = vport->adapter; + struct virtchnl2_alloc_vectors *alloc_vec; + struct idpf_cmd_info args; + int err, len; + + len = sizeof(struct virtchnl2_alloc_vectors) + + (num_vectors - 1) * sizeof(struct virtchnl2_vector_chunk); + alloc_vec = rte_zmalloc("alloc_vec", len, 0); + if (alloc_vec == NULL) + return -ENOMEM; + + alloc_vec->num_vectors = num_vectors; + + args.ops = VIRTCHNL2_OP_ALLOC_VECTORS; + args.in_args = (u8 *)alloc_vec; + args.in_args_size = sizeof(struct virtchnl2_alloc_vectors); + args.out_buffer = adapter->mbx_resp; + args.out_size = IDPF_DFLT_MBX_BUF_SIZE; + err = idpf_execute_vc_cmd(adapter, &args); + if (err != 0) + PMD_DRV_LOG(ERR, "Failed to execute command VIRTCHNL2_OP_ALLOC_VECTORS"); + + if (vport->recv_vectors == NULL) { + vport->recv_vectors = rte_zmalloc("recv_vectors", len, 0); + if (vport->recv_vectors == NULL) { + rte_free(alloc_vec); + return -ENOMEM; + } + } + + rte_memcpy(vport->recv_vectors, args.out_buffer, len); + rte_free(alloc_vec); + return err; +} + +int +idpf_vc_dealloc_vectors(struct idpf_vport *vport) +{ + struct idpf_adapter *adapter = vport->adapter; + struct virtchnl2_alloc_vectors *alloc_vec; + struct virtchnl2_vector_chunks *vcs; + struct idpf_cmd_info args; + int err, len; + + alloc_vec = vport->recv_vectors; + vcs = &alloc_vec->vchunks; + + len = sizeof(struct virtchnl2_vector_chunks) + + (vcs->num_vchunks - 1) * sizeof(struct virtchnl2_vector_chunk); + + args.ops = VIRTCHNL2_OP_DEALLOC_VECTORS; + args.in_args = (u8 *)vcs; + args.in_args_size = len; + args.out_buffer = adapter->mbx_resp; + args.out_size = IDPF_DFLT_MBX_BUF_SIZE; + err = idpf_execute_vc_cmd(adapter, &args); + if (err != 0) + PMD_DRV_LOG(ERR, "Failed to execute command VIRTCHNL2_OP_DEALLOC_VECTORS"); + + return err; +} + static int idpf_vc_ena_dis_one_queue(struct idpf_vport *vport, uint16_t qid, uint32_t type, bool on) From patchwork Mon Oct 24 13:01:30 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 119000 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id A9591A034C; Mon, 24 Oct 2022 15:05:05 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 0C31342BBC; Mon, 24 Oct 2022 15:04:41 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id 5C18A42BAB for ; Mon, 24 Oct 2022 15:04:32 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616672; x=1698152672; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=r9s3/m98Fll1KOLsv2GLgZXr1wxsr6AvlN0A6lXk76Q=; b=DZoiUkC8MDZFaFtvfGf0W6Z+crDjlCMHOxFR/NIDH5EDq6q10QtR42QG uxv19tT0Mdo1THf8jRS58aOde22+lDGz0IbJ9LotmSY7jnC0Pc2YyRD7P 9WnSby7iehV5KJB41fG4yr+Y9+7kZH8kOtfIa8MU/M3O/tBT0AujUNMG1 XpPrKOamPpVNayFKarUu+KWnWVw/qouWVrWIwe8AlmZvSMQn9ht9uGdVP V3IqRUVDdbx30w+yg+IbrJRltUp4C9Z+oM3mYe3JDQQw9p1wnX1lFNrz4 bwvdgV//0zWr3CdkI7XMWioGQvOj80blEzFB+6Nv9OMaqkHCEJ8o+H42N w==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305018877" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018877" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:32 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248420" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248420" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:30 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo Subject: [PATCH v10 10/14] net/idpf: add support for RSS Date: Mon, 24 Oct 2022 21:01:30 +0800 Message-Id: <20221024130134.1046536-11-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add RSS support. Signed-off-by: Beilei Xing Signed-off-by: Junfeng Guo --- drivers/net/idpf/idpf_ethdev.c | 116 ++++++++++++++++++++++++++++++++- drivers/net/idpf/idpf_ethdev.h | 26 ++++++++ drivers/net/idpf/idpf_vchnl.c | 97 +++++++++++++++++++++++++++ 3 files changed, 237 insertions(+), 2 deletions(-) diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index 57e4bf0d67..739cf31d65 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -90,6 +90,7 @@ idpf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->max_mtu = dev_info->max_rx_pktlen - IDPF_ETH_OVERHEAD; dev_info->min_mtu = RTE_ETHER_MIN_MTU; + dev_info->flow_type_rss_offloads = IDPF_RSS_OFFLOAD_ALL; dev_info->max_mac_addrs = IDPF_NUM_MACADDR_MAX; dev_info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP | RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP; @@ -226,6 +227,10 @@ idpf_init_vport(struct rte_eth_dev *dev) vport->max_mtu = vport_info->max_mtu; rte_memcpy(vport->default_mac_addr, vport_info->default_mac_addr, ETH_ALEN); + vport->rss_algorithm = vport_info->rss_algorithm; + vport->rss_key_size = RTE_MIN(IDPF_RSS_KEY_LEN, + vport_info->rss_key_size); + vport->rss_lut_size = vport_info->rss_lut_size; vport->sw_idx = idx; for (i = 0; i < vport_info->chunks.num_chunks; i++) { @@ -271,13 +276,114 @@ idpf_init_vport(struct rte_eth_dev *dev) return 0; } + +static int +idpf_config_rss(struct idpf_vport *vport) +{ + int ret; + + ret = idpf_vc_set_rss_key(vport); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Failed to configure RSS key"); + return ret; + } + + ret = idpf_vc_set_rss_lut(vport); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Failed to configure RSS lut"); + return ret; + } + + ret = idpf_vc_set_rss_hash(vport); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Failed to configure RSS hash"); + return ret; + } + + return ret; +} + +static int +idpf_init_rss(struct idpf_vport *vport) +{ + struct rte_eth_rss_conf *rss_conf; + uint16_t i, nb_q, lut_size; + int ret = 0; + + rss_conf = &vport->dev_data->dev_conf.rx_adv_conf.rss_conf; + nb_q = vport->dev_data->nb_rx_queues; + + vport->rss_key = (uint8_t *)rte_zmalloc("rss_key", + vport->rss_key_size, 0); + if (vport->rss_key == NULL) { + PMD_INIT_LOG(ERR, "Failed to allocate RSS key"); + ret = -ENOMEM; + goto err_alloc_key; + } + + lut_size = vport->rss_lut_size; + vport->rss_lut = (uint32_t *)rte_zmalloc("rss_lut", + sizeof(uint32_t) * lut_size, 0); + if (vport->rss_lut == NULL) { + PMD_INIT_LOG(ERR, "Failed to allocate RSS lut"); + ret = -ENOMEM; + goto err_alloc_lut; + } + + if (rss_conf->rss_key == NULL) { + for (i = 0; i < vport->rss_key_size; i++) + vport->rss_key[i] = (uint8_t)rte_rand(); + } else if (rss_conf->rss_key_len != vport->rss_key_size) { + PMD_INIT_LOG(ERR, "Invalid RSS key length in RSS configuration, should be %d", + vport->rss_key_size); + ret = -EINVAL; + goto err_cfg_key; + } else { + rte_memcpy(vport->rss_key, rss_conf->rss_key, + vport->rss_key_size); + } + + for (i = 0; i < lut_size; i++) + vport->rss_lut[i] = i % nb_q; + + vport->rss_hf = IDPF_DEFAULT_RSS_HASH_EXPANDED; + + ret = idpf_config_rss(vport); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Failed to configure RSS"); + goto err_cfg_key; + } + + return ret; + +err_cfg_key: + rte_free(vport->rss_lut); + vport->rss_lut = NULL; +err_alloc_lut: + rte_free(vport->rss_key); + vport->rss_key = NULL; +err_alloc_key: + return ret; +} + static int idpf_dev_configure(struct rte_eth_dev *dev) { - if ((dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) != 0) + struct idpf_vport *vport = dev->data->dev_private; + struct idpf_adapter *adapter = vport->adapter; + int ret = 0; + + if (adapter->caps->rss_caps != 0 && dev->data->nb_rx_queues != 0) { + ret = idpf_init_rss(vport); + if (ret != 0) { + PMD_INIT_LOG(ERR, "Failed to init rss"); + return ret; + } + } else if ((dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) != 0) { PMD_INIT_LOG(ERR, "RSS is not supported."); + } - return 0; + return ret; } static int @@ -474,6 +580,12 @@ idpf_dev_close(struct rte_eth_dev *dev) idpf_vc_destroy_vport(vport); + rte_free(vport->rss_lut); + vport->rss_lut = NULL; + + rte_free(vport->rss_key); + vport->rss_key = NULL; + rte_free(vport->recv_vectors); vport->recv_vectors = NULL; diff --git a/drivers/net/idpf/idpf_ethdev.h b/drivers/net/idpf/idpf_ethdev.h index d89a06d239..7fe2647956 100644 --- a/drivers/net/idpf/idpf_ethdev.h +++ b/drivers/net/idpf/idpf_ethdev.h @@ -57,6 +57,20 @@ #define IDPF_ETH_OVERHEAD \ (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + IDPF_VLAN_TAG_SIZE * 2) +#define IDPF_RSS_OFFLOAD_ALL ( \ + RTE_ETH_RSS_IPV4 | \ + RTE_ETH_RSS_FRAG_IPV4 | \ + RTE_ETH_RSS_NONFRAG_IPV4_TCP | \ + RTE_ETH_RSS_NONFRAG_IPV4_UDP | \ + RTE_ETH_RSS_NONFRAG_IPV4_SCTP | \ + RTE_ETH_RSS_NONFRAG_IPV4_OTHER | \ + RTE_ETH_RSS_IPV6 | \ + RTE_ETH_RSS_FRAG_IPV6 | \ + RTE_ETH_RSS_NONFRAG_IPV6_TCP | \ + RTE_ETH_RSS_NONFRAG_IPV6_UDP | \ + RTE_ETH_RSS_NONFRAG_IPV6_SCTP | \ + RTE_ETH_RSS_NONFRAG_IPV6_OTHER) + #ifndef ETH_ADDR_LEN #define ETH_ADDR_LEN 6 #endif @@ -103,11 +117,20 @@ struct idpf_vport { uint16_t max_mtu; uint8_t default_mac_addr[VIRTCHNL_ETH_LENGTH_OF_ADDRESS]; + enum virtchnl_rss_algorithm rss_algorithm; + uint16_t rss_key_size; + uint16_t rss_lut_size; + uint16_t sw_idx; /* SW idx */ struct rte_eth_dev_data *dev_data; /* Pointer to the device data */ uint16_t max_pkt_len; /* Maximum packet length */ + /* RSS info */ + uint32_t *rss_lut; + uint8_t *rss_key; + uint64_t rss_hf; + /* MSIX info*/ struct virtchnl2_queue_vector *qv_map; /* queue vector mapping */ uint16_t max_vectors; @@ -221,6 +244,9 @@ int idpf_get_pkt_type(struct idpf_adapter *adapter); int idpf_vc_get_caps(struct idpf_adapter *adapter); int idpf_vc_create_vport(struct idpf_adapter *adapter); int idpf_vc_destroy_vport(struct idpf_vport *vport); +int idpf_vc_set_rss_key(struct idpf_vport *vport); +int idpf_vc_set_rss_lut(struct idpf_vport *vport); +int idpf_vc_set_rss_hash(struct idpf_vport *vport); int idpf_vc_config_rxqs(struct idpf_vport *vport); int idpf_vc_config_rxq(struct idpf_vport *vport, uint16_t rxq_id); int idpf_vc_config_txqs(struct idpf_vport *vport); diff --git a/drivers/net/idpf/idpf_vchnl.c b/drivers/net/idpf/idpf_vchnl.c index 34369cfc96..63605f2f64 100644 --- a/drivers/net/idpf/idpf_vchnl.c +++ b/drivers/net/idpf/idpf_vchnl.c @@ -228,6 +228,9 @@ idpf_execute_vc_cmd(struct idpf_adapter *adapter, struct idpf_cmd_info *args) case VIRTCHNL2_OP_GET_CAPS: case VIRTCHNL2_OP_CREATE_VPORT: case VIRTCHNL2_OP_DESTROY_VPORT: + case VIRTCHNL2_OP_SET_RSS_KEY: + case VIRTCHNL2_OP_SET_RSS_LUT: + case VIRTCHNL2_OP_SET_RSS_HASH: case VIRTCHNL2_OP_CONFIG_RX_QUEUES: case VIRTCHNL2_OP_CONFIG_TX_QUEUES: case VIRTCHNL2_OP_ENABLE_QUEUES: @@ -667,6 +670,100 @@ idpf_vc_destroy_vport(struct idpf_vport *vport) return err; } +int +idpf_vc_set_rss_key(struct idpf_vport *vport) +{ + struct idpf_adapter *adapter = vport->adapter; + struct virtchnl2_rss_key *rss_key; + struct idpf_cmd_info args; + int len, err; + + len = sizeof(*rss_key) + sizeof(rss_key->key[0]) * + (vport->rss_key_size - 1); + rss_key = rte_zmalloc("rss_key", len, 0); + if (rss_key == NULL) + return -ENOMEM; + + rss_key->vport_id = vport->vport_id; + rss_key->key_len = vport->rss_key_size; + rte_memcpy(rss_key->key, vport->rss_key, + sizeof(rss_key->key[0]) * vport->rss_key_size); + + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL2_OP_SET_RSS_KEY; + args.in_args = (uint8_t *)rss_key; + args.in_args_size = len; + args.out_buffer = adapter->mbx_resp; + args.out_size = IDPF_DFLT_MBX_BUF_SIZE; + + err = idpf_execute_vc_cmd(adapter, &args); + if (err != 0) + PMD_DRV_LOG(ERR, "Failed to execute command of VIRTCHNL2_OP_SET_RSS_KEY"); + + rte_free(rss_key); + return err; +} + +int +idpf_vc_set_rss_lut(struct idpf_vport *vport) +{ + struct idpf_adapter *adapter = vport->adapter; + struct virtchnl2_rss_lut *rss_lut; + struct idpf_cmd_info args; + int len, err; + + len = sizeof(*rss_lut) + sizeof(rss_lut->lut[0]) * + (vport->rss_lut_size - 1); + rss_lut = rte_zmalloc("rss_lut", len, 0); + if (rss_lut == NULL) + return -ENOMEM; + + rss_lut->vport_id = vport->vport_id; + rss_lut->lut_entries = vport->rss_lut_size; + rte_memcpy(rss_lut->lut, vport->rss_lut, + sizeof(rss_lut->lut[0]) * vport->rss_lut_size); + + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL2_OP_SET_RSS_LUT; + args.in_args = (uint8_t *)rss_lut; + args.in_args_size = len; + args.out_buffer = adapter->mbx_resp; + args.out_size = IDPF_DFLT_MBX_BUF_SIZE; + + err = idpf_execute_vc_cmd(adapter, &args); + if (err != 0) + PMD_DRV_LOG(ERR, "Failed to execute command of VIRTCHNL2_OP_SET_RSS_LUT"); + + rte_free(rss_lut); + return err; +} + +int +idpf_vc_set_rss_hash(struct idpf_vport *vport) +{ + struct idpf_adapter *adapter = vport->adapter; + struct virtchnl2_rss_hash rss_hash; + struct idpf_cmd_info args; + int err; + + memset(&rss_hash, 0, sizeof(rss_hash)); + rss_hash.ptype_groups = vport->rss_hf; + rss_hash.vport_id = vport->vport_id; + + memset(&args, 0, sizeof(args)); + args.ops = VIRTCHNL2_OP_SET_RSS_HASH; + args.in_args = (uint8_t *)&rss_hash; + args.in_args_size = sizeof(rss_hash); + args.out_buffer = adapter->mbx_resp; + args.out_size = IDPF_DFLT_MBX_BUF_SIZE; + + err = idpf_execute_vc_cmd(adapter, &args); + if (err != 0) + PMD_DRV_LOG(ERR, "Failed to execute command of OP_SET_RSS_HASH"); + + return err; +} + #define IDPF_RX_BUF_STRIDE 64 int idpf_vc_config_rxqs(struct idpf_vport *vport) From patchwork Mon Oct 24 13:01:31 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 119001 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 75E13A034C; Mon, 24 Oct 2022 15:05:13 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 3391C42BC3; Mon, 24 Oct 2022 15:04:42 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id 0267B42B94 for ; Mon, 24 Oct 2022 15:04:34 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616675; x=1698152675; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=VHcA6tYScxx0YTmFOmv6t49iT8VX8BXx/IpwpcIBV/A=; b=HRn4IlWjYGKEtTT+OL0oZKfIxkkatWqSlY9Pog3KpRQiZAuRqdaRX5GR /eA9eOZvjS276eqwXuDOwIBjrXNrp99sIhi6oJjP4g2a/kiMvry78ybgP dvU0cTZBlMO421lh8RRK0aeRSzZ4aKdz4ofHk0msPIB/0Ax7BLQTXqntg s/uwPn4u/we0CW2DOYsixSJDeT1uIgTsaLH3tvEFtmwxVSgfNBZ7cpM9L SPr5/z4+w4N4U2K8U6qv/ztAv05Thy04AMbARz3oJqyrU/vr9Dk0MjQN5 ydNj/bxoEu48lrB0KnaYB4AcK64YjiN5h0AdMVl/HYDdKGyQVLV0Sdorq g==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305018899" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018899" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:34 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248441" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248441" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:32 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo , Xiaoyun Li Subject: [PATCH v10 11/14] net/idpf: add support for Rx offloading Date: Mon, 24 Oct 2022 21:01:31 +0800 Message-Id: <20221024130134.1046536-12-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add Rx offloading support: - support CHKSUM and RSS offload for split queue model - support CHKSUM offload for single queue model Signed-off-by: Beilei Xing Signed-off-by: Xiaoyun Li Signed-off-by: Junfeng Guo --- doc/guides/nics/features/idpf.ini | 2 + drivers/net/idpf/idpf_ethdev.c | 9 ++- drivers/net/idpf/idpf_rxtx.c | 122 ++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) diff --git a/doc/guides/nics/features/idpf.ini b/doc/guides/nics/features/idpf.ini index d4eb9b374c..c86d9378ea 100644 --- a/doc/guides/nics/features/idpf.ini +++ b/doc/guides/nics/features/idpf.ini @@ -9,5 +9,7 @@ [Features] Queue start/stop = Y MTU update = Y +L3 checksum offload = P +L4 checksum offload = P Packet type parsing = Y Linux = Y diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index 739cf31d65..d8cc423a23 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -94,7 +94,14 @@ idpf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->max_mac_addrs = IDPF_NUM_MACADDR_MAX; dev_info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP | RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP; - dev_info->rx_offload_capa = 0; + + dev_info->rx_offload_capa = + RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | + RTE_ETH_RX_OFFLOAD_UDP_CKSUM | + RTE_ETH_RX_OFFLOAD_TCP_CKSUM | + RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM | + RTE_ETH_RX_OFFLOAD_RSS_HASH; + dev_info->tx_offload_capa = RTE_ETH_TX_OFFLOAD_MULTI_SEGS; dev_info->default_rxconf = (struct rte_eth_rxconf) { diff --git a/drivers/net/idpf/idpf_rxtx.c b/drivers/net/idpf/idpf_rxtx.c index fd203c985d..143c8b69f3 100644 --- a/drivers/net/idpf/idpf_rxtx.c +++ b/drivers/net/idpf/idpf_rxtx.c @@ -1250,6 +1250,72 @@ idpf_stop_queues(struct rte_eth_dev *dev) } } +#define IDPF_RX_FLEX_DESC_ADV_STATUS0_XSUM_S \ + (RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_IPE_S) | \ + RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_L4E_S) | \ + RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_EIPE_S) | \ + RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_EUDPE_S)) + +static inline uint64_t +idpf_splitq_rx_csum_offload(uint8_t err) +{ + uint64_t flags = 0; + + if (unlikely((err & RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_L3L4P_S)) == 0)) + return flags; + + if (likely((err & IDPF_RX_FLEX_DESC_ADV_STATUS0_XSUM_S) == 0)) { + flags |= (RTE_MBUF_F_RX_IP_CKSUM_GOOD | + RTE_MBUF_F_RX_L4_CKSUM_GOOD); + return flags; + } + + if (unlikely((err & RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_IPE_S)) != 0)) + flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; + else + flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; + + if (unlikely((err & RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_L4E_S)) != 0)) + flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; + else + flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; + + if (unlikely((err & RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_EIPE_S)) != 0)) + flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; + + if (unlikely((err & RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_EUDPE_S)) != 0)) + flags |= RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD; + else + flags |= RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD; + + return flags; +} + +#define IDPF_RX_FLEX_DESC_ADV_HASH1_S 0 +#define IDPF_RX_FLEX_DESC_ADV_HASH2_S 16 +#define IDPF_RX_FLEX_DESC_ADV_HASH3_S 24 + +static inline uint64_t +idpf_splitq_rx_rss_offload(struct rte_mbuf *mb, + volatile struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) +{ + uint8_t status_err0_qw0; + uint64_t flags = 0; + + status_err0_qw0 = rx_desc->status_err0_qw0; + + if ((status_err0_qw0 & RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_RSS_VALID_S)) != 0) { + flags |= RTE_MBUF_F_RX_RSS_HASH; + mb->hash.rss = (rte_le_to_cpu_16(rx_desc->hash1) << + IDPF_RX_FLEX_DESC_ADV_HASH1_S) | + ((uint32_t)(rx_desc->ff2_mirrid_hash2.hash2) << + IDPF_RX_FLEX_DESC_ADV_HASH2_S) | + ((uint32_t)(rx_desc->hash3) << + IDPF_RX_FLEX_DESC_ADV_HASH3_S); + } + + return flags; +} static void idpf_split_rx_bufq_refill(struct idpf_rx_queue *rx_bufq) @@ -1325,9 +1391,11 @@ idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pktlen_gen_bufq_id; struct idpf_rx_queue *rxq; const uint32_t *ptype_tbl; + uint8_t status_err0_qw1; struct rte_mbuf *rxm; uint16_t rx_id_bufq1; uint16_t rx_id_bufq2; + uint64_t pkt_flags; uint16_t pkt_len; uint16_t bufq_id; uint16_t gen_id; @@ -1393,11 +1461,18 @@ idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, rxm->next = NULL; rxm->nb_segs = 1; rxm->port = rxq->port_id; + rxm->ol_flags = 0; rxm->packet_type = ptype_tbl[(rte_le_to_cpu_16(rx_desc->ptype_err_fflags0) & VIRTCHNL2_RX_FLEX_DESC_ADV_PTYPE_M) >> VIRTCHNL2_RX_FLEX_DESC_ADV_PTYPE_S]; + status_err0_qw1 = rx_desc->status_err0_qw1; + pkt_flags = idpf_splitq_rx_csum_offload(status_err0_qw1); + pkt_flags |= idpf_splitq_rx_rss_offload(rxm, rx_desc); + + rxm->ol_flags |= pkt_flags; + rx_pkts[nb_rx++] = rxm; } @@ -1557,6 +1632,48 @@ idpf_splitq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, return nb_tx; } +#define IDPF_RX_FLEX_DESC_STATUS0_XSUM_S \ + (RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) | \ + RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_STATUS0_XSUM_L4E_S) | \ + RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S) | \ + RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S)) + +/* Translate the rx descriptor status and error fields to pkt flags */ +static inline uint64_t +idpf_rxd_to_pkt_flags(uint16_t status_error) +{ + uint64_t flags = 0; + + if (unlikely((status_error & RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_STATUS0_L3L4P_S)) == 0)) + return flags; + + if (likely((status_error & IDPF_RX_FLEX_DESC_STATUS0_XSUM_S) == 0)) { + flags |= (RTE_MBUF_F_RX_IP_CKSUM_GOOD | + RTE_MBUF_F_RX_L4_CKSUM_GOOD); + return flags; + } + + if (unlikely((status_error & RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_STATUS0_XSUM_IPE_S)) != 0)) + flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; + else + flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; + + if (unlikely((status_error & RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_STATUS0_XSUM_L4E_S)) != 0)) + flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; + else + flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; + + if (unlikely((status_error & RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S)) != 0)) + flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; + + if (unlikely((status_error & RTE_BIT32(VIRTCHNL2_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S)) != 0)) + flags |= RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD; + else + flags |= RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD; + + return flags; +} + static inline void idpf_update_rx_tail(struct idpf_rx_queue *rxq, uint16_t nb_hold, uint16_t rx_id) @@ -1590,6 +1707,7 @@ idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, struct rte_mbuf *rxm; struct rte_mbuf *nmb; uint16_t rx_status0; + uint64_t pkt_flags; uint64_t dma_addr; uint16_t nb_rx; @@ -1655,10 +1773,14 @@ idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, rxm->pkt_len = rx_packet_len; rxm->data_len = rx_packet_len; rxm->port = rxq->port_id; + rxm->ol_flags = 0; + pkt_flags = idpf_rxd_to_pkt_flags(rx_status0); rxm->packet_type = ptype_tbl[(uint8_t)(rte_cpu_to_le_16(rxd.flex_nic_wb.ptype_flex_flags0) & VIRTCHNL2_RX_FLEX_DESC_PTYPE_M)]; + rxm->ol_flags |= pkt_flags; + rx_pkts[nb_rx++] = rxm; } rxq->rx_tail = rx_id; From patchwork Mon Oct 24 13:01:32 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 119002 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 02F9CA034C; Mon, 24 Oct 2022 15:05:22 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 8329842BAF; Mon, 24 Oct 2022 15:04:49 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id BE03A42B8A for ; Mon, 24 Oct 2022 15:04:47 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616688; x=1698152688; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=Dwkh6vObO9Rg2seQBJah7nude05Ie2zWClgvmv3vlZ0=; b=TdQ66HdCUA3mFKsDDg8prZHL6aBnr+z+Z/jjU+4lb+tN42ZPcGD2zuUT v2NknMov73zUm8tuoGbNqWU3cQKrF1wG1f2K6ER6g+5I3yVRkMG1nErDW BRKN/b3H+kbwUeWCD+wSjMfYMGsEN+1F4YCk9AIA/VsKvgAS/lNzU80VA xAe8l8Vxezy4C/vgotPmpijgkD8exVaEUkF0TLd3nvSeyWyeem1nQSYmi SRDaf2mQoS7NzfWmdWYndRdpUfKqtI+GNTFiXOjMpRq8pd/SZokwjKQno amijGVXPt5cRqaV/Nok7IqO2UoGmNEPF4oSQb+7IIHAnXN6QTGBH0uPIQ w==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305018925" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018925" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:38 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248460" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248460" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:34 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo , Xiaoyun Li Subject: [PATCH v10 12/14] net/idpf: add support for Tx offloading Date: Mon, 24 Oct 2022 21:01:32 +0800 Message-Id: <20221024130134.1046536-13-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add Tx offloading support: - support TSO Signed-off-by: Beilei Xing Signed-off-by: Xiaoyun Li Signed-off-by: Junfeng Guo --- doc/guides/nics/features/idpf.ini | 1 + drivers/net/idpf/idpf_ethdev.c | 4 +- drivers/net/idpf/idpf_rxtx.c | 134 +++++++++++++++++++++++++++++- drivers/net/idpf/idpf_rxtx.h | 22 +++++ 4 files changed, 158 insertions(+), 3 deletions(-) diff --git a/doc/guides/nics/features/idpf.ini b/doc/guides/nics/features/idpf.ini index c86d9378ea..47c686762d 100644 --- a/doc/guides/nics/features/idpf.ini +++ b/doc/guides/nics/features/idpf.ini @@ -9,6 +9,7 @@ [Features] Queue start/stop = Y MTU update = Y +TSO = P L3 checksum offload = P L4 checksum offload = P Packet type parsing = Y diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index d8cc423a23..21315866bf 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -102,7 +102,9 @@ idpf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM | RTE_ETH_RX_OFFLOAD_RSS_HASH; - dev_info->tx_offload_capa = RTE_ETH_TX_OFFLOAD_MULTI_SEGS; + dev_info->tx_offload_capa = + RTE_ETH_TX_OFFLOAD_TCP_TSO | + RTE_ETH_TX_OFFLOAD_MULTI_SEGS; dev_info->default_rxconf = (struct rte_eth_rxconf) { .rx_free_thresh = IDPF_DEFAULT_RX_FREE_THRESH, diff --git a/drivers/net/idpf/idpf_rxtx.c b/drivers/net/idpf/idpf_rxtx.c index 143c8b69f3..8f82cf1b59 100644 --- a/drivers/net/idpf/idpf_rxtx.c +++ b/drivers/net/idpf/idpf_rxtx.c @@ -1549,6 +1549,49 @@ idpf_split_tx_free(struct idpf_tx_queue *cq) cq->tx_tail = next; } +/* Check if the context descriptor is needed for TX offloading */ +static inline uint16_t +idpf_calc_context_desc(uint64_t flags) +{ + if ((flags & RTE_MBUF_F_TX_TCP_SEG) != 0) + return 1; + + return 0; +} + +/* set TSO context descriptor + */ +static inline void +idpf_set_splitq_tso_ctx(struct rte_mbuf *mbuf, + union idpf_tx_offload tx_offload, + volatile union idpf_flex_tx_ctx_desc *ctx_desc) +{ + uint16_t cmd_dtype; + uint32_t tso_len; + uint8_t hdr_len; + + if (tx_offload.l4_len == 0) { + PMD_TX_LOG(DEBUG, "L4 length set to 0"); + return; + } + + hdr_len = tx_offload.l2_len + + tx_offload.l3_len + + tx_offload.l4_len; + cmd_dtype = IDPF_TX_DESC_DTYPE_FLEX_TSO_CTX | + IDPF_TX_FLEX_CTX_DESC_CMD_TSO; + tso_len = mbuf->pkt_len - hdr_len; + + ctx_desc->tso.qw1.cmd_dtype = rte_cpu_to_le_16(cmd_dtype); + ctx_desc->tso.qw0.hdr_len = hdr_len; + ctx_desc->tso.qw0.mss_rt = + rte_cpu_to_le_16((uint16_t)mbuf->tso_segsz & + IDPF_TXD_FLEX_CTX_MSS_RT_M); + ctx_desc->tso.qw0.flex_tlen = + rte_cpu_to_le_32(tso_len & + IDPF_TXD_FLEX_CTX_MSS_RT_M); +} + uint16_t idpf_splitq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) @@ -1557,11 +1600,14 @@ idpf_splitq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, volatile struct idpf_flex_tx_sched_desc *txr; volatile struct idpf_flex_tx_sched_desc *txd; struct idpf_tx_entry *sw_ring; + union idpf_tx_offload tx_offload = {0}; struct idpf_tx_entry *txe, *txn; uint16_t nb_used, tx_id, sw_id; struct rte_mbuf *tx_pkt; uint16_t nb_to_clean; uint16_t nb_tx = 0; + uint64_t ol_flags; + uint16_t nb_ctx; if (unlikely(txq == NULL) || unlikely(!txq->q_started)) return nb_tx; @@ -1591,7 +1637,29 @@ idpf_splitq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, if (txq->nb_free < tx_pkt->nb_segs) break; - nb_used = tx_pkt->nb_segs; + + ol_flags = tx_pkt->ol_flags; + tx_offload.l2_len = tx_pkt->l2_len; + tx_offload.l3_len = tx_pkt->l3_len; + tx_offload.l4_len = tx_pkt->l4_len; + tx_offload.tso_segsz = tx_pkt->tso_segsz; + /* Calculate the number of context descriptors needed. */ + nb_ctx = idpf_calc_context_desc(ol_flags); + nb_used = tx_pkt->nb_segs + nb_ctx; + + /* context descriptor */ + if (nb_ctx != 0) { + volatile union idpf_flex_tx_ctx_desc *ctx_desc = + (volatile union idpf_flex_tx_ctx_desc *)&txr[tx_id]; + + if ((ol_flags & RTE_MBUF_F_TX_TCP_SEG) != 0) + idpf_set_splitq_tso_ctx(tx_pkt, tx_offload, + ctx_desc); + + tx_id++; + if (tx_id == txq->nb_tx_desc) + tx_id = 0; + } do { txd = &txr[tx_id]; @@ -1842,14 +1910,17 @@ idpf_singleq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, { volatile struct idpf_flex_tx_desc *txd; volatile struct idpf_flex_tx_desc *txr; + union idpf_tx_offload tx_offload = {0}; struct idpf_tx_entry *txe, *txn; struct idpf_tx_entry *sw_ring; struct idpf_tx_queue *txq; struct rte_mbuf *tx_pkt; struct rte_mbuf *m_seg; uint64_t buf_dma_addr; + uint64_t ol_flags; uint16_t tx_last; uint16_t nb_used; + uint16_t nb_ctx; uint16_t td_cmd; uint16_t tx_id; uint16_t nb_tx; @@ -1876,11 +1947,19 @@ idpf_singleq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, tx_pkt = *tx_pkts++; RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf); + ol_flags = tx_pkt->ol_flags; + tx_offload.l2_len = tx_pkt->l2_len; + tx_offload.l3_len = tx_pkt->l3_len; + tx_offload.l4_len = tx_pkt->l4_len; + tx_offload.tso_segsz = tx_pkt->tso_segsz; + /* Calculate the number of context descriptors needed. */ + nb_ctx = idpf_calc_context_desc(ol_flags); + /* The number of descriptors that must be allocated for * a packet equals to the number of the segments of that * packet plus 1 context descriptor if needed. */ - nb_used = (uint16_t)(tx_pkt->nb_segs); + nb_used = (uint16_t)(tx_pkt->nb_segs + nb_ctx); tx_last = (uint16_t)(tx_id + nb_used - 1); /* Circular ring */ @@ -1908,6 +1987,29 @@ idpf_singleq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } } + if (nb_ctx != 0) { + /* Setup TX context descriptor if required */ + volatile union idpf_flex_tx_ctx_desc *ctx_txd = + (volatile union idpf_flex_tx_ctx_desc *) + &txr[tx_id]; + + txn = &sw_ring[txe->next_id]; + RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf); + if (txe->mbuf != NULL) { + rte_pktmbuf_free_seg(txe->mbuf); + txe->mbuf = NULL; + } + + /* TSO enabled */ + if ((ol_flags & RTE_MBUF_F_TX_TCP_SEG) != 0) + idpf_set_splitq_tso_ctx(tx_pkt, tx_offload, + ctx_txd); + + txe->last_id = tx_last; + tx_id = txe->next_id; + txe = txn; + } + m_seg = tx_pkt; do { txd = &txr[tx_id]; @@ -1968,16 +2070,44 @@ idpf_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { int i, ret; + uint64_t ol_flags; struct rte_mbuf *m; for (i = 0; i < nb_pkts; i++) { m = tx_pkts[i]; + ol_flags = m->ol_flags; + + /* Check condition for nb_segs > IDPF_TX_MAX_MTU_SEG. */ + if ((ol_flags & RTE_MBUF_F_TX_TCP_SEG) == 0) { + if (m->nb_segs > IDPF_TX_MAX_MTU_SEG) { + rte_errno = EINVAL; + return i; + } + } else if ((m->tso_segsz < IDPF_MIN_TSO_MSS) || + (m->tso_segsz > IDPF_MAX_TSO_MSS) || + (m->pkt_len > IDPF_MAX_TSO_FRAME_SIZE)) { + /* MSS outside the range are considered malicious */ + rte_errno = EINVAL; + return i; + } + + if ((ol_flags & IDPF_TX_OFFLOAD_NOTSUP_MASK) != 0) { + rte_errno = ENOTSUP; + return i; + } if (m->pkt_len < IDPF_MIN_FRAME_SIZE) { rte_errno = EINVAL; return i; } +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + ret = rte_validate_tx_offload(m); + if (ret != 0) { + rte_errno = -ret; + return i; + } +#endif ret = rte_net_intel_cksum_prepare(m); if (ret != 0) { rte_errno = -ret; diff --git a/drivers/net/idpf/idpf_rxtx.h b/drivers/net/idpf/idpf_rxtx.h index eb0b230d3a..efb2734d85 100644 --- a/drivers/net/idpf/idpf_rxtx.h +++ b/drivers/net/idpf/idpf_rxtx.h @@ -21,6 +21,16 @@ #define IDPF_DEFAULT_TX_RS_THRESH 32 #define IDPF_DEFAULT_TX_FREE_THRESH 32 +#define IDPF_MIN_TSO_MSS 88 +#define IDPF_MAX_TSO_MSS 9728 +#define IDPF_MAX_TSO_FRAME_SIZE 262143 +#define IDPF_TX_MAX_MTU_SEG 10 + +#define IDPF_TX_OFFLOAD_MASK RTE_MBUF_F_TX_TCP_SEG + +#define IDPF_TX_OFFLOAD_NOTSUP_MASK \ + (RTE_MBUF_F_TX_OFFLOAD_MASK ^ IDPF_TX_OFFLOAD_MASK) + #define IDPF_GET_PTYPE_SIZE(p) \ (sizeof(struct virtchnl2_ptype) + \ (((p)->proto_id_count ? ((p)->proto_id_count - 1) : 0) * sizeof((p)->proto_id[0]))) @@ -115,6 +125,18 @@ struct idpf_tx_queue { struct idpf_tx_queue *complq; }; +/* Offload features */ +union idpf_tx_offload { + uint64_t data; + struct { + uint64_t l2_len:7; /* L2 (MAC) Header Length. */ + uint64_t l3_len:9; /* L3 (IP) Header Length. */ + uint64_t l4_len:8; /* L4 Header Length. */ + uint64_t tso_segsz:16; /* TCP TSO segment size */ + /* uint64_t unused : 24; */ + }; +}; + struct idpf_rxq_ops { void (*release_mbufs)(struct idpf_rx_queue *rxq); }; From patchwork Mon Oct 24 13:01:33 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 119003 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 3721CA034C; Mon, 24 Oct 2022 15:05:28 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 81DD942B92; Mon, 24 Oct 2022 15:04:54 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id 415A242B8E for ; Mon, 24 Oct 2022 15:04:52 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616692; x=1698152692; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=NteOEebsOBCh22LdS2wpSQ1pmNg5yCYcqngTUdTKLTI=; b=Yvle3GvYxdk1jpbGkTvI/3ZOu93kSJq8Evf9nttEzO4TBWgiIlInn3BB S7ZlqPRRRa5CYEqA3xNEWJdUjdP6VZszGsI8r2vxsa33DAXGpL3+jNrHI DcnQhZxUUBt7Hp6i79jee3cdhrVZU8e1ME7cFTIL3Ks02jEhQiZJ71cZ+ lF/pUzQzEaxpVjEGrqbf1PRtrnU5/gvCM38Miv/4UDQC4JgWY0e9816N7 jRmX0tUZ7/sB5dtGCuMP0Bt2N5I17IifqzxpFohT5MIE5bHFNwVsVCi+V jc1Ujn9xg/qUBIqpf7mbpjySiPzIU39Kb3qB6hXaI1U7WyFxha7wi1IRd Q==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305018946" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018946" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:40 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248481" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248481" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:38 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo , Wenjun Wu Subject: [PATCH v10 13/14] net/idpf: add AVX512 data path for single queue model Date: Mon, 24 Oct 2022 21:01:33 +0800 Message-Id: <20221024130134.1046536-14-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add support of AVX512 vector data path for single queue model. Signed-off-by: Wenjun Wu Signed-off-by: Junfeng Guo --- doc/guides/nics/idpf.rst | 19 + drivers/net/idpf/idpf_ethdev.h | 5 + drivers/net/idpf/idpf_rxtx.c | 145 ++++ drivers/net/idpf/idpf_rxtx.h | 22 + drivers/net/idpf/idpf_rxtx_vec_avx512.c | 871 ++++++++++++++++++++++++ drivers/net/idpf/idpf_rxtx_vec_common.h | 100 +++ drivers/net/idpf/meson.build | 28 + 7 files changed, 1190 insertions(+) create mode 100644 drivers/net/idpf/idpf_rxtx_vec_avx512.c create mode 100644 drivers/net/idpf/idpf_rxtx_vec_common.h diff --git a/doc/guides/nics/idpf.rst b/doc/guides/nics/idpf.rst index c1001d5d0c..3039c61748 100644 --- a/doc/guides/nics/idpf.rst +++ b/doc/guides/nics/idpf.rst @@ -64,3 +64,22 @@ Refer to the document :ref:`compiling and testing a PMD for a NIC #include +#include #include "idpf_ethdev.h" #include "idpf_rxtx.h" +#include "idpf_rxtx_vec_common.h" const uint32_t * idpf_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) @@ -271,6 +273,8 @@ reset_single_rx_queue(struct idpf_rx_queue *rxq) rxq->pkt_first_seg = NULL; rxq->pkt_last_seg = NULL; + rxq->rxrearm_start = 0; + rxq->rxrearm_nb = 0; } static inline void @@ -2118,25 +2122,166 @@ idpf_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, return i; } +static void __rte_cold +release_rxq_mbufs_vec(struct idpf_rx_queue *rxq) +{ + const uint16_t mask = rxq->nb_rx_desc - 1; + uint16_t i; + + if (rxq->sw_ring == NULL || rxq->rxrearm_nb >= rxq->nb_rx_desc) + return; + + /* free all mbufs that are valid in the ring */ + if (rxq->rxrearm_nb == 0) { + for (i = 0; i < rxq->nb_rx_desc; i++) { + if (rxq->sw_ring[i] != NULL) + rte_pktmbuf_free_seg(rxq->sw_ring[i]); + } + } else { + for (i = rxq->rx_tail; i != rxq->rxrearm_start; i = (i + 1) & mask) { + if (rxq->sw_ring[i] != NULL) + rte_pktmbuf_free_seg(rxq->sw_ring[i]); + } + } + + rxq->rxrearm_nb = rxq->nb_rx_desc; + + /* set all entries to NULL */ + memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc); +} + +static const struct idpf_rxq_ops def_singleq_rx_ops_vec = { + .release_mbufs = release_rxq_mbufs_vec, +}; + +static inline int +idpf_singleq_rx_vec_setup_default(struct idpf_rx_queue *rxq) +{ + uintptr_t p; + struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */ + + mb_def.nb_segs = 1; + mb_def.data_off = RTE_PKTMBUF_HEADROOM; + mb_def.port = rxq->port_id; + rte_mbuf_refcnt_set(&mb_def, 1); + + /* prevent compiler reordering: rearm_data covers previous fields */ + rte_compiler_barrier(); + p = (uintptr_t)&mb_def.rearm_data; + rxq->mbuf_initializer = *(uint64_t *)p; + return 0; +} + +int __rte_cold +idpf_singleq_rx_vec_setup(struct idpf_rx_queue *rxq) +{ + rxq->ops = &def_singleq_rx_ops_vec; + return idpf_singleq_rx_vec_setup_default(rxq); +} + void idpf_set_rx_function(struct rte_eth_dev *dev) { struct idpf_vport *vport = dev->data->dev_private; +#ifdef RTE_ARCH_X86 + struct idpf_adapter *ad = vport->adapter; + struct idpf_rx_queue *rxq; + int i; + + if (idpf_rx_vec_dev_check_default(dev) == IDPF_VECTOR_PATH && + rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { + ad->rx_vec_allowed = true; + + if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) +#ifdef CC_AVX512_SUPPORT + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1) + ad->rx_use_avx512 = true; +#else + PMD_DRV_LOG(NOTICE, + "AVX512 is not supported in build env"); +#endif /* CC_AVX512_SUPPORT */ + } else { + ad->rx_vec_allowed = false; + } +#endif /* RTE_ARCH_X86 */ + +#ifdef RTE_ARCH_X86 + if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) { + dev->rx_pkt_burst = idpf_splitq_recv_pkts; + } else { + if (ad->rx_vec_allowed) { + for (i = 0; i < dev->data->nb_tx_queues; i++) { + rxq = dev->data->rx_queues[i]; + (void)idpf_singleq_rx_vec_setup(rxq); + } +#ifdef CC_AVX512_SUPPORT + if (ad->rx_use_avx512) { + dev->rx_pkt_burst = idpf_singleq_recv_pkts_avx512; + return; + } +#endif /* CC_AVX512_SUPPORT */ + } + dev->rx_pkt_burst = idpf_singleq_recv_pkts; + } +#else if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) dev->rx_pkt_burst = idpf_splitq_recv_pkts; else dev->rx_pkt_burst = idpf_singleq_recv_pkts; +#endif /* RTE_ARCH_X86 */ } void idpf_set_tx_function(struct rte_eth_dev *dev) { struct idpf_vport *vport = dev->data->dev_private; +#ifdef RTE_ARCH_X86 + struct idpf_adapter *ad = vport->adapter; +#ifdef CC_AVX512_SUPPORT + struct idpf_tx_queue *txq; + int i; +#endif /* CC_AVX512_SUPPORT */ + + if (idpf_rx_vec_dev_check_default(dev) == IDPF_VECTOR_PATH && + rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { + ad->tx_vec_allowed = true; + if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512) +#ifdef CC_AVX512_SUPPORT + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1) + ad->tx_use_avx512 = true; +#else + PMD_DRV_LOG(NOTICE, + "AVX512 is not supported in build env"); +#endif /* CC_AVX512_SUPPORT */ + } else { + ad->tx_vec_allowed = false; + } +#endif /* RTE_ARCH_X86 */ + if (vport->txq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) { dev->tx_pkt_burst = idpf_splitq_xmit_pkts; dev->tx_pkt_prepare = idpf_prep_pkts; } else { +#ifdef RTE_ARCH_X86 + if (ad->tx_vec_allowed) { +#ifdef CC_AVX512_SUPPORT + if (ad->tx_use_avx512) { + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + if (txq == NULL) + continue; + idpf_singleq_tx_vec_setup_avx512(txq); + } + dev->tx_pkt_burst = idpf_singleq_xmit_pkts_avx512; + dev->tx_pkt_prepare = idpf_prep_pkts; + return; + } +#endif /* CC_AVX512_SUPPORT */ + } +#endif /* RTE_ARCH_X86 */ dev->tx_pkt_burst = idpf_singleq_xmit_pkts; dev->tx_pkt_prepare = idpf_prep_pkts; } diff --git a/drivers/net/idpf/idpf_rxtx.h b/drivers/net/idpf/idpf_rxtx.h index efb2734d85..e808710b41 100644 --- a/drivers/net/idpf/idpf_rxtx.h +++ b/drivers/net/idpf/idpf_rxtx.h @@ -18,6 +18,12 @@ #define IDPF_RX_MAX_BURST 32 #define IDPF_DEFAULT_RX_FREE_THRESH 32 +/* used for Vector PMD */ +#define IDPF_VPMD_RX_MAX_BURST 32 +#define IDPF_VPMD_TX_MAX_BURST 32 +#define IDPF_VPMD_DESCS_PER_LOOP 4 +#define IDPF_RXQ_REARM_THRESH 64 + #define IDPF_DEFAULT_TX_RS_THRESH 32 #define IDPF_DEFAULT_TX_FREE_THRESH 32 @@ -52,6 +58,11 @@ struct idpf_rx_queue { struct rte_mbuf *pkt_last_seg; /* last segment of current packet */ struct rte_mbuf fake_mbuf; /* dummy mbuf */ + /* used for VPMD */ + uint16_t rxrearm_nb; /* number of remaining to be re-armed */ + uint16_t rxrearm_start; /* the idx we start the re-arming from */ + uint64_t mbuf_initializer; /* value to init mbufs */ + uint16_t rx_nb_avail; uint16_t rx_next_avail; @@ -82,6 +93,10 @@ struct idpf_tx_entry { uint16_t last_id; }; +struct idpf_tx_vec_entry { + struct rte_mbuf *mbuf; +}; + /* Structure associated with each TX queue. */ struct idpf_tx_queue { const struct rte_memzone *mz; /* memzone for Tx ring */ @@ -166,12 +181,19 @@ uint16_t idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); uint16_t idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +uint16_t idpf_singleq_recv_pkts_avx512(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); uint16_t idpf_singleq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); uint16_t idpf_splitq_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); +uint16_t idpf_singleq_xmit_pkts_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); +int idpf_singleq_tx_vec_setup_avx512(struct idpf_tx_queue *txq); uint16_t idpf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); +int idpf_singleq_rx_vec_setup(struct idpf_rx_queue *rxq); + void idpf_stop_queues(struct rte_eth_dev *dev); void idpf_set_rx_function(struct rte_eth_dev *dev); diff --git a/drivers/net/idpf/idpf_rxtx_vec_avx512.c b/drivers/net/idpf/idpf_rxtx_vec_avx512.c new file mode 100644 index 0000000000..2e8b52b795 --- /dev/null +++ b/drivers/net/idpf/idpf_rxtx_vec_avx512.c @@ -0,0 +1,871 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2022 Intel Corporation + */ + +#include "idpf_rxtx_vec_common.h" + +#include + +#ifndef __INTEL_COMPILER +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif + +#define IDPF_DESCS_PER_LOOP_AVX 8 +#define PKTLEN_SHIFT 10 + +/****************************************************************************** + * If user knows a specific offload is not enabled by APP, + * the macro can be commented to save the effort of fast path. + * Currently below 1 feature is supported in RX path, + * 1, packet type analysis + ******************************************************************************/ +#define IDPF_RX_PTYPE_OFFLOAD + +static __rte_always_inline void +idpf_singleq_rearm_common(struct idpf_rx_queue *rxq) +{ + struct rte_mbuf **rxp = &rxq->sw_ring[rxq->rxrearm_start]; + volatile union virtchnl2_rx_desc *rxdp = rxq->rx_ring; + uint16_t rx_id; + int i; + + rxdp += rxq->rxrearm_start; + + /* Pull 'n' more MBUFs into the software ring */ + if (rte_mempool_get_bulk(rxq->mp, + (void *)rxp, + IDPF_RXQ_REARM_THRESH) < 0) { + if (rxq->rxrearm_nb + IDPF_RXQ_REARM_THRESH >= + rxq->nb_rx_desc) { + __m128i dma_addr0; + + dma_addr0 = _mm_setzero_si128(); + for (i = 0; i < IDPF_VPMD_DESCS_PER_LOOP; i++) { + rxp[i] = &rxq->fake_mbuf; + _mm_store_si128((__m128i *)&rxdp[i].read, + dma_addr0); + } + } + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += + IDPF_RXQ_REARM_THRESH; + return; + } + struct rte_mbuf *mb0, *mb1, *mb2, *mb3; + struct rte_mbuf *mb4, *mb5, *mb6, *mb7; + __m512i dma_addr0_3, dma_addr4_7; + __m512i hdr_room = _mm512_set1_epi64(RTE_PKTMBUF_HEADROOM); + /* Initialize the mbufs in vector, process 8 mbufs in one loop */ + for (i = 0; i < IDPF_RXQ_REARM_THRESH; + i += 8, rxp += 8, rxdp += 8) { + __m128i vaddr0, vaddr1, vaddr2, vaddr3; + __m128i vaddr4, vaddr5, vaddr6, vaddr7; + __m256i vaddr0_1, vaddr2_3; + __m256i vaddr4_5, vaddr6_7; + __m512i vaddr0_3, vaddr4_7; + + mb0 = rxp[0]; + mb1 = rxp[1]; + mb2 = rxp[2]; + mb3 = rxp[3]; + mb4 = rxp[4]; + mb5 = rxp[5]; + mb6 = rxp[6]; + mb7 = rxp[7]; + + /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) != + offsetof(struct rte_mbuf, buf_addr) + 8); + vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr); + vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr); + vaddr2 = _mm_loadu_si128((__m128i *)&mb2->buf_addr); + vaddr3 = _mm_loadu_si128((__m128i *)&mb3->buf_addr); + vaddr4 = _mm_loadu_si128((__m128i *)&mb4->buf_addr); + vaddr5 = _mm_loadu_si128((__m128i *)&mb5->buf_addr); + vaddr6 = _mm_loadu_si128((__m128i *)&mb6->buf_addr); + vaddr7 = _mm_loadu_si128((__m128i *)&mb7->buf_addr); + + /** + * merge 0 & 1, by casting 0 to 256-bit and inserting 1 + * into the high lanes. Similarly for 2 & 3, and so on. + */ + vaddr0_1 = + _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr0), + vaddr1, 1); + vaddr2_3 = + _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr2), + vaddr3, 1); + vaddr4_5 = + _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr4), + vaddr5, 1); + vaddr6_7 = + _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr6), + vaddr7, 1); + vaddr0_3 = + _mm512_inserti64x4(_mm512_castsi256_si512(vaddr0_1), + vaddr2_3, 1); + vaddr4_7 = + _mm512_inserti64x4(_mm512_castsi256_si512(vaddr4_5), + vaddr6_7, 1); + + /* convert pa to dma_addr hdr/data */ + dma_addr0_3 = _mm512_unpackhi_epi64(vaddr0_3, vaddr0_3); + dma_addr4_7 = _mm512_unpackhi_epi64(vaddr4_7, vaddr4_7); + + /* add headroom to pa values */ + dma_addr0_3 = _mm512_add_epi64(dma_addr0_3, hdr_room); + dma_addr4_7 = _mm512_add_epi64(dma_addr4_7, hdr_room); + + /* flush desc with pa dma_addr */ + _mm512_store_si512((__m512i *)&rxdp->read, dma_addr0_3); + _mm512_store_si512((__m512i *)&(rxdp + 4)->read, dma_addr4_7); + } + + rxq->rxrearm_start += IDPF_RXQ_REARM_THRESH; + if (rxq->rxrearm_start >= rxq->nb_rx_desc) + rxq->rxrearm_start = 0; + + rxq->rxrearm_nb -= IDPF_RXQ_REARM_THRESH; + + rx_id = (uint16_t)((rxq->rxrearm_start == 0) ? + (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); + + /* Update the tail pointer on the NIC */ + IDPF_PCI_REG_WRITE(rxq->qrx_tail, rx_id); +} + +static __rte_always_inline void +idpf_singleq_rearm(struct idpf_rx_queue *rxq) +{ + int i; + uint16_t rx_id; + volatile union virtchnl2_rx_desc *rxdp = rxq->rx_ring; + struct rte_mempool_cache *cache = + rte_mempool_default_cache(rxq->mp, rte_lcore_id()); + struct rte_mbuf **rxp = &rxq->sw_ring[rxq->rxrearm_start]; + + rxdp += rxq->rxrearm_start; + + if (unlikely(cache == NULL)) + return idpf_singleq_rearm_common(rxq); + + /* We need to pull 'n' more MBUFs into the software ring from mempool + * We inline the mempool function here, so we can vectorize the copy + * from the cache into the shadow ring. + */ + + /* Can this be satisfied from the cache? */ + if (cache->len < IDPF_RXQ_REARM_THRESH) { + /* No. Backfill the cache first, and then fill from it */ + uint32_t req = IDPF_RXQ_REARM_THRESH + (cache->size - + cache->len); + + /* How many do we require i.e. number to fill the cache + the request */ + int ret = rte_mempool_ops_dequeue_bulk + (rxq->mp, &cache->objs[cache->len], req); + if (ret == 0) { + cache->len += req; + } else { + if (rxq->rxrearm_nb + IDPF_RXQ_REARM_THRESH >= + rxq->nb_rx_desc) { + __m128i dma_addr0; + + dma_addr0 = _mm_setzero_si128(); + for (i = 0; i < IDPF_VPMD_DESCS_PER_LOOP; i++) { + rxp[i] = &rxq->fake_mbuf; + _mm_storeu_si128((__m128i *)&rxdp[i].read, + dma_addr0); + } + } + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += + IDPF_RXQ_REARM_THRESH; + return; + } + } + + const __m512i iova_offsets = _mm512_set1_epi64(offsetof + (struct rte_mbuf, buf_iova)); + const __m512i headroom = _mm512_set1_epi64(RTE_PKTMBUF_HEADROOM); + + /* to shuffle the addresses to correct slots. Values 4-7 will contain + * zeros, so use 7 for a zero-value. + */ + const __m512i permute_idx = _mm512_set_epi64(7, 7, 3, 1, 7, 7, 2, 0); + + /* Initialize the mbufs in vector, process 8 mbufs in one loop, taking + * from mempool cache and populating both shadow and HW rings + */ + for (i = 0; i < IDPF_RXQ_REARM_THRESH / IDPF_DESCS_PER_LOOP_AVX; i++) { + const __m512i mbuf_ptrs = _mm512_loadu_si512 + (&cache->objs[cache->len - IDPF_DESCS_PER_LOOP_AVX]); + _mm512_storeu_si512(rxp, mbuf_ptrs); + + const __m512i iova_base_addrs = _mm512_i64gather_epi64 + (_mm512_add_epi64(mbuf_ptrs, iova_offsets), + 0, /* base */ + 1 /* scale */); + const __m512i iova_addrs = _mm512_add_epi64(iova_base_addrs, + headroom); + const __m512i iovas0 = _mm512_castsi256_si512 + (_mm512_extracti64x4_epi64(iova_addrs, 0)); + const __m512i iovas1 = _mm512_castsi256_si512 + (_mm512_extracti64x4_epi64(iova_addrs, 1)); + + /* permute leaves desc 2-3 addresses in header address slots 0-1 + * but these are ignored by driver since header split not + * enabled. Similarly for desc 6 & 7. + */ + const __m512i desc0_1 = _mm512_permutexvar_epi64 + (permute_idx, + iovas0); + const __m512i desc2_3 = _mm512_bsrli_epi128(desc0_1, 8); + + const __m512i desc4_5 = _mm512_permutexvar_epi64 + (permute_idx, + iovas1); + const __m512i desc6_7 = _mm512_bsrli_epi128(desc4_5, 8); + + _mm512_storeu_si512((void *)rxdp, desc0_1); + _mm512_storeu_si512((void *)(rxdp + 2), desc2_3); + _mm512_storeu_si512((void *)(rxdp + 4), desc4_5); + _mm512_storeu_si512((void *)(rxdp + 6), desc6_7); + + rxp += IDPF_DESCS_PER_LOOP_AVX; + rxdp += IDPF_DESCS_PER_LOOP_AVX; + cache->len -= IDPF_DESCS_PER_LOOP_AVX; + } + + rxq->rxrearm_start += IDPF_RXQ_REARM_THRESH; + if (rxq->rxrearm_start >= rxq->nb_rx_desc) + rxq->rxrearm_start = 0; + + rxq->rxrearm_nb -= IDPF_RXQ_REARM_THRESH; + + rx_id = (uint16_t)((rxq->rxrearm_start == 0) ? + (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); + + /* Update the tail pointer on the NIC */ + IDPF_PCI_REG_WRITE(rxq->qrx_tail, rx_id); +} + +#define IDPF_RX_LEN_MASK 0x80808080 +static __rte_always_inline uint16_t +_idpf_singleq_recv_raw_pkts_avx512(struct idpf_rx_queue *rxq, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ +#ifdef IDPF_RX_PTYPE_OFFLOAD + const uint32_t *type_table = rxq->adapter->ptype_tbl; +#endif + + const __m256i mbuf_init = _mm256_set_epi64x(0, 0, 0, + rxq->mbuf_initializer); + struct rte_mbuf **sw_ring = &rxq->sw_ring[rxq->rx_tail]; + volatile union virtchnl2_rx_desc *rxdp = rxq->rx_ring; + + rxdp += rxq->rx_tail; + + rte_prefetch0(rxdp); + + /* nb_pkts has to be floor-aligned to IDPF_DESCS_PER_LOOP_AVX */ + nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, IDPF_DESCS_PER_LOOP_AVX); + + /* See if we need to rearm the RX queue - gives the prefetch a bit + * of time to act + */ + if (rxq->rxrearm_nb > IDPF_RXQ_REARM_THRESH) + idpf_singleq_rearm(rxq); + + /* Before we start moving massive data around, check to see if + * there is actually a packet available + */ + if ((rxdp->flex_nic_wb.status_error0 & + rte_cpu_to_le_32(1 << VIRTCHNL2_RX_FLEX_DESC_STATUS0_DD_S)) == 0) + return 0; + + /* 8 packets DD mask, LSB in each 32-bit value */ + const __m256i dd_check = _mm256_set1_epi32(1); + + /* mask to shuffle from desc. to mbuf (4 descriptors)*/ + const __m512i shuf_msk = + _mm512_set_epi32 + (/* 1st descriptor */ + 0xFFFFFFFF, /* rss set as unknown */ + 0xFFFF0504, /* vlan_macip set as unknown */ + /* octet 15~14, 16 bits data_len */ + 0xFFFF0504, /* skip high 16 bits pkt_len, zero out */ + /* octet 15~14, low 16 bits pkt_len */ + 0xFFFFFFFF, /* pkt_type set as unknown */ + /* 2nd descriptor */ + 0xFFFFFFFF, /* rss set as unknown */ + 0xFFFF0504, /* vlan_macip set as unknown */ + /* octet 15~14, 16 bits data_len */ + 0xFFFF0504, /* skip high 16 bits pkt_len, zero out */ + /* octet 15~14, low 16 bits pkt_len */ + 0xFFFFFFFF, /* pkt_type set as unknown */ + /* 3rd descriptor */ + 0xFFFFFFFF, /* rss set as unknown */ + 0xFFFF0504, /* vlan_macip set as unknown */ + /* octet 15~14, 16 bits data_len */ + 0xFFFF0504, /* skip high 16 bits pkt_len, zero out */ + /* octet 15~14, low 16 bits pkt_len */ + 0xFFFFFFFF, /* pkt_type set as unknown */ + /* 4th descriptor */ + 0xFFFFFFFF, /* rss set as unknown */ + 0xFFFF0504, /* vlan_macip set as unknown */ + /* octet 15~14, 16 bits data_len */ + 0xFFFF0504, /* skip high 16 bits pkt_len, zero out */ + /* octet 15~14, low 16 bits pkt_len */ + 0xFFFFFFFF /* pkt_type set as unknown */ + ); + /** + * compile-time check the shuffle layout is correct. + * NOTE: the first field (lowest address) is given last in set_epi + * calls above. + */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) != + offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12); + + uint16_t i, received; + + for (i = 0, received = 0; i < nb_pkts; + i += IDPF_DESCS_PER_LOOP_AVX, + rxdp += IDPF_DESCS_PER_LOOP_AVX) { + /* step 1, copy over 8 mbuf pointers to rx_pkts array */ + _mm256_storeu_si256((void *)&rx_pkts[i], + _mm256_loadu_si256((void *)&sw_ring[i])); +#ifdef RTE_ARCH_X86_64 + _mm256_storeu_si256 + ((void *)&rx_pkts[i + 4], + _mm256_loadu_si256((void *)&sw_ring[i + 4])); +#endif + + __m512i raw_desc0_3, raw_desc4_7; + const __m128i raw_desc7 = + _mm_load_si128((void *)(rxdp + 7)); + rte_compiler_barrier(); + const __m128i raw_desc6 = + _mm_load_si128((void *)(rxdp + 6)); + rte_compiler_barrier(); + const __m128i raw_desc5 = + _mm_load_si128((void *)(rxdp + 5)); + rte_compiler_barrier(); + const __m128i raw_desc4 = + _mm_load_si128((void *)(rxdp + 4)); + rte_compiler_barrier(); + const __m128i raw_desc3 = + _mm_load_si128((void *)(rxdp + 3)); + rte_compiler_barrier(); + const __m128i raw_desc2 = + _mm_load_si128((void *)(rxdp + 2)); + rte_compiler_barrier(); + const __m128i raw_desc1 = + _mm_load_si128((void *)(rxdp + 1)); + rte_compiler_barrier(); + const __m128i raw_desc0 = + _mm_load_si128((void *)(rxdp + 0)); + + raw_desc4_7 = _mm512_broadcast_i32x4(raw_desc4); + raw_desc4_7 = _mm512_inserti32x4(raw_desc4_7, raw_desc5, 1); + raw_desc4_7 = _mm512_inserti32x4(raw_desc4_7, raw_desc6, 2); + raw_desc4_7 = _mm512_inserti32x4(raw_desc4_7, raw_desc7, 3); + raw_desc0_3 = _mm512_broadcast_i32x4(raw_desc0); + raw_desc0_3 = _mm512_inserti32x4(raw_desc0_3, raw_desc1, 1); + raw_desc0_3 = _mm512_inserti32x4(raw_desc0_3, raw_desc2, 2); + raw_desc0_3 = _mm512_inserti32x4(raw_desc0_3, raw_desc3, 3); + + /** + * convert descriptors 4-7 into mbufs, adjusting length and + * re-arranging fields. Then write into the mbuf + */ + const __m512i len4_7 = _mm512_slli_epi32(raw_desc4_7, + PKTLEN_SHIFT); + const __m512i desc4_7 = _mm512_mask_blend_epi16(IDPF_RX_LEN_MASK, + raw_desc4_7, + len4_7); + __m512i mb4_7 = _mm512_shuffle_epi8(desc4_7, shuf_msk); + +#ifdef IDPF_RX_PTYPE_OFFLOAD + /** + * to get packet types, shift 64-bit values down 30 bits + * and so ptype is in lower 8-bits in each + */ + const __m512i ptypes4_7 = _mm512_srli_epi64(desc4_7, 16); + const __m256i ptypes6_7 = _mm512_extracti64x4_epi64(ptypes4_7, 1); + const __m256i ptypes4_5 = _mm512_extracti64x4_epi64(ptypes4_7, 0); + const uint8_t ptype7 = _mm256_extract_epi8(ptypes6_7, 16); + const uint8_t ptype6 = _mm256_extract_epi8(ptypes6_7, 0); + const uint8_t ptype5 = _mm256_extract_epi8(ptypes4_5, 16); + const uint8_t ptype4 = _mm256_extract_epi8(ptypes4_5, 0); + + const __m512i ptype4_7 = _mm512_set_epi32 + (0, 0, 0, type_table[ptype7], + 0, 0, 0, type_table[ptype6], + 0, 0, 0, type_table[ptype5], + 0, 0, 0, type_table[ptype4]); + mb4_7 = _mm512_mask_blend_epi32(0x1111, mb4_7, ptype4_7); +#endif + + /** + * convert descriptors 0-3 into mbufs, adjusting length and + * re-arranging fields. Then write into the mbuf + */ + const __m512i len0_3 = _mm512_slli_epi32(raw_desc0_3, + PKTLEN_SHIFT); + const __m512i desc0_3 = _mm512_mask_blend_epi16(IDPF_RX_LEN_MASK, + raw_desc0_3, + len0_3); + __m512i mb0_3 = _mm512_shuffle_epi8(desc0_3, shuf_msk); + +#ifdef IDPF_RX_PTYPE_OFFLOAD + /* get the packet types */ + const __m512i ptypes0_3 = _mm512_srli_epi64(desc0_3, 16); + const __m256i ptypes2_3 = _mm512_extracti64x4_epi64(ptypes0_3, 1); + const __m256i ptypes0_1 = _mm512_extracti64x4_epi64(ptypes0_3, 0); + const uint8_t ptype3 = _mm256_extract_epi8(ptypes2_3, 16); + const uint8_t ptype2 = _mm256_extract_epi8(ptypes2_3, 0); + const uint8_t ptype1 = _mm256_extract_epi8(ptypes0_1, 16); + const uint8_t ptype0 = _mm256_extract_epi8(ptypes0_1, 0); + + const __m512i ptype0_3 = _mm512_set_epi32 + (0, 0, 0, type_table[ptype3], + 0, 0, 0, type_table[ptype2], + 0, 0, 0, type_table[ptype1], + 0, 0, 0, type_table[ptype0]); + mb0_3 = _mm512_mask_blend_epi32(0x1111, mb0_3, ptype0_3); +#endif + + /** + * use permute/extract to get status content + * After the operations, the packets status flags are in the + * order (hi->lo): [1, 3, 5, 7, 0, 2, 4, 6] + */ + /* merge the status bits into one register */ + const __m512i status_permute_msk = _mm512_set_epi32 + (0, 0, 0, 0, + 0, 0, 0, 0, + 22, 30, 6, 14, + 18, 26, 2, 10); + const __m512i raw_status0_7 = _mm512_permutex2var_epi32 + (raw_desc4_7, status_permute_msk, raw_desc0_3); + __m256i status0_7 = _mm512_extracti64x4_epi64 + (raw_status0_7, 0); + + /* now do flag manipulation */ + + /** + * At this point, we have the 8 sets of flags in the low 16-bits + * of each 32-bit value. + * We want to extract these, and merge them with the mbuf init + * data so we can do a single write to the mbuf to set the flags + * and all the other initialization fields. Extracting the + * appropriate flags means that we have to do a shift and blend + * for each mbuf before we do the write. However, we can also + * add in the previously computed rx_descriptor fields to + * make a single 256-bit write per mbuf + */ + /* check the structure matches expectations */ + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) != + offsetof(struct rte_mbuf, rearm_data) + 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) != + RTE_ALIGN(offsetof(struct rte_mbuf, + rearm_data), + 16)); + /* build up data and do writes */ + __m256i rearm0, rearm1, rearm2, rearm3, rearm4, rearm5, + rearm6, rearm7; + const __m256i mb4_5 = _mm512_extracti64x4_epi64(mb4_7, 0); + const __m256i mb6_7 = _mm512_extracti64x4_epi64(mb4_7, 1); + const __m256i mb0_1 = _mm512_extracti64x4_epi64(mb0_3, 0); + const __m256i mb2_3 = _mm512_extracti64x4_epi64(mb0_3, 1); + + rearm6 = _mm256_permute2f128_si256(mbuf_init, mb6_7, 0x20); + rearm4 = _mm256_permute2f128_si256(mbuf_init, mb4_5, 0x20); + rearm2 = _mm256_permute2f128_si256(mbuf_init, mb2_3, 0x20); + rearm0 = _mm256_permute2f128_si256(mbuf_init, mb0_1, 0x20); + + /* write to mbuf */ + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, + rearm6); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, + rearm4); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, + rearm2); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, + rearm0); + + rearm7 = _mm256_blend_epi32(mbuf_init, mb6_7, 0xF0); + rearm5 = _mm256_blend_epi32(mbuf_init, mb4_5, 0xF0); + rearm3 = _mm256_blend_epi32(mbuf_init, mb2_3, 0xF0); + rearm1 = _mm256_blend_epi32(mbuf_init, mb0_1, 0xF0); + + /* again write to mbufs */ + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, + rearm7); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, + rearm5); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, + rearm3); + _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, + rearm1); + + /* perform dd_check */ + status0_7 = _mm256_and_si256(status0_7, dd_check); + status0_7 = _mm256_packs_epi32(status0_7, + _mm256_setzero_si256()); + + uint64_t burst = __builtin_popcountll + (_mm_cvtsi128_si64 + (_mm256_extracti128_si256 + (status0_7, 1))); + burst += __builtin_popcountll + (_mm_cvtsi128_si64 + (_mm256_castsi256_si128(status0_7))); + received += burst; + if (burst != IDPF_DESCS_PER_LOOP_AVX) + break; + } + + /* update tail pointers */ + rxq->rx_tail += received; + rxq->rx_tail &= (rxq->nb_rx_desc - 1); + if ((rxq->rx_tail & 1) == 1 && received > 1) { /* keep aligned */ + rxq->rx_tail--; + received--; + } + rxq->rxrearm_nb += received; + return received; +} + +/** + * Notice: + * - nb_pkts < IDPF_DESCS_PER_LOOP, just return no packet + */ +uint16_t +idpf_singleq_recv_pkts_avx512(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + return _idpf_singleq_recv_raw_pkts_avx512(rx_queue, rx_pkts, nb_pkts); +} + +static __rte_always_inline int +idpf_tx_free_bufs_avx512(struct idpf_tx_queue *txq) +{ + struct idpf_tx_vec_entry *txep; + uint32_t n; + uint32_t i; + int nb_free = 0; + struct rte_mbuf *m, *free[txq->rs_thresh]; + + /* check DD bits on threshold descriptor */ + if ((txq->tx_ring[txq->next_dd].qw1.cmd_dtype & + rte_cpu_to_le_64(IDPF_TXD_QW1_DTYPE_M)) != + rte_cpu_to_le_64(IDPF_TX_DESC_DTYPE_DESC_DONE)) + return 0; + + n = txq->rs_thresh; + + /* first buffer to free from S/W ring is at index + * tx_next_dd - (tx_rs_thresh-1) + */ + txep = (void *)txq->sw_ring; + txep += txq->next_dd - (n - 1); + + if (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE && (n & 31) == 0) { + struct rte_mempool *mp = txep[0].mbuf->pool; + struct rte_mempool_cache *cache = rte_mempool_default_cache(mp, + rte_lcore_id()); + void **cache_objs; + + if (cache == NULL || cache->len == 0) + goto normal; + + cache_objs = &cache->objs[cache->len]; + + if (n > RTE_MEMPOOL_CACHE_MAX_SIZE) { + rte_mempool_ops_enqueue_bulk(mp, (void *)txep, n); + goto done; + } + + /* The cache follows the following algorithm + * 1. Add the objects to the cache + * 2. Anything greater than the cache min value (if it crosses the + * cache flush threshold) is flushed to the ring. + */ + /* Add elements back into the cache */ + uint32_t copied = 0; + /* n is multiple of 32 */ + while (copied < n) { + const __m512i a = _mm512_loadu_si512(&txep[copied]); + const __m512i b = _mm512_loadu_si512(&txep[copied + 8]); + const __m512i c = _mm512_loadu_si512(&txep[copied + 16]); + const __m512i d = _mm512_loadu_si512(&txep[copied + 24]); + + _mm512_storeu_si512(&cache_objs[copied], a); + _mm512_storeu_si512(&cache_objs[copied + 8], b); + _mm512_storeu_si512(&cache_objs[copied + 16], c); + _mm512_storeu_si512(&cache_objs[copied + 24], d); + copied += 32; + } + cache->len += n; + + if (cache->len >= cache->flushthresh) { + rte_mempool_ops_enqueue_bulk(mp, + &cache->objs[cache->size], + cache->len - cache->size); + cache->len = cache->size; + } + goto done; + } + +normal: + m = rte_pktmbuf_prefree_seg(txep[0].mbuf); + if (likely(m != NULL)) { + free[0] = m; + nb_free = 1; + for (i = 1; i < n; i++) { + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); + if (likely(m != NULL)) { + if (likely(m->pool == free[0]->pool)) { + free[nb_free++] = m; + } else { + rte_mempool_put_bulk(free[0]->pool, + (void *)free, + nb_free); + free[0] = m; + nb_free = 1; + } + } + } + rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); + } else { + for (i = 1; i < n; i++) { + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); + if (m != NULL) + rte_mempool_put(m->pool, m); + } + } + +done: + /* buffers were freed, update counters */ + txq->nb_free = (uint16_t)(txq->nb_free + txq->rs_thresh); + txq->next_dd = (uint16_t)(txq->next_dd + txq->rs_thresh); + if (txq->next_dd >= txq->nb_tx_desc) + txq->next_dd = (uint16_t)(txq->rs_thresh - 1); + + return txq->rs_thresh; +} + +static __rte_always_inline void +tx_backlog_entry_avx512(struct idpf_tx_vec_entry *txep, + struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + int i; + + for (i = 0; i < (int)nb_pkts; ++i) + txep[i].mbuf = tx_pkts[i]; +} + +#define IDPF_FLEX_TXD_QW1_BUF_SZ_S 48 +static __rte_always_inline void +idpf_vtx1(volatile struct idpf_flex_tx_desc *txdp, + struct rte_mbuf *pkt, uint64_t flags) +{ + uint64_t high_qw = + (IDPF_TX_DESC_DTYPE_FLEX_DATA << IDPF_FLEX_TXD_QW1_DTYPE_S | + ((uint64_t)flags << IDPF_FLEX_TXD_QW1_CMD_S) | + ((uint64_t)pkt->data_len << IDPF_FLEX_TXD_QW1_BUF_SZ_S)); + + __m128i descriptor = _mm_set_epi64x(high_qw, + pkt->buf_iova + pkt->data_off); + _mm_storeu_si128((__m128i *)txdp, descriptor); +} + +#define IDPF_TX_LEN_MASK 0xAA +#define IDPF_TX_OFF_MASK 0x55 +static __rte_always_inline void +idpf_vtx(volatile struct idpf_flex_tx_desc *txdp, + struct rte_mbuf **pkt, uint16_t nb_pkts, uint64_t flags) +{ + const uint64_t hi_qw_tmpl = (IDPF_TX_DESC_DTYPE_FLEX_DATA | + ((uint64_t)flags << IDPF_FLEX_TXD_QW1_CMD_S)); + + /* if unaligned on 32-bit boundary, do one to align */ + if (((uintptr_t)txdp & 0x1F) != 0 && nb_pkts != 0) { + idpf_vtx1(txdp, *pkt, flags); + nb_pkts--, txdp++, pkt++; + } + + /* do 4 at a time while possible, in bursts */ + for (; nb_pkts > 3; txdp += 4, pkt += 4, nb_pkts -= 4) { + uint64_t hi_qw3 = + hi_qw_tmpl | + ((uint64_t)pkt[3]->data_len << + IDPF_FLEX_TXD_QW1_BUF_SZ_S); + uint64_t hi_qw2 = + hi_qw_tmpl | + ((uint64_t)pkt[2]->data_len << + IDPF_FLEX_TXD_QW1_BUF_SZ_S); + uint64_t hi_qw1 = + hi_qw_tmpl | + ((uint64_t)pkt[1]->data_len << + IDPF_FLEX_TXD_QW1_BUF_SZ_S); + uint64_t hi_qw0 = + hi_qw_tmpl | + ((uint64_t)pkt[0]->data_len << + IDPF_FLEX_TXD_QW1_BUF_SZ_S); + + __m512i desc0_3 = + _mm512_set_epi64 + (hi_qw3, + pkt[3]->buf_iova + pkt[3]->data_off, + hi_qw2, + pkt[2]->buf_iova + pkt[2]->data_off, + hi_qw1, + pkt[1]->buf_iova + pkt[1]->data_off, + hi_qw0, + pkt[0]->buf_iova + pkt[0]->data_off); + _mm512_storeu_si512((void *)txdp, desc0_3); + } + + /* do any last ones */ + while (nb_pkts) { + idpf_vtx1(txdp, *pkt, flags); + txdp++, pkt++, nb_pkts--; + } +} + +static __rte_always_inline uint16_t +idpf_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct idpf_tx_queue *txq = (struct idpf_tx_queue *)tx_queue; + volatile struct idpf_flex_tx_desc *txdp; + struct idpf_tx_vec_entry *txep; + uint16_t n, nb_commit, tx_id; + uint64_t flags = IDPF_TX_FLEX_DESC_CMD_EOP; + uint64_t rs = IDPF_TX_FLEX_DESC_CMD_RS | flags; + + /* cross rx_thresh boundary is not allowed */ + nb_pkts = RTE_MIN(nb_pkts, txq->rs_thresh); + + if (txq->nb_free < txq->free_thresh) + idpf_tx_free_bufs_avx512(txq); + + nb_pkts = (uint16_t)RTE_MIN(txq->nb_free, nb_pkts); + nb_commit = nb_pkts; + if (unlikely(nb_pkts == 0)) + return 0; + + tx_id = txq->tx_tail; + txdp = &txq->tx_ring[tx_id]; + txep = (void *)txq->sw_ring; + txep += tx_id; + + txq->nb_free = (uint16_t)(txq->nb_free - nb_pkts); + + n = (uint16_t)(txq->nb_tx_desc - tx_id); + if (nb_commit >= n) { + tx_backlog_entry_avx512(txep, tx_pkts, n); + + idpf_vtx(txdp, tx_pkts, n - 1, flags); + tx_pkts += (n - 1); + txdp += (n - 1); + + idpf_vtx1(txdp, *tx_pkts++, rs); + + nb_commit = (uint16_t)(nb_commit - n); + + tx_id = 0; + txq->next_rs = (uint16_t)(txq->rs_thresh - 1); + + /* avoid reach the end of ring */ + txdp = &txq->tx_ring[tx_id]; + txep = (void *)txq->sw_ring; + txep += tx_id; + } + + tx_backlog_entry_avx512(txep, tx_pkts, nb_commit); + + idpf_vtx(txdp, tx_pkts, nb_commit, flags); + + tx_id = (uint16_t)(tx_id + nb_commit); + if (tx_id > txq->next_rs) { + txq->tx_ring[txq->next_rs].qw1.cmd_dtype |= + rte_cpu_to_le_64(((uint64_t)IDPF_TX_FLEX_DESC_CMD_RS) << + IDPF_FLEX_TXD_QW1_CMD_S); + txq->next_rs = + (uint16_t)(txq->next_rs + txq->rs_thresh); + } + + txq->tx_tail = tx_id; + + IDPF_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + + return nb_pkts; +} + +static __rte_always_inline uint16_t +idpf_xmit_pkts_vec_avx512_cmn(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + uint16_t nb_tx = 0; + struct idpf_tx_queue *txq = (struct idpf_tx_queue *)tx_queue; + + while (nb_pkts) { + uint16_t ret, num; + + num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh); + ret = idpf_xmit_fixed_burst_vec_avx512(tx_queue, &tx_pkts[nb_tx], + num); + nb_tx += ret; + nb_pkts -= ret; + if (ret < num) + break; + } + + return nb_tx; +} + +uint16_t +idpf_singleq_xmit_pkts_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + return idpf_xmit_pkts_vec_avx512_cmn(tx_queue, tx_pkts, nb_pkts); +} + +static inline void +idpf_singleq_tx_release_mbufs_avx512(struct idpf_tx_queue *txq) +{ + unsigned int i; + const uint16_t max_desc = (uint16_t)(txq->nb_tx_desc - 1); + struct idpf_tx_vec_entry *swr = (void *)txq->sw_ring; + + if (txq->sw_ring == NULL || txq->nb_free == max_desc) + return; + + i = txq->next_dd - txq->rs_thresh + 1; + if (txq->tx_tail < i) { + for (; i < txq->nb_tx_desc; i++) { + rte_pktmbuf_free_seg(swr[i].mbuf); + swr[i].mbuf = NULL; + } + i = 0; + } +} + +static const struct idpf_txq_ops avx512_singleq_tx_vec_ops = { + .release_mbufs = idpf_singleq_tx_release_mbufs_avx512, +}; + +int __rte_cold +idpf_singleq_tx_vec_setup_avx512(struct idpf_tx_queue *txq) +{ + txq->ops = &avx512_singleq_tx_vec_ops; + return 0; +} diff --git a/drivers/net/idpf/idpf_rxtx_vec_common.h b/drivers/net/idpf/idpf_rxtx_vec_common.h new file mode 100644 index 0000000000..0f4e10e154 --- /dev/null +++ b/drivers/net/idpf/idpf_rxtx_vec_common.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2022 Intel Corporation + */ + +#ifndef _IDPF_RXTX_VEC_COMMON_H_ +#define _IDPF_RXTX_VEC_COMMON_H_ +#include +#include +#include + +#include "idpf_ethdev.h" +#include "idpf_rxtx.h" + +#ifndef __INTEL_COMPILER +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif + +#define IDPF_VECTOR_PATH 0 +#define ICE_RX_NO_VECTOR_FLAGS ( \ + RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | \ + RTE_ETH_RX_OFFLOAD_UDP_CKSUM | \ + RTE_ETH_RX_OFFLOAD_TCP_CKSUM | \ + RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM | \ + RTE_ETH_RX_OFFLOAD_TIMESTAMP) +#define ICE_TX_NO_VECTOR_FLAGS ( \ + RTE_ETH_TX_OFFLOAD_TCP_TSO | \ + RTE_ETH_TX_OFFLOAD_MULTI_SEGS) + +static inline int +idpf_rx_vec_queue_default(struct idpf_rx_queue *rxq) +{ + if (rxq == NULL) + return -1; + + if (rte_is_power_of_2(rxq->nb_rx_desc) == 0) + return -1; + + if (rxq->rx_free_thresh < IDPF_VPMD_RX_MAX_BURST) + return -1; + + if ((rxq->nb_rx_desc % rxq->rx_free_thresh) != 0) + return -1; + + if ((rxq->offloads & ICE_RX_NO_VECTOR_FLAGS) != 0) + return -1; + + return IDPF_VECTOR_PATH; +} + +static inline int +idpf_tx_vec_queue_default(struct idpf_tx_queue *txq) +{ + if (txq == NULL) + return -1; + + if (txq->rs_thresh < IDPF_VPMD_TX_MAX_BURST || + (txq->rs_thresh & 3) != 0) + return -1; + + if ((txq->offloads & ICE_TX_NO_VECTOR_FLAGS) != 0) + return -1; + + return IDPF_VECTOR_PATH; +} + +static inline int +idpf_rx_vec_dev_check_default(struct rte_eth_dev *dev) +{ + int i; + struct idpf_rx_queue *rxq; + int ret = 0; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + ret = (idpf_rx_vec_queue_default(rxq)); + if (ret < 0) + return -1; + } + + return IDPF_VECTOR_PATH; +} + +static inline int +idpf_tx_vec_dev_check_default(struct rte_eth_dev *dev) +{ + int i; + struct idpf_tx_queue *txq; + int ret = 0; + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + ret = idpf_tx_vec_queue_default(txq); + if (ret < 0) + return -1; + } + + return IDPF_VECTOR_PATH; +} + +#endif /*_IDPF_RXTX_VEC_COMMON_H_*/ diff --git a/drivers/net/idpf/meson.build b/drivers/net/idpf/meson.build index b632b76656..da99c098ab 100644 --- a/drivers/net/idpf/meson.build +++ b/drivers/net/idpf/meson.build @@ -14,3 +14,31 @@ sources = files( 'idpf_rxtx.c', 'idpf_vchnl.c', ) + +if arch_subdir == 'x86' + idpf_avx512_cpu_support = ( + cc.get_define('__AVX512F__', args: machine_args) != '' and + cc.get_define('__AVX512BW__', args: machine_args) != '' + ) + + idpf_avx512_cc_support = ( + not machine_args.contains('-mno-avx512f') and + cc.has_argument('-mavx512f') and + cc.has_argument('-mavx512bw') + ) + + if idpf_avx512_cpu_support == true or idpf_avx512_cc_support == true + cflags += ['-DCC_AVX512_SUPPORT'] + avx512_args = [cflags, '-mavx512f', '-mavx512bw'] + if cc.has_argument('-march=skylake-avx512') + avx512_args += '-march=skylake-avx512' + endif + idpf_avx512_lib = static_library('idpf_avx512_lib', + 'idpf_rxtx_vec_avx512.c', + dependencies: [static_rte_common_idpf, static_rte_ethdev, static_rte_bus_pci, + static_rte_kvargs, static_rte_hash], + include_directories: includes, + c_args: avx512_args) + objs += idpf_avx512_lib.extract_objects('idpf_rxtx_vec_avx512.c') + endif +endif From patchwork Mon Oct 24 13:01:34 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 119004 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 774D4A034C; Mon, 24 Oct 2022 15:05:37 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id EF07342BB9; Mon, 24 Oct 2022 15:04:57 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id 4031E42B8E for ; Mon, 24 Oct 2022 15:04:54 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666616694; x=1698152694; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=VhJvx0pGpx3HVYR4cZYRjx8EnojXaUiG0rnziT7m9xg=; b=HafcKWP9/DiQgB9/W49asdcdUiJx0LaebqkoBDNxhYJV4Sp7YnDSqpt4 zZg0Mum0gA+1DoY5M238XyIk5uCRV54c1z1EmDu/Ufs8bH3F4iGL9tD6A NbbM2nFgPK8/rCAEylAMgR7/gzWZ7QDlIlasrED3yqpKM/pKNOg9jhyLz Cm57KCcDj19dpMMFJ+T4yij75ty42ctyrziVHdwz/8w8E7HVLzvQJ7yX2 b8xY2oxyHheHOFo7RCDZnmRlnNHFLSQN0ff4QEkbdYWBPpqtW4XFgmKLs 7tRuZtDFGhL7CNw9iMWjArUMYgooSD5RBVUJvl8oshLpp00Gy53NtBTgk g==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305018962" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="305018962" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 06:03:43 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="631248537" X-IronPort-AV: E=Sophos;i="5.95,209,1661842800"; d="scan'208";a="631248537" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.111.104]) by orsmga002.jf.intel.com with ESMTP; 24 Oct 2022 06:03:40 -0700 From: Junfeng Guo To: andrew.rybchenko@oktetlabs.ru, qi.z.zhang@intel.com, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo , Wenjing Qiao Subject: [PATCH v10 14/14] net/idpf: add support for timestamp offload Date: Mon, 24 Oct 2022 21:01:34 +0800 Message-Id: <20221024130134.1046536-15-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20221024130134.1046536-1-junfeng.guo@intel.com> References: <20221021051821.2164939-2-junfeng.guo@intel.com> <20221024130134.1046536-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add support for timestamp offload. Signed-off-by: Wenjing Qiao Signed-off-by: Junfeng Guo --- doc/guides/nics/features/idpf.ini | 1 + drivers/net/idpf/idpf_ethdev.c | 5 +- drivers/net/idpf/idpf_ethdev.h | 3 ++ drivers/net/idpf/idpf_rxtx.c | 65 ++++++++++++++++++++++ drivers/net/idpf/idpf_rxtx.h | 90 +++++++++++++++++++++++++++++++ 5 files changed, 163 insertions(+), 1 deletion(-) diff --git a/doc/guides/nics/features/idpf.ini b/doc/guides/nics/features/idpf.ini index 47c686762d..0a0ffc2c29 100644 --- a/doc/guides/nics/features/idpf.ini +++ b/doc/guides/nics/features/idpf.ini @@ -12,5 +12,6 @@ MTU update = Y TSO = P L3 checksum offload = P L4 checksum offload = P +Timestamp offload = P Packet type parsing = Y Linux = Y diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index 21315866bf..bd33fd1797 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -21,6 +21,8 @@ rte_spinlock_t idpf_adapter_lock; struct idpf_adapter_list idpf_adapter_list; bool idpf_adapter_list_init; +uint64_t idpf_timestamp_dynflag; + static const char * const idpf_valid_args[] = { IDPF_TX_SINGLE_Q, IDPF_RX_SINGLE_Q, @@ -100,7 +102,8 @@ idpf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) RTE_ETH_RX_OFFLOAD_UDP_CKSUM | RTE_ETH_RX_OFFLOAD_TCP_CKSUM | RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM | - RTE_ETH_RX_OFFLOAD_RSS_HASH; + RTE_ETH_RX_OFFLOAD_RSS_HASH | + RTE_ETH_RX_OFFLOAD_TIMESTAMP; dev_info->tx_offload_capa = RTE_ETH_TX_OFFLOAD_TCP_TSO | diff --git a/drivers/net/idpf/idpf_ethdev.h b/drivers/net/idpf/idpf_ethdev.h index 2485b3a784..d6080aff81 100644 --- a/drivers/net/idpf/idpf_ethdev.h +++ b/drivers/net/idpf/idpf_ethdev.h @@ -185,6 +185,9 @@ struct idpf_adapter { bool tx_vec_allowed; bool rx_use_avx512; bool tx_use_avx512; + + /* For PTP */ + uint64_t time_hw; }; TAILQ_HEAD(idpf_adapter_list, idpf_adapter); diff --git a/drivers/net/idpf/idpf_rxtx.c b/drivers/net/idpf/idpf_rxtx.c index abef84b3b0..90b2111781 100644 --- a/drivers/net/idpf/idpf_rxtx.c +++ b/drivers/net/idpf/idpf_rxtx.c @@ -10,6 +10,8 @@ #include "idpf_rxtx.h" #include "idpf_rxtx_vec_common.h" +static int idpf_timestamp_dynfield_offset = -1; + const uint32_t * idpf_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) { @@ -941,6 +943,24 @@ idpf_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, return idpf_tx_split_queue_setup(dev, queue_idx, nb_desc, socket_id, tx_conf); } + +static int +idpf_register_ts_mbuf(struct idpf_rx_queue *rxq) +{ + int err; + if ((rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) != 0) { + /* Register mbuf field and flag for Rx timestamp */ + err = rte_mbuf_dyn_rx_timestamp_register(&idpf_timestamp_dynfield_offset, + &idpf_timestamp_dynflag); + if (err != 0) { + PMD_DRV_LOG(ERR, + "Cannot register mbuf field/flag for timestamp"); + return -EINVAL; + } + } + return 0; +} + static int idpf_alloc_single_rxq_mbufs(struct idpf_rx_queue *rxq) { @@ -1034,6 +1054,13 @@ idpf_rx_queue_init(struct rte_eth_dev *dev, uint16_t rx_queue_id) return -EINVAL; } + err = idpf_register_ts_mbuf(rxq); + if (err != 0) { + PMD_DRV_LOG(ERR, "fail to regidter timestamp mbuf %u", + rx_queue_id); + return -EIO; + } + if (rxq->bufq1 == NULL) { /* Single queue */ err = idpf_alloc_single_rxq_mbufs(rxq); @@ -1396,6 +1423,7 @@ idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, struct idpf_rx_queue *rxq; const uint32_t *ptype_tbl; uint8_t status_err0_qw1; + struct idpf_adapter *ad; struct rte_mbuf *rxm; uint16_t rx_id_bufq1; uint16_t rx_id_bufq2; @@ -1405,9 +1433,11 @@ idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t gen_id; uint16_t rx_id; uint16_t nb_rx; + uint64_t ts_ns; nb_rx = 0; rxq = (struct idpf_rx_queue *)rx_queue; + ad = rxq->adapter; if (unlikely(rxq == NULL) || unlikely(!rxq->q_started)) return nb_rx; @@ -1419,6 +1449,9 @@ idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, (volatile struct virtchnl2_rx_flex_desc_adv_nic_3 *)rxq->rx_ring; ptype_tbl = rxq->adapter->ptype_tbl; + if ((rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) != 0) + rxq->hw_register_set = 1; + while (nb_rx < nb_pkts) { rx_desc = &rx_desc_ring[rx_id]; @@ -1474,6 +1507,18 @@ idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, status_err0_qw1 = rx_desc->status_err0_qw1; pkt_flags = idpf_splitq_rx_csum_offload(status_err0_qw1); pkt_flags |= idpf_splitq_rx_rss_offload(rxm, rx_desc); + if (idpf_timestamp_dynflag > 0 && + (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) { + /* timestamp */ + ts_ns = idpf_tstamp_convert_32b_64b(ad, + rxq->hw_register_set, + rte_le_to_cpu_32(rx_desc->ts_high)); + rxq->hw_register_set = 0; + *RTE_MBUF_DYNFIELD(rxm, + idpf_timestamp_dynfield_offset, + rte_mbuf_timestamp_t *) = ts_ns; + rxm->ol_flags |= idpf_timestamp_dynflag; + } rxm->ol_flags |= pkt_flags; @@ -1775,18 +1820,22 @@ idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, const uint32_t *ptype_tbl; uint16_t rx_id, nb_hold; struct rte_eth_dev *dev; + struct idpf_adapter *ad; uint16_t rx_packet_len; struct rte_mbuf *rxm; struct rte_mbuf *nmb; uint16_t rx_status0; uint64_t pkt_flags; uint64_t dma_addr; + uint64_t ts_ns; uint16_t nb_rx; nb_rx = 0; nb_hold = 0; rxq = rx_queue; + ad = rxq->adapter; + if (unlikely(rxq == NULL) || unlikely(!rxq->q_started)) return nb_rx; @@ -1794,6 +1843,9 @@ idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, rx_ring = rxq->rx_ring; ptype_tbl = rxq->adapter->ptype_tbl; + if ((rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) != 0) + rxq->hw_register_set = 1; + while (nb_rx < nb_pkts) { rxdp = &rx_ring[rx_id]; rx_status0 = rte_le_to_cpu_16(rxdp->flex_nic_wb.status_error0); @@ -1853,6 +1905,19 @@ idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, rxm->ol_flags |= pkt_flags; + if (idpf_timestamp_dynflag > 0 && + (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) != 0) { + /* timestamp */ + ts_ns = idpf_tstamp_convert_32b_64b(ad, + rxq->hw_register_set, + rte_le_to_cpu_32(rxd.flex_nic_wb.flex_ts.ts_high)); + rxq->hw_register_set = 0; + *RTE_MBUF_DYNFIELD(rxm, + idpf_timestamp_dynfield_offset, + rte_mbuf_timestamp_t *) = ts_ns; + rxm->ol_flags |= idpf_timestamp_dynflag; + } + rx_pkts[nb_rx++] = rxm; } rxq->rx_tail = rx_id; diff --git a/drivers/net/idpf/idpf_rxtx.h b/drivers/net/idpf/idpf_rxtx.h index e808710b41..6e7fbaf7ef 100644 --- a/drivers/net/idpf/idpf_rxtx.h +++ b/drivers/net/idpf/idpf_rxtx.h @@ -7,6 +7,41 @@ #include "idpf_ethdev.h" +/* MTS */ +#define GLTSYN_CMD_SYNC_0_0 (PF_TIMESYNC_BASE + 0x0) +#define PF_GLTSYN_SHTIME_0_0 (PF_TIMESYNC_BASE + 0x4) +#define PF_GLTSYN_SHTIME_L_0 (PF_TIMESYNC_BASE + 0x8) +#define PF_GLTSYN_SHTIME_H_0 (PF_TIMESYNC_BASE + 0xC) +#define GLTSYN_ART_L_0 (PF_TIMESYNC_BASE + 0x10) +#define GLTSYN_ART_H_0 (PF_TIMESYNC_BASE + 0x14) +#define PF_GLTSYN_SHTIME_0_1 (PF_TIMESYNC_BASE + 0x24) +#define PF_GLTSYN_SHTIME_L_1 (PF_TIMESYNC_BASE + 0x28) +#define PF_GLTSYN_SHTIME_H_1 (PF_TIMESYNC_BASE + 0x2C) +#define PF_GLTSYN_SHTIME_0_2 (PF_TIMESYNC_BASE + 0x44) +#define PF_GLTSYN_SHTIME_L_2 (PF_TIMESYNC_BASE + 0x48) +#define PF_GLTSYN_SHTIME_H_2 (PF_TIMESYNC_BASE + 0x4C) +#define PF_GLTSYN_SHTIME_0_3 (PF_TIMESYNC_BASE + 0x64) +#define PF_GLTSYN_SHTIME_L_3 (PF_TIMESYNC_BASE + 0x68) +#define PF_GLTSYN_SHTIME_H_3 (PF_TIMESYNC_BASE + 0x6C) + +#define PF_TIMESYNC_BAR4_BASE 0x0E400000 +#define GLTSYN_ENA (PF_TIMESYNC_BAR4_BASE + 0x90) +#define GLTSYN_CMD (PF_TIMESYNC_BAR4_BASE + 0x94) +#define GLTSYC_TIME_L (PF_TIMESYNC_BAR4_BASE + 0x104) +#define GLTSYC_TIME_H (PF_TIMESYNC_BAR4_BASE + 0x108) + +#define GLTSYN_CMD_SYNC_0_4 (PF_TIMESYNC_BAR4_BASE + 0x110) +#define PF_GLTSYN_SHTIME_L_4 (PF_TIMESYNC_BAR4_BASE + 0x118) +#define PF_GLTSYN_SHTIME_H_4 (PF_TIMESYNC_BAR4_BASE + 0x11C) +#define GLTSYN_INCVAL_L (PF_TIMESYNC_BAR4_BASE + 0x150) +#define GLTSYN_INCVAL_H (PF_TIMESYNC_BAR4_BASE + 0x154) +#define GLTSYN_SHADJ_L (PF_TIMESYNC_BAR4_BASE + 0x158) +#define GLTSYN_SHADJ_H (PF_TIMESYNC_BAR4_BASE + 0x15C) + +#define GLTSYN_CMD_SYNC_0_5 (PF_TIMESYNC_BAR4_BASE + 0x130) +#define PF_GLTSYN_SHTIME_L_5 (PF_TIMESYNC_BAR4_BASE + 0x138) +#define PF_GLTSYN_SHTIME_H_5 (PF_TIMESYNC_BAR4_BASE + 0x13C) + /* In QLEN must be whole number of 32 descriptors. */ #define IDPF_ALIGN_RING_DESC 32 #define IDPF_MIN_RING_DESC 32 @@ -41,6 +76,8 @@ (sizeof(struct virtchnl2_ptype) + \ (((p)->proto_id_count ? ((p)->proto_id_count - 1) : 0) * sizeof((p)->proto_id[0]))) +extern uint64_t idpf_timestamp_dynflag; + struct idpf_rx_queue { struct idpf_adapter *adapter; /* the adapter this queue belongs to */ struct rte_mempool *mp; /* mbuf pool to populate Rx ring */ @@ -201,4 +238,57 @@ void idpf_set_tx_function(struct rte_eth_dev *dev); const uint32_t *idpf_dev_supported_ptypes_get(struct rte_eth_dev *dev); +#define IDPF_TIMESYNC_REG_WRAP_GUARD_BAND 10000 +/* Helper function to convert a 32b nanoseconds timestamp to 64b. */ +static inline uint64_t + +idpf_tstamp_convert_32b_64b(struct idpf_adapter *ad, uint32_t flag, + uint32_t in_timestamp) +{ +#ifdef RTE_ARCH_X86_64 + struct idpf_hw *hw = &ad->hw; + const uint64_t mask = 0xFFFFFFFF; + uint32_t hi, lo, lo2, delta; + uint64_t ns; + + if (flag != 0) { + IDPF_WRITE_REG(hw, GLTSYN_CMD_SYNC_0_0, PF_GLTSYN_CMD_SYNC_SHTIME_EN_M); + IDPF_WRITE_REG(hw, GLTSYN_CMD_SYNC_0_0, PF_GLTSYN_CMD_SYNC_EXEC_CMD_M | + PF_GLTSYN_CMD_SYNC_SHTIME_EN_M); + lo = IDPF_READ_REG(hw, PF_GLTSYN_SHTIME_L_0); + hi = IDPF_READ_REG(hw, PF_GLTSYN_SHTIME_H_0); + /* + * On typical system, the delta between lo and lo2 is ~1000ns, + * so 10000 seems a large-enough but not overly-big guard band. + */ + if (lo > (UINT32_MAX - IDPF_TIMESYNC_REG_WRAP_GUARD_BAND)) + lo2 = IDPF_READ_REG(hw, PF_GLTSYN_SHTIME_L_0); + else + lo2 = lo; + + if (lo2 < lo) { + lo = IDPF_READ_REG(hw, PF_GLTSYN_SHTIME_L_0); + hi = IDPF_READ_REG(hw, PF_GLTSYN_SHTIME_H_0); + } + + ad->time_hw = ((uint64_t)hi << 32) | lo; + } + + delta = (in_timestamp - (uint32_t)(ad->time_hw & mask)); + if (delta > (mask / 2)) { + delta = ((uint32_t)(ad->time_hw & mask) - in_timestamp); + ns = ad->time_hw - delta; + } else { + ns = ad->time_hw + delta; + } + + return ns; +#else /* !RTE_ARCH_X86_64 */ + RTE_SET_USED(ad); + RTE_SET_USED(flag); + RTE_SET_USED(in_timestamp); + return 0; +#endif /* RTE_ARCH_X86_64 */ +} + #endif /* _IDPF_RXTX_H_ */