From patchwork Wed Jan 4 07:30:41 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Feifei Wang X-Patchwork-Id: 121564 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 5A2F4A00C2; Wed, 4 Jan 2023 08:30:57 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id C7E4D42B8E; Wed, 4 Jan 2023 08:30:56 +0100 (CET) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 8FB86410EA for ; Wed, 4 Jan 2023 08:30:55 +0100 (CET) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id AD46B1596; Tue, 3 Jan 2023 23:31:36 -0800 (PST) Received: from net-x86-dell-8268.shanghai.arm.com (net-x86-dell-8268.shanghai.arm.com [10.169.210.116]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 2581A3F71A; Tue, 3 Jan 2023 23:30:51 -0800 (PST) From: Feifei Wang To: Thomas Monjalon , Ferruh Yigit , Andrew Rybchenko Cc: dev@dpdk.org, konstantin.v.ananyev@yandex.ru, nd@arm.com, Feifei Wang , Honnappa Nagarahalli , Ruifeng Wang Subject: [PATCH v3 1/3] ethdev: enable direct rearm with separate API Date: Wed, 4 Jan 2023 15:30:41 +0800 Message-Id: <20230104073043.1120168-2-feifei.wang2@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20230104073043.1120168-1-feifei.wang2@arm.com> References: <20220420081650.2043183-1-feifei.wang2@arm.com> <20230104073043.1120168-1-feifei.wang2@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add 'tx_fill_sw_ring' and 'rx_flush_descriptor' API into direct rearm mode for separate Rx and Tx Operation. And this can support different multiple sources in direct rearm mode. For examples, Rx driver is ixgbe, and Tx driver is i40e. Suggested-by: Honnappa Nagarahalli Suggested-by: Ruifeng Wang Signed-off-by: Feifei Wang Reviewed-by: Ruifeng Wang Reviewed-by: Honnappa Nagarahalli --- lib/ethdev/ethdev_driver.h | 10 ++ lib/ethdev/ethdev_private.c | 2 + lib/ethdev/rte_ethdev.c | 52 +++++++++++ lib/ethdev/rte_ethdev.h | 174 +++++++++++++++++++++++++++++++++++ lib/ethdev/rte_ethdev_core.h | 11 +++ lib/ethdev/version.map | 6 ++ 6 files changed, 255 insertions(+) diff --git a/lib/ethdev/ethdev_driver.h b/lib/ethdev/ethdev_driver.h index 6a550cfc83..bc539ec862 100644 --- a/lib/ethdev/ethdev_driver.h +++ b/lib/ethdev/ethdev_driver.h @@ -59,6 +59,10 @@ struct rte_eth_dev { eth_rx_descriptor_status_t rx_descriptor_status; /** Check the status of a Tx descriptor */ eth_tx_descriptor_status_t tx_descriptor_status; + /** Fill Rx sw-ring with Tx buffers in direct rearm mode */ + eth_tx_fill_sw_ring_t tx_fill_sw_ring; + /** Flush Rx descriptor in direct rearm mode */ + eth_rx_flush_descriptor_t rx_flush_descriptor; /** * Device data that is shared between primary and secondary processes @@ -504,6 +508,10 @@ typedef void (*eth_rxq_info_get_t)(struct rte_eth_dev *dev, typedef void (*eth_txq_info_get_t)(struct rte_eth_dev *dev, uint16_t tx_queue_id, struct rte_eth_txq_info *qinfo); +/**< @internal Get rearm data for a receive queue of an Ethernet device. */ +typedef void (*eth_rxq_rearm_data_get_t)(struct rte_eth_dev *dev, + uint16_t tx_queue_id, struct rte_eth_rxq_rearm_data *rxq_rearm_data); + typedef int (*eth_burst_mode_get_t)(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_burst_mode *mode); @@ -1215,6 +1223,8 @@ struct eth_dev_ops { eth_rxq_info_get_t rxq_info_get; /** Retrieve Tx queue information */ eth_txq_info_get_t txq_info_get; + /** Get Rx queue rearm data */ + eth_rxq_rearm_data_get_t rxq_rearm_data_get; eth_burst_mode_get_t rx_burst_mode_get; /**< Get Rx burst mode */ eth_burst_mode_get_t tx_burst_mode_get; /**< Get Tx burst mode */ eth_fw_version_get_t fw_version_get; /**< Get firmware version */ diff --git a/lib/ethdev/ethdev_private.c b/lib/ethdev/ethdev_private.c index 48090c879a..c5dd5e30f6 100644 --- a/lib/ethdev/ethdev_private.c +++ b/lib/ethdev/ethdev_private.c @@ -276,6 +276,8 @@ eth_dev_fp_ops_setup(struct rte_eth_fp_ops *fpo, fpo->rx_queue_count = dev->rx_queue_count; fpo->rx_descriptor_status = dev->rx_descriptor_status; fpo->tx_descriptor_status = dev->tx_descriptor_status; + fpo->tx_fill_sw_ring = dev->tx_fill_sw_ring; + fpo->rx_flush_descriptor = dev->rx_flush_descriptor; fpo->rxq.data = dev->data->rx_queues; fpo->rxq.clbk = (void **)(uintptr_t)dev->post_rx_burst_cbs; diff --git a/lib/ethdev/rte_ethdev.c b/lib/ethdev/rte_ethdev.c index 5d5e18db1e..2af5cb42fe 100644 --- a/lib/ethdev/rte_ethdev.c +++ b/lib/ethdev/rte_ethdev.c @@ -3282,6 +3282,21 @@ rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id, stat_idx, STAT_QMAP_RX)); } +int +rte_eth_dev_direct_rearm(uint16_t rx_port_id, uint16_t rx_queue_id, + uint16_t tx_port_id, uint16_t tx_rx_queue_id, + struct rte_eth_rxq_rearm_data *rxq_rearm_data) +{ + int nb_rearm = 0; + + nb_rearm = rte_eth_tx_fill_sw_ring(tx_port_id, tx_rx_queue_id, rxq_rearm_data); + + if (nb_rearm > 0) + return rte_eth_rx_flush_descriptor(rx_port_id, rx_queue_id, nb_rearm); + + return 0; +} + int rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size) { @@ -5323,6 +5338,43 @@ rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id, return 0; } +int +rte_eth_rx_queue_rearm_data_get(uint16_t port_id, uint16_t queue_id, + struct rte_eth_rxq_rearm_data *rxq_rearm_data) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + dev = &rte_eth_devices[port_id]; + + if (queue_id >= dev->data->nb_rx_queues) { + RTE_ETHDEV_LOG(ERR, "Invalid Rx queue_id=%u\n", queue_id); + return -EINVAL; + } + + if (rxq_rearm_data == NULL) { + RTE_ETHDEV_LOG(ERR, "Cannot get ethdev port %u Rx queue %u rearm data to NULL\n", + port_id, queue_id); + return -EINVAL; + } + + if (dev->data->rx_queues == NULL || + dev->data->rx_queues[queue_id] == NULL) { + RTE_ETHDEV_LOG(ERR, + "Rx queue %"PRIu16" of device with port_id=%" + PRIu16" has not been setup\n", + queue_id, port_id); + return -EINVAL; + } + + if (*dev->dev_ops->rxq_rearm_data_get == NULL) + return -ENOTSUP; + + dev->dev_ops->rxq_rearm_data_get(dev, queue_id, rxq_rearm_data); + + return 0; +} + int rte_eth_rx_burst_mode_get(uint16_t port_id, uint16_t queue_id, struct rte_eth_burst_mode *mode) diff --git a/lib/ethdev/rte_ethdev.h b/lib/ethdev/rte_ethdev.h index c129ca1eaf..381c3d535f 100644 --- a/lib/ethdev/rte_ethdev.h +++ b/lib/ethdev/rte_ethdev.h @@ -1818,6 +1818,17 @@ struct rte_eth_txq_info { uint8_t queue_state; /**< one of RTE_ETH_QUEUE_STATE_*. */ } __rte_cache_min_aligned; +/** + * @internal + * Structure used to hold pointers to internal ethdev Rx rearm data. + * The main purpose is to load Rx queue rearm data in direct rearm mode. + */ +struct rte_eth_rxq_rearm_data { + void *rx_sw_ring; + uint16_t *rearm_start; + uint16_t *rearm_nb; +} __rte_cache_min_aligned; + /* Generic Burst mode flag definition, values can be ORed. */ /** @@ -3184,6 +3195,34 @@ int rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id, uint8_t stat_idx); +/** + * @warning + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice + * + * Directly put Tx freed buffers into Rx sw-ring and flush desc. + * + * @param rx_port_id + * Port identifying the receive side. + * @param rx_queue_id + * The index of the receive queue identifying the receive side. + * The value must be in the range [0, nb_rx_queue - 1] previously supplied + * to rte_eth_dev_configure(). + * @param tx_port_id + * Port identifying the transmit side. + * @param tx_queue_id + * The index of the transmit queue identifying the transmit side. + * The value must be in the range [0, nb_tx_queue - 1] previously supplied + * to rte_eth_dev_configure(). + * @param rxq_rearm_data + * A pointer to a structure of type *rte_eth_txq_rearm_data* to be filled. + * @return + * - 0: Success + */ +__rte_experimental +int rte_eth_dev_direct_rearm(uint16_t rx_port_id, uint16_t rx_queue_id, + uint16_t tx_port_id, uint16_t tx_queue_id, + struct rte_eth_rxq_rearm_data *rxq_rearm_data); + /** * Retrieve the Ethernet address of an Ethernet device. * @@ -4782,6 +4821,27 @@ int rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id, int rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id, struct rte_eth_txq_info *qinfo); +/** + * Get rearm data about given ports's Rx queue. + * + * @param port_id + * The port identifier of the Ethernet device. + * @param queue_id + * The Rx queue on the Ethernet device for which rearm data + * will be got. + * @param rxq_rearm_data + * A pointer to a structure of type *rte_eth_txq_rearm_data* to be filled. + * + * @return + * - 0: Success + * - -ENODEV: If *port_id* is invalid. + * - -ENOTSUP: routine is not supported by the device PMD. + * - -EINVAL: The queue_id is out of range. + */ +__rte_experimental +int rte_eth_rx_queue_rearm_data_get(uint16_t port_id, uint16_t queue_id, + struct rte_eth_rxq_rearm_data *rxq_rearm_data); + /** * Retrieve information about the Rx packet burst mode. * @@ -6103,6 +6163,120 @@ static inline int rte_eth_tx_descriptor_status(uint16_t port_id, return (*p->tx_descriptor_status)(qd, offset); } +/** + * @warning + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice + * + * Fill Rx sw-ring with Tx buffers in direct rearm mode. + * + * @param port_id + * The port identifier of the Ethernet device. + * @param queue_id + * The index of the transmit queue. + * The value must be in the range [0, nb_tx_queue - 1] previously supplied + * to rte_eth_dev_configure(). + * @param rxq_rearm_data + * A pointer to a structure of type *rte_eth_rxq_rearm_data* to be filled with + * the rearm data of a receive queue. + * @return + * - The number buffers correct to be filled in the Rx sw-ring. + * - (-EINVAL) bad port or queue. + * - (-ENODEV) bad port. + * - (-ENOTSUP) if the device does not support this function. + * + */ +__rte_experimental +static inline int rte_eth_tx_fill_sw_ring(uint16_t port_id, + uint16_t queue_id, struct rte_eth_rxq_rearm_data *rxq_rearm_data) +{ + struct rte_eth_fp_ops *p; + void *qd; + +#ifdef RTE_ETHDEV_DEBUG_TX + if (port_id >= RTE_MAX_ETHPORTS || + queue_id >= RTE_MAX_QUEUES_PER_PORT) { + RTE_ETHDEV_LOG(ERR, + "Invalid port_id=%u or queue_id=%u\n", + port_id, queue_id); + return -EINVAL; + } +#endif + + p = &rte_eth_fp_ops[port_id]; + qd = p->txq.data[queue_id]; + +#ifdef RTE_ETHDEV_DEBUG_TX + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0); + + if (qd == NULL) { + RTE_ETHDEV_LOG(ERR, "Invalid Tx queue_id=%u for port_id=%u\n", + queue_id, port_id); + return -ENODEV; + } +#endif + + if (p->tx_fill_sw_ring == NULL) + return -ENOTSUP; + + return p->tx_fill_sw_ring(qd, rxq_rearm_data); +} + +/** + * @warning + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice + * + * Flush Rx descriptor in direct rearm mode. + * + * @param port_id + * The port identifier of the Ethernet device. + * @param queue_id + * The index of the receive queue. + * The value must be in the range [0, nb_rx_queue - 1] previously supplied + * to rte_eth_dev_configure(). + *@param nb_rearm + * The number of Rx sw-ring buffers need to be flushed. + * @return + * - (0) if successful. + * - (-EINVAL) bad port or queue. + * - (-ENODEV) bad port. + * - (-ENOTSUP) if the device does not support this function. + */ +__rte_experimental +static inline int rte_eth_rx_flush_descriptor(uint16_t port_id, + uint16_t queue_id, uint16_t nb_rearm) +{ + struct rte_eth_fp_ops *p; + void *qd; + +#ifdef RTE_ETHDEV_DEBUG_RX + if (port_id >= RTE_MAX_ETHPORTS || + queue_id >= RTE_MAX_QUEUES_PER_PORT) { + RTE_ETHDEV_LOG(ERR, + "Invalid port_id=%u or queue_id=%u\n", + port_id, queue_id); + return -EINVAL; + } +#endif + + p = &rte_eth_fp_ops[port_id]; + qd = p->rxq.data[queue_id]; + +#ifdef RTE_ETHDEV_DEBUG_RX + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0); + + if (qd == NULL) { + RTE_ETHDEV_LOG(ERR, "Invalid Rx queue_id=%u for port_id=%u\n", + queue_id, port_id); + return -ENODEV; + } +#endif + + if (p->rx_flush_descriptor == NULL) + return -ENOTSUP; + + return p->rx_flush_descriptor(qd, nb_rearm); +} + /** * @internal * Helper routine for rte_eth_tx_burst(). diff --git a/lib/ethdev/rte_ethdev_core.h b/lib/ethdev/rte_ethdev_core.h index dcf8adab92..5ecb57f6f0 100644 --- a/lib/ethdev/rte_ethdev_core.h +++ b/lib/ethdev/rte_ethdev_core.h @@ -56,6 +56,13 @@ typedef int (*eth_rx_descriptor_status_t)(void *rxq, uint16_t offset); /** @internal Check the status of a Tx descriptor */ typedef int (*eth_tx_descriptor_status_t)(void *txq, uint16_t offset); +/** @internal Fill Rx sw-ring with Tx buffers in direct rearm mode */ +typedef int (*eth_tx_fill_sw_ring_t)(void *txq, + struct rte_eth_rxq_rearm_data *rxq_rearm_data); + +/** @internal Flush Rx descriptor in direct rearm mode */ +typedef int (*eth_rx_flush_descriptor_t)(void *rxq, uint16_t nb_rearm); + /** * @internal * Structure used to hold opaque pointers to internal ethdev Rx/Tx @@ -90,6 +97,8 @@ struct rte_eth_fp_ops { eth_rx_queue_count_t rx_queue_count; /** Check the status of a Rx descriptor. */ eth_rx_descriptor_status_t rx_descriptor_status; + /** Flush Rx descriptor in direct rearm mode */ + eth_rx_flush_descriptor_t rx_flush_descriptor; /** Rx queues data. */ struct rte_ethdev_qdata rxq; uintptr_t reserved1[3]; @@ -106,6 +115,8 @@ struct rte_eth_fp_ops { eth_tx_prep_t tx_pkt_prepare; /** Check the status of a Tx descriptor. */ eth_tx_descriptor_status_t tx_descriptor_status; + /** Fill Rx sw-ring with Tx buffers in direct rearm mode */ + eth_tx_fill_sw_ring_t tx_fill_sw_ring; /** Tx queues data. */ struct rte_ethdev_qdata txq; uintptr_t reserved2[3]; diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map index 17201fbe0f..f39f02a69b 100644 --- a/lib/ethdev/version.map +++ b/lib/ethdev/version.map @@ -298,6 +298,12 @@ EXPERIMENTAL { rte_flow_get_q_aged_flows; rte_mtr_meter_policy_get; rte_mtr_meter_profile_get; + + # added in 23.03 + rte_eth_dev_direct_rearm; + rte_eth_rx_flush_descriptor; + rte_eth_rx_queue_rearm_data_get; + rte_eth_tx_fill_sw_ring; }; INTERNAL { From patchwork Wed Jan 4 07:30:42 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Feifei Wang X-Patchwork-Id: 121565 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id AF14BA00C2; Wed, 4 Jan 2023 08:31:03 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 9F7E842D23; Wed, 4 Jan 2023 08:31:00 +0100 (CET) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 9F0EF42D23 for ; Wed, 4 Jan 2023 08:30:58 +0100 (CET) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id DDF801063; Tue, 3 Jan 2023 23:31:39 -0800 (PST) Received: from net-x86-dell-8268.shanghai.arm.com (net-x86-dell-8268.shanghai.arm.com [10.169.210.116]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 988123F71A; Tue, 3 Jan 2023 23:30:55 -0800 (PST) From: Feifei Wang To: Yuying Zhang , Beilei Xing , Ruifeng Wang Cc: dev@dpdk.org, konstantin.v.ananyev@yandex.ru, nd@arm.com, Feifei Wang , Honnappa Nagarahalli Subject: [PATCH v3 2/3] net/i40e: enable direct rearm with separate API Date: Wed, 4 Jan 2023 15:30:42 +0800 Message-Id: <20230104073043.1120168-3-feifei.wang2@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20230104073043.1120168-1-feifei.wang2@arm.com> References: <20220420081650.2043183-1-feifei.wang2@arm.com> <20230104073043.1120168-1-feifei.wang2@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add internal API to separate direct rearm operations between Rx and Tx. Suggested-by: Honnappa Nagarahalli Signed-off-by: Feifei Wang Reviewed-by: Ruifeng Wang Reviewed-by: Honnappa Nagarahalli --- drivers/net/i40e/i40e_ethdev.c | 1 + drivers/net/i40e/i40e_ethdev.h | 2 + drivers/net/i40e/i40e_rxtx.c | 19 +++++++++ drivers/net/i40e/i40e_rxtx.h | 4 ++ drivers/net/i40e/i40e_rxtx_vec_common.h | 54 +++++++++++++++++++++++++ drivers/net/i40e/i40e_rxtx_vec_neon.c | 42 +++++++++++++++++++ 6 files changed, 122 insertions(+) diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c index 7726a89d99..29c1ce2470 100644 --- a/drivers/net/i40e/i40e_ethdev.c +++ b/drivers/net/i40e/i40e_ethdev.c @@ -497,6 +497,7 @@ static const struct eth_dev_ops i40e_eth_dev_ops = { .flow_ops_get = i40e_dev_flow_ops_get, .rxq_info_get = i40e_rxq_info_get, .txq_info_get = i40e_txq_info_get, + .rxq_rearm_data_get = i40e_rxq_rearm_data_get, .rx_burst_mode_get = i40e_rx_burst_mode_get, .tx_burst_mode_get = i40e_tx_burst_mode_get, .timesync_enable = i40e_timesync_enable, diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h index fe943a45ff..6a6a2a6d3c 100644 --- a/drivers/net/i40e/i40e_ethdev.h +++ b/drivers/net/i40e/i40e_ethdev.h @@ -1352,6 +1352,8 @@ void i40e_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_rxq_info *qinfo); void i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_txq_info *qinfo); +void i40e_rxq_rearm_data_get(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_rxq_rearm_data *rxq_rearm_data); int i40e_rx_burst_mode_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_burst_mode *mode); int i40e_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t queue_id, diff --git a/drivers/net/i40e/i40e_rxtx.c b/drivers/net/i40e/i40e_rxtx.c index 788ffb51c2..d8d801acaf 100644 --- a/drivers/net/i40e/i40e_rxtx.c +++ b/drivers/net/i40e/i40e_rxtx.c @@ -3197,6 +3197,19 @@ i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->conf.offloads = txq->offloads; } +void +i40e_rxq_rearm_data_get(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_rxq_rearm_data *rxq_rearm_data) +{ + struct i40e_rx_queue *rxq; + + rxq = dev->data->rx_queues[queue_id]; + + rxq_rearm_data->rx_sw_ring = rxq->sw_ring; + rxq_rearm_data->rearm_start = &rxq->rxrearm_start; + rxq_rearm_data->rearm_nb = &rxq->rxrearm_nb; +} + #ifdef RTE_ARCH_X86 static inline bool get_avx_supported(bool request_avx512) @@ -3321,6 +3334,9 @@ i40e_set_rx_function(struct rte_eth_dev *dev) PMD_INIT_LOG(DEBUG, "Using Vector Rx (port %d).", dev->data->port_id); dev->rx_pkt_burst = i40e_recv_pkts_vec; +#ifdef RTE_ARCH_ARM64 + dev->rx_flush_descriptor = i40e_rx_flush_descriptor_vec; +#endif } #endif /* RTE_ARCH_X86 */ } else if (!dev->data->scattered_rx && ad->rx_bulk_alloc_allowed) { @@ -3484,6 +3500,9 @@ i40e_set_tx_function(struct rte_eth_dev *dev) PMD_INIT_LOG(DEBUG, "Using Vector Tx (port %d).", dev->data->port_id); dev->tx_pkt_burst = i40e_xmit_pkts_vec; +#ifdef RTE_ARCH_ARM64 + dev->tx_fill_sw_ring = i40e_tx_fill_sw_ring; +#endif #endif /* RTE_ARCH_X86 */ } else { PMD_INIT_LOG(DEBUG, "Simple tx finally be used."); diff --git a/drivers/net/i40e/i40e_rxtx.h b/drivers/net/i40e/i40e_rxtx.h index 5e6eecc501..8a29bd89df 100644 --- a/drivers/net/i40e/i40e_rxtx.h +++ b/drivers/net/i40e/i40e_rxtx.h @@ -233,6 +233,10 @@ uint32_t i40e_dev_rx_queue_count(void *rx_queue); int i40e_dev_rx_descriptor_status(void *rx_queue, uint16_t offset); int i40e_dev_tx_descriptor_status(void *tx_queue, uint16_t offset); +int i40e_tx_fill_sw_ring(void *tx_queue, + struct rte_eth_rxq_rearm_data *rxq_rearm_data); +int i40e_rx_flush_descriptor_vec(void *rx_queue, uint16_t nb_rearm); + uint16_t i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); uint16_t i40e_recv_scattered_pkts_vec(void *rx_queue, diff --git a/drivers/net/i40e/i40e_rxtx_vec_common.h b/drivers/net/i40e/i40e_rxtx_vec_common.h index fe1a6ec75e..eb96301a43 100644 --- a/drivers/net/i40e/i40e_rxtx_vec_common.h +++ b/drivers/net/i40e/i40e_rxtx_vec_common.h @@ -146,6 +146,60 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq) return txq->tx_rs_thresh; } +int +i40e_tx_fill_sw_ring(void *tx_queue, + struct rte_eth_rxq_rearm_data *rxq_rearm_data) +{ + struct i40e_tx_queue *txq = tx_queue; + struct i40e_tx_entry *txep; + void **rxep; + struct rte_mbuf *m; + int i, n; + int nb_rearm = 0; + + if (*rxq_rearm_data->rearm_nb < txq->tx_rs_thresh || + txq->nb_tx_free > txq->tx_free_thresh) + return 0; + + /* check DD bits on threshold descriptor */ + if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & + rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) != + rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) + return 0; + + n = txq->tx_rs_thresh; + + /* first buffer to free from S/W ring is at index + * tx_next_dd - (tx_rs_thresh-1) + */ + txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)]; + rxep = rxq_rearm_data->rx_sw_ring; + rxep += *rxq_rearm_data->rearm_start; + + if (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) { + /* directly put mbufs from Tx to Rx */ + for (i = 0; i < n; i++, rxep++, txep++) + *rxep = txep[0].mbuf; + } else { + for (i = 0; i < n; i++, rxep++) { + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); + if (m != NULL) { + *rxep = m; + nb_rearm++; + } + } + n = nb_rearm; + } + + /* update counters for Tx */ + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); + txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); + if (txq->tx_next_dd >= txq->nb_tx_desc) + txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); + + return n; +} + static __rte_always_inline void tx_backlog_entry(struct i40e_tx_entry *txep, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) diff --git a/drivers/net/i40e/i40e_rxtx_vec_neon.c b/drivers/net/i40e/i40e_rxtx_vec_neon.c index 12e6f1cbcb..1509d3223b 100644 --- a/drivers/net/i40e/i40e_rxtx_vec_neon.c +++ b/drivers/net/i40e/i40e_rxtx_vec_neon.c @@ -739,6 +739,48 @@ i40e_xmit_fixed_burst_vec(void *__rte_restrict tx_queue, return nb_pkts; } +int +i40e_rx_flush_descriptor_vec(void *rx_queue, uint16_t nb_rearm) +{ + struct i40e_rx_queue *rxq = rx_queue; + struct i40e_rx_entry *rxep; + volatile union i40e_rx_desc *rxdp; + uint16_t rx_id; + uint64x2_t dma_addr; + uint64_t paddr; + uint16_t i; + + rxdp = rxq->rx_ring + rxq->rxrearm_start; + rxep = &rxq->sw_ring[rxq->rxrearm_start]; + + for (i = 0; i < nb_rearm; i++) { + /* Initialize rxdp descs */ + paddr = (rxep[i].mbuf)->buf_iova + RTE_PKTMBUF_HEADROOM; + dma_addr = vdupq_n_u64(paddr); + /* flush desc with pa dma_addr */ + vst1q_u64((uint64_t *)&rxdp++->read, dma_addr); + } + + /* Update the descriptor initializer index */ + rxq->rxrearm_start += nb_rearm; + rx_id = rxq->rxrearm_start - 1; + + if (unlikely(rxq->rxrearm_start >= rxq->nb_rx_desc)) { + rxq->rxrearm_start = rxq->rxrearm_start - rxq->nb_rx_desc; + if (!rxq->rxrearm_start) + rx_id = rxq->nb_rx_desc - 1; + else + rx_id = rxq->rxrearm_start - 1; + } + rxq->rxrearm_nb -= nb_rearm; + + rte_io_wmb(); + /* Update the tail pointer on the NIC */ + I40E_PCI_REG_WRITE_RELAXED(rxq->qrx_tail, rx_id); + + return 0; +} + void __rte_cold i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq) { From patchwork Wed Jan 4 07:30:43 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Feifei Wang X-Patchwork-Id: 121566 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id CB044A00C2; Wed, 4 Jan 2023 08:31:11 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id E66E142D34; Wed, 4 Jan 2023 08:31:02 +0100 (CET) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id CB2D242D27 for ; Wed, 4 Jan 2023 08:31:01 +0100 (CET) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 1EEC61063; Tue, 3 Jan 2023 23:31:43 -0800 (PST) Received: from net-x86-dell-8268.shanghai.arm.com (net-x86-dell-8268.shanghai.arm.com [10.169.210.116]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id CB7083F71A; Tue, 3 Jan 2023 23:30:58 -0800 (PST) From: Feifei Wang To: Qiming Yang , Wenjun Wu , Ruifeng Wang Cc: dev@dpdk.org, konstantin.v.ananyev@yandex.ru, nd@arm.com, Feifei Wang , Honnappa Nagarahalli Subject: [PATCH v3 3/3] net/ixgbe: enable direct rearm with separate API Date: Wed, 4 Jan 2023 15:30:43 +0800 Message-Id: <20230104073043.1120168-4-feifei.wang2@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20230104073043.1120168-1-feifei.wang2@arm.com> References: <20220420081650.2043183-1-feifei.wang2@arm.com> <20230104073043.1120168-1-feifei.wang2@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add internal API to separate direct rearm operations between Rx and Tx. Signed-off-by: Feifei Wang Reviewed-by: Ruifeng Wang Reviewed-by: Honnappa Nagarahalli --- drivers/net/ixgbe/ixgbe_ethdev.c | 1 + drivers/net/ixgbe/ixgbe_ethdev.h | 3 ++ drivers/net/ixgbe/ixgbe_rxtx.c | 19 +++++++++ drivers/net/ixgbe/ixgbe_rxtx.h | 4 ++ drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 48 +++++++++++++++++++++ drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 52 +++++++++++++++++++++++ 6 files changed, 127 insertions(+) diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c b/drivers/net/ixgbe/ixgbe_ethdev.c index ae9f65b334..e5383d7dbc 100644 --- a/drivers/net/ixgbe/ixgbe_ethdev.c +++ b/drivers/net/ixgbe/ixgbe_ethdev.c @@ -542,6 +542,7 @@ static const struct eth_dev_ops ixgbe_eth_dev_ops = { .set_mc_addr_list = ixgbe_dev_set_mc_addr_list, .rxq_info_get = ixgbe_rxq_info_get, .txq_info_get = ixgbe_txq_info_get, + .rxq_rearm_data_get = ixgbe_rxq_rearm_data_get, .timesync_enable = ixgbe_timesync_enable, .timesync_disable = ixgbe_timesync_disable, .timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp, diff --git a/drivers/net/ixgbe/ixgbe_ethdev.h b/drivers/net/ixgbe/ixgbe_ethdev.h index 48290af512..2a8ae5af7a 100644 --- a/drivers/net/ixgbe/ixgbe_ethdev.h +++ b/drivers/net/ixgbe/ixgbe_ethdev.h @@ -625,6 +625,9 @@ void ixgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, void ixgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_txq_info *qinfo); +void ixgbe_rxq_rearm_data_get(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_rxq_rearm_data *rxq_rearm_data); + int ixgbevf_dev_rx_init(struct rte_eth_dev *dev); void ixgbevf_dev_tx_init(struct rte_eth_dev *dev); diff --git a/drivers/net/ixgbe/ixgbe_rxtx.c b/drivers/net/ixgbe/ixgbe_rxtx.c index c9d6ca9efe..2d7fe710e4 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx.c +++ b/drivers/net/ixgbe/ixgbe_rxtx.c @@ -2559,6 +2559,9 @@ ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq) ixgbe_txq_vec_setup(txq) == 0)) { PMD_INIT_LOG(DEBUG, "Vector tx enabled."); dev->tx_pkt_burst = ixgbe_xmit_pkts_vec; +#ifdef RTE_ARCH_ARM64 + dev->tx_fill_sw_ring = ixgbe_tx_fill_sw_ring; +#endif } else dev->tx_pkt_burst = ixgbe_xmit_pkts_simple; } else { @@ -4853,6 +4856,9 @@ ixgbe_set_rx_function(struct rte_eth_dev *dev) dev->data->port_id); dev->rx_pkt_burst = ixgbe_recv_pkts_vec; +#ifdef RTE_ARCH_ARM64 + dev->rx_flush_descriptor = ixgbe_rx_flush_descriptor_vec; +#endif } else if (adapter->rx_bulk_alloc_allowed) { PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " "satisfied. Rx Burst Bulk Alloc function " @@ -5623,6 +5629,19 @@ ixgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->conf.tx_deferred_start = txq->tx_deferred_start; } +void +ixgbe_rxq_rearm_data_get(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_rxq_rearm_data *rxq_rearm_data) +{ + struct ixgbe_rx_queue *rxq; + + rxq = dev->data->rx_queues[queue_id]; + + rxq_rearm_data->rx_sw_ring = rxq->sw_ring; + rxq_rearm_data->rearm_start = &rxq->rxrearm_start; + rxq_rearm_data->rearm_nb = &rxq->rxrearm_nb; +} + /* * [VF] Initializes Receive Unit. */ diff --git a/drivers/net/ixgbe/ixgbe_rxtx.h b/drivers/net/ixgbe/ixgbe_rxtx.h index 668a5b9814..7c90426f49 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx.h +++ b/drivers/net/ixgbe/ixgbe_rxtx.h @@ -295,6 +295,10 @@ int ixgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt); extern const uint32_t ptype_table[IXGBE_PACKET_TYPE_MAX]; extern const uint32_t ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX]; +int ixgbe_tx_fill_sw_ring(void *tx_queue, + struct rte_eth_rxq_rearm_data *rxq_rearm_data); +int ixgbe_rx_flush_descriptor_vec(void *rx_queue, uint16_t nb_rearm); + uint16_t ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); int ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq); diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h b/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h index a4d9ec9b08..36799dd7f5 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h +++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_common.h @@ -129,6 +129,54 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) return txq->tx_rs_thresh; } +int +ixgbe_tx_fill_sw_ring(void *tx_queue, + struct rte_eth_rxq_rearm_data *rxq_rearm_data) +{ + struct ixgbe_tx_queue *txq = tx_queue; + struct ixgbe_tx_entry_v *txep; + void **rxep; + uint32_t status; + struct rte_mbuf *m; + int i, n; + int nb_rearm = 0; + + if (*rxq_rearm_data->rearm_nb < txq->tx_rs_thresh || + txq->nb_tx_free > txq->tx_free_thresh) + return 0; + + /* check DD bits on threshold descriptor */ + status = txq->tx_ring[txq->tx_next_dd].wb.status; + if (!(status & IXGBE_ADVTXD_STAT_DD)) + return 0; + + n = txq->tx_rs_thresh; + + /* first buffer to free from S/W ring is at index + * tx_next_dd - (tx_rs_thresh-1) + */ + txep = &txq->sw_ring_v[txq->tx_next_dd - (n - 1)]; + rxep = rxq_rearm_data->rx_sw_ring; + rxep += *rxq_rearm_data->rearm_start; + + for (i = 0; i < n; i++, rxep++) { + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); + if (m != NULL) { + *rxep = m; + nb_rearm++; + } + } + n = nb_rearm; + + /* update counters for Tx */ + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); + txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); + if (txq->tx_next_dd >= txq->nb_tx_desc) + txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); + + return n; +} + static __rte_always_inline void tx_backlog_entry(struct ixgbe_tx_entry_v *txep, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c index 90b254ea26..af6a66c2d5 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c +++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c @@ -633,6 +633,58 @@ ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, return nb_pkts; } +int +ixgbe_rx_flush_descriptor_vec(void *rx_queue, uint16_t nb_rearm) +{ + struct ixgbe_rx_queue *rxq = rx_queue; + struct ixgbe_rx_entry *rxep; + volatile union ixgbe_adv_rx_desc *rxdp; + struct rte_mbuf *mb; + uint16_t rx_id; + uint64x2_t dma_addr; + uint64x2_t zero = vdupq_n_u64(0); + uint64_t paddr; + uint8x8_t p; + uint16_t i; + + rxdp = rxq->rx_ring + rxq->rxrearm_start; + rxep = &rxq->sw_ring[rxq->rxrearm_start]; + + p = vld1_u8((uint8_t *)&rxq->mbuf_initializer); + + for (i = 0; i < nb_rearm; i++) { + mb = rxep[i].mbuf; + /* + * Flush mbuf with pkt template. + * Data to be rearmed is 6 bytes long. + */ + vst1_u8((uint8_t *)&mb->rearm_data, p); + /* Initialize rxdp descs */ + paddr = mb->buf_iova + RTE_PKTMBUF_HEADROOM; + dma_addr = vsetq_lane_u64(paddr, zero, 0); + /* flush desc with pa dma_addr */ + vst1q_u64((uint64_t *)&rxdp++->read, dma_addr); + } + + /* Update the descriptor initializer index */ + rxq->rxrearm_start += nb_rearm; + rx_id = rxq->rxrearm_start - 1; + + if (unlikely(rxq->rxrearm_start >= rxq->nb_rx_desc)) { + rxq->rxrearm_start = rxq->rxrearm_start - rxq->nb_rx_desc; + if (!rxq->rxrearm_start) + rx_id = rxq->nb_rx_desc - 1; + else + rx_id = rxq->rxrearm_start - 1; + } + rxq->rxrearm_nb -= nb_rearm; + + /* Update the tail pointer on the NIC */ + IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id); + + return 0; +} + static void __rte_cold ixgbe_tx_queue_release_mbufs_vec(struct ixgbe_tx_queue *txq) {