From patchwork Sat Sep 24 02:45:46 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Long Li X-Patchwork-Id: 116798 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 7A650A054F; Sat, 24 Sep 2022 04:47:41 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id BD8FE42C3F; Sat, 24 Sep 2022 04:46:44 +0200 (CEST) Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by mails.dpdk.org (Postfix) with ESMTP id AA25C42BD8 for ; Sat, 24 Sep 2022 04:46:01 +0200 (CEST) Received: by linux.microsoft.com (Postfix, from userid 1004) id 15FAD204B532; Fri, 23 Sep 2022 19:46:01 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com 15FAD204B532 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxonhyperv.com; s=default; t=1663987561; bh=UGCu4QQWpSGShxINCyntdxY0dUG3voszfnBB1iMwvK4=; h=From:To:Cc:Subject:Date:In-Reply-To:References:Reply-To:From; b=HVEiSLka4ty756ScmDteyEHVFktFP3yX7ilXiiPFqBRd7fs2UYQcgciCWqF4TGRFf /DTqfDcZ2G638VuQINd0cU7hen/ZB5jZ8lyHYb1JsnLZyjjo6YGQ8KMLsi4xkYiWQ+ tQWjuF/vRwxILM1lf7qbMSQxYIAZujEw763+Suu8= From: longli@linuxonhyperv.com To: Ferruh Yigit Cc: dev@dpdk.org, Ajay Sharma , Stephen Hemminger , Long Li Subject: [Patch v9 18/18] net/mana: support Rx interrupts Date: Fri, 23 Sep 2022 19:45:46 -0700 Message-Id: <1663987546-15982-19-git-send-email-longli@linuxonhyperv.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1663987546-15982-1-git-send-email-longli@linuxonhyperv.com> References: <1662674189-29524-1-git-send-email-longli@linuxonhyperv.com> <1663987546-15982-1-git-send-email-longli@linuxonhyperv.com> X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: longli@microsoft.com Errors-To: dev-bounces@dpdk.org From: Long Li mana can receive Rx interrupts from kernel through RDMA verbs interface. Implement Rx interrupts in the driver. Signed-off-by: Long Li --- Change log: v5: New patch added to the series v8: Fix coding style on function definitions. doc/guides/nics/features/mana.ini | 1 + drivers/net/mana/gdma.c | 10 +-- drivers/net/mana/mana.c | 128 ++++++++++++++++++++++++++---- drivers/net/mana/mana.h | 9 ++- drivers/net/mana/rx.c | 94 +++++++++++++++++++--- drivers/net/mana/tx.c | 3 +- 6 files changed, 211 insertions(+), 34 deletions(-) diff --git a/doc/guides/nics/features/mana.ini b/doc/guides/nics/features/mana.ini index 81ebc9c365..5fb62ea85d 100644 --- a/doc/guides/nics/features/mana.ini +++ b/doc/guides/nics/features/mana.ini @@ -14,6 +14,7 @@ Multiprocess aware = Y Queue start/stop = Y Removal event = Y RSS hash = Y +Rx interrupt = Y Speed capabilities = P Usage doc = Y x86-64 = Y diff --git a/drivers/net/mana/gdma.c b/drivers/net/mana/gdma.c index 3f937d6c93..c67c5af2f9 100644 --- a/drivers/net/mana/gdma.c +++ b/drivers/net/mana/gdma.c @@ -213,7 +213,7 @@ union gdma_doorbell_entry { */ int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type, - uint32_t queue_id, uint32_t tail) + uint32_t queue_id, uint32_t tail, uint8_t arm) { uint8_t *addr = db_page; union gdma_doorbell_entry e = {}; @@ -228,14 +228,14 @@ mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type, case GDMA_QUEUE_RECEIVE: e.rq.id = queue_id; e.rq.tail_ptr = tail; - e.rq.wqe_cnt = 1; + e.rq.wqe_cnt = arm; addr += DOORBELL_OFFSET_RQ; break; case GDMA_QUEUE_COMPLETION: e.cq.id = queue_id; e.cq.tail_ptr = tail; - e.cq.arm = 1; + e.cq.arm = arm; addr += DOORBELL_OFFSET_CQ; break; @@ -247,8 +247,8 @@ mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type, /* Ensure all writes are done before ringing doorbell */ rte_wmb(); - DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u", - db_page, addr, queue_id, queue_type, tail); + DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u arm %u", + db_page, addr, queue_id, queue_type, tail, arm); rte_write64(e.as_uint64, addr); return 0; diff --git a/drivers/net/mana/mana.c b/drivers/net/mana/mana.c index d247b930db..d277a35dae 100644 --- a/drivers/net/mana/mana.c +++ b/drivers/net/mana/mana.c @@ -103,7 +103,72 @@ mana_dev_configure(struct rte_eth_dev *dev) return 0; } -static int mana_intr_uninstall(struct mana_priv *priv); +static void +rx_intr_vec_disable(struct mana_priv *priv) +{ + struct rte_intr_handle *intr_handle = priv->intr_handle; + + rte_intr_free_epoll_fd(intr_handle); + rte_intr_vec_list_free(intr_handle); + rte_intr_nb_efd_set(intr_handle, 0); +} + +static int +rx_intr_vec_enable(struct mana_priv *priv) +{ + unsigned int i; + unsigned int rxqs_n = priv->dev_data->nb_rx_queues; + unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); + struct rte_intr_handle *intr_handle = priv->intr_handle; + int ret; + + rx_intr_vec_disable(priv); + + if (rte_intr_vec_list_alloc(intr_handle, NULL, n)) { + DRV_LOG(ERR, "Failed to allocate memory for interrupt vector"); + return -ENOMEM; + } + + for (i = 0; i < n; i++) { + struct mana_rxq *rxq = priv->dev_data->rx_queues[i]; + + ret = rte_intr_vec_list_index_set(intr_handle, i, + RTE_INTR_VEC_RXTX_OFFSET + i); + if (ret) { + DRV_LOG(ERR, "Failed to set intr vec %u", i); + return ret; + } + + ret = rte_intr_efds_index_set(intr_handle, i, rxq->channel->fd); + if (ret) { + DRV_LOG(ERR, "Failed to set FD at intr %u", i); + return ret; + } + } + + return rte_intr_nb_efd_set(intr_handle, n); +} + +static void +rxq_intr_disable(struct mana_priv *priv) +{ + int err = rte_errno; + + rx_intr_vec_disable(priv); + rte_errno = err; +} + +static int +rxq_intr_enable(struct mana_priv *priv) +{ + const struct rte_eth_intr_conf *const intr_conf = + &priv->dev_data->dev_conf.intr_conf; + + if (!intr_conf->rxq) + return 0; + + return rx_intr_vec_enable(priv); +} static int mana_dev_start(struct rte_eth_dev *dev) @@ -141,8 +206,17 @@ mana_dev_start(struct rte_eth_dev *dev) /* Enable datapath for secondary processes */ mana_mp_req_on_rxtx(dev, MANA_MP_REQ_START_RXTX); + ret = rxq_intr_enable(priv); + if (ret) { + DRV_LOG(ERR, "Failed to enable RX interrupts"); + goto failed_intr; + } + return 0; +failed_intr: + mana_stop_rx_queues(dev); + failed_rx: mana_stop_tx_queues(dev); @@ -153,9 +227,12 @@ mana_dev_start(struct rte_eth_dev *dev) } static int -mana_dev_stop(struct rte_eth_dev *dev __rte_unused) +mana_dev_stop(struct rte_eth_dev *dev) { int ret; + struct mana_priv *priv = dev->data->dev_private; + + rxq_intr_disable(priv); dev->tx_pkt_burst = mana_tx_burst_removed; dev->rx_pkt_burst = mana_rx_burst_removed; @@ -180,6 +257,8 @@ mana_dev_stop(struct rte_eth_dev *dev __rte_unused) return 0; } +static int mana_intr_uninstall(struct mana_priv *priv); + static int mana_dev_close(struct rte_eth_dev *dev) { @@ -614,6 +693,8 @@ static const struct eth_dev_ops mana_dev_ops = { .tx_queue_release = mana_dev_tx_queue_release, .rx_queue_setup = mana_dev_rx_queue_setup, .rx_queue_release = mana_dev_rx_queue_release, + .rx_queue_intr_enable = mana_rx_intr_enable, + .rx_queue_intr_disable = mana_rx_intr_disable, .link_update = mana_dev_link_update, .stats_get = mana_dev_stats_get, .stats_reset = mana_dev_stats_reset, @@ -849,10 +930,22 @@ mana_intr_uninstall(struct mana_priv *priv) return 0; } +int +mana_fd_set_non_blocking(int fd) +{ + int ret = fcntl(fd, F_GETFL); + + if (ret != -1 && !fcntl(fd, F_SETFL, ret | O_NONBLOCK)) + return 0; + + rte_errno = errno; + return -rte_errno; +} + static int -mana_intr_install(struct mana_priv *priv) +mana_intr_install(struct rte_eth_dev *eth_dev, struct mana_priv *priv) { - int ret, flags; + int ret; struct ibv_context *ctx = priv->ib_ctx; priv->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED); @@ -862,31 +955,35 @@ mana_intr_install(struct mana_priv *priv) return -ENOMEM; } - rte_intr_fd_set(priv->intr_handle, -1); + ret = rte_intr_fd_set(priv->intr_handle, -1); + if (ret) + goto free_intr; - flags = fcntl(ctx->async_fd, F_GETFL); - ret = fcntl(ctx->async_fd, F_SETFL, flags | O_NONBLOCK); + ret = mana_fd_set_non_blocking(ctx->async_fd); if (ret) { DRV_LOG(ERR, "Failed to change async_fd to NONBLOCK"); goto free_intr; } - rte_intr_fd_set(priv->intr_handle, ctx->async_fd); - rte_intr_type_set(priv->intr_handle, RTE_INTR_HANDLE_EXT); + ret = rte_intr_fd_set(priv->intr_handle, ctx->async_fd); + if (ret) + goto free_intr; + + ret = rte_intr_type_set(priv->intr_handle, RTE_INTR_HANDLE_EXT); + if (ret) + goto free_intr; ret = rte_intr_callback_register(priv->intr_handle, mana_intr_handler, priv); if (ret) { DRV_LOG(ERR, "Failed to register intr callback"); rte_intr_fd_set(priv->intr_handle, -1); - goto restore_fd; + goto free_intr; } + eth_dev->intr_handle = priv->intr_handle; return 0; -restore_fd: - fcntl(ctx->async_fd, F_SETFL, flags); - free_intr: rte_intr_instance_free(priv->intr_handle); priv->intr_handle = NULL; @@ -1224,8 +1321,10 @@ mana_pci_probe_mac(struct rte_pci_device *pci_dev, name, priv->max_rx_queues, priv->max_rx_desc, priv->max_send_sge); + rte_eth_copy_pci_info(eth_dev, pci_dev); + /* Create async interrupt handler */ - ret = mana_intr_install(priv); + ret = mana_intr_install(eth_dev, priv); if (ret) { DRV_LOG(ERR, "Failed to install intr handler"); goto failed; @@ -1246,7 +1345,6 @@ mana_pci_probe_mac(struct rte_pci_device *pci_dev, eth_dev->tx_pkt_burst = mana_tx_burst_removed; eth_dev->dev_ops = &mana_dev_ops; - rte_eth_copy_pci_info(eth_dev, pci_dev); rte_eth_dev_probing_finish(eth_dev); } diff --git a/drivers/net/mana/mana.h b/drivers/net/mana/mana.h index 9c576a82fa..14d813466e 100644 --- a/drivers/net/mana/mana.h +++ b/drivers/net/mana/mana.h @@ -425,6 +425,7 @@ struct mana_rxq { uint32_t num_desc; struct rte_mempool *mp; struct ibv_cq *cq; + struct ibv_comp_channel *channel; struct ibv_wq *wq; /* For storing pending requests */ @@ -458,8 +459,8 @@ extern int mana_logtype_init; #define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type, - uint32_t queue_id, uint32_t tail); -int mana_rq_ring_doorbell(struct mana_rxq *rxq); + uint32_t queue_id, uint32_t tail, uint8_t arm); +int mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm); int gdma_post_work_request(struct mana_gdma_queue *queue, struct gdma_work_request *work_req, @@ -539,4 +540,8 @@ void mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type); void *mana_alloc_verbs_buf(size_t size, void *data); void mana_free_verbs_buf(void *ptr, void *data __rte_unused); +int mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id); +int mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id); +int mana_fd_set_non_blocking(int fd); + #endif diff --git a/drivers/net/mana/rx.c b/drivers/net/mana/rx.c index b80a5d1c7a..57dfae7bcd 100644 --- a/drivers/net/mana/rx.c +++ b/drivers/net/mana/rx.c @@ -22,7 +22,7 @@ static uint8_t mana_rss_hash_key_default[TOEPLITZ_HASH_KEY_SIZE_IN_BYTES] = { }; int -mana_rq_ring_doorbell(struct mana_rxq *rxq) +mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm) { struct mana_priv *priv = rxq->priv; int ret; @@ -37,9 +37,9 @@ mana_rq_ring_doorbell(struct mana_rxq *rxq) } ret = mana_ring_doorbell(db_page, GDMA_QUEUE_RECEIVE, - rxq->gdma_rq.id, - rxq->gdma_rq.head * - GDMA_WQE_ALIGNMENT_UNIT_SIZE); + rxq->gdma_rq.id, + rxq->gdma_rq.head * GDMA_WQE_ALIGNMENT_UNIT_SIZE, + arm); if (ret) DRV_LOG(ERR, "failed to ring RX doorbell ret %d", ret); @@ -121,7 +121,7 @@ mana_alloc_and_post_rx_wqes(struct mana_rxq *rxq) } } - mana_rq_ring_doorbell(rxq); + mana_rq_ring_doorbell(rxq, rxq->num_desc); return ret; } @@ -163,6 +163,14 @@ mana_stop_rx_queues(struct rte_eth_dev *dev) DRV_LOG(ERR, "rx_queue destroy_cq failed %d", ret); rxq->cq = NULL; + + if (rxq->channel) { + ret = ibv_destroy_comp_channel(rxq->channel); + if (ret) + DRV_LOG(ERR, "failed destroy comp %d", + ret); + rxq->channel = NULL; + } } /* Drain and free posted WQEs */ @@ -204,8 +212,24 @@ mana_start_rx_queues(struct rte_eth_dev *dev) .data = (void *)(uintptr_t)rxq->socket, })); + if (dev->data->dev_conf.intr_conf.rxq) { + rxq->channel = ibv_create_comp_channel(priv->ib_ctx); + if (!rxq->channel) { + ret = -errno; + DRV_LOG(ERR, "Queue %d comp channel failed", i); + goto fail; + } + + ret = mana_fd_set_non_blocking(rxq->channel->fd); + if (ret) { + DRV_LOG(ERR, "Failed to set comp non-blocking"); + goto fail; + } + } + rxq->cq = ibv_create_cq(priv->ib_ctx, rxq->num_desc, - NULL, NULL, 0); + NULL, rxq->channel, + rxq->channel ? i : 0); if (!rxq->cq) { ret = -errno; DRV_LOG(ERR, "failed to create rx cq queue %d", i); @@ -356,7 +380,8 @@ mana_start_rx_queues(struct rte_eth_dev *dev) uint16_t mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) { - uint16_t pkt_received = 0, cqe_processed = 0; + uint16_t pkt_received = 0; + uint8_t wqe_posted = 0; struct mana_rxq *rxq = dpdk_rxq; struct mana_priv *priv = rxq->priv; struct gdma_comp comp; @@ -442,18 +467,65 @@ mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) if (rxq->desc_ring_tail >= rxq->num_desc) rxq->desc_ring_tail = 0; - cqe_processed++; - /* Post another request */ ret = mana_alloc_and_post_rx_wqe(rxq); if (ret) { DRV_LOG(ERR, "failed to post rx wqe ret=%d", ret); break; } + + wqe_posted++; } - if (cqe_processed) - mana_rq_ring_doorbell(rxq); + if (wqe_posted) + mana_rq_ring_doorbell(rxq, wqe_posted); return pkt_received; } + +static int +mana_arm_cq(struct mana_rxq *rxq, uint8_t arm) +{ + struct mana_priv *priv = rxq->priv; + uint32_t head = rxq->gdma_cq.head % + (rxq->gdma_cq.count << COMPLETION_QUEUE_ENTRY_OWNER_BITS_SIZE); + + DRV_LOG(ERR, "Ringing completion queue ID %u head %u arm %d", + rxq->gdma_cq.id, head, arm); + + return mana_ring_doorbell(priv->db_page, GDMA_QUEUE_COMPLETION, + rxq->gdma_cq.id, head, arm); +} + +int +mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct mana_rxq *rxq = dev->data->rx_queues[rx_queue_id]; + + return mana_arm_cq(rxq, 1); +} + +int +mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct mana_rxq *rxq = dev->data->rx_queues[rx_queue_id]; + struct ibv_cq *ev_cq; + void *ev_ctx; + int ret; + + ret = ibv_get_cq_event(rxq->channel, &ev_cq, &ev_ctx); + if (ret) + ret = errno; + else if (ev_cq != rxq->cq) + ret = EINVAL; + + if (ret) { + if (ret != EAGAIN) + DRV_LOG(ERR, "Can't disable RX intr queue %d", + rx_queue_id); + } else { + ibv_ack_cq_events(rxq->cq, 1); + } + + return -ret; +} diff --git a/drivers/net/mana/tx.c b/drivers/net/mana/tx.c index 0884681c30..a92d895e54 100644 --- a/drivers/net/mana/tx.c +++ b/drivers/net/mana/tx.c @@ -406,7 +406,8 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) ret = mana_ring_doorbell(db_page, GDMA_QUEUE_SEND, txq->gdma_sq.id, txq->gdma_sq.head * - GDMA_WQE_ALIGNMENT_UNIT_SIZE); + GDMA_WQE_ALIGNMENT_UNIT_SIZE, + 0); if (ret) DRV_LOG(ERR, "mana_ring_doorbell failed ret %d", ret);