From patchwork Sun Jan 8 15:42:02 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Elad Persiko X-Patchwork-Id: 19007 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [IPv6:::1]) by dpdk.org (Postfix) with ESMTP id 4927CF919; Sun, 8 Jan 2017 16:43:02 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 5A22BF612 for ; Sun, 8 Jan 2017 16:42:39 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from eladpe@mellanox.com) with ESMTPS (AES256-SHA encrypted); 8 Jan 2017 17:42:37 +0200 Received: from r-aa-dragon21.mtr.labs.mlnx (r-aa-dragon21.mtr.labs.mlnx [10.209.68.158]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id v08FgbX7007977; Sun, 8 Jan 2017 17:42:37 +0200 Received: from r-aa-dragon21.mtr.labs.mlnx (localhost [127.0.0.1]) by r-aa-dragon21.mtr.labs.mlnx (8.14.7/8.14.7) with ESMTP id v08FgbXT004917; Sun, 8 Jan 2017 15:42:37 GMT Received: (from eladpe@localhost) by r-aa-dragon21.mtr.labs.mlnx (8.14.7/8.14.7/Submit) id v08FgbXL004916; Sun, 8 Jan 2017 15:42:37 GMT From: Elad Persiko To: dev@dpdk.org Cc: Elad Persiko Date: Sun, 8 Jan 2017 15:42:02 +0000 Message-Id: <1483890123-4854-4-git-send-email-eladpe@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1483890123-4854-1-git-send-email-eladpe@mellanox.com> References: <1483890123-4854-1-git-send-email-eladpe@mellanox.com> Subject: [dpdk-dev] [PATCH 4/5] net/mlx5: implement TSO data path X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Signed-off-by: Elad Persiko --- drivers/net/mlx5/mlx5_ethdev.c | 2 + drivers/net/mlx5/mlx5_rxtx.c | 246 +++++++++++++++++++++++++++++++---------- 2 files changed, 187 insertions(+), 61 deletions(-) diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index fbb1b65..ea5ab02 100644 --- a/drivers/net/mlx5/mlx5_ethdev.c +++ b/drivers/net/mlx5/mlx5_ethdev.c @@ -589,6 +589,8 @@ struct priv * (priv->hw_vlan_strip ? DEV_RX_OFFLOAD_VLAN_STRIP : 0); if (!priv->mps) info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; + if (priv->lso) + info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; if (priv->hw_csum) info->tx_offload_capa |= (DEV_TX_OFFLOAD_IPV4_CKSUM | diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index 1560530..4940dc1 100644 --- a/drivers/net/mlx5/mlx5_rxtx.c +++ b/drivers/net/mlx5/mlx5_rxtx.c @@ -388,6 +388,8 @@ uint16_t pkt_inline_sz = MLX5_WQE_DWORD_SIZE; uint16_t ehdr; uint8_t cs_flags = 0; + uint8_t header_sum; + uint8_t tso; #ifdef MLX5_PMD_SOFT_COUNTERS uint32_t total_length = 0; #endif @@ -429,37 +431,29 @@ pkt_addr = rte_pktmbuf_mtod(*pkts, volatile void *); rte_prefetch0(pkt_addr); } - /* Should we enable HW CKSUM offload */ - if (buf->ol_flags & - (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { - cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; - } - raw = ((uint8_t *)(uintptr_t)wqe) + 2 * MLX5_WQE_DWORD_SIZE; - /* - * Start by copying the Ethernet header minus the first two - * bytes which will be appended at the end of the Ethernet - * segment. - */ - memcpy((uint8_t *)raw, ((uint8_t *)addr) + 2, 16); - length -= MLX5_WQE_DWORD_SIZE; - addr += MLX5_WQE_DWORD_SIZE; - /* Replace the Ethernet type by the VLAN if necessary. */ - if (buf->ol_flags & PKT_TX_VLAN_PKT) { - uint32_t vlan = htonl(0x81000000 | buf->vlan_tci); - - memcpy((uint8_t *)(raw + MLX5_WQE_DWORD_SIZE - 2 - - sizeof(vlan)), - &vlan, sizeof(vlan)); - addr -= sizeof(vlan); - length += sizeof(vlan); - } - /* Inline if enough room. */ - if (txq->max_inline != 0) { + tso = buf->tso_segsz && buf->l4_len; + if (tso) { + /* + * After copying the ETH seg we need to copy 16 bytes + * less. + */ + header_sum = buf->l2_len + buf->l3_len + buf->l4_len + - MLX5_WQE_DWORD_SIZE; + raw = ((uint8_t *)(uintptr_t)wqe) + + 2 * MLX5_WQE_DWORD_SIZE; + /* + * Start by copying the Ethernet header minus the + * first two bytes which will be appended at the + * end of the Ethernet segment. + */ + memcpy((uint8_t *)raw, ((uint8_t *)addr) + 2, + MLX5_WQE_DWORD_SIZE); + length -= MLX5_WQE_DWORD_SIZE; + addr += MLX5_WQE_DWORD_SIZE; + uintptr_t end = (uintptr_t) (((uintptr_t)txq->wqes) + (1 << txq->wqe_n) * MLX5_WQE_SIZE); - uint16_t max_inline = - txq->max_inline * RTE_CACHE_LINE_SIZE; uint16_t room; /* @@ -468,12 +462,9 @@ */ raw += MLX5_WQE_DWORD_SIZE - 2; room = end - (uintptr_t)raw; - if (room > max_inline) { - uintptr_t addr_end = (addr + max_inline) & - ~(RTE_CACHE_LINE_SIZE - 1); - uint16_t copy_b = ((addr_end - addr) > length) ? - length : - (addr_end - addr); + if (room > header_sum) { + uintptr_t addr_end = addr + header_sum; + uint16_t copy_b = addr_end - addr; rte_memcpy((void *)raw, (void *)addr, copy_b); addr += copy_b; @@ -488,10 +479,16 @@ 0, 0, }; + wqe->eseg = (rte_v128u32_t){ + 0, + 0, + 0, + 0}; length = 0; buf = *(pkts--); ds = 1; - goto next_pkt_part; + elts_head = (elts_head - 1) & (elts_n - 1); + goto next_pkt_end; } /* * 2 DWORDs consumed by the WQE header + ETH segment + @@ -500,28 +497,138 @@ ds = 2 + MLX5_WQE_DS(pkt_inline_sz - 2); if (length > 0) { dseg = (volatile rte_v128u32_t *) - ((uintptr_t)wqe + - (ds * MLX5_WQE_DWORD_SIZE)); - if ((uintptr_t)dseg >= end) - dseg = (volatile rte_v128u32_t *) - txq->wqes; + ((uintptr_t)wqe + + (ds * MLX5_WQE_DWORD_SIZE)); + if ((uintptr_t)dseg >= end) + dseg = (volatile rte_v128u32_t *) + txq->wqes; } else if (!segs_n) { goto next_pkt; } else { /* dseg will be advance as part of next_seg */ dseg = (volatile rte_v128u32_t *) - ((uintptr_t)wqe + - ((ds - 1) * MLX5_WQE_DWORD_SIZE)); + ((uintptr_t)wqe + + ((ds - 1) * MLX5_WQE_DWORD_SIZE)); goto next_seg; } } else { + /* Should we enable HW CKSUM offload */ + if (buf->ol_flags & + (PKT_TX_IP_CKSUM | + PKT_TX_TCP_CKSUM | + PKT_TX_UDP_CKSUM)) { + cs_flags = MLX5_ETH_WQE_L3_CSUM | + MLX5_ETH_WQE_L4_CSUM; + } + raw = ((uint8_t *)(uintptr_t)wqe) + + 2 * MLX5_WQE_DWORD_SIZE; /* - * No inline has been done in the packet, only the - * Ethernet Header as been stored. + * Start by copying the Ethernet header minus the + * first two bytes which will be appended at the end + * of the Ethernet segment. */ - dseg = (volatile rte_v128u32_t *) - ((uintptr_t)wqe + (3 * MLX5_WQE_DWORD_SIZE)); - ds = 3; + memcpy((uint8_t *)raw, ((uint8_t *)addr) + 2, 16); + length -= MLX5_WQE_DWORD_SIZE; + addr += MLX5_WQE_DWORD_SIZE; + /* Replace the Ethernet type by the VLAN if necessary. + */ + if (buf->ol_flags & PKT_TX_VLAN_PKT) { + uint32_t vlan = htonl(0x81000000 | + buf->vlan_tci); + + memcpy((uint8_t *)(raw + MLX5_WQE_DWORD_SIZE - + 2 - sizeof(vlan)), + &vlan, sizeof(vlan)); + addr -= sizeof(vlan); + length += sizeof(vlan); + } + /* Inline if enough room. */ + if (txq->max_inline != 0) { + uintptr_t end = (uintptr_t) + (((uintptr_t)txq->wqes) + + (1 << txq->wqe_n) * MLX5_WQE_SIZE); + uint16_t max_inline = + txq->max_inline * RTE_CACHE_LINE_SIZE; + uint16_t room; + + /* + * raw starts two bytes before the boundary to + * continue the above copy of packet data. + */ + raw += MLX5_WQE_DWORD_SIZE - 2; + room = end - (uintptr_t)raw; + if (room > max_inline) { + uintptr_t addr_end = + (addr + max_inline) & + ~(RTE_CACHE_LINE_SIZE - 1); + uint16_t copy_b = ((addr_end - addr) + > length) ? + length : + (addr_end - addr); + + rte_memcpy((void *)raw, (void *)addr, + copy_b); + addr += copy_b; + length -= copy_b; + pkt_inline_sz += copy_b; + /* Sanity check. */ + assert(addr <= addr_end); + } else { + wqe->ctrl = (rte_v128u32_t){ + htonl(txq->wqe_ci << 8), + htonl(txq->qp_num_8s | 1), + 0, + 0, + }; + wqe->eseg = (rte_v128u32_t){ + 0, + 0, + 0, + 0}; + length = 0; + buf = *(pkts--); + ds = 1; + elts_head = (elts_head - 1) & + (elts_n - 1); + goto next_pkt_end; + } + /* + * 2 DWORDs consumed by the WQE header + * + ETH segment + 1 DSEG + + * the size of the inline part of the packet. + */ + ds = 2 + MLX5_WQE_DS(pkt_inline_sz - 2); + if (length > 0) { + dseg = (volatile rte_v128u32_t *) + ((uintptr_t)wqe + + (ds * MLX5_WQE_DWORD_SIZE)); + if ((uintptr_t)dseg >= end) + dseg = + (volatile rte_v128u32_t *) + txq->wqes; + } else if (!segs_n) { + goto next_pkt; + } else { + /* + * dseg will be advance as part + * of next_seg. + */ + dseg = (volatile rte_v128u32_t *) + ((uintptr_t)wqe + + ((ds - 1) * + MLX5_WQE_DWORD_SIZE)); + goto next_seg; + } + } else { + /* + * No inline has been done in the packet, + * only the Ethernet Header as been stored. + */ + dseg = (volatile rte_v128u32_t *) + ((uintptr_t)wqe + + (3 * MLX5_WQE_DWORD_SIZE)); + ds = 3; + } } /* Add the remaining packet as a simple ds. */ addr = htonll(addr); @@ -578,21 +685,38 @@ else --pkts_n; next_pkt: - ++i; /* Initialize known and common part of the WQE structure. */ - wqe->ctrl = (rte_v128u32_t){ - htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND), - htonl(txq->qp_num_8s | ds), - 0, - 0, - }; -next_pkt_part: - wqe->eseg = (rte_v128u32_t){ - 0, - cs_flags, - 0, - (ehdr << 16) | htons(pkt_inline_sz), - }; + if (tso) { + wqe->ctrl = (rte_v128u32_t){ + htonl((txq->wqe_ci << 8) | MLX5_OPCODE_TSO), + htonl(txq->qp_num_8s | ds), + 0, + 0, + }; + wqe->eseg = (rte_v128u32_t){ + 0, + MLX5_ETH_WQE_L3_CSUM | + MLX5_ETH_WQE_L4_CSUM | + htons(buf->tso_segsz) << 16, + 0, + (ehdr << 16) | htons(pkt_inline_sz), + }; + } else { + wqe->ctrl = (rte_v128u32_t){ + htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND), + htonl(txq->qp_num_8s | ds), + 0, + 0, + }; + wqe->eseg = (rte_v128u32_t){ + 0, + cs_flags, + 0, + (ehdr << 16) | htons(pkt_inline_sz), + }; + } +next_pkt_end: + ++i; txq->wqe_ci += (ds + 3) / 4; #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment sent bytes counter. */