diff mbox

[dpdk-dev,v2,11/13] ixgbe: support TCP segmentation offload

Message ID 1415984609-2484-12-git-send-email-olivier.matz@6wind.com (mailing list archive)
State Superseded, archived
Headers show

Commit Message

Olivier Matz Nov. 14, 2014, 5:03 p.m. UTC
Implement TSO (TCP segmentation offload) in ixgbe driver. The driver is
now able to use PKT_TX_TCP_SEG mbuf flag and mbuf hardware offload infos
(l2_len, l3_len, l4_len, tso_segsz) to configure the hardware support of
TCP segmentation.

In ixgbe, when doing TSO, the IP length must not be included in the TCP
pseudo header checksum. A new function ixgbe_fix_tcp_phdr_cksum() is
used to fix the pseudo header checksum of the packet before giving it to
the hardware.

In the patch, the tx_desc_cksum_flags_to_olinfo() and
tx_desc_ol_flags_to_cmdtype() functions have been reworked to make them
clearer. This should not impact performance as gcc (version 4.8 in my
case) is smart enough to convert the tests into a code that does not
contain any branch instruction.

Signed-off-by: Olivier Matz <olivier.matz@6wind.com>
---
 lib/librte_pmd_ixgbe/ixgbe_ethdev.c |   3 +-
 lib/librte_pmd_ixgbe/ixgbe_rxtx.c   | 169 ++++++++++++++++++++++--------------
 lib/librte_pmd_ixgbe/ixgbe_rxtx.h   |  19 ++--
 3 files changed, 117 insertions(+), 74 deletions(-)

Comments

Ananyev, Konstantin Nov. 17, 2014, 6:26 p.m. UTC | #1
> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Olivier Matz
> Sent: Friday, November 14, 2014 5:03 PM
> To: dev@dpdk.org
> Cc: jigsaw@gmail.com
> Subject: [dpdk-dev] [PATCH v2 11/13] ixgbe: support TCP segmentation offload
> 
> Implement TSO (TCP segmentation offload) in ixgbe driver. The driver is
> now able to use PKT_TX_TCP_SEG mbuf flag and mbuf hardware offload infos
> (l2_len, l3_len, l4_len, tso_segsz) to configure the hardware support of
> TCP segmentation.
> 
> In ixgbe, when doing TSO, the IP length must not be included in the TCP
> pseudo header checksum. A new function ixgbe_fix_tcp_phdr_cksum() is
> used to fix the pseudo header checksum of the packet before giving it to
> the hardware.
> 
> In the patch, the tx_desc_cksum_flags_to_olinfo() and
> tx_desc_ol_flags_to_cmdtype() functions have been reworked to make them
> clearer. This should not impact performance as gcc (version 4.8 in my
> case) is smart enough to convert the tests into a code that does not
> contain any branch instruction.
> 
> Signed-off-by: Olivier Matz <olivier.matz@6wind.com>

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

Just one thing - double semicolon -  looks like a typo:
> +	/* check if TCP segmentation required for this packet */
> +	if (ol_flags & PKT_TX_TCP_SEG) {
> +		/* implies IP cksum and TCP cksum */
> +		type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4 |
> +			IXGBE_ADVTXD_TUCMD_L4T_TCP |
> +			IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;;
 

> ---
>  lib/librte_pmd_ixgbe/ixgbe_ethdev.c |   3 +-
>  lib/librte_pmd_ixgbe/ixgbe_rxtx.c   | 169 ++++++++++++++++++++++--------------
>  lib/librte_pmd_ixgbe/ixgbe_rxtx.h   |  19 ++--
>  3 files changed, 117 insertions(+), 74 deletions(-)
> 
> diff --git a/lib/librte_pmd_ixgbe/ixgbe_ethdev.c b/lib/librte_pmd_ixgbe/ixgbe_ethdev.c
> index 2eb609c..2c2ecc0 100644
> --- a/lib/librte_pmd_ixgbe/ixgbe_ethdev.c
> +++ b/lib/librte_pmd_ixgbe/ixgbe_ethdev.c
> @@ -1964,7 +1964,8 @@ ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
>  		DEV_TX_OFFLOAD_IPV4_CKSUM  |
>  		DEV_TX_OFFLOAD_UDP_CKSUM   |
>  		DEV_TX_OFFLOAD_TCP_CKSUM   |
> -		DEV_TX_OFFLOAD_SCTP_CKSUM;
> +		DEV_TX_OFFLOAD_SCTP_CKSUM  |
> +		DEV_TX_OFFLOAD_TCP_TSO;
> 
>  	dev_info->default_rxconf = (struct rte_eth_rxconf) {
>  			.rx_thresh = {
> diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
> index 2df3385..19e3b73 100644
> --- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
> +++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
> @@ -94,7 +94,8 @@
>  #define IXGBE_TX_OFFLOAD_MASK (			 \
>  		PKT_TX_VLAN_PKT |		 \
>  		PKT_TX_IP_CKSUM |		 \
> -		PKT_TX_L4_MASK)
> +		PKT_TX_L4_MASK |		 \
> +		PKT_TX_TCP_SEG)
> 
>  static inline struct rte_mbuf *
>  rte_rxmbuf_alloc(struct rte_mempool *mp)
> @@ -363,59 +364,84 @@ ixgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
>  static inline void
>  ixgbe_set_xmit_ctx(struct igb_tx_queue* txq,
>  		volatile struct ixgbe_adv_tx_context_desc *ctx_txd,
> -		uint64_t ol_flags, uint32_t vlan_macip_lens)
> +		uint64_t ol_flags, union ixgbe_tx_offload tx_offload)
>  {
>  	uint32_t type_tucmd_mlhl;
> -	uint32_t mss_l4len_idx;
> +	uint32_t mss_l4len_idx = 0;
>  	uint32_t ctx_idx;
> -	uint32_t cmp_mask;
> +	uint32_t vlan_macip_lens;
> +	union ixgbe_tx_offload tx_offload_mask;
> 
>  	ctx_idx = txq->ctx_curr;
> -	cmp_mask = 0;
> +	tx_offload_mask.data = 0;
>  	type_tucmd_mlhl = 0;
> 
> +	/* Specify which HW CTX to upload. */
> +	mss_l4len_idx |= (ctx_idx << IXGBE_ADVTXD_IDX_SHIFT);
> +
>  	if (ol_flags & PKT_TX_VLAN_PKT) {
> -		cmp_mask |= TX_VLAN_CMP_MASK;
> +		tx_offload_mask.vlan_tci = ~0;
>  	}
> 
> -	if (ol_flags & PKT_TX_IP_CKSUM) {
> -		type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4;
> -		cmp_mask |= TX_MACIP_LEN_CMP_MASK;
> -	}
> +	/* check if TCP segmentation required for this packet */
> +	if (ol_flags & PKT_TX_TCP_SEG) {
> +		/* implies IP cksum and TCP cksum */
> +		type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4 |
> +			IXGBE_ADVTXD_TUCMD_L4T_TCP |
> +			IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;;
> +
> +		tx_offload_mask.l2_len = ~0;
> +		tx_offload_mask.l3_len = ~0;
> +		tx_offload_mask.l4_len = ~0;
> +		tx_offload_mask.tso_segsz = ~0;
> +		mss_l4len_idx |= tx_offload.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT;
> +		mss_l4len_idx |= tx_offload.l4_len << IXGBE_ADVTXD_L4LEN_SHIFT;
> +	} else { /* no TSO, check if hardware checksum is needed */
> +		if (ol_flags & PKT_TX_IP_CKSUM) {
> +			type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4;
> +			tx_offload_mask.l2_len = ~0;
> +			tx_offload_mask.l3_len = ~0;
> +		}
> 
> -	/* Specify which HW CTX to upload. */
> -	mss_l4len_idx = (ctx_idx << IXGBE_ADVTXD_IDX_SHIFT);
> -	switch (ol_flags & PKT_TX_L4_MASK) {
> -	case PKT_TX_UDP_CKSUM:
> -		type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP |
> +		switch (ol_flags & PKT_TX_L4_MASK) {
> +		case PKT_TX_UDP_CKSUM:
> +			type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP |
>  				IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
> -		mss_l4len_idx |= sizeof(struct udp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
> -		cmp_mask |= TX_MACIP_LEN_CMP_MASK;
> -		break;
> -	case PKT_TX_TCP_CKSUM:
> -		type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP |
> +			mss_l4len_idx |= sizeof(struct udp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
> +			tx_offload_mask.l2_len = ~0;
> +			tx_offload_mask.l3_len = ~0;
> +			break;
> +		case PKT_TX_TCP_CKSUM:
> +			type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP |
>  				IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
> -		mss_l4len_idx |= sizeof(struct tcp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
> -		cmp_mask |= TX_MACIP_LEN_CMP_MASK;
> -		break;
> -	case PKT_TX_SCTP_CKSUM:
> -		type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP |
> +			mss_l4len_idx |= sizeof(struct tcp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
> +			tx_offload_mask.l2_len = ~0;
> +			tx_offload_mask.l3_len = ~0;
> +			tx_offload_mask.l4_len = ~0;
> +			break;
> +		case PKT_TX_SCTP_CKSUM:
> +			type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP |
>  				IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
> -		mss_l4len_idx |= sizeof(struct sctp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
> -		cmp_mask |= TX_MACIP_LEN_CMP_MASK;
> -		break;
> -	default:
> -		type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_RSV |
> +			mss_l4len_idx |= sizeof(struct sctp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
> +			tx_offload_mask.l2_len = ~0;
> +			tx_offload_mask.l3_len = ~0;
> +			break;
> +		default:
> +			type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_RSV |
>  				IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
> -		break;
> +			break;
> +		}
>  	}
> 
>  	txq->ctx_cache[ctx_idx].flags = ol_flags;
> -	txq->ctx_cache[ctx_idx].cmp_mask = cmp_mask;
> -	txq->ctx_cache[ctx_idx].vlan_macip_lens.data =
> -		vlan_macip_lens & cmp_mask;
> +	txq->ctx_cache[ctx_idx].tx_offload.data  =
> +		tx_offload_mask.data & tx_offload.data;
> +	txq->ctx_cache[ctx_idx].tx_offload_mask    = tx_offload_mask;
> 
>  	ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl);
> +	vlan_macip_lens = tx_offload.l3_len;
> +	vlan_macip_lens |= (tx_offload.l2_len << IXGBE_ADVTXD_MACLEN_SHIFT);
> +	vlan_macip_lens |= ((uint32_t)tx_offload.vlan_tci << IXGBE_ADVTXD_VLAN_SHIFT);
>  	ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens);
>  	ctx_txd->mss_l4len_idx   = rte_cpu_to_le_32(mss_l4len_idx);
>  	ctx_txd->seqnum_seed     = 0;
> @@ -427,20 +453,20 @@ ixgbe_set_xmit_ctx(struct igb_tx_queue* txq,
>   */
>  static inline uint32_t
>  what_advctx_update(struct igb_tx_queue *txq, uint64_t flags,
> -		uint32_t vlan_macip_lens)
> +		union ixgbe_tx_offload tx_offload)
>  {
>  	/* If match with the current used context */
>  	if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
> -		(txq->ctx_cache[txq->ctx_curr].vlan_macip_lens.data ==
> -		(txq->ctx_cache[txq->ctx_curr].cmp_mask & vlan_macip_lens)))) {
> +		(txq->ctx_cache[txq->ctx_curr].tx_offload.data ==
> +		(txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data & tx_offload.data)))) {
>  			return txq->ctx_curr;
>  	}
> 
>  	/* What if match with the next context  */
>  	txq->ctx_curr ^= 1;
>  	if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
> -		(txq->ctx_cache[txq->ctx_curr].vlan_macip_lens.data ==
> -		(txq->ctx_cache[txq->ctx_curr].cmp_mask & vlan_macip_lens)))) {
> +		(txq->ctx_cache[txq->ctx_curr].tx_offload.data ==
> +		(txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data & tx_offload.data)))) {
>  			return txq->ctx_curr;
>  	}
> 
> @@ -451,20 +477,25 @@ what_advctx_update(struct igb_tx_queue *txq, uint64_t flags,
>  static inline uint32_t
>  tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
>  {
> -	static const uint32_t l4_olinfo[2] = {0, IXGBE_ADVTXD_POPTS_TXSM};
> -	static const uint32_t l3_olinfo[2] = {0, IXGBE_ADVTXD_POPTS_IXSM};
> -	uint32_t tmp;
> -
> -	tmp  = l4_olinfo[(ol_flags & PKT_TX_L4_MASK)  != PKT_TX_L4_NO_CKSUM];
> -	tmp |= l3_olinfo[(ol_flags & PKT_TX_IP_CKSUM) != 0];
> +	uint32_t tmp = 0;
> +	if ((ol_flags & PKT_TX_L4_MASK) != PKT_TX_L4_NO_CKSUM)
> +		tmp |= IXGBE_ADVTXD_POPTS_TXSM;
> +	if (ol_flags & PKT_TX_IP_CKSUM)
> +		tmp |= IXGBE_ADVTXD_POPTS_IXSM;
> +	if (ol_flags & PKT_TX_TCP_SEG)
> +		tmp |= IXGBE_ADVTXD_POPTS_TXSM;
>  	return tmp;
>  }
> 
>  static inline uint32_t
> -tx_desc_vlan_flags_to_cmdtype(uint64_t ol_flags)
> +tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
>  {
> -	static const uint32_t vlan_cmd[2] = {0, IXGBE_ADVTXD_DCMD_VLE};
> -	return vlan_cmd[(ol_flags & PKT_TX_VLAN_PKT) != 0];
> +	uint32_t cmdtype = 0;
> +	if (ol_flags & PKT_TX_VLAN_PKT)
> +		cmdtype |= IXGBE_ADVTXD_DCMD_VLE;
> +	if (ol_flags & PKT_TX_TCP_SEG)
> +		cmdtype |= IXGBE_ADVTXD_DCMD_TSE;
> +	return cmdtype;
>  }
> 
>  /* Default RS bit threshold values */
> @@ -545,14 +576,6 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
>  	volatile union ixgbe_adv_tx_desc *txd;
>  	struct rte_mbuf     *tx_pkt;
>  	struct rte_mbuf     *m_seg;
> -	union ixgbe_vlan_macip vlan_macip_lens;
> -	union {
> -		uint16_t u16;
> -		struct {
> -			uint16_t l3_len:9;
> -			uint16_t l2_len:7;
> -		};
> -	} l2_l3_len;
>  	uint64_t buf_dma_addr;
>  	uint32_t olinfo_status;
>  	uint32_t cmd_type_len;
> @@ -566,6 +589,7 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
>  	uint64_t tx_ol_req;
>  	uint32_t ctx = 0;
>  	uint32_t new_ctx;
> +	union ixgbe_tx_offload tx_offload = { .data = 0 };
> 
>  	txq = tx_queue;
>  	sw_ring = txq->sw_ring;
> @@ -595,14 +619,15 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
>  		/* If hardware offload required */
>  		tx_ol_req = ol_flags & IXGBE_TX_OFFLOAD_MASK;
>  		if (tx_ol_req) {
> -			l2_l3_len.l2_len = tx_pkt->l2_len;
> -			l2_l3_len.l3_len = tx_pkt->l3_len;
> -			vlan_macip_lens.f.vlan_tci = tx_pkt->vlan_tci;
> -			vlan_macip_lens.f.l2_l3_len = l2_l3_len.u16;
> +			tx_offload.l2_len = tx_pkt->l2_len;
> +			tx_offload.l3_len = tx_pkt->l3_len;
> +			tx_offload.l4_len = tx_pkt->l4_len;
> +			tx_offload.vlan_tci = tx_pkt->vlan_tci;
> +			tx_offload.tso_segsz = tx_pkt->tso_segsz;
> 
>  			/* If new context need be built or reuse the exist ctx. */
>  			ctx = what_advctx_update(txq, tx_ol_req,
> -				vlan_macip_lens.data);
> +				tx_offload);
>  			/* Only allocate context descriptor if required*/
>  			new_ctx = (ctx == IXGBE_CTX_NUM);
>  			ctx = txq->ctx_curr;
> @@ -717,13 +742,22 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
>  		 */
>  		cmd_type_len = IXGBE_ADVTXD_DTYP_DATA |
>  			IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
> -		olinfo_status = (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
> +
>  #ifdef RTE_LIBRTE_IEEE1588
>  		if (ol_flags & PKT_TX_IEEE1588_TMST)
>  			cmd_type_len |= IXGBE_ADVTXD_MAC_1588;
>  #endif
> 
> +		olinfo_status = 0;
>  		if (tx_ol_req) {
> +
> +			if (ol_flags & PKT_TX_TCP_SEG) {
> +				/* when TSO is on, paylen in descriptor is the
> +				 * not the packet len but the tcp payload len */
> +				pkt_len -= (tx_offload.l2_len +
> +					tx_offload.l3_len + tx_offload.l4_len);
> +			}
> +
>  			/*
>  			 * Setup the TX Advanced Context Descriptor if required
>  			 */
> @@ -744,7 +778,7 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
>  				}
> 
>  				ixgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
> -				    vlan_macip_lens.data);
> +					tx_offload);
> 
>  				txe->last_id = tx_last;
>  				tx_id = txe->next_id;
> @@ -756,11 +790,13 @@ ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
>  			 * This path will go through
>  			 * whatever new/reuse the context descriptor
>  			 */
> -			cmd_type_len  |= tx_desc_vlan_flags_to_cmdtype(ol_flags);
> +			cmd_type_len  |= tx_desc_ol_flags_to_cmdtype(ol_flags);
>  			olinfo_status |= tx_desc_cksum_flags_to_olinfo(ol_flags);
>  			olinfo_status |= ctx << IXGBE_ADVTXD_IDX_SHIFT;
>  		}
> 
> +		olinfo_status |= (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
> +
>  		m_seg = tx_pkt;
>  		do {
>  			txd = &txr[tx_id];
> @@ -3611,9 +3647,10 @@ ixgbe_dev_tx_init(struct rte_eth_dev *dev)
>  	PMD_INIT_FUNC_TRACE();
>  	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> 
> -	/* Enable TX CRC (checksum offload requirement) */
> +	/* Enable TX CRC (checksum offload requirement) and hw padding
> +	 * (TSO requirement) */
>  	hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
> -	hlreg0 |= IXGBE_HLREG0_TXCRCEN;
> +	hlreg0 |= (IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN);
>  	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
> 
>  	/* Setup the Base and Length of the Tx Descriptor Rings */
> diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
> index eb89715..13099af 100644
> --- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
> +++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
> @@ -145,13 +145,16 @@ enum ixgbe_advctx_num {
>  };
> 
>  /** Offload features */
> -union ixgbe_vlan_macip {
> -	uint32_t data;
> +union ixgbe_tx_offload {
> +	uint64_t data;
>  	struct {
> -		uint16_t l2_l3_len; /**< combined 9-bit l3, 7-bit l2 lengths */
> -		uint16_t vlan_tci;
> +		uint64_t l2_len:7; /**< L2 (MAC) Header Length. */
> +		uint64_t l3_len:9; /**< L3 (IP) Header Length. */
> +		uint64_t l4_len:8; /**< L4 (TCP/UDP) Header Length. */
> +		uint64_t tso_segsz:16; /**< TCP TSO segment size */
> +		uint64_t vlan_tci:16;
>  		/**< VLAN Tag Control Identifier (CPU order). */
> -	} f;
> +	};
>  };
> 
>  /*
> @@ -170,8 +173,10 @@ union ixgbe_vlan_macip {
> 
>  struct ixgbe_advctx_info {
>  	uint64_t flags;           /**< ol_flags for context build. */
> -	uint32_t cmp_mask;        /**< compare mask for vlan_macip_lens */
> -	union ixgbe_vlan_macip vlan_macip_lens; /**< vlan, mac ip length. */
> +	/**< tx offload: vlan, tso, l2-l3-l4 lengths. */
> +	union ixgbe_tx_offload tx_offload;
> +	/** compare mask for tx offload. */
> +	union ixgbe_tx_offload tx_offload_mask;
>  };
> 
>  /**
> --
> 2.1.0
Olivier Matz Nov. 18, 2014, 9:11 a.m. UTC | #2
Hi Konstantin,

On 11/17/2014 07:26 PM, Ananyev, Konstantin wrote:
> Just one thing - double semicolon -  looks like a typo:
>> +	/* check if TCP segmentation required for this packet */
>> +	if (ol_flags & PKT_TX_TCP_SEG) {
>> +		/* implies IP cksum and TCP cksum */
>> +		type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4 |
>> +			IXGBE_ADVTXD_TUCMD_L4T_TCP |
>> +			IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;;

Good catch, I'll fix this too.


Olivier
diff mbox

Patch

diff --git a/lib/librte_pmd_ixgbe/ixgbe_ethdev.c b/lib/librte_pmd_ixgbe/ixgbe_ethdev.c
index 2eb609c..2c2ecc0 100644
--- a/lib/librte_pmd_ixgbe/ixgbe_ethdev.c
+++ b/lib/librte_pmd_ixgbe/ixgbe_ethdev.c
@@ -1964,7 +1964,8 @@  ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 		DEV_TX_OFFLOAD_IPV4_CKSUM  |
 		DEV_TX_OFFLOAD_UDP_CKSUM   |
 		DEV_TX_OFFLOAD_TCP_CKSUM   |
-		DEV_TX_OFFLOAD_SCTP_CKSUM;
+		DEV_TX_OFFLOAD_SCTP_CKSUM  |
+		DEV_TX_OFFLOAD_TCP_TSO;
 
 	dev_info->default_rxconf = (struct rte_eth_rxconf) {
 			.rx_thresh = {
diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
index 2df3385..19e3b73 100644
--- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
+++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
@@ -94,7 +94,8 @@ 
 #define IXGBE_TX_OFFLOAD_MASK (			 \
 		PKT_TX_VLAN_PKT |		 \
 		PKT_TX_IP_CKSUM |		 \
-		PKT_TX_L4_MASK)
+		PKT_TX_L4_MASK |		 \
+		PKT_TX_TCP_SEG)
 
 static inline struct rte_mbuf *
 rte_rxmbuf_alloc(struct rte_mempool *mp)
@@ -363,59 +364,84 @@  ixgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
 static inline void
 ixgbe_set_xmit_ctx(struct igb_tx_queue* txq,
 		volatile struct ixgbe_adv_tx_context_desc *ctx_txd,
-		uint64_t ol_flags, uint32_t vlan_macip_lens)
+		uint64_t ol_flags, union ixgbe_tx_offload tx_offload)
 {
 	uint32_t type_tucmd_mlhl;
-	uint32_t mss_l4len_idx;
+	uint32_t mss_l4len_idx = 0;
 	uint32_t ctx_idx;
-	uint32_t cmp_mask;
+	uint32_t vlan_macip_lens;
+	union ixgbe_tx_offload tx_offload_mask;
 
 	ctx_idx = txq->ctx_curr;
-	cmp_mask = 0;
+	tx_offload_mask.data = 0;
 	type_tucmd_mlhl = 0;
 
+	/* Specify which HW CTX to upload. */
+	mss_l4len_idx |= (ctx_idx << IXGBE_ADVTXD_IDX_SHIFT);
+
 	if (ol_flags & PKT_TX_VLAN_PKT) {
-		cmp_mask |= TX_VLAN_CMP_MASK;
+		tx_offload_mask.vlan_tci = ~0;
 	}
 
-	if (ol_flags & PKT_TX_IP_CKSUM) {
-		type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4;
-		cmp_mask |= TX_MACIP_LEN_CMP_MASK;
-	}
+	/* check if TCP segmentation required for this packet */
+	if (ol_flags & PKT_TX_TCP_SEG) {
+		/* implies IP cksum and TCP cksum */
+		type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4 |
+			IXGBE_ADVTXD_TUCMD_L4T_TCP |
+			IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;;
+
+		tx_offload_mask.l2_len = ~0;
+		tx_offload_mask.l3_len = ~0;
+		tx_offload_mask.l4_len = ~0;
+		tx_offload_mask.tso_segsz = ~0;
+		mss_l4len_idx |= tx_offload.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT;
+		mss_l4len_idx |= tx_offload.l4_len << IXGBE_ADVTXD_L4LEN_SHIFT;
+	} else { /* no TSO, check if hardware checksum is needed */
+		if (ol_flags & PKT_TX_IP_CKSUM) {
+			type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4;
+			tx_offload_mask.l2_len = ~0;
+			tx_offload_mask.l3_len = ~0;
+		}
 
-	/* Specify which HW CTX to upload. */
-	mss_l4len_idx = (ctx_idx << IXGBE_ADVTXD_IDX_SHIFT);
-	switch (ol_flags & PKT_TX_L4_MASK) {
-	case PKT_TX_UDP_CKSUM:
-		type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP |
+		switch (ol_flags & PKT_TX_L4_MASK) {
+		case PKT_TX_UDP_CKSUM:
+			type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP |
 				IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
-		mss_l4len_idx |= sizeof(struct udp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
-		cmp_mask |= TX_MACIP_LEN_CMP_MASK;
-		break;
-	case PKT_TX_TCP_CKSUM:
-		type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP |
+			mss_l4len_idx |= sizeof(struct udp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
+			tx_offload_mask.l2_len = ~0;
+			tx_offload_mask.l3_len = ~0;
+			break;
+		case PKT_TX_TCP_CKSUM:
+			type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP |
 				IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
-		mss_l4len_idx |= sizeof(struct tcp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
-		cmp_mask |= TX_MACIP_LEN_CMP_MASK;
-		break;
-	case PKT_TX_SCTP_CKSUM:
-		type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP |
+			mss_l4len_idx |= sizeof(struct tcp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
+			tx_offload_mask.l2_len = ~0;
+			tx_offload_mask.l3_len = ~0;
+			tx_offload_mask.l4_len = ~0;
+			break;
+		case PKT_TX_SCTP_CKSUM:
+			type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP |
 				IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
-		mss_l4len_idx |= sizeof(struct sctp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
-		cmp_mask |= TX_MACIP_LEN_CMP_MASK;
-		break;
-	default:
-		type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_RSV |
+			mss_l4len_idx |= sizeof(struct sctp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
+			tx_offload_mask.l2_len = ~0;
+			tx_offload_mask.l3_len = ~0;
+			break;
+		default:
+			type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_RSV |
 				IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
-		break;
+			break;
+		}
 	}
 
 	txq->ctx_cache[ctx_idx].flags = ol_flags;
-	txq->ctx_cache[ctx_idx].cmp_mask = cmp_mask;
-	txq->ctx_cache[ctx_idx].vlan_macip_lens.data =
-		vlan_macip_lens & cmp_mask;
+	txq->ctx_cache[ctx_idx].tx_offload.data  =
+		tx_offload_mask.data & tx_offload.data;
+	txq->ctx_cache[ctx_idx].tx_offload_mask    = tx_offload_mask;
 
 	ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl);
+	vlan_macip_lens = tx_offload.l3_len;
+	vlan_macip_lens |= (tx_offload.l2_len << IXGBE_ADVTXD_MACLEN_SHIFT);
+	vlan_macip_lens |= ((uint32_t)tx_offload.vlan_tci << IXGBE_ADVTXD_VLAN_SHIFT);
 	ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens);
 	ctx_txd->mss_l4len_idx   = rte_cpu_to_le_32(mss_l4len_idx);
 	ctx_txd->seqnum_seed     = 0;
@@ -427,20 +453,20 @@  ixgbe_set_xmit_ctx(struct igb_tx_queue* txq,
  */
 static inline uint32_t
 what_advctx_update(struct igb_tx_queue *txq, uint64_t flags,
-		uint32_t vlan_macip_lens)
+		union ixgbe_tx_offload tx_offload)
 {
 	/* If match with the current used context */
 	if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
-		(txq->ctx_cache[txq->ctx_curr].vlan_macip_lens.data ==
-		(txq->ctx_cache[txq->ctx_curr].cmp_mask & vlan_macip_lens)))) {
+		(txq->ctx_cache[txq->ctx_curr].tx_offload.data ==
+		(txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data & tx_offload.data)))) {
 			return txq->ctx_curr;
 	}
 
 	/* What if match with the next context  */
 	txq->ctx_curr ^= 1;
 	if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
-		(txq->ctx_cache[txq->ctx_curr].vlan_macip_lens.data ==
-		(txq->ctx_cache[txq->ctx_curr].cmp_mask & vlan_macip_lens)))) {
+		(txq->ctx_cache[txq->ctx_curr].tx_offload.data ==
+		(txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data & tx_offload.data)))) {
 			return txq->ctx_curr;
 	}
 
@@ -451,20 +477,25 @@  what_advctx_update(struct igb_tx_queue *txq, uint64_t flags,
 static inline uint32_t
 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
 {
-	static const uint32_t l4_olinfo[2] = {0, IXGBE_ADVTXD_POPTS_TXSM};
-	static const uint32_t l3_olinfo[2] = {0, IXGBE_ADVTXD_POPTS_IXSM};
-	uint32_t tmp;
-
-	tmp  = l4_olinfo[(ol_flags & PKT_TX_L4_MASK)  != PKT_TX_L4_NO_CKSUM];
-	tmp |= l3_olinfo[(ol_flags & PKT_TX_IP_CKSUM) != 0];
+	uint32_t tmp = 0;
+	if ((ol_flags & PKT_TX_L4_MASK) != PKT_TX_L4_NO_CKSUM)
+		tmp |= IXGBE_ADVTXD_POPTS_TXSM;
+	if (ol_flags & PKT_TX_IP_CKSUM)
+		tmp |= IXGBE_ADVTXD_POPTS_IXSM;
+	if (ol_flags & PKT_TX_TCP_SEG)
+		tmp |= IXGBE_ADVTXD_POPTS_TXSM;
 	return tmp;
 }
 
 static inline uint32_t
-tx_desc_vlan_flags_to_cmdtype(uint64_t ol_flags)
+tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
 {
-	static const uint32_t vlan_cmd[2] = {0, IXGBE_ADVTXD_DCMD_VLE};
-	return vlan_cmd[(ol_flags & PKT_TX_VLAN_PKT) != 0];
+	uint32_t cmdtype = 0;
+	if (ol_flags & PKT_TX_VLAN_PKT)
+		cmdtype |= IXGBE_ADVTXD_DCMD_VLE;
+	if (ol_flags & PKT_TX_TCP_SEG)
+		cmdtype |= IXGBE_ADVTXD_DCMD_TSE;
+	return cmdtype;
 }
 
 /* Default RS bit threshold values */
@@ -545,14 +576,6 @@  ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	volatile union ixgbe_adv_tx_desc *txd;
 	struct rte_mbuf     *tx_pkt;
 	struct rte_mbuf     *m_seg;
-	union ixgbe_vlan_macip vlan_macip_lens;
-	union {
-		uint16_t u16;
-		struct {
-			uint16_t l3_len:9;
-			uint16_t l2_len:7;
-		};
-	} l2_l3_len;
 	uint64_t buf_dma_addr;
 	uint32_t olinfo_status;
 	uint32_t cmd_type_len;
@@ -566,6 +589,7 @@  ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	uint64_t tx_ol_req;
 	uint32_t ctx = 0;
 	uint32_t new_ctx;
+	union ixgbe_tx_offload tx_offload = { .data = 0 };
 
 	txq = tx_queue;
 	sw_ring = txq->sw_ring;
@@ -595,14 +619,15 @@  ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		/* If hardware offload required */
 		tx_ol_req = ol_flags & IXGBE_TX_OFFLOAD_MASK;
 		if (tx_ol_req) {
-			l2_l3_len.l2_len = tx_pkt->l2_len;
-			l2_l3_len.l3_len = tx_pkt->l3_len;
-			vlan_macip_lens.f.vlan_tci = tx_pkt->vlan_tci;
-			vlan_macip_lens.f.l2_l3_len = l2_l3_len.u16;
+			tx_offload.l2_len = tx_pkt->l2_len;
+			tx_offload.l3_len = tx_pkt->l3_len;
+			tx_offload.l4_len = tx_pkt->l4_len;
+			tx_offload.vlan_tci = tx_pkt->vlan_tci;
+			tx_offload.tso_segsz = tx_pkt->tso_segsz;
 
 			/* If new context need be built or reuse the exist ctx. */
 			ctx = what_advctx_update(txq, tx_ol_req,
-				vlan_macip_lens.data);
+				tx_offload);
 			/* Only allocate context descriptor if required*/
 			new_ctx = (ctx == IXGBE_CTX_NUM);
 			ctx = txq->ctx_curr;
@@ -717,13 +742,22 @@  ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 		 */
 		cmd_type_len = IXGBE_ADVTXD_DTYP_DATA |
 			IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
-		olinfo_status = (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
+
 #ifdef RTE_LIBRTE_IEEE1588
 		if (ol_flags & PKT_TX_IEEE1588_TMST)
 			cmd_type_len |= IXGBE_ADVTXD_MAC_1588;
 #endif
 
+		olinfo_status = 0;
 		if (tx_ol_req) {
+
+			if (ol_flags & PKT_TX_TCP_SEG) {
+				/* when TSO is on, paylen in descriptor is the
+				 * not the packet len but the tcp payload len */
+				pkt_len -= (tx_offload.l2_len +
+					tx_offload.l3_len + tx_offload.l4_len);
+			}
+
 			/*
 			 * Setup the TX Advanced Context Descriptor if required
 			 */
@@ -744,7 +778,7 @@  ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 				}
 
 				ixgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
-				    vlan_macip_lens.data);
+					tx_offload);
 
 				txe->last_id = tx_last;
 				tx_id = txe->next_id;
@@ -756,11 +790,13 @@  ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 			 * This path will go through
 			 * whatever new/reuse the context descriptor
 			 */
-			cmd_type_len  |= tx_desc_vlan_flags_to_cmdtype(ol_flags);
+			cmd_type_len  |= tx_desc_ol_flags_to_cmdtype(ol_flags);
 			olinfo_status |= tx_desc_cksum_flags_to_olinfo(ol_flags);
 			olinfo_status |= ctx << IXGBE_ADVTXD_IDX_SHIFT;
 		}
 
+		olinfo_status |= (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
+
 		m_seg = tx_pkt;
 		do {
 			txd = &txr[tx_id];
@@ -3611,9 +3647,10 @@  ixgbe_dev_tx_init(struct rte_eth_dev *dev)
 	PMD_INIT_FUNC_TRACE();
 	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
-	/* Enable TX CRC (checksum offload requirement) */
+	/* Enable TX CRC (checksum offload requirement) and hw padding
+	 * (TSO requirement) */
 	hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
-	hlreg0 |= IXGBE_HLREG0_TXCRCEN;
+	hlreg0 |= (IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN);
 	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
 
 	/* Setup the Base and Length of the Tx Descriptor Rings */
diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
index eb89715..13099af 100644
--- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
+++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
@@ -145,13 +145,16 @@  enum ixgbe_advctx_num {
 };
 
 /** Offload features */
-union ixgbe_vlan_macip {
-	uint32_t data;
+union ixgbe_tx_offload {
+	uint64_t data;
 	struct {
-		uint16_t l2_l3_len; /**< combined 9-bit l3, 7-bit l2 lengths */
-		uint16_t vlan_tci;
+		uint64_t l2_len:7; /**< L2 (MAC) Header Length. */
+		uint64_t l3_len:9; /**< L3 (IP) Header Length. */
+		uint64_t l4_len:8; /**< L4 (TCP/UDP) Header Length. */
+		uint64_t tso_segsz:16; /**< TCP TSO segment size */
+		uint64_t vlan_tci:16;
 		/**< VLAN Tag Control Identifier (CPU order). */
-	} f;
+	};
 };
 
 /*
@@ -170,8 +173,10 @@  union ixgbe_vlan_macip {
 
 struct ixgbe_advctx_info {
 	uint64_t flags;           /**< ol_flags for context build. */
-	uint32_t cmp_mask;        /**< compare mask for vlan_macip_lens */
-	union ixgbe_vlan_macip vlan_macip_lens; /**< vlan, mac ip length. */
+	/**< tx offload: vlan, tso, l2-l3-l4 lengths. */
+	union ixgbe_tx_offload tx_offload;
+	/** compare mask for tx offload. */
+	union ixgbe_tx_offload tx_offload_mask;
 };
 
 /**