@@ -72,8 +72,10 @@ nix_tx_offload_flags(struct rte_eth_dev *eth_dev)
offsetof(struct rte_mbuf, buf_addr) + 24);
RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
offsetof(struct rte_mbuf, ol_flags) + 12);
+/*
RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, tx_offload) !=
offsetof(struct rte_mbuf, pool) + 2 * sizeof(void *));
+*/
if (conf & RTE_ETH_TX_OFFLOAD_VLAN_INSERT ||
conf & RTE_ETH_TX_OFFLOAD_QINQ_INSERT)
@@ -72,8 +72,10 @@ nix_tx_offload_flags(struct rte_eth_dev *eth_dev)
offsetof(struct rte_mbuf, buf_addr) + 24);
RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
offsetof(struct rte_mbuf, ol_flags) + 12);
+/*
RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, tx_offload) !=
offsetof(struct rte_mbuf, pool) + 2 * sizeof(void *));
+*/
if (conf & RTE_ETH_TX_OFFLOAD_VLAN_INSERT ||
conf & RTE_ETH_TX_OFFLOAD_QINQ_INSERT)
@@ -920,8 +920,9 @@ i40e_recv_scattered_pkts(void *rx_queue,
first_seg->pkt_len =
(uint16_t)(first_seg->pkt_len +
rx_packet_len);
- first_seg->nb_segs++;
+ last_seg->nb_segs = 2;
last_seg->next = rxm;
+ first_seg->nb_segs++;
}
/**
@@ -944,6 +945,7 @@ i40e_recv_scattered_pkts(void *rx_queue,
* the length of that CRC part from the data length of the
* previous mbuf.
*/
+ rxm->nb_segs = 1;
rxm->next = NULL;
if (unlikely(rxq->crc_len > 0)) {
first_seg->pkt_len -= RTE_ETHER_CRC_LEN;
@@ -953,6 +955,7 @@ i40e_recv_scattered_pkts(void *rx_queue,
last_seg->data_len =
(uint16_t)(last_seg->data_len -
(RTE_ETHER_CRC_LEN - rx_packet_len));
+ last_seg->nb_segs = 1;
last_seg->next = NULL;
} else
rxm->data_len = (uint16_t)(rx_packet_len -
@@ -1065,7 +1068,7 @@ i40e_calc_pkt_desc(struct rte_mbuf *tx_pkt)
while (txd != NULL) {
count += DIV_ROUND_UP(txd->data_len, I40E_MAX_DATA_PER_TXD);
- txd = txd->next;
+ txd = (txd->nb_segs == 1) ? NULL : txd->next;
}
return count;
@@ -1282,7 +1285,7 @@ i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
txe->last_id = tx_last;
tx_id = txe->next_id;
txe = txn;
- m_seg = m_seg->next;
+ m_seg = (m_seg->nb_segs == 1) ? NULL : m_seg->next;
} while (m_seg != NULL);
/* The last packet data descriptor needs End Of Packet (EOP) */
@@ -410,6 +410,10 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
split_packet += RTE_I40E_DESCS_PER_LOOP;
/* zero-out next pointers */
+ rx_pkts[pos]->nb_segs = 1;
+ rx_pkts[pos + 1]->nb_segs = 1;
+ rx_pkts[pos + 2]->nb_segs = 1;
+ rx_pkts[pos + 3]->nb_segs = 1;
rx_pkts[pos]->next = NULL;
rx_pkts[pos + 1]->next = NULL;
rx_pkts[pos + 2]->next = NULL;
@@ -27,6 +27,7 @@ reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs,
for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
if (end != NULL) {
/* processing a split packet */
+ end->nb_segs = 2;
end->next = rx_bufs[buf_idx];
rx_bufs[buf_idx]->data_len += rxq->crc_len;
@@ -52,6 +53,7 @@ reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs,
secondlast = secondlast->next;
secondlast->data_len -= (rxq->crc_len -
end->data_len);
+ secondlast->nb_segs = 1;
secondlast->next = NULL;
rte_pktmbuf_free_seg(end);
}
@@ -541,6 +541,10 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *__rte_restrict rxq,
split_packet += RTE_I40E_DESCS_PER_LOOP;
/* zero-out next pointers */
+ rx_pkts[pos]->nb_segs = 1;
+ rx_pkts[pos + 1]->nb_segs = 1;
+ rx_pkts[pos + 2]->nb_segs = 1;
+ rx_pkts[pos + 3]->nb_segs = 1;
rx_pkts[pos]->next = NULL;
rx_pkts[pos + 1]->next = NULL;
rx_pkts[pos + 2]->next = NULL;
@@ -123,10 +123,10 @@ rte_pktmbuf_free_pinned_extmem(void *addr, void *opaque)
rte_mbuf_ext_refcnt_set(m->shinfo, 1);
m->ol_flags = RTE_MBUF_F_EXTERNAL;
- if (m->next != NULL)
- m->next = NULL;
- if (m->nb_segs != 1)
+ if (m->nb_segs != 1) {
m->nb_segs = 1;
+ m->next = NULL;
+ }
rte_mbuf_raw_free(m);
}
@@ -427,7 +427,7 @@ int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
}
nb_segs -= 1;
pkt_len -= m->data_len;
- } while ((m = m->next) != NULL);
+ } while ((m = ((m->nb_segs == 1) ? NULL : m->next)) != NULL);
if (nb_segs) {
*reason = "bad nb_segs";
@@ -495,7 +495,7 @@ void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
__rte_mbuf_sanity_check(m, 1);
do {
- m_next = m->next;
+ m_next = (m->nb_segs == 1) ? NULL : m->next;
__rte_pktmbuf_free_seg_via_array(m,
pending, &nb_pending,
RTE_PKTMBUF_FREE_PENDING_SZ);
@@ -511,7 +511,7 @@ void rte_pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int count)
struct rte_mbuf *
rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
{
- struct rte_mbuf *mc, *mi, **prev;
+ struct rte_mbuf *mc, *mi, *prev;
uint32_t pktlen;
uint16_t nseg;
@@ -520,19 +520,21 @@ rte_pktmbuf_clone(struct rte_mbuf *md, struct rte_mempool *mp)
return NULL;
mi = mc;
- prev = &mi->next;
+ prev = mi;
pktlen = md->pkt_len;
nseg = 0;
do {
nseg++;
rte_pktmbuf_attach(mi, md);
- *prev = mi;
- prev = &mi->next;
- } while ((md = md->next) != NULL &&
+ prev->nb_segs = 2;
+ prev->next = mi;
+ prev = mi;
+ } while ((md = ((md->nb_segs == 1) ? NULL : md->next)) != NULL &&
(mi = rte_pktmbuf_alloc(mp)) != NULL);
- *prev = NULL;
+ prev->nb_segs = 1;
+ prev->next = NULL;
mc->nb_segs = nseg;
mc->pkt_len = pktlen;
@@ -565,9 +567,9 @@ __rte_pktmbuf_linearize(struct rte_mbuf *mbuf)
mbuf->data_len = (uint16_t)(mbuf->pkt_len);
/* Append data from next segments to the first one */
- m = mbuf->next;
+ m = (m->nb_segs == 1) ? NULL : m->next;
while (m != NULL) {
- m_next = m->next;
+ m_next = (m->nb_segs == 1) ? NULL : m->next;
seg_len = rte_pktmbuf_data_len(m);
rte_memcpy(buffer, rte_pktmbuf_mtod(m, char *), seg_len);
@@ -589,7 +591,7 @@ rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
uint32_t off, uint32_t len)
{
const struct rte_mbuf *seg = m;
- struct rte_mbuf *mc, *m_last, **prev;
+ struct rte_mbuf *mc, *m_last, *prev;
/* garbage in check */
__rte_mbuf_sanity_check(m, 1);
@@ -611,7 +613,7 @@ rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
/* copied mbuf is not indirect or external */
mc->ol_flags = m->ol_flags & ~(RTE_MBUF_F_INDIRECT|RTE_MBUF_F_EXTERNAL);
- prev = &mc->next;
+ prev = mc;
m_last = mc;
while (len > 0) {
uint32_t copy_len;
@@ -629,9 +631,10 @@ rte_pktmbuf_copy(const struct rte_mbuf *m, struct rte_mempool *mp,
rte_pktmbuf_free(mc);
return NULL;
}
+ prev->nb_segs = 2;
+ prev->next = m_last;
++mc->nb_segs;
- *prev = m_last;
- prev = &m_last->next;
+ prev = m_last;
}
/*
@@ -697,7 +700,7 @@ rte_pktmbuf_dump(FILE *f, const struct rte_mbuf *m, unsigned dump_len)
if (len != 0)
rte_hexdump(f, NULL, rte_pktmbuf_mtod(m, void *), len);
dump_len -= len;
- m = m->next;
+ m = (m->nb_segs == 1) ? NULL : m->next;
nb_segs --;
}
}
@@ -1353,10 +1353,10 @@ rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
return NULL;
}
- if (m->next != NULL)
- m->next = NULL;
- if (m->nb_segs != 1)
+ if (m->nb_segs != 1) {
m->nb_segs = 1;
+ m->next = NULL;
+ }
return m;
@@ -1370,10 +1370,10 @@ rte_pktmbuf_prefree_seg(struct rte_mbuf *m)
return NULL;
}
- if (m->next != NULL)
- m->next = NULL;
- if (m->nb_segs != 1)
+ if (m->nb_segs != 1) {
m->nb_segs = 1;
+ m->next = NULL;
+ }
rte_mbuf_refcnt_set(m, 1);
return m;
@@ -1415,7 +1415,7 @@ static inline void rte_pktmbuf_free(struct rte_mbuf *m)
__rte_mbuf_sanity_check(m, 1);
while (m != NULL) {
- m_next = m->next;
+ m_next = (m->nb_segs == 1) ? NULL : m->next;
rte_pktmbuf_free_seg(m);
m = m_next;
}
@@ -1497,7 +1497,7 @@ static inline void rte_pktmbuf_refcnt_update(struct rte_mbuf *m, int16_t v)
do {
rte_mbuf_refcnt_update(m, v);
- } while ((m = m->next) != NULL);
+ } while ((m = ((m->nb_segs == 1) ? NULL : m->next)) != NULL);
}
/**
@@ -1540,7 +1540,7 @@ static inline uint16_t rte_pktmbuf_tailroom(const struct rte_mbuf *m)
static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m)
{
__rte_mbuf_sanity_check(m, 1);
- while (m->next != NULL)
+ while (m->nb_segs != 1)
m = m->next;
return m;
}
@@ -1765,6 +1765,7 @@ static inline int rte_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *tail
/* Chain 'tail' onto the old tail */
cur_tail = rte_pktmbuf_lastseg(head);
+ cur_tail->nb_segs = 2;
cur_tail->next = tail;
/* accumulate number of segments and total length.
@@ -594,25 +594,6 @@ struct rte_mbuf {
uint16_t buf_len; /**< Length of segment buffer. */
- struct rte_mempool *pool; /**< Pool from which mbuf was allocated. */
-
- /* second cache line - fields only used in slow path or on TX */
- RTE_MARKER cacheline1 __rte_cache_min_aligned;
-
-#if RTE_IOVA_AS_PA
- /**
- * Next segment of scattered packet. Must be NULL in the last
- * segment or in case of non-segmented packet.
- */
- struct rte_mbuf *next;
-#else
- /**
- * Reserved for dynamic fields
- * when the next pointer is in first cache line (i.e. RTE_IOVA_AS_PA is 0).
- */
- uint64_t dynfield2;
-#endif
-
/* fields to support TX offloads */
RTE_STD_C11
union {
@@ -651,6 +632,25 @@ struct rte_mbuf {
};
};
+ /* second cache line - fields only used in slow path or on TX */
+ RTE_MARKER cacheline1 __rte_cache_min_aligned;
+
+#if RTE_IOVA_AS_PA
+ /**
+ * Next segment of scattered packet. Must be NULL in the last
+ * segment or in case of non-segmented packet.
+ */
+ struct rte_mbuf *next;
+#else
+ /**
+ * Reserved for dynamic fields
+ * when the next pointer is in first cache line (i.e. RTE_IOVA_AS_PA is 0).
+ */
+ uint64_t dynfield2;
+#endif
+
+ struct rte_mempool *pool; /**< Pool from which mbuf was allocated. */
+
/** Shared data for external buffer attached to mbuf. See
* rte_pktmbuf_attach_extbuf().
*/