@@ -852,10 +852,15 @@ em_event_loop_vector(struct l3fwd_event_resources *evt_rsrc,
int i, nb_enq = 0, nb_deq = 0;
struct lcore_conf *lconf;
unsigned int lcore_id;
+ uint16_t *dst_ports;
if (event_p_id < 0)
return;
+ dst_ports = rte_zmalloc("", sizeof(uint16_t) * evt_rsrc->vector_size,
+ RTE_CACHE_LINE_SIZE);
+ if (dst_ports == NULL)
+ return;
lcore_id = rte_lcore_id();
lconf = &lcore_conf[lcore_id];
@@ -877,13 +882,12 @@ em_event_loop_vector(struct l3fwd_event_resources *evt_rsrc,
}
#if defined RTE_ARCH_X86 || defined __ARM_NEON
- l3fwd_em_process_event_vector(events[i].vec, lconf);
+ l3fwd_em_process_event_vector(events[i].vec, lconf,
+ dst_ports);
#else
l3fwd_em_no_opt_process_event_vector(events[i].vec,
- lconf);
+ lconf, dst_ports);
#endif
- if (flags & L3FWD_EVENT_TX_DIRECT)
- event_vector_txq_set(events[i].vec, 0);
}
if (flags & L3FWD_EVENT_TX_ENQ) {
@@ -100,7 +100,7 @@ l3fwd_em_simple_forward(struct rte_mbuf *m, uint16_t portid,
}
}
-static __rte_always_inline void
+static __rte_always_inline uint16_t
l3fwd_em_simple_process(struct rte_mbuf *m, struct lcore_conf *qconf)
{
struct rte_ether_hdr *eth_hdr;
@@ -117,6 +117,8 @@ l3fwd_em_simple_process(struct rte_mbuf *m, struct lcore_conf *qconf)
m->port = l3fwd_em_handle_ipv6(m, m->port, eth_hdr, qconf);
else
m->port = BAD_PORT;
+
+ return m->port;
}
/*
@@ -179,7 +181,8 @@ l3fwd_em_no_opt_process_events(int nb_rx, struct rte_event **events,
static inline void
l3fwd_em_no_opt_process_event_vector(struct rte_event_vector *vec,
- struct lcore_conf *qconf)
+ struct lcore_conf *qconf,
+ uint16_t *dst_ports)
{
struct rte_mbuf **mbufs = vec->mbufs;
int32_t i;
@@ -188,30 +191,20 @@ l3fwd_em_no_opt_process_event_vector(struct rte_event_vector *vec,
for (i = 0; i < PREFETCH_OFFSET && i < vec->nb_elem; i++)
rte_prefetch0(rte_pktmbuf_mtod(mbufs[i], void *));
- /* Process first packet to init vector attributes */
- l3fwd_em_simple_process(mbufs[0], qconf);
- if (vec->attr_valid) {
- if (mbufs[0]->port != BAD_PORT)
- vec->port = mbufs[0]->port;
- else
- vec->attr_valid = 0;
- }
-
/*
* Prefetch and forward already prefetched packets.
*/
- for (i = 1; i < (vec->nb_elem - PREFETCH_OFFSET); i++) {
+ for (i = 0; i < (vec->nb_elem - PREFETCH_OFFSET); i++) {
rte_prefetch0(
rte_pktmbuf_mtod(mbufs[i + PREFETCH_OFFSET], void *));
- l3fwd_em_simple_process(mbufs[i], qconf);
- event_vector_attr_validate(vec, mbufs[i]);
+ dst_ports[i] = l3fwd_em_simple_process(mbufs[i], qconf);
}
/* Forward remaining prefetched packets */
- for (; i < vec->nb_elem; i++) {
- l3fwd_em_simple_process(mbufs[i], qconf);
- event_vector_attr_validate(vec, mbufs[i]);
- }
+ for (; i < vec->nb_elem; i++)
+ dst_ports[i] = l3fwd_em_simple_process(mbufs[i], qconf);
+
+ process_event_vector(vec, dst_ports);
}
#endif /* __L3FWD_EM_H__ */
@@ -332,70 +332,20 @@ l3fwd_em_process_events(int nb_rx, struct rte_event **ev,
static inline void
l3fwd_em_process_event_vector(struct rte_event_vector *vec,
- struct lcore_conf *qconf)
+ struct lcore_conf *qconf, uint16_t *dst_port)
{
- struct rte_mbuf **mbufs = vec->mbufs;
- uint16_t dst_port[MAX_PKT_BURST];
- int32_t i, j, n, pos;
-
- for (j = 0; j < EM_HASH_LOOKUP_COUNT && j < vec->nb_elem; j++)
- rte_prefetch0(
- rte_pktmbuf_mtod(mbufs[j], struct rte_ether_hdr *) + 1);
+ uint16_t i;
if (vec->attr_valid)
- vec->port = em_get_dst_port(qconf, mbufs[0], mbufs[0]->port);
-
- n = RTE_ALIGN_FLOOR(vec->nb_elem, EM_HASH_LOOKUP_COUNT);
- for (j = 0; j < n; j += EM_HASH_LOOKUP_COUNT) {
- uint32_t pkt_type =
- RTE_PTYPE_L3_MASK | RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP;
- uint32_t l3_type, tcp_or_udp;
-
- for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++)
- pkt_type &= mbufs[j + i]->packet_type;
-
- l3_type = pkt_type & RTE_PTYPE_L3_MASK;
- tcp_or_udp = pkt_type & (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP);
-
- for (i = 0, pos = j + EM_HASH_LOOKUP_COUNT;
- i < EM_HASH_LOOKUP_COUNT && pos < vec->nb_elem;
- i++, pos++) {
- rte_prefetch0(rte_pktmbuf_mtod(mbufs[pos],
- struct rte_ether_hdr *) +
- 1);
- }
-
- if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV4)) {
- em_get_dst_port_ipv4xN_events(qconf, &mbufs[j],
- &dst_port[j]);
- } else if (tcp_or_udp && (l3_type == RTE_PTYPE_L3_IPV6)) {
- em_get_dst_port_ipv6xN_events(qconf, &mbufs[j],
- &dst_port[j]);
- } else {
- for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) {
- mbufs[j + i]->port =
- em_get_dst_port(qconf, mbufs[j + i],
- mbufs[j + i]->port);
- process_packet(mbufs[j + i],
- &mbufs[j + i]->port);
- event_vector_attr_validate(vec, mbufs[j + i]);
- }
- continue;
- }
- processx4_step3(&mbufs[j], &dst_port[j]);
-
- for (i = 0; i < EM_HASH_LOOKUP_COUNT; i++) {
- mbufs[j + i]->port = dst_port[j + i];
- event_vector_attr_validate(vec, mbufs[j + i]);
- }
- }
-
- for (; j < vec->nb_elem; j++) {
- mbufs[j]->port =
- em_get_dst_port(qconf, mbufs[j], mbufs[j]->port);
- process_packet(mbufs[j], &mbufs[j]->port);
- event_vector_attr_validate(vec, mbufs[j]);
- }
+ l3fwd_em_process_packets(vec->nb_elem, vec->mbufs, dst_port,
+ vec->port, qconf, 1);
+ else
+ for (i = 0; i < vec->nb_elem; i++)
+ l3fwd_em_process_packets(1, &vec->mbufs[i],
+ &dst_port[i],
+ vec->mbufs[i]->port, qconf, 1);
+
+ process_event_vector(vec, dst_port);
}
#endif /* __L3FWD_EM_HLM_H__ */
@@ -113,39 +113,48 @@ l3fwd_em_process_events(int nb_rx, struct rte_event **events,
for (i = 1, j = 0; j < nb_rx; i++, j++) {
struct rte_mbuf *mbuf = events[j]->mbuf;
+ uint16_t port;
if (i < nb_rx) {
rte_prefetch0(rte_pktmbuf_mtod(
events[i]->mbuf,
struct rte_ether_hdr *) + 1);
}
+ port = mbuf->port;
mbuf->port = em_get_dst_port(qconf, mbuf, mbuf->port);
process_packet(mbuf, &mbuf->port);
+ if (mbuf->port == BAD_PORT)
+ mbuf->port = port;
}
}
static inline void
l3fwd_em_process_event_vector(struct rte_event_vector *vec,
- struct lcore_conf *qconf)
+ struct lcore_conf *qconf, uint16_t *dst_ports)
{
+ const uint8_t attr_valid = vec->attr_valid;
struct rte_mbuf **mbufs = vec->mbufs;
int32_t i, j;
rte_prefetch0(rte_pktmbuf_mtod(mbufs[0], struct rte_ether_hdr *) + 1);
- if (vec->attr_valid)
- vec->port = em_get_dst_port(qconf, mbufs[0], mbufs[0]->port);
-
for (i = 0, j = 1; i < vec->nb_elem; i++, j++) {
if (j < vec->nb_elem)
rte_prefetch0(rte_pktmbuf_mtod(mbufs[j],
struct rte_ether_hdr *) +
1);
- mbufs[i]->port =
- em_get_dst_port(qconf, mbufs[i], mbufs[i]->port);
- process_packet(mbufs[i], &mbufs[i]->port);
- event_vector_attr_validate(vec, mbufs[i]);
+ dst_ports[i] = em_get_dst_port(qconf, mbufs[i],
+ attr_valid ? vec->port :
+ mbufs[i]->port);
}
+ j = RTE_ALIGN_FLOOR(vec->nb_elem, FWDSTEP);
+
+ for (i = 0; i != j; i += FWDSTEP)
+ processx4_step3(&vec->mbufs[i], &dst_ports[i]);
+ for (; i < vec->nb_elem; i++)
+ process_packet(vec->mbufs[i], &dst_ports[i]);
+
+ process_event_vector(vec, dst_ports);
}
#endif /* __L3FWD_EM_SEQUENTIAL_H__ */
@@ -103,27 +103,6 @@ process_dst_port(uint16_t *dst_ports, uint16_t nb_elem)
}
#endif
-static inline void
-event_vector_attr_validate(struct rte_event_vector *vec, struct rte_mbuf *mbuf)
-{
- /* l3fwd application only changes mbuf port while processing */
- if (vec->attr_valid && (vec->port != mbuf->port))
- vec->attr_valid = 0;
-}
-
-static inline void
-event_vector_txq_set(struct rte_event_vector *vec, uint16_t txq)
-{
- if (vec->attr_valid) {
- vec->queue = txq;
- } else {
- int i;
-
- for (i = 0; i < vec->nb_elem; i++)
- rte_event_eth_tx_adapter_txq_set(vec->mbufs[i], txq);
- }
-}
-
static inline uint16_t
filter_bad_packets(struct rte_mbuf **mbufs, uint16_t *dst_port,
uint16_t nb_pkts)