@@ -49,6 +49,7 @@ SRCS-y += parameters.c
SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += cmdline.c
SRCS-y += config.c
SRCS-y += iofwd.c
+SRCS-y += portfwd.c
SRCS-y += macfwd.c
SRCS-y += macfwd-retry.c
SRCS-y += macswap.c
@@ -92,6 +92,8 @@
#include <rte_ether.h>
#include <rte_ethdev.h>
#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_malloc.h>
#include "testpmd.h"
@@ -150,6 +152,11 @@ print_ethaddr(const char *name, struct ether_addr *eth_addr)
void
nic_stats_display(portid_t port_id)
{
+ static uint64_t cnt_rx[RTE_MAX_ETHPORTS];
+ static uint64_t cnt_tx[RTE_MAX_ETHPORTS];
+ static uint64_t cycle[RTE_MAX_ETHPORTS];
+ uint64_t crx, ctx, c;
+
struct rte_eth_stats stats;
struct rte_port *port = &ports[port_id];
uint8_t i;
@@ -209,6 +216,20 @@ nic_stats_display(portid_t port_id)
}
}
+ c = cycle[port_id];
+ cycle[port_id] = rte_rdtsc();
+ if (c > 0)
+ c = cycle[port_id] - c;
+
+ crx = stats.ipackets - cnt_rx[port_id];
+ ctx = stats.opackets - cnt_tx[port_id];
+ cnt_rx[port_id] = stats.ipackets;
+ cnt_tx[port_id] = stats.opackets;
+ printf(" Throughput (since last show):\n");
+ printf(" RX PPS: %12"PRIu64"\n TX PPS: %12"PRIu64"\n",
+ c > 0 ? crx * rte_get_tsc_hz() / c : 0,
+ c > 0 ? ctx * rte_get_tsc_hz() / c : 0);
+
printf(" %s############################%s\n",
nic_stats_border, nic_stats_border);
}
@@ -1087,6 +1108,178 @@ setup_fwd_config_of_each_lcore(struct fwd_config *cfg)
}
static void
+copy_fwd_stream(struct fwd_stream *src, struct fwd_stream *dst)
+{
+ rte_memcpy(dst, src, sizeof(struct fwd_stream));
+}
+
+int
+set_fwd_stream_affinity(unsigned int idx, unsigned int core)
+{
+ struct fwd_stream **fwd_streams_tmp;
+ struct fwd_stream *fs;
+ unsigned int lc_id_dst;
+ unsigned int lc_id_src;
+ unsigned int fs_id;
+ unsigned int i, j, ci, cj;
+
+ if (cur_fwd_eng != &port_fwd_engine)
+ return 0;
+ if (test_done == 0) {
+ printf("please stop forwarding first\n");
+ return 0;
+ }
+ for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
+ if (fwd_lcores_cpuids[i] == core) {
+ lc_id_dst = i;
+ break;
+ }
+ }
+ if (i >= cur_fwd_config.nb_fwd_lcores)
+ return -1;
+ for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) {
+ if (fwd_streams[i]->idx == idx) {
+ fs_id = i;
+ break;
+ }
+ }
+ if (i >= cur_fwd_config.nb_fwd_streams)
+ return -1;
+ for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
+ for (j = 0; j < fwd_lcores[i]->stream_nb; j++) {
+ fs = fwd_streams[fwd_lcores[i]->stream_idx + j];
+ if (idx == fs->idx) {
+ lc_id_src = i;
+ break;
+ }
+ }
+ if (j < fwd_lcores[i]->stream_nb)
+ break;
+ }
+ if (i >= cur_fwd_config.nb_fwd_lcores)
+ return -1;
+ if (lc_id_src == lc_id_dst)
+ return 0;
+ fwd_streams_tmp = rte_zmalloc("testpmd: fwd_streams_tmp",
+ sizeof(struct fwd_stream *) * cur_fwd_config.nb_fwd_streams,
+ RTE_CACHE_LINE_SIZE);
+ if (fwd_streams_tmp == NULL)
+ rte_exit(EXIT_FAILURE,
+ "rte_zmalloc(%d (struct fwd_stream *)) "
+ "failed\n", nb_fwd_streams);
+
+ for (i = 0; i < cur_fwd_config.nb_fwd_streams; i++) {
+ fwd_streams_tmp[i] =
+ rte_zmalloc("testpmd: struct fwd_stream",
+ sizeof(struct fwd_stream),
+ RTE_CACHE_LINE_SIZE);
+ if (fwd_streams_tmp[i] == NULL)
+ rte_exit(EXIT_FAILURE,
+ "rte_zmalloc(struct fwd_stream)"
+ " failed\n");
+ }
+ ci = 0;
+ cj = 0;
+ for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
+ if (i == lc_id_src) {
+ fwd_lcores[i]->stream_idx = cj;
+ for (j = 0; j < fwd_lcores[i]->stream_nb; j++) {
+ if (ci == fs_id) {
+ ci++;
+ continue;
+ }
+ copy_fwd_stream(fwd_streams[ci],
+ fwd_streams_tmp[cj]);
+ ci++;
+ cj++;
+ }
+ fwd_lcores[i]->stream_nb -= 1;
+ } else if (i == lc_id_dst) {
+ fwd_lcores[i]->stream_idx = cj;
+ for (j = 0; j < fwd_lcores[i]->stream_nb; j++) {
+ copy_fwd_stream(fwd_streams[ci],
+ fwd_streams_tmp[cj]);
+ ci++;
+ cj++;
+ }
+ copy_fwd_stream(fwd_streams[fs_id],
+ fwd_streams_tmp[cj]);
+ cj++;
+ fwd_lcores[i]->stream_nb += 1;
+ } else {
+ fwd_lcores[i]->stream_idx = cj;
+ for (j = 0; j < fwd_lcores[i]->stream_nb; j++) {
+ copy_fwd_stream(fwd_streams[ci],
+ fwd_streams_tmp[cj]);
+ ci++;
+ cj++;
+ }
+ }
+ }
+ if (fwd_streams != NULL) {
+ for (i = 0; i < nb_fwd_streams; i++) {
+ if (fwd_streams[i] == NULL)
+ continue;
+ rte_free(fwd_streams[i]);
+ fwd_streams[i] = NULL;
+ }
+ rte_free(fwd_streams);
+ fwd_streams = NULL;
+ }
+ fwd_streams = fwd_streams_tmp;
+
+ return 0;
+}
+
+static void
+port_fwd_config_setup(void)
+{
+ portid_t rxp;
+ queueid_t rxq;
+ lcoreid_t lc_id;
+
+ if (fwd_config_init == 1)
+ return;
+
+ fwd_config_init = 1;
+ cur_fwd_config.nb_fwd_lcores = (lcoreid_t) nb_fwd_lcores;
+ cur_fwd_config.nb_fwd_ports = nb_fwd_ports;
+ cur_fwd_config.nb_fwd_streams =
+ (streamid_t) (nb_rxq * nb_fwd_ports);
+ if (cur_fwd_config.nb_fwd_lcores > cur_fwd_config.nb_fwd_streams)
+ cur_fwd_config.nb_fwd_lcores =
+ (lcoreid_t)cur_fwd_config.nb_fwd_streams;
+ init_fwd_streams();
+ setup_fwd_config_of_each_lcore(&cur_fwd_config);
+
+ for (rxp = 0; rxp < nb_ports; rxp++)
+ fwd_ports_ids[rxp] = rxp;
+
+ rxp = 0;
+ rxq = 0;
+ for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_streams; lc_id++) {
+ struct fwd_stream *fs = fwd_streams[lc_id];
+
+ fs->idx = lc_id;
+ fs->rx_port = fwd_ports_ids[rxp];
+ fs->rx_queue = rxq;
+ fs->tx_port = 0;
+ fs->tx_queue = 0;
+ fs->peer_addr = fs->tx_port;
+ rxq = (queueid_t) (rxq + 1);
+ if (rxq < nb_rxq)
+ continue;
+ rxq = 0;
+ if (numa_support && (nb_fwd_ports <= (nb_ports >> 1)))
+ rxp = (portid_t) (rxp +
+ ((nb_ports >> 1) / nb_fwd_ports));
+ else
+ rxp = (portid_t) (rxp + 1);
+ }
+ clear_perf_stats();
+}
+
+static void
simple_fwd_config_setup(void)
{
portid_t i;
@@ -1371,14 +1564,17 @@ fwd_config_setup(void)
icmp_echo_config_setup();
return;
}
- if ((nb_rxq > 1) && (nb_txq > 1)){
- if (dcb_config)
- dcb_fwd_config_setup();
- else
- rss_fwd_config_setup();
+ if (cur_fwd_eng == &port_fwd_engine)
+ port_fwd_config_setup();
+ else {
+ if ((nb_rxq > 1) && (nb_txq > 1)) {
+ if (dcb_config)
+ dcb_fwd_config_setup();
+ else
+ rss_fwd_config_setup();
+ } else
+ simple_fwd_config_setup();
}
- else
- simple_fwd_config_setup();
}
static void
@@ -1406,8 +1602,9 @@ pkt_fwd_config_display(struct fwd_config *cfg)
fwd_lcores[lc_id]->stream_nb);
for (sm_id = 0; sm_id < fwd_lcores[lc_id]->stream_nb; sm_id++) {
fs = fwd_streams[fwd_lcores[lc_id]->stream_idx + sm_id];
- printf("\n RX P=%d/Q=%d (socket %u) -> TX "
+ printf("\n %2u: RX P=%d/Q=%d (socket %u) -> TX "
"P=%d/Q=%d (socket %u) ",
+ fs->idx,
fs->rx_port, fs->rx_queue,
ports[fs->rx_port].socket_id,
fs->tx_port, fs->tx_queue,
@@ -1688,12 +1885,17 @@ set_pkt_forwarding_mode(const char *fwd_mode_name)
struct fwd_engine *fwd_eng;
unsigned i;
+ if (test_done == 0) {
+ printf("Please stop forwarding first\n");
+ return;
+ }
i = 0;
while ((fwd_eng = fwd_engines[i]) != NULL) {
if (! strcmp(fwd_eng->fwd_mode_name, fwd_mode_name)) {
printf("Set %s packet forwarding mode\n",
fwd_mode_name);
cur_fwd_eng = fwd_eng;
+ fwd_config_init = 0;
return;
}
i++;
@@ -2479,3 +2681,193 @@ port_dcb_info_display(uint8_t port_id)
printf("\t%4d", dcb_info.tc_queue.tc_txq[0][i].nb_queue);
printf("\n");
}
+
+void set_fixed_route(uint16_t srcp, uint16_t dstp)
+{
+ uint32_t error = 0;
+
+ if (cur_fwd_eng != &port_fwd_engine)
+ printf("warning: current forward engine is not portfwd!\n");
+ if (srcp >= nb_fwd_ports) {
+ printf("error: invalid srcport!\n");
+ error = 1;
+ }
+ if (dstp >= nb_fwd_ports) {
+ printf("error: invalid dstport!\n");
+ error = 1;
+ }
+ if (error == 0)
+ route_table[srcp] = dstp;
+}
+
+void set_ip_route(uint16_t srcp)
+{
+ if (cur_fwd_eng != &port_fwd_engine)
+ printf("warning: current forward engine is not portfwd!\n");
+ if (srcp >= nb_fwd_ports) {
+ printf("error: invalid srcport!\n");
+ return;
+ }
+ route_table[srcp] = PORT_ROUTE_IP;
+}
+
+void show_route(void)
+{
+ portid_t i;
+
+ if (cur_fwd_eng != &port_fwd_engine)
+ printf("warning: current forward engine is not portfwd!\n");
+ if (1 != ipv4_route_available) {
+ printf("warning: ipv4 route not available!\n");
+ printf(" try increase IPV4_ROUTE_MASK\n");
+ printf(" and/or set different ip ends\n");
+ }
+ printf("route table:\n");
+ printf(" srcport dstport\n");
+ for (i = 0; i < nb_fwd_ports; ++i) {
+ if (route_table[i] == PORT_ROUTE_IP)
+ printf("%8d ip\n", i);
+ else
+ printf("%8d %8d\n", i, route_table[i]);
+ }
+}
+
+void
+init_txq_lock(void)
+{
+ uint16_t portid, queueid;
+
+ for (portid = 0; portid < RTE_MAX_ETHPORTS; ++portid)
+ for (queueid = 0; queueid < MAX_TX_QUEUE; ++queueid)
+ rte_spinlock_init(
+ &(txq_lock[portid][queueid].spinlock));
+}
+
+void build_ipv4_route(void)
+{
+ uint16_t i, j;
+
+ printf("--------------------------------\n");
+ printf("building ipv4 route table...\n");
+ printf("ipv4 route mask: 0x%x\n", IPV4_ROUTE_MASK);
+ for (i = 0; i < MAX_IP_DIFF; ++i)
+ ipv4_route[i] = MAX_IP_DIFF;
+ for (i = 0; i < nb_fwd_ports; ++i) {
+ j = ipv4_table[i] & IPV4_ROUTE_MASK;
+ if (ipv4_route[j] < MAX_IP_DIFF) {
+ ipv4_route_available = 0;
+ printf("warning: ipv4 route failed\n");
+ printf(" try increase IPV4_ROUTE_MASK\n");
+ printf(" and/or set different ip ends\n");
+ printf("--------------------------------\n");
+ return;
+ }
+ ipv4_route[j] = i;
+ printf("route: suffix: %d -> port: %d\n", j, i);
+ }
+ ipv4_route_available = 1;
+ for (i = 0; i < MAX_IP_DIFF; ++i)
+ if (ipv4_route[i] == MAX_IP_DIFF)
+ ipv4_route[i] = 0;
+ printf("ipv4 route available\n");
+ printf("--------------------------------\n");
+}
+
+void add_port_addr(void)
+{
+ unsigned long i, j, k;
+
+ printf("--------------------------------\n");
+ printf("adding port addr...\n");
+ printf("port number: %d, rxq nubmer: %d, txq number: %d\n",
+ nb_fwd_ports, nb_rxq, nb_txq);
+ for (i = 0; i < nb_fwd_streams; ++i)
+ fwd_streams[i]->rx_port = i;
+ for (i = 0; i < nb_fwd_ports; ++i) {
+ /* assume less than 256 ports */
+ ipv4_table[i] = (192 << 24) + (168 << 16) + (1 << 8) + (1 + i);
+ printf("port %lu: ipv4: %d.%d.%d.%d\n",
+ i,
+ (ipv4_table[i] & 0xFF000000) >> 24,
+ (ipv4_table[i] & 0x00FF0000) >> 16,
+ (ipv4_table[i] & 0x0000FF00) >> 8,
+ (ipv4_table[i] & 0x000000FF));
+ for (j = 0; j < nb_rxq; ++j) {
+ for (k = 0; k < nb_fwd_ports; ++k)
+ fs_buf[i][j].tx_mbufs[k].len = 0;
+ fs_buf[i][j].dst_queue = (i * nb_rxq + j) % nb_txq;
+ printf("port %lu: rxq: %lu -> txq: %d\n",
+ i, j, fs_buf[i][j].dst_queue);
+ }
+ route_table[i] = PORT_ROUTE_IP;
+ }
+ printf("--------------------------------\n");
+ for (i = 0; i < RTE_MAX_ETHPORTS; ++i)
+ for (j = 0; j < MAX_RX_QUEUE; ++j) {
+ drainer[i][j].cycle_last = 0;
+ drainer[i][j].cycle_now = 0;
+ }
+
+ build_ipv4_route();
+}
+
+void set_ip(uint32_t srcp, uint32_t num0, uint32_t num1,
+ uint32_t num2, uint32_t num3)
+{
+ if (cur_fwd_eng != &port_fwd_engine)
+ printf("warning: current forward engine is not portfwd!\n");
+ if (srcp >= nb_fwd_ports) {
+ printf("error: invalid srcport!\n");
+ return;
+ }
+ if (num0 > 255 || num1 > 255 || num2 > 255 || num3 > 255) {
+ printf("error: invalid ip address!\n");
+ return;
+ }
+ ipv4_table[srcp] = (num0 << 24) + (num1 << 16) + (num2 << 8) + num3;
+ printf("port %u: ipv4: %d.%d.%d.%d\n",
+ srcp,
+ (ipv4_table[srcp] & 0xFF000000) >> 24,
+ (ipv4_table[srcp] & 0x00FF0000) >> 16,
+ (ipv4_table[srcp] & 0x0000FF00) >> 8,
+ (ipv4_table[srcp] & 0x000000FF));
+
+ build_ipv4_route();
+}
+
+void set_drain_interval_ns(unsigned long drain_ns)
+{
+ if (cur_fwd_eng != &port_fwd_engine)
+ printf("warning: current forward engine is not portfwd!\n");
+ drain_cycle = rte_get_tsc_hz() / 1E9 * drain_ns;
+ if (drain_cycle > 0)
+ printf("portfwd drain interval: %lu ns, %lu cycles\n",
+ drain_ns, drain_cycle);
+ else
+ printf("portfwd drain disabled\n");
+}
+
+void print_port_info(void)
+{
+ portid_t pid;
+ struct rte_port *port;
+
+ printf("port mac ip name\n");
+ FOREACH_PORT(pid, ports) {
+ port = &ports[pid];
+ printf("%4u", pid);
+ printf(" %02x:%02x:%02x:%02x:%02x:%02x",
+ (unsigned int)port->eth_addr.addr_bytes[0],
+ (unsigned int)port->eth_addr.addr_bytes[1],
+ (unsigned int)port->eth_addr.addr_bytes[2],
+ (unsigned int)port->eth_addr.addr_bytes[3],
+ (unsigned int)port->eth_addr.addr_bytes[4],
+ (unsigned int)port->eth_addr.addr_bytes[5]);
+ printf(" %3d.%3d.%3d.%3d"
+ , (ipv4_table[pid] & 0xFF000000) >> 24
+ , (ipv4_table[pid] & 0x00FF0000) >> 16
+ , (ipv4_table[pid] & 0x0000FF00) >> 8
+ , (ipv4_table[pid] & 0x000000FF));
+ printf(" %s\n", port->dev_info.driver_name);
+ }
+}
new file mode 100644
@@ -0,0 +1,418 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <sys/queue.h>
+#include <sys/stat.h>
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_per_lcore.h>
+#include <rte_lcore.h>
+#include <rte_atomic.h>
+#include <rte_branch_prediction.h>
+#include <rte_ring.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_interrupts.h>
+#include <rte_pci.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_string_fns.h>
+#include <rte_ip.h>
+#include "testpmd.h"
+
+#define PORTFWD_PERF_STATS 1
+
+struct op_stat {
+ uint64_t run[DEF_PKT_BURST+1];
+ uint64_t cycle[DEF_PKT_BURST+1];
+ uint64_t cycle_min[DEF_PKT_BURST+1];
+ uint64_t cycle_max[DEF_PKT_BURST+1];
+} __rte_cache_aligned;
+
+struct port_perf_stat {
+ struct op_stat rx_stat[MAX_RX_QUEUE];
+ struct op_stat tx_stat[MAX_TX_QUEUE];
+ uint64_t pkt_loss[MAX_TX_QUEUE];
+};
+
+struct port_perf_stat port_stat[RTE_MAX_ETHPORTS];
+uint64_t print_stat_cycle = 1e10;
+uint64_t last_stat_cycle;
+
+inline void __attribute__((always_inline))
+clear_perf_stats(void)
+{
+ uint16_t i, j, k;
+ struct op_stat *ops;
+
+ for (i = 0; i < nb_fwd_ports; ++i) {
+ for (j = 0; j < nb_rxq; ++j) {
+ for (k = 0; k <= DEF_PKT_BURST; ++k) {
+ ops = &port_stat[i].rx_stat[j];
+ ops->run[k] = 0;
+ ops->cycle[k] = 0;
+ ops->cycle_min[k] = print_stat_cycle;
+ ops->cycle_max[k] = 0;
+ }
+ }
+ for (j = 0; j < nb_txq; ++j) {
+ for (k = 0; k <= DEF_PKT_BURST; ++k) {
+ ops = &port_stat[i].tx_stat[j];
+ ops->run[k] = 0;
+ ops->cycle[k] = 0;
+ ops->cycle_min[k] = print_stat_cycle;
+ ops->cycle_max[k] = 0;
+ }
+ }
+ for (j = 0; j < nb_txq; ++j)
+ port_stat[i].pkt_loss[j] = 0;
+ }
+}
+
+inline void __attribute__((always_inline))
+print_perf_stats(void)
+{
+#if PORTFWD_PERF_STATS
+ uint16_t i, j, k;
+ struct op_stat *ops;
+
+ printf("\ncycle stat (since last show)\n");
+ printf("--------------------------------\n");
+ for (i = 0; i < nb_fwd_ports; ++i) {
+ printf("port %3d, burst %3d,\n", i, DEF_PKT_BURST);
+ printf(" rx,");
+ for (j = 0; j < nb_txq; ++j)
+ printf(" run, min,"
+ " avg, max,");
+ printf("\n");
+ for (k = 0; k <= DEF_PKT_BURST; ++k) {
+ printf("%12d,", k);
+ for (j = 0; j < nb_rxq; ++j) {
+ ops = &port_stat[i].rx_stat[j];
+ printf("%12"PRIu64",%12"PRIu64","
+ "%12"PRIu64",%12"PRIu64",",
+ ops->run[k],
+ ops->run[k] > 0 ?
+ ops->cycle_min[k] : 0,
+ ops->run[k] > 0 ?
+ ops->cycle[k] /
+ ops->run[k] : 0,
+ ops->cycle_max[k]);
+ }
+ printf("\n");
+ }
+ printf(" tx,");
+ for (j = 0; j < nb_txq; ++j)
+ printf(" run, min,"
+ " avg, max,");
+ printf("\n");
+ for (k = 0; k <= DEF_PKT_BURST; ++k) {
+ printf("%12d,", k);
+ for (j = 0; j < nb_txq; ++j) {
+ ops = &port_stat[i].tx_stat[j];
+ printf("%12"PRIu64",%12"PRIu64","
+ "%12"PRIu64",%12"PRIu64",",
+ ops->run[k],
+ ops->run[k] > 0 ?
+ ops->cycle_min[k] : 0,
+ ops->run[k] > 0 ?
+ ops->cycle[k] /
+ ops->run[k] : 0,
+ ops->cycle_max[k]);
+ }
+ printf("\n");
+ }
+ }
+ printf("\ntx pktloss (since last show)\n");
+ printf("--------------------------------\n");
+ printf(" ");
+ for (i = 0; i < nb_txq; ++i)
+ printf(" txq %3d,", i);
+ printf("\n");
+ for (i = 0; i < nb_fwd_ports; ++i) {
+ printf("port %3d,", i);
+ for (j = 0; j < nb_txq; ++j)
+ printf("%12"PRIu64",", port_stat[i].pkt_loss[j]);
+ printf("\n");
+ }
+ printf("\n");
+ clear_perf_stats();
+#else
+ printf("warning: portfwd perf stats not enabled!\n");
+#endif
+}
+
+static inline uint32_t __attribute__((always_inline))
+get_ipv4_dst_port(void *ipv4_hdr, uint16_t portid)
+{
+ uint32_t route;
+ uint32_t ip;
+ uint32_t i;
+
+ route = route_table[portid];
+ if (route == PORT_ROUTE_IP) {
+ ip = rte_be_to_cpu_32(((struct ipv4_hdr *)ipv4_hdr)->dst_addr);
+ i = ipv4_route[ip & IPV4_ROUTE_MASK];
+ if (likely(ipv4_route_available == 1 && ipv4_table[i] == ip))
+ return i;
+ if (likely(ipv4_route_available == 1))
+ return portid;
+ for (i = 0; i < nb_fwd_ports; ++i)
+ if (ip == ipv4_table[i])
+ return i;
+ return portid;
+ }
+ return route;
+}
+
+static inline void __attribute__((always_inline))
+log_stat(uint16_t port, uint16_t queue, uint64_t pkt,
+ uint64_t pktloss, uint64_t t, uint16_t is_tx)
+{
+ if (is_tx) {
+ port_stat[port].pkt_loss[queue] += pktloss;
+ port_stat[port].tx_stat[queue].run[pkt]++;
+ port_stat[port].tx_stat[queue].cycle[pkt] += t;
+ if (port_stat[port].tx_stat[queue].cycle_min[pkt] > t)
+ port_stat[port].tx_stat[queue].cycle_min[pkt] = t;
+ if (port_stat[port].tx_stat[queue].cycle_max[pkt] < t)
+ port_stat[port].tx_stat[queue].cycle_max[pkt] = t;
+ } else {
+ port_stat[port].rx_stat[queue].run[pkt]++;
+ port_stat[port].rx_stat[queue].cycle[pkt] += t;
+ if (port_stat[port].rx_stat[queue].cycle_min[pkt] > t)
+ port_stat[port].rx_stat[queue].cycle_min[pkt] = t;
+ if (port_stat[port].rx_stat[queue].cycle_max[pkt] < t)
+ port_stat[port].rx_stat[queue].cycle_max[pkt] = t;
+ }
+}
+
+static inline uint32_t __attribute__((always_inline))
+send_single_packet(uint16_t dst_port, uint16_t src_port,
+ uint16_t src_queue, struct rte_mbuf *m)
+{
+ struct rte_mbuf **pkts_burst;
+ struct fwd_stream_buffer *fsb;
+ uint32_t len;
+ uint16_t dst_queue;
+ uint32_t nb_tx;
+ uint32_t ret;
+#if PORTFWD_PERF_STATS
+ uint64_t t[2];
+#endif
+
+ ret = 0;
+ fsb = &fs_buf[src_port][src_queue];
+ len = fsb->tx_mbufs[dst_port].len;
+ fsb->tx_mbufs[dst_port].m_table[len] = m;
+ ++len;
+ if (len == DEF_PKT_BURST) {
+ dst_queue = fsb->dst_queue;
+ pkts_burst = fsb->tx_mbufs[dst_port].m_table;
+ rte_spinlock_lock(&(txq_lock[dst_port][dst_queue].spinlock));
+#if PORTFWD_PERF_STATS
+ t[0] = rte_rdtsc();
+#endif
+ nb_tx = rte_eth_tx_burst(dst_port, dst_queue,
+ pkts_burst, DEF_PKT_BURST);
+#if PORTFWD_PERF_STATS
+ t[1] = rte_rdtsc();
+ log_stat(dst_port, dst_queue, nb_tx, DEF_PKT_BURST - nb_tx,
+ t[1] - t[0], 1);
+#endif
+ rte_spinlock_unlock(&(txq_lock[dst_port][dst_queue].spinlock));
+ ret = nb_tx;
+ if (nb_tx < DEF_PKT_BURST)
+ do {
+ rte_pktmbuf_free(pkts_burst[nb_tx]);
+ } while (++nb_tx < DEF_PKT_BURST);
+ len = 0;
+ }
+ fsb->tx_mbufs[dst_port].len = len;
+ return ret;
+}
+
+static inline uint32_t __attribute__((always_inline))
+drain_packets(uint16_t src_port, uint16_t src_queue)
+{
+ struct rte_mbuf **pkts_burst;
+ struct fwd_stream_buffer *fsb;
+ uint16_t dst_port;
+ uint16_t dst_queue;
+ uint16_t len;
+ uint16_t nb_tx;
+ uint32_t ret;
+#if PORTFWD_PERF_STATS
+ uint64_t t[2];
+#endif
+
+ ret = 0;
+ for (dst_port = 0; dst_port < nb_fwd_ports; ++dst_port) {
+ fsb = &fs_buf[src_port][src_queue];
+ len = fsb->tx_mbufs[dst_port].len;
+ if (len > 0) {
+ dst_queue = fsb->dst_queue;
+ pkts_burst =
+ fsb->tx_mbufs[dst_port].m_table;
+ rte_spinlock_lock(
+ &(txq_lock[dst_port][dst_queue].spinlock));
+#if PORTFWD_PERF_STATS
+ t[0] = rte_rdtsc();
+#endif
+ nb_tx = rte_eth_tx_burst(dst_port, dst_queue,
+ pkts_burst, len);
+#if PORTFWD_PERF_STATS
+ t[1] = rte_rdtsc();
+ log_stat(dst_port, dst_queue, nb_tx, len - nb_tx,
+ t[1] - t[0], 1);
+#endif
+ rte_spinlock_unlock(
+ &(txq_lock[dst_port][dst_queue].spinlock));
+ ret += nb_tx;
+ if (nb_tx < len)
+ do {
+ rte_pktmbuf_free(pkts_burst[nb_tx]);
+ } while (++nb_tx < len);
+ fsb->tx_mbufs[dst_port].len = 0;
+ }
+ }
+ return ret;
+}
+
+static inline uint32_t __attribute__((always_inline))
+batch_send_ipv4_packets(uint16_t src_port, uint16_t src_queue,
+ struct rte_mbuf *m[], uint16_t len)
+{
+ struct ipv4_hdr *ipv4_hdr[2];
+ struct rte_mbuf **mp;
+ uint16_t dst_port[2];
+ uint16_t nb_tx;
+
+ nb_tx = 0;
+ mp = m;
+ while (len >= 4) {
+ rte_prefetch0((void *)mp[2]);
+ rte_prefetch0((void *)mp[3]);
+ ipv4_hdr[0] = rte_pktmbuf_mtod_offset(mp[0], struct ipv4_hdr *,
+ sizeof(struct ether_hdr));
+ ipv4_hdr[1] = rte_pktmbuf_mtod_offset(mp[1], struct ipv4_hdr *,
+ sizeof(struct ether_hdr));
+ dst_port[0] = get_ipv4_dst_port(ipv4_hdr[0], src_port);
+ dst_port[1] = get_ipv4_dst_port(ipv4_hdr[1], src_port);
+ nb_tx += send_single_packet(dst_port[0], src_port,
+ src_queue, mp[0]);
+ nb_tx += send_single_packet(dst_port[1], src_port,
+ src_queue, mp[1]);
+ mp += 2;
+ len -= 2;
+ }
+ while (len > 0) {
+ ipv4_hdr[0] = rte_pktmbuf_mtod_offset(mp[0], struct ipv4_hdr *,
+ sizeof(struct ether_hdr));
+ dst_port[0] = get_ipv4_dst_port(ipv4_hdr[0], src_port);
+ nb_tx += send_single_packet(dst_port[0], src_port,
+ src_queue, mp[0]);
+ mp += 1;
+ len -= 1;
+ }
+
+ return nb_tx;
+}
+
+static void
+pkt_burst_ip_forward(struct fwd_stream *fs)
+{
+ struct rte_mbuf *pkts_burst[DEF_PKT_BURST];
+ uint32_t nb_rx;
+ uint32_t nb_tx;
+ uint16_t src_port = fs->rx_port;
+ uint16_t src_queue = fs->rx_queue;
+#if PORTFWD_PERF_STATS
+ uint64_t t[2];
+#endif
+
+ /* send timeout packets */
+ if (drain_cycle > 0) {
+ drainer[src_port][src_queue].cycle_now = rte_rdtsc();
+ if (drainer[src_port][src_queue].cycle_now -
+ drainer[src_port][src_queue].cycle_last >
+ drain_cycle) {
+ nb_tx = drain_packets(src_port, src_queue);
+ fs->tx_packets += nb_tx;
+ drainer[src_port][src_queue].cycle_last =
+ drainer[src_port][src_queue].cycle_now;
+ }
+ }
+
+#if PORTFWD_PERF_STATS
+ t[0] = rte_rdtsc();
+#endif
+ nb_rx = rte_eth_rx_burst(src_port, src_queue,
+ pkts_burst, nb_pkt_per_burst);
+#if PORTFWD_PERF_STATS
+ t[1] = rte_rdtsc();
+ log_stat(src_port, src_queue, nb_rx, 0, t[1] - t[0], 0);
+#endif
+ if (unlikely(nb_rx == 0))
+ return;
+ fs->rx_packets += nb_rx;
+ nb_tx = 0;
+ /* assume ipv4 packet */
+ nb_tx += batch_send_ipv4_packets(src_port, src_queue,
+ pkts_burst, nb_rx);
+ fs->tx_packets += nb_tx;
+ fs->fwd_dropped = fs->rx_packets - fs->tx_packets;
+}
+
+struct fwd_engine port_fwd_engine = {
+ .fwd_mode_name = "port",
+ .port_fwd_begin = NULL,
+ .port_fwd_end = NULL,
+ .packet_fwd = pkt_burst_ip_forward,
+};
@@ -138,11 +138,26 @@ portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */
struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */
+unsigned long drain_cycle = 10000;
+struct drain_counter drainer[RTE_MAX_ETHPORTS][MAX_RX_QUEUE];
+
+uint32_t ipv4_table[RTE_MAX_ETHPORTS];
+
+uint32_t ipv4_route[MAX_IP_DIFF];
+uint32_t ipv4_route_available;
+uint32_t route_table[RTE_MAX_ETHPORTS];
+
+struct fwd_stream_buffer fs_buf[RTE_MAX_ETHPORTS][MAX_RX_QUEUE];
+struct queue_lock txq_lock[RTE_MAX_ETHPORTS][MAX_TX_QUEUE];
+
+volatile int fwd_config_init;
+
/*
* Forwarding engines.
*/
struct fwd_engine * fwd_engines[] = {
&io_fwd_engine,
+ &port_fwd_engine,
&mac_fwd_engine,
&mac_retry_fwd_engine,
&mac_swap_engine,
@@ -1002,6 +1017,7 @@ start_packet_forwarding(int with_tx_first)
if(!no_flush_rx)
flush_fwd_rx_queues();
+ init_txq_lock();
fwd_config_setup();
rxtx_config_display();
@@ -2063,6 +2079,9 @@ main(int argc, char** argv)
"but nb_txq=%d will prevent to fully test it.\n",
nb_rxq, nb_txq);
+ add_port_addr();
+ set_drain_interval_ns(DRAIN_INTERVAL_NS);
+
init_config();
if (start_port(RTE_PORT_ALL) != 0)
rte_exit(EXIT_FAILURE, "Start ports failed\n");
@@ -34,6 +34,8 @@
#ifndef _TESTPMD_H_
#define _TESTPMD_H_
+#include <rte_spinlock.h>
+
#define RTE_PORT_ALL (~(portid_t)0x0)
#define RTE_TEST_RX_DESC_MAX 2048
@@ -103,6 +105,8 @@ struct fwd_stream {
queueid_t tx_queue; /**< TX queue to send forwarded packets */
streamid_t peer_addr; /**< index of peer ethernet address of packets */
+ unsigned int idx;
+
/* "read-write" results */
unsigned int rx_packets; /**< received packets */
unsigned int tx_packets; /**< received packets transmitted */
@@ -221,6 +225,7 @@ struct fwd_engine {
};
extern struct fwd_engine io_fwd_engine;
+extern struct fwd_engine port_fwd_engine;
extern struct fwd_engine mac_fwd_engine;
extern struct fwd_engine mac_retry_fwd_engine;
extern struct fwd_engine mac_swap_engine;
@@ -329,6 +334,48 @@ extern portid_t nb_fwd_ports; /**< Number of forwarding ports. */
extern portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];
extern struct rte_port *ports;
+#define MAX_RX_QUEUE 32
+#define MAX_TX_QUEUE 32
+#define PORT_ROUTE_IP 65536
+#define IPV4_ROUTE_MASK 0x3F
+#define MAX_IP_DIFF ((IPV4_ROUTE_MASK) + 1)
+#define DRAIN_INTERVAL_NS 5000
+
+struct mbuf_table {
+ uint16_t len;
+ struct rte_mbuf *m_table[DEF_PKT_BURST];
+};
+
+struct fwd_stream_buffer {
+ struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS];
+ uint16_t dst_queue;
+} __rte_cache_aligned;
+
+struct drain_counter {
+ unsigned long cycle_now;
+ unsigned long cycle_last;
+} __rte_cache_aligned;
+
+struct queue_lock {
+ rte_spinlock_t spinlock;
+} __rte_cache_aligned;
+
+extern unsigned long drain_cycle;
+extern struct drain_counter drainer[RTE_MAX_ETHPORTS][MAX_RX_QUEUE];
+
+extern streamid_t nb_fwd_streams;
+
+extern uint32_t ipv4_table[RTE_MAX_ETHPORTS];
+
+extern uint32_t route_table[RTE_MAX_ETHPORTS];
+extern uint32_t ipv4_route[MAX_IP_DIFF];
+extern uint32_t ipv4_route_available;
+
+extern struct fwd_stream_buffer fs_buf[RTE_MAX_ETHPORTS][MAX_RX_QUEUE];
+extern struct queue_lock txq_lock[RTE_MAX_ETHPORTS][MAX_TX_QUEUE];
+
+extern volatile int fwd_config_init;
+
extern struct rte_eth_rxmode rx_mode;
extern uint64_t rss_hf;
@@ -575,6 +622,21 @@ void mcast_addr_add(uint8_t port_id, struct ether_addr *mc_addr);
void mcast_addr_remove(uint8_t port_id, struct ether_addr *mc_addr);
void port_dcb_info_display(uint8_t port_id);
+void set_fixed_route(uint16_t srcp, uint16_t dstp);
+void set_ip_route(uint16_t srcp);
+void show_route(void);
+void init_txq_lock(void);
+void set_ip(uint32_t srcp, uint32_t num0, uint32_t num1,
+ uint32_t num2, uint32_t num3);
+void add_port_addr(void);
+void build_ipv4_route(void);
+void set_drain_interval_ns(unsigned long drain_ns);
+void print_perf_stats(void);
+void clear_perf_stats(void);
+int set_fwd_stream_affinity(unsigned int idx, unsigned int core);
+void print_port_info(void);
+
+
enum print_warning {
ENABLED_WARN = 0,
DISABLED_WARN