[dpdk-dev,memnic,1/7] guest: memnic-tester: PMD benchmark in guest

diff --git a/guest/Makefile b/guest/Makefile
new file mode 100644
index 0000000..3c90350
--- /dev/null
+++ b/guest/Makefile
@@ -0,0 +1,20 @@ 
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overriden by command line or environment
+ifeq ($(RTE_TARGET),)
+$(error "Please define RTE_TARGET environment variable")
+endif
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+COMMON_INC_OPT = -I $(PWD)/../common
+
+APP = memnic-tester
+
+CFLAGS += -Wall -g -O3 $(COMMON_INC_OPT)
+
+SRCS-y := memnic-tester.c
+
+include $(RTE_SDK)/mk/rte.extapp.mk
diff --git a/guest/README.rst b/guest/README.rst
new file mode 100644
index 0000000..760014e
--- /dev/null
+++ b/guest/README.rst
@@ -0,0 +1,94 @@ 
+.. Copyright 2014 NEC
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+   - Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+   - Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in
+     the documentation and/or other materials provided with the
+     distribution.
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+   FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+   COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+   INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+   (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+   HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+   STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+   OF THE POSSIBILITY OF SUCH DAMAGE.
+
+MEMNIC TESTER
+=============
+
+DESCRIPTION
+-----------
+
+It is a simple benchmark test of MEMNIC PMD in guest.
+
+It have two threads, one thread produces and consumes packets,
+other thread receives packets and directly transmits the received
+packets back in MEMNIC interface. This evaluates MEMNIC PMD running cost.
+
+The master thread does rx_burst and tx_burst through MEMNIC PMD.
+            +---------+
+            | master  |
+            +---------+
+     rx_burst ^     | tx_burst
+              |     V
+          +------+------+
+          |  up  | down | MEMNIC shared memory
+          +------+------+
+     set flag ^     | unset flag
+              |     V
+            +---------+
+            |  slave  |
+            +---------+
+The slave thread emulates packet-in/out by setting flag on/off.
+
+Measuring how many sets in the certain period, that represents
+the MEMNIC PMD performance. The master workload must be very low.
+
+     master |<- put packets ->|     |<- get packets ->|
+     slave  |   |<- rx packets ->|<- tx packets ->|   |
+            |<----------------- set ----------------->|
+
+Like RFC2544, evaluations are performed the below frame size packets.
+  64, 128, 256, 512, 1024, 1280, 1518
+
+It shows the result as packets per second number of each frame size.
+
+HOW TO BUILD
+------------
+
+DPDK and DPDK MEMNIC PMD must be built first like below::
+
+  cd /path/to/dpdk
+  make install T=x86_64-native-linuxapp-gcc
+  cd /path/to/memnic/pmd
+  make RTE_INCLUDE=/path/to/dpdk/x86_64-native-linuxapp-gcc/include
+  cd /path/to/memnic/guest
+  make RTE_SDK=/path/to/dpdk RTE_TARGET=x86_64-native-linuxapp-gcc
+
+The file ``memnic-tester`` is generated under ``build`` directory.
+
+HOW TO RUN
+----------
+
+On host the MEMNIC device must be initialized with proper program.
+``memnic-host-sim`` should take care about it::
+
+  [host]# ./memnic-host-sim /dev/shm/ivshm
+
+Then stop the ``memnic-host-sim`` by CTRL-C.
+
+Run ``memnic-tester`` in guest::
+
+  [guest]# echo 64 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
+  [guest]# mount -t hugetlbfs nodev /mnt/huge
+  [guest]# ./build/memnic-tester -c 0x6 -n 4 -d /path/to/librte_pmd_memnic_copy.so
+
+The result shows how much packets are handled by MEMNIC PMD per second.
diff --git a/guest/memnic-tester.c b/guest/memnic-tester.c
new file mode 100644
index 0000000..10e304b
--- /dev/null
+++ b/guest/memnic-tester.c
@@ -0,0 +1,281 @@ 
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2014 NEC All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <rte_eal.h>
+#include <rte_ethdev.h>
+#include <rte_pci.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_byteorder.h>
+#include <rte_ether.h>
+#include <rte_ring.h>
+
+#include "memnic.h"
+
+#define PKTS_BURST_SIZE	(32)
+#define TEST_DURATION	(10)
+
+static const struct rte_eth_conf port_conf = {
+	.rxmode = {
+		.jumbo_frame = 0,
+	},
+	.txmode = {
+		.mq_mode = ETH_MQ_TX_NONE,
+	},
+};
+
+static const struct rte_eth_rxconf rx_conf = {
+	.rx_thresh = {
+		.pthresh = 8,
+		.hthresh = 8,
+		.wthresh = 4,
+	},
+};
+
+static const struct rte_eth_txconf tx_conf = {
+	.tx_thresh = {
+		.pthresh = 36,
+		.hthresh = 0,
+		.wthresh = 0,
+	},
+};
+
+static const unsigned nr_rxdesc = 128;
+static const unsigned nr_txdesc = 512;
+
+static struct rte_mempool *mempool;
+
+static struct memnic_area *memnic;
+
+static void init_port(unsigned portid)
+{
+	if (rte_eth_dev_configure(portid, 1, 1, &port_conf) < 0) {
+		rte_exit(EXIT_FAILURE, "failed to configure port %u\n",
+			portid);
+	}
+	if (rte_eth_rx_queue_setup(portid, 0, nr_rxdesc,
+			rte_eth_dev_socket_id(portid), &rx_conf,
+			mempool) < 0) {
+		rte_exit(EXIT_FAILURE,
+			"failed to configure rx queue port %u\n",
+			portid);
+	}
+	if (rte_eth_tx_queue_setup(portid, 0, nr_txdesc,
+			rte_eth_dev_socket_id(portid), &tx_conf) < 0) {
+		rte_exit(EXIT_FAILURE,
+			"failed to configure tx queue port %u\n",
+			portid);
+	}
+
+	rte_eth_promiscuous_enable(portid);
+}
+
+static void reset_memnic(void)
+{
+	struct memnic_header *hdr = &memnic->hdr;
+	struct memnic_data *up = &memnic->up;
+	struct memnic_data *down = &memnic->down;
+	int i;
+
+	/* prepare packet data */
+	for (i = 0; i < MEMNIC_NR_PACKET; i++) {
+		struct memnic_packet *p = &up->packets[i];
+
+		p->status = MEMNIC_PKT_ST_FREE;
+		p->len = 60; /* short packet */
+
+		/* don't care about content */
+	}
+
+	/* clear packet data */
+	for (i = 0; i < MEMNIC_NR_PACKET; i++) {
+		struct memnic_packet *p = &down->packets[i];
+
+		p->status = MEMNIC_PKT_ST_FREE;
+	}
+
+	/* use default framesz */
+	hdr->framesz = MEMNIC_MAX_FRAME_LEN;
+
+	rte_compiler_barrier();
+
+	hdr->reset = 0;
+	hdr->valid = 1;
+}
+
+static void slave(void)
+{
+	struct memnic_header *hdr = &memnic->hdr;
+	int up_idx, down_idx;
+	uint64_t hz = rte_get_tsc_hz(), next;
+	uint64_t count;
+	/* RFC2544 like */
+	uint32_t testset[] = {64, 128, 256, 512, 1024, 1280, 1518};
+	int n, nr_tests = sizeof(testset) / sizeof(uint32_t);
+
+	/* wait to turn reset flag on */
+	while (hdr->reset == 0)
+		rte_pause();
+
+	/* wait a sec to confirm no one handles this MEMNIC in host side */
+	next = rte_rdtsc() + hz;
+	while (next < rte_rdtsc()) {
+		if (ACCESS_ONCE(hdr->valid))
+			rte_exit(EXIT_FAILURE, "MEMNIC is active\n");
+	}
+
+	reset_memnic();
+
+	up_idx = down_idx = 0;
+
+	for (n = 0; n < nr_tests; n++) {
+		struct memnic_data *up = &memnic->up;
+		struct memnic_data *down = &memnic->down;
+		struct memnic_packet *p;
+		int i;
+
+		/* prepare incoming packet */
+		for (i = 0; i < MEMNIC_NR_PACKET; i++) {
+			p = &up->packets[i];
+			p->len = testset[n] - 4; /* remove FCS */
+		}
+
+		count = 0;
+		next = rte_rdtsc() + hz * TEST_DURATION;
+		while (next > rte_rdtsc()) {
+			/* put packets */
+			for (i = 0; i < PKTS_BURST_SIZE; i++) {
+xmit_retry:
+				p = &up->packets[up_idx];
+				if (ACCESS_ONCE(p->status) != MEMNIC_PKT_ST_FREE)
+					goto xmit_retry;
+				if (++up_idx >= MEMNIC_NR_PACKET)
+					up_idx = 0;
+				p->status = MEMNIC_PKT_ST_FILLED;
+			}
+			/* get packets */
+			for (i = 0; i < PKTS_BURST_SIZE; i++) {
+recv_retry:
+				p = &down->packets[down_idx];
+				if (ACCESS_ONCE(p->status) != MEMNIC_PKT_ST_FILLED)
+					goto recv_retry;
+				if (++down_idx >= MEMNIC_NR_PACKET)
+					down_idx = 0;
+				p->status = MEMNIC_PKT_ST_FREE;
+			}
+			++count;
+		}
+		printf("frame size %u throughput %lu pps\n",
+			testset[n], (count * PKTS_BURST_SIZE) / TEST_DURATION);
+	}
+
+	/* finish the test */
+	rte_exit(EXIT_SUCCESS, "Test done\n");
+}
+
+static void master(void)
+{
+	if (rte_eth_dev_start(0) < 0)
+		rte_exit(EXIT_FAILURE, "failed to start device\n");
+
+	/* infinity loop back */
+	for (;;) {
+		struct rte_mbuf *bufs[PKTS_BURST_SIZE];
+		int rx, tx;
+
+		rx = rte_eth_rx_burst(0, 0, bufs, PKTS_BURST_SIZE);
+		tx = 0;
+		while (rx != tx)
+			tx += rte_eth_tx_burst(0, 0, &bufs[tx], rx - tx);
+	}
+}
+
+static int lcore_main(void *p)
+{
+	if (rte_lcore_id() == rte_get_master_lcore())
+		master();
+	else
+		slave();
+
+	/* never reach here */
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+	struct rte_eth_dev *dev;
+	struct memnic_adapter {
+		struct memnic_area *nic;
+	} *adapter;
+	int ret;
+	unsigned lcore_id;
+
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		exit(1);
+
+	argc -= ret;
+	argv += ret;
+
+	if (rte_lcore_count() != 2)
+		rte_exit(EXIT_FAILURE, "Need just 2 lcores\n");
+
+	/* alloc mempool */
+	mempool = rte_mempool_create("pkt_mempool", 8192, 2048, 32,
+				sizeof(struct rte_pktmbuf_pool_private),
+				rte_pktmbuf_pool_init, NULL,
+				rte_pktmbuf_init, NULL,
+				rte_socket_id(), 0);
+
+	if (rte_eal_pci_probe() < 0)
+		rte_exit(EXIT_FAILURE, "failed to probe PCI\n");
+
+	/* get MEMNIC data from ether device */
+	dev = &rte_eth_devices[0];
+	adapter = (struct memnic_adapter *)(dev->data->dev_private);
+
+	memnic = adapter->nic;
+
+	if (memnic->hdr.magic != MEMNIC_MAGIC)
+		rte_exit(EXIT_FAILURE, "Not a MEMNIC device\n");
+
+	/* port 0 must initialize MEMNIC */
+	init_port(0);
+
+	rte_eal_mp_remote_launch(lcore_main, NULL, CALL_MASTER);
+	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
+		if (rte_eal_wait_lcore(lcore_id) < 0)
+			return -1;
+	}
+
+	return 0;
+}