[v1] examples/l3fwd-power: add telemetry mode support

diff --git a/doc/guides/rel_notes/release_19_08.rst b/doc/guides/rel_notes/release_19_08.rst
index b9510f93a..e318e400a 100644
--- a/doc/guides/rel_notes/release_19_08.rst
+++ b/doc/guides/rel_notes/release_19_08.rst
@@ -54,6 +54,11 @@  New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+* **Added new telemetry mode for l3fwd-power application.**
+
+  Added telemetry mode to l3fwd-power application to report
+  application level busyness, empty and full polls of rte_eth_rx_burst().
+
 
 Removed Items
 -------------
diff --git a/doc/guides/sample_app_ug/l3_forward_power_man.rst b/doc/guides/sample_app_ug/l3_forward_power_man.rst
index e44a11b2c..7b0fc8d2b 100644
--- a/doc/guides/sample_app_ug/l3_forward_power_man.rst
+++ b/doc/guides/sample_app_ug/l3_forward_power_man.rst
@@ -107,6 +107,8 @@  where,
 
 *   --empty-poll: Traffic Aware power management. See below for details
 
+*   --telemetry:  Telemetry mode.
+
 See :doc:`l3_forward` for details.
 The L3fwd-power example reuses the L3fwd command line options.
 
@@ -431,3 +433,22 @@  then be started without the training mode so traffic can start immediately.
 .. code-block:: console
 
         ./examples/l3fwd-power/build/l3fwd-power -l 1-3 -- -p 0x0f --config="(0,0,2),(0,1,3)" --empty-poll "0,340000,540000" –P
+
+Telemetry Mode
+--------------
+
+The telemetry mode support for ``l3fwd-power`` is a standalone mode, in this mode
+``l3fwd-power`` does simple l3fwding along with calculating empty polls, full polls,
+and busy percentage for each forwarding core. The aggregation of these
+values of all cores is reported as application level telemetry to metric
+library for every 10ms from the master core.
+
+The busy percentage is calculated by recording the poll_count
+and when the count reaches a defined value the total
+cycles it took is measured and compared with minimum and maximum
+reference cycles and accordingly busy rate is set  to either 0% or
+50% or 100%.
+
+.. code-block:: console
+
+        ./examples/l3fwd-power/build/l3fwd-power -l 1-3 -- -p 0x0f --config="(0,0,2),(0,1,3)" --telemetry
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 3b448acc4..70a4451e1 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -14,6 +14,7 @@ 
 #include <getopt.h>
 #include <unistd.h>
 #include <signal.h>
+#include <math.h>
 
 #include <rte_common.h>
 #include <rte_byteorder.h>
@@ -44,6 +45,7 @@ 
 #include <rte_power.h>
 #include <rte_spinlock.h>
 #include <rte_power_empty_poll.h>
+#include <rte_metrics.h>
 
 #include "perf_core.h"
 #include "main.h"
@@ -146,17 +148,56 @@  static uint32_t enabled_port_mask = 0;
 static int promiscuous_on = 0;
 /* NUMA is enabled by default. */
 static int numa_on = 1;
-/* emptypoll is disabled by default. */
-static bool empty_poll_on;
+static bool empty_poll_stop;
 static bool empty_poll_train;
-volatile bool empty_poll_stop;
+volatile bool quit_signal;
 static struct  ep_params *ep_params;
 static struct  ep_policy policy;
 static long  ep_med_edpi, ep_hgh_edpi;
+/* timer to update telemetry every 10ms */
+static struct rte_timer telemetry_timer;
+
+/* stats index returned by metrics lib */
+int telstats_index;
+
+struct telstats_name {
+	char name[RTE_ETH_XSTATS_NAME_SIZE];
+};
+
+/* telemetry stats to be reported */
+const struct telstats_name telstats_strings[] = {
+	{"empty_poll"},
+	{"full_poll"},
+	{"busy_percent"}
+};
+
+/* core busyness in percentage */
+enum busy_rate {
+	ZERO = 0,
+	PARTIAL = 50,
+	FULL = 100
+};
+
+/* reference poll count to measure core busyness */
+#define DEFAULT_COUNT 10000
+/*
+ * reference CYCLES to be used to
+ * measure core busyness based on poll count
+ */
+#define MIN_CYCLES 1500000ULL
+#define MAX_CYCLES 2500000ULL
 
 static int parse_ptype; /**< Parse packet type using rx callback, and */
 			/**< disabled by default */
 
+enum appmode {
+	APP_MODE_LEGACY = 0,
+	APP_MODE_EMPTY_POLL,
+	APP_MODE_TELEMETRY
+};
+
+enum appmode app_mode;
+
 enum freq_scale_hint_t
 {
 	FREQ_LOWER    =      -1,
@@ -341,7 +382,26 @@  struct lcore_stats {
 	uint64_t nb_rx_processed;
 	/* total iterations looped recently */
 	uint64_t nb_iteration_looped;
-	uint32_t padding[9];
+	/*
+	 * Represents empty and non empty polls
+	 * of rte_eth_rx_burst();
+	 * ep_nep[0] holds non empty polls
+	 * i.e. 0 < nb_rx <= MAX_BURST
+	 * ep_nep[1] holds empty polls.
+	 * i.e. nb_rx == 0
+	 */
+	uint64_t ep_nep[2];
+	/*
+	 * Represents full and empty+partial
+	 * polls of rte_eth_rx_burst();
+	 * ep_nep[0] holds empty+partial polls.
+	 * i.e. 0 <= nb_rx < MAX_BURST
+	 * ep_nep[1] holds full polls
+	 * i.e. nb_rx == MAX_BURST
+	 */
+	uint64_t fp_nfp[2];
+	enum busy_rate br;
+	rte_spinlock_t telemetry_lock;
 } __rte_cache_aligned;
 
 static struct lcore_conf lcore_conf[RTE_MAX_LCORE] __rte_cache_aligned;
@@ -362,7 +422,7 @@  static uint8_t  freq_tlb[] = {14, 9, 1};
 
 static int is_done(void)
 {
-	return empty_poll_stop;
+	return quit_signal;
 }
 
 /* exit signal handler */
@@ -374,8 +434,9 @@  signal_exit_now(int sigtype)
 	int ret;
 
 	if (sigtype == SIGINT) {
-		if (empty_poll_on)
-			empty_poll_stop = true;
+		if (app_mode == APP_MODE_EMPTY_POLL ||
+				app_mode == APP_MODE_TELEMETRY)
+			quit_signal = true;
 
 
 		for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
@@ -390,7 +451,7 @@  signal_exit_now(int sigtype)
 							"core%u\n", lcore_id);
 		}
 
-		if (!empty_poll_on) {
+		if (app_mode != APP_MODE_EMPTY_POLL) {
 			RTE_ETH_FOREACH_DEV(portid) {
 				if ((enabled_port_mask & (1 << portid)) == 0)
 					continue;
@@ -401,7 +462,7 @@  signal_exit_now(int sigtype)
 		}
 	}
 
-	if (!empty_poll_on)
+	if (app_mode != APP_MODE_EMPTY_POLL)
 		rte_exit(EXIT_SUCCESS, "User forced exit\n");
 }
 
@@ -869,6 +930,126 @@  static int event_register(struct lcore_conf *qconf)
 }
 /* main processing loop */
 static int
+main_telemetry_loop(__attribute__((unused)) void *dummy)
+{
+	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
+	unsigned int lcore_id;
+	uint64_t prev_tsc, diff_tsc, cur_tsc, prev_tel_tsc;
+	int i, j, nb_rx;
+	uint8_t queueid;
+	uint16_t portid;
+	struct lcore_conf *qconf;
+	struct lcore_rx_queue *rx_queue;
+	uint64_t ep_nep[2] = {0}, fp_nfp[2] = {0};
+	uint64_t poll_count;
+	enum busy_rate br;
+
+	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
+					US_PER_S * BURST_TX_DRAIN_US;
+
+	poll_count = 0;
+	prev_tsc = 0;
+	prev_tel_tsc = 0;
+
+	lcore_id = rte_lcore_id();
+	qconf = &lcore_conf[lcore_id];
+
+	if (qconf->n_rx_queue == 0) {
+		RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n",
+			lcore_id);
+		return 0;
+	}
+
+	RTE_LOG(INFO, L3FWD_POWER, "entering main telemetry loop on lcore %u\n",
+		lcore_id);
+
+	for (i = 0; i < qconf->n_rx_queue; i++) {
+		portid = qconf->rx_queue_list[i].port_id;
+		queueid = qconf->rx_queue_list[i].queue_id;
+		RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u "
+			"rxqueueid=%hhu\n", lcore_id, portid, queueid);
+	}
+
+	while (!is_done()) {
+
+		cur_tsc = rte_rdtsc();
+		/*
+		 * TX burst queue drain
+		 */
+		diff_tsc = cur_tsc - prev_tsc;
+		if (unlikely(diff_tsc > drain_tsc)) {
+			for (i = 0; i < qconf->n_tx_port; ++i) {
+				portid = qconf->tx_port_id[i];
+				rte_eth_tx_buffer_flush(portid,
+						qconf->tx_queue_id[portid],
+						qconf->tx_buffer[portid]);
+			}
+			prev_tsc = cur_tsc;
+		}
+
+		/*
+		 * Read packet from RX queues
+		 */
+		for (i = 0; i < qconf->n_rx_queue; ++i) {
+			rx_queue = &(qconf->rx_queue_list[i]);
+			portid = rx_queue->port_id;
+			queueid = rx_queue->queue_id;
+
+			nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst,
+								MAX_PKT_BURST);
+			ep_nep[nb_rx == 0]++;
+			fp_nfp[nb_rx == MAX_PKT_BURST]++;
+			poll_count++;
+			if (unlikely(nb_rx == 0))
+				continue;
+
+			/* Prefetch first packets */
+			for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
+				rte_prefetch0(rte_pktmbuf_mtod(
+						pkts_burst[j], void *));
+			}
+
+			/* Prefetch and forward already prefetched packets */
+			for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
+				rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[
+						j + PREFETCH_OFFSET], void *));
+				l3fwd_simple_forward(pkts_burst[j], portid,
+								qconf);
+			}
+
+			/* Forward remaining prefetched packets */
+			for (; j < nb_rx; j++) {
+				l3fwd_simple_forward(pkts_burst[j], portid,
+								qconf);
+			}
+		}
+		if (unlikely(poll_count >= DEFAULT_COUNT)) {
+			diff_tsc = cur_tsc - prev_tel_tsc;
+			if (diff_tsc >= MAX_CYCLES) {
+				br = FULL;
+			} else if (diff_tsc > MIN_CYCLES &&
+					diff_tsc < MAX_CYCLES) {
+				br = PARTIAL;
+			} else {
+				br = ZERO;
+			}
+			poll_count = 0;
+			prev_tel_tsc = cur_tsc;
+			/* update stats for telemetry */
+			rte_spinlock_lock(&stats[lcore_id].telemetry_lock);
+			stats[lcore_id].ep_nep[0] = ep_nep[0];
+			stats[lcore_id].ep_nep[1] = ep_nep[1];
+			stats[lcore_id].fp_nfp[0] = fp_nfp[0];
+			stats[lcore_id].fp_nfp[1] = fp_nfp[1];
+			stats[lcore_id].br = br;
+			rte_spinlock_unlock(&stats[lcore_id].telemetry_lock);
+		}
+	}
+
+	return 0;
+}
+/* main processing loop */
+static int
 main_empty_poll_loop(__attribute__((unused)) void *dummy)
 {
 	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
@@ -1274,7 +1455,9 @@  print_usage(const char *prgname)
 		" which max packet len is PKTLEN in decimal (64-9600)\n"
 		"  --parse-ptype: parse packet type by software\n"
 		"  --empty-poll: enable empty poll detection"
-		" follow (training_flag, high_threshold, med_threshold)\n",
+		" follow (training_flag, high_threshold, med_threshold)\n"
+		" --telemetry: enable telemetry mode, to upadte"
+		" empty polls, full polls, and core busyness to telemetry\n",
 		prgname);
 }
 
@@ -1417,6 +1600,7 @@  parse_ep_config(const char *q_arg)
 
 }
 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype"
+#define CMD_LINE_OPT_TELEMETRY "telemetry"
 
 /* Parse the argument given in the command line of the application */
 static int
@@ -1435,6 +1619,7 @@  parse_args(int argc, char **argv)
 		{"enable-jumbo", 0, 0, 0},
 		{"empty-poll", 1, 0, 0},
 		{CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0},
+		{CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
 		{NULL, 0, 0, 0}
 	};
 
@@ -1508,8 +1693,11 @@  parse_args(int argc, char **argv)
 
 			if (!strncmp(lgopts[option_index].name,
 						"empty-poll", 10)) {
-				printf("empty-poll is enabled\n");
-				empty_poll_on = true;
+				if (app_mode == APP_MODE_TELEMETRY) {
+					printf(" empty-poll cannot be enabled as telemetry mode is enabled\n");
+					return -1;
+				}
+				app_mode = APP_MODE_EMPTY_POLL;
 				ret = parse_ep_config(optarg);
 
 				if (ret) {
@@ -1517,7 +1705,18 @@  parse_args(int argc, char **argv)
 					print_usage(prgname);
 					return -1;
 				}
+				printf("empty-poll is enabled\n");
+			}
 
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_TELEMETRY,
+					sizeof(CMD_LINE_OPT_TELEMETRY))) {
+				if (app_mode == APP_MODE_EMPTY_POLL) {
+					printf("telemetry mode cannot be enabled as empty poll mode is enabled\n");
+					return -1;
+				}
+				app_mode = APP_MODE_TELEMETRY;
+				printf("telemetry mode is enabled\n");
 			}
 
 			if (!strncmp(lgopts[option_index].name,
@@ -1869,6 +2068,52 @@  init_power_library(void)
 	return ret;
 }
 static void
+update_telemetry(__attribute__((unused)) struct rte_timer *tim,
+		__attribute__((unused)) void *arg)
+{
+	unsigned int lcore_id = rte_lcore_id();
+	struct lcore_conf *qconf;
+	uint64_t app_eps = 0, app_fps = 0, app_br = 0;
+	uint64_t values[3] = {0};
+	int ret;
+	uint64_t count = 0;
+
+	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
+		qconf = &lcore_conf[lcore_id];
+		if (qconf->n_rx_queue != 0) {
+			count++;
+			rte_spinlock_lock(&stats[lcore_id].telemetry_lock);
+			app_eps += stats[lcore_id].ep_nep[1];
+			app_fps += stats[lcore_id].fp_nfp[1];
+			app_br += stats[lcore_id].br;
+			rte_spinlock_unlock(&stats[lcore_id].telemetry_lock);
+		}
+	}
+
+	values[0] = round(app_eps/count);
+	values[1] = round(app_fps/count);
+	values[2] = round(app_br/count);
+	ret = rte_metrics_update_values(RTE_METRICS_GLOBAL, telstats_index,
+					values, RTE_DIM(values));
+	if (ret < 0)
+		RTE_LOG(WARNING, POWER, "failed to update metrcis\n");
+}
+static void
+telemetry_setup_timer(void)
+{
+	int lcore_id = rte_lcore_id();
+	uint64_t hz = rte_get_timer_hz();
+	uint64_t ticks;
+
+	ticks = hz / INTERVALS_PER_SECOND;
+	rte_timer_reset_sync(&telemetry_timer,
+			ticks,
+			PERIODICAL,
+			lcore_id,
+			update_telemetry,
+			NULL);
+}
+static void
 empty_poll_setup_timer(void)
 {
 	int lcore_id = rte_lcore_id();
@@ -1902,7 +2147,10 @@  launch_timer(unsigned int lcore_id)
 
 	RTE_LOG(INFO, POWER, "Bring up the Timer\n");
 
-	empty_poll_setup_timer();
+	if (app_mode == APP_MODE_EMPTY_POLL)
+		empty_poll_setup_timer();
+	else
+		telemetry_setup_timer();
 
 	cycles_10ms = rte_get_timer_hz() / 100;
 
@@ -1937,6 +2185,8 @@  main(int argc, char **argv)
 	uint32_t dev_rxq_num, dev_txq_num;
 	uint8_t nb_rx_queue, queue, socketid;
 	uint16_t portid;
+	uint8_t num_telstats = RTE_DIM(telstats_strings);
+	const char *ptr_strings[num_telstats];
 
 	/* catch SIGINT and restore cpufreq governor to ondemand */
 	signal(SIGINT, signal_exit_now);
@@ -2103,7 +2353,7 @@  main(int argc, char **argv)
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
 
-		if (empty_poll_on == false) {
+		if (app_mode == APP_MODE_LEGACY) {
 			/* init timer structures for each enabled lcore */
 			rte_timer_init(&power_timers[lcore_id]);
 			hz = rte_get_timer_hz();
@@ -2182,7 +2432,7 @@  main(int argc, char **argv)
 
 	check_all_ports_link_status(enabled_port_mask);
 
-	if (empty_poll_on == true) {
+	if (app_mode == APP_MODE_EMPTY_POLL) {
 
 		if (empty_poll_train) {
 			policy.state = TRAINING;
@@ -2201,15 +2451,34 @@  main(int argc, char **argv)
 
 
 	/* launch per-lcore init on every lcore */
-	if (empty_poll_on == false) {
+	if (app_mode == APP_MODE_LEGACY) {
 		rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
-	} else {
+	} else if (app_mode == APP_MODE_EMPTY_POLL) {
 		empty_poll_stop = false;
 		rte_eal_mp_remote_launch(main_empty_poll_loop, NULL,
 				SKIP_MASTER);
+	} else {
+		/* Init metrics library */
+		rte_metrics_init(rte_socket_id());
+		/** Register latency stats with stats library */
+		for (unsigned int i = 0; i < num_telstats; i++)
+			ptr_strings[i] = telstats_strings[i].name;
+
+		ret = rte_metrics_reg_names(ptr_strings, num_telstats);
+		if (ret >= 0)
+			telstats_index = ret;
+		else
+			rte_exit(EXIT_FAILURE, "failed to register metrics names");
+
+		RTE_LCORE_FOREACH_SLAVE(lcore_id) {
+			rte_spinlock_init(&stats[lcore_id].telemetry_lock);
+		}
+		rte_timer_init(&telemetry_timer);
+		rte_eal_mp_remote_launch(main_telemetry_loop, NULL,
+						CALL_MASTER);
 	}
 
-	if (empty_poll_on == true)
+	if (app_mode == APP_MODE_EMPTY_POLL || app_mode == APP_MODE_TELEMETRY)
 		launch_timer(rte_lcore_id());
 
 	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
@@ -2217,7 +2486,7 @@  main(int argc, char **argv)
 			return -1;
 	}
 
-	if (empty_poll_on)
+	if (app_mode == APP_MODE_EMPTY_POLL)
 		rte_power_empty_poll_stat_free();
 
 	return 0;