[3/3] l3fwd-power: add interrupt-only mode
diff mbox series

Message ID ceb97619dcaa91188757e6cf330870a6dfe97de6.1590656906.git.anatoly.burakov@intel.com
State Superseded
Headers show
Series
  • Add interrupt-only mode to l3fwd-power
Related show

Checks

Context Check Description
ci/Intel-compilation success Compilation OK
ci/travis-robot success Travis build: passed
ci/checkpatch success coding style OK

Commit Message

Anatoly Burakov May 28, 2020, 9:13 a.m. UTC
In addition to existing modes, add a mode which is very similar to
legacy mode, but does not do frequency scaling, and thus does not
depend on the power library.

Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
---
 examples/l3fwd-power/main.c | 215 +++++++++++++++++++++++++++++++++---
 1 file changed, 202 insertions(+), 13 deletions(-)

Comments

Harman Kalra May 29, 2020, 1:19 p.m. UTC | #1
On Thu, May 28, 2020 at 10:13:51AM +0100, Anatoly Burakov wrote:
> In addition to existing modes, add a mode which is very similar to
> legacy mode, but does not do frequency scaling, and thus does not
> depend on the power library.
> 
> Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
> ---
>  examples/l3fwd-power/main.c | 215 +++++++++++++++++++++++++++++++++---
>  1 file changed, 202 insertions(+), 13 deletions(-)
> 
> diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
> index 5cee9d5387..4161e01974 100644
> --- a/examples/l3fwd-power/main.c
> +++ b/examples/l3fwd-power/main.c
> @@ -195,9 +195,11 @@ static int parse_ptype; /**< Parse packet type using rx callback, and */
>  			/**< disabled by default */
>  
>  enum appmode {
> -	APP_MODE_LEGACY = 0,
> +	APP_MODE_DEFAULT = 0,
> +	APP_MODE_LEGACY,
>  	APP_MODE_EMPTY_POLL,
> -	APP_MODE_TELEMETRY
> +	APP_MODE_TELEMETRY,
> +	APP_MODE_INTERRUPT
>  };
>  
>  enum appmode app_mode;
> @@ -900,6 +902,170 @@ static int event_register(struct lcore_conf *qconf)
>  
>  	return 0;
>  }
> +
> +/* main processing loop */
> +static int main_intr_loop(__rte_unused void *dummy)
> +{
> +	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
> +	unsigned int lcore_id;
> +	uint64_t prev_tsc, diff_tsc, cur_tsc;
> +	int i, j, nb_rx;
> +	uint8_t queueid;
> +	uint16_t portid;
> +	struct lcore_conf *qconf;
> +	struct lcore_rx_queue *rx_queue;
> +	uint32_t lcore_rx_idle_count = 0;
> +	uint32_t lcore_idle_hint = 0;
> +	int intr_en = 0;
> +
> +	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
> +				   US_PER_S * BURST_TX_DRAIN_US;
> +
> +	prev_tsc = 0;
> +
> +	lcore_id = rte_lcore_id();
> +	qconf = &lcore_conf[lcore_id];
> +
> +	if (qconf->n_rx_queue == 0) {
> +		RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n",
> +				lcore_id);
> +		return 0;
> +	}
> +
> +	RTE_LOG(INFO, L3FWD_POWER, "entering main interrupt loop on lcore %u\n",
> +			lcore_id);
> +
> +	for (i = 0; i < qconf->n_rx_queue; i++) {
> +		portid = qconf->rx_queue_list[i].port_id;
> +		queueid = qconf->rx_queue_list[i].queue_id;
> +		RTE_LOG(INFO, L3FWD_POWER,
> +				" -- lcoreid=%u portid=%u rxqueueid=%hhu\n",
> +				lcore_id, portid, queueid);
> +	}
> +
> +	/* add into event wait list */
> +	if (event_register(qconf) == 0)
> +		intr_en = 1;
> +	else
> +		RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n");
> +
> +	while (!is_done()) {
> +		stats[lcore_id].nb_iteration_looped++;
> +
> +		cur_tsc = rte_rdtsc();
> +
> +		/*
> +		 * TX burst queue drain
> +		 */
> +		diff_tsc = cur_tsc - prev_tsc;
> +		if (unlikely(diff_tsc > drain_tsc)) {
> +			for (i = 0; i < qconf->n_tx_port; ++i) {
> +				portid = qconf->tx_port_id[i];
> +				rte_eth_tx_buffer_flush(portid,
> +						qconf->tx_queue_id[portid],
> +						qconf->tx_buffer[portid]);
> +			}
> +			prev_tsc = cur_tsc;
> +		}
> +
> +start_rx:
> +		/*
> +		 * Read packet from RX queues
> +		 */
> +		lcore_rx_idle_count = 0;
> +		for (i = 0; i < qconf->n_rx_queue; ++i) {
> +			rx_queue = &(qconf->rx_queue_list[i]);
> +			rx_queue->idle_hint = 0;
> +			portid = rx_queue->port_id;
> +			queueid = rx_queue->queue_id;
> +
> +			nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst,
> +					MAX_PKT_BURST);
> +
> +			stats[lcore_id].nb_rx_processed += nb_rx;
> +			if (unlikely(nb_rx == 0)) {
> +				/**
> +				 * no packet received from rx queue, try to
> +				 * sleep for a while forcing CPU enter deeper
> +				 * C states.
> +				 */
> +				rx_queue->zero_rx_packet_count++;
> +
> +				if (rx_queue->zero_rx_packet_count <=
> +						MIN_ZERO_POLL_COUNT)
> +					continue;
> +
> +				rx_queue->idle_hint = power_idle_heuristic(
> +						rx_queue->zero_rx_packet_count);
> +				lcore_rx_idle_count++;
> +			} else {
> +				rx_queue->zero_rx_packet_count = 0;
> +			}
> +
> +			/* Prefetch first packets */
> +			for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
> +				rte_prefetch0(rte_pktmbuf_mtod(
> +						pkts_burst[j], void *));
> +			}
> +
> +			/* Prefetch and forward already prefetched packets */
> +			for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
> +				rte_prefetch0(rte_pktmbuf_mtod(
> +						pkts_burst[j + PREFETCH_OFFSET],
> +						void *));
> +				l3fwd_simple_forward(
> +						pkts_burst[j], portid, qconf);
> +			}
> +
> +			/* Forward remaining prefetched packets */
> +			for (; j < nb_rx; j++) {
> +				l3fwd_simple_forward(
> +						pkts_burst[j], portid, qconf);
> +			}
> +		}
> +
> +		if (unlikely(lcore_rx_idle_count == qconf->n_rx_queue)) {
> +			/**
> +			 * All Rx queues empty in recent consecutive polls,
> +			 * sleep in a conservative manner, meaning sleep as
> +			 * less as possible.
> +			 */
> +			for (i = 1,
> +			    lcore_idle_hint = qconf->rx_queue_list[0].idle_hint;
> +					i < qconf->n_rx_queue; ++i) {
> +				rx_queue = &(qconf->rx_queue_list[i]);
> +				if (rx_queue->idle_hint < lcore_idle_hint)
> +					lcore_idle_hint = rx_queue->idle_hint;
> +			}
> +
> +			if (lcore_idle_hint < SUSPEND_THRESHOLD)
> +				/**
> +				 * execute "pause" instruction to avoid context
> +				 * switch which generally take hundred of
> +				 * microseconds for short sleep.
> +				 */
> +				rte_delay_us(lcore_idle_hint);
> +			else {
> +				/* suspend until rx interrupt triggers */
> +				if (intr_en) {
> +					turn_on_off_intr(qconf, 1);
> +					sleep_until_rx_interrupt(
> +							qconf->n_rx_queue);
> +					turn_on_off_intr(qconf, 0);
> +					/**
> +					 * start receiving packets immediately
> +					 */
> +					if (likely(!is_done()))
> +						goto start_rx;
> +				}
> +			}
> +			stats[lcore_id].sleep_time += lcore_idle_hint;
> +		}
> +	}
> +
> +	return 0;
> +}
> +
>  /* main processing loop */
>  static int
>  main_telemetry_loop(__rte_unused void *dummy)
> @@ -1126,7 +1292,7 @@ main_empty_poll_loop(__rte_unused void *dummy)
>  }
>  /* main processing loop */
>  static int
> -main_loop(__rte_unused void *dummy)
> +main_legacy_loop(__rte_unused void *dummy)
>  {
>  	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
>  	unsigned lcore_id;
> @@ -1438,7 +1604,8 @@ print_usage(const char *prgname)
>  		"  --empty-poll: enable empty poll detection"
>  		" follow (training_flag, high_threshold, med_threshold)\n"
>  		" --telemetry: enable telemetry mode, to update"
> -		" empty polls, full polls, and core busyness to telemetry\n",
> +		" empty polls, full polls, and core busyness to telemetry\n"
> +		" --interrupt-only: enable interrupt-only mode\n",
>  		prgname);
>  }
>  
> @@ -1582,6 +1749,7 @@ parse_ep_config(const char *q_arg)
>  }
>  #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype"
>  #define CMD_LINE_OPT_TELEMETRY "telemetry"
> +#define CMD_LINE_OPT_INTERRUPT_ONLY "interrupt-only"
>  
>  /* Parse the argument given in the command line of the application */
>  static int
> @@ -1601,6 +1769,7 @@ parse_args(int argc, char **argv)
>  		{"empty-poll", 1, 0, 0},
>  		{CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0},
>  		{CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
> +		{CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0},
>  		{NULL, 0, 0, 0}
>  	};
>  
> @@ -1674,8 +1843,8 @@ parse_args(int argc, char **argv)
>  
>  			if (!strncmp(lgopts[option_index].name,
>  						"empty-poll", 10)) {
> -				if (app_mode == APP_MODE_TELEMETRY) {
> -					printf(" empty-poll cannot be enabled as telemetry mode is enabled\n");
> +				if (app_mode != APP_MODE_DEFAULT) {
> +					printf(" empty-poll mode is mutually exclusive with other modes\n");
>  					return -1;
>  				}
>  				app_mode = APP_MODE_EMPTY_POLL;
> @@ -1692,14 +1861,25 @@ parse_args(int argc, char **argv)
>  			if (!strncmp(lgopts[option_index].name,
>  					CMD_LINE_OPT_TELEMETRY,
>  					sizeof(CMD_LINE_OPT_TELEMETRY))) {
> -				if (app_mode == APP_MODE_EMPTY_POLL) {
> -					printf("telemetry mode cannot be enabled as empty poll mode is enabled\n");
> +				if (app_mode != APP_MODE_DEFAULT) {
> +					printf(" telemetry mode is mutually exclusive with other modes\n");
>  					return -1;
>  				}
>  				app_mode = APP_MODE_TELEMETRY;
>  				printf("telemetry mode is enabled\n");
>  			}
>  
> +			if (!strncmp(lgopts[option_index].name,
> +					CMD_LINE_OPT_INTERRUPT_ONLY,
> +					sizeof(CMD_LINE_OPT_INTERRUPT_ONLY))) {
> +				if (app_mode != APP_MODE_DEFAULT) {
> +					printf(" interrupt-only mode is mutually exclusive with other modes\n");
> +					return -1;
> +				}
> +				app_mode = APP_MODE_INTERRUPT;
> +				printf("interrupt-only mode is enabled\n");
> +			}
> +
>  			if (!strncmp(lgopts[option_index].name,
>  					"enable-jumbo", 12)) {
>  				struct option lenopts =
> @@ -2253,7 +2433,12 @@ main(int argc, char **argv)
>  	if (ret < 0)
>  		rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
>  
> -	if (app_mode != APP_MODE_TELEMETRY && init_power_library())
> +	if (app_mode == APP_MODE_DEFAULT)
> +		app_mode = APP_MODE_LEGACY;
> +
> +	/* only legacy and empty poll mode rely on power library */
> +	if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
> +			init_power_library())
>  		rte_exit(EXIT_FAILURE, "init_power_library failed\n");
>  
Hi,

Rather than just exiting from here can we have a else condition to
automatically enter into the "interrupt only" mode.
Please correct me if I am missing something.

Thanks
Harman
	
>  	if (update_lcore_params() < 0)
> @@ -2277,7 +2462,8 @@ main(int argc, char **argv)
>  	RTE_ETH_FOREACH_DEV(portid) {
>  		struct rte_eth_conf local_port_conf = port_conf;
>  		/* not all app modes need interrupts */
> -		bool need_intr = app_mode == APP_MODE_LEGACY;
> +		bool need_intr = app_mode == APP_MODE_LEGACY ||
> +				app_mode == APP_MODE_INTERRUPT;
>  
>  		/* skip ports that are not enabled */
>  		if ((enabled_port_mask & (1 << portid)) == 0) {
> @@ -2526,12 +2712,12 @@ main(int argc, char **argv)
>  
>  	/* launch per-lcore init on every lcore */
>  	if (app_mode == APP_MODE_LEGACY) {
> -		rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
> +		rte_eal_mp_remote_launch(main_legacy_loop, NULL, CALL_MASTER);
>  	} else if (app_mode == APP_MODE_EMPTY_POLL) {
>  		empty_poll_stop = false;
>  		rte_eal_mp_remote_launch(main_empty_poll_loop, NULL,
>  				SKIP_MASTER);
> -	} else {
> +	} else if (app_mode == APP_MODE_TELEMETRY) {
>  		unsigned int i;
>  
>  		/* Init metrics library */
> @@ -2555,6 +2741,8 @@ main(int argc, char **argv)
>  				"Returns global power stats. Parameters: None");
>  		rte_eal_mp_remote_launch(main_telemetry_loop, NULL,
>  						SKIP_MASTER);
> +	} else if (app_mode == APP_MODE_INTERRUPT) {
> +		rte_eal_mp_remote_launch(main_intr_loop, NULL, CALL_MASTER);
>  	}
>  
>  	if (app_mode == APP_MODE_EMPTY_POLL || app_mode == APP_MODE_TELEMETRY)
> @@ -2577,7 +2765,8 @@ main(int argc, char **argv)
>  	if (app_mode == APP_MODE_EMPTY_POLL)
>  		rte_power_empty_poll_stat_free();
>  
> -	if (app_mode != APP_MODE_TELEMETRY && deinit_power_library())
> +	if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
> +			deinit_power_library())
>  		rte_exit(EXIT_FAILURE, "deinit_power_library failed\n");
>  
>  	if (rte_eal_cleanup() < 0)
> -- 
> 2.17.1
Anatoly Burakov May 29, 2020, 2:19 p.m. UTC | #2
On 29-May-20 2:19 PM, Harman Kalra wrote:

>>   	if (ret < 0)
>>   		rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
>>   
>> -	if (app_mode != APP_MODE_TELEMETRY && init_power_library())
>> +	if (app_mode == APP_MODE_DEFAULT)
>> +		app_mode = APP_MODE_LEGACY;
>> +
>> +	/* only legacy and empty poll mode rely on power library */
>> +	if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
>> +			init_power_library())
>>   		rte_exit(EXIT_FAILURE, "init_power_library failed\n");
>>   
> Hi,
> 
> Rather than just exiting from here can we have a else condition to
> automatically enter into the "interrupt only" mode.
> Please correct me if I am missing something.

Hi,

Thanks for your review. I don't think silently proceeding is a good 
idea. If the user wants interrupt-only mode, they should request it. 
Silently falling back to interrupt-only mode will create an illusion of 
successful initialization and set the wrong expectation for how the 
application will behave.
Harman Kalra May 30, 2020, 10:02 a.m. UTC | #3
On Fri, May 29, 2020 at 03:19:45PM +0100, Burakov, Anatoly wrote:
> External Email
> 
> ----------------------------------------------------------------------
> On 29-May-20 2:19 PM, Harman Kalra wrote:
> 
> > >   	if (ret < 0)
> > >   		rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
> > > -	if (app_mode != APP_MODE_TELEMETRY && init_power_library())
> > > +	if (app_mode == APP_MODE_DEFAULT)
> > > +		app_mode = APP_MODE_LEGACY;
> > > +
> > > +	/* only legacy and empty poll mode rely on power library */
> > > +	if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
> > > +			init_power_library())
> > >   		rte_exit(EXIT_FAILURE, "init_power_library failed\n");
> > Hi,
> > 
> > Rather than just exiting from here can we have a else condition to
> > automatically enter into the "interrupt only" mode.
> > Please correct me if I am missing something.
> 
> Hi,
> 
> Thanks for your review. I don't think silently proceeding is a good idea. If
> the user wants interrupt-only mode, they should request it. Silently falling
> back to interrupt-only mode will create an illusion of successful
> initialization and set the wrong expectation for how the application will
> behave.
> 

Hi,

Thanks for the explanation which even I also believe is logically perfect.

But since l3fwd-power is an old application and has many users around
which are currently using this app in interrupt only mode without giving
an extra argument. But suddenly they will start getting failure messages with
the new patchset.

My only intent with else condition was backward compatibility.
Or may be we can have more descriptive failure message, something like
"init_power_library failed, check manual for other modes".

Thanks
Harman


> -- 
> Thanks,
> Anatoly
Anatoly Burakov June 1, 2020, 12:50 p.m. UTC | #4
On 30-May-20 11:02 AM, Harman Kalra wrote:
> On Fri, May 29, 2020 at 03:19:45PM +0100, Burakov, Anatoly wrote:
>> External Email
>>
>> ----------------------------------------------------------------------
>> On 29-May-20 2:19 PM, Harman Kalra wrote:
>>
>>>>    	if (ret < 0)
>>>>    		rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
>>>> -	if (app_mode != APP_MODE_TELEMETRY && init_power_library())
>>>> +	if (app_mode == APP_MODE_DEFAULT)
>>>> +		app_mode = APP_MODE_LEGACY;
>>>> +
>>>> +	/* only legacy and empty poll mode rely on power library */
>>>> +	if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
>>>> +			init_power_library())
>>>>    		rte_exit(EXIT_FAILURE, "init_power_library failed\n");
>>> Hi,
>>>
>>> Rather than just exiting from here can we have a else condition to
>>> automatically enter into the "interrupt only" mode.
>>> Please correct me if I am missing something.
>>
>> Hi,
>>
>> Thanks for your review. I don't think silently proceeding is a good idea. If
>> the user wants interrupt-only mode, they should request it. Silently falling
>> back to interrupt-only mode will create an illusion of successful
>> initialization and set the wrong expectation for how the application will
>> behave.
>>
> 
> Hi,
> 
> Thanks for the explanation which even I also believe is logically perfect.
> 
> But since l3fwd-power is an old application and has many users around
> which are currently using this app in interrupt only mode without giving
> an extra argument. But suddenly they will start getting failure messages with
> the new patchset.
> 
> My only intent with else condition was backward compatibility.
> Or may be we can have more descriptive failure message, something like
> "init_power_library failed, check manual for other modes".
> 
> Thanks
> Harman
> 
> 

I think we can compormise on an informative log message suggesting to 
use interrupt mode. I'm not keen on reverting to previous buggy behavior :)

>> -- 
>> Thanks,
>> Anatoly
Harman Kalra June 2, 2020, 10:23 a.m. UTC | #5
On Mon, Jun 01, 2020 at 01:50:26PM +0100, Burakov, Anatoly wrote:
> On 30-May-20 11:02 AM, Harman Kalra wrote:
> > On Fri, May 29, 2020 at 03:19:45PM +0100, Burakov, Anatoly wrote:
> > > External Email
> > > 
> > > ----------------------------------------------------------------------
> > > On 29-May-20 2:19 PM, Harman Kalra wrote:
> > > 
> > > > >    	if (ret < 0)
> > > > >    		rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
> > > > > -	if (app_mode != APP_MODE_TELEMETRY && init_power_library())
> > > > > +	if (app_mode == APP_MODE_DEFAULT)
> > > > > +		app_mode = APP_MODE_LEGACY;
> > > > > +
> > > > > +	/* only legacy and empty poll mode rely on power library */
> > > > > +	if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
> > > > > +			init_power_library())
> > > > >    		rte_exit(EXIT_FAILURE, "init_power_library failed\n");
> > > > Hi,
> > > > 
> > > > Rather than just exiting from here can we have a else condition to
> > > > automatically enter into the "interrupt only" mode.
> > > > Please correct me if I am missing something.
> > > 
> > > Hi,
> > > 
> > > Thanks for your review. I don't think silently proceeding is a good idea. If
> > > the user wants interrupt-only mode, they should request it. Silently falling
> > > back to interrupt-only mode will create an illusion of successful
> > > initialization and set the wrong expectation for how the application will
> > > behave.
> > > 
> > 
> > Hi,
> > 
> > Thanks for the explanation which even I also believe is logically perfect.
> > 
> > But since l3fwd-power is an old application and has many users around
> > which are currently using this app in interrupt only mode without giving
> > an extra argument. But suddenly they will start getting failure messages with
> > the new patchset.
> > 
> > My only intent with else condition was backward compatibility.
> > Or may be we can have more descriptive failure message, something like
> > "init_power_library failed, check manual for other modes".
> > 
> > Thanks
> > Harman
> > 
> > 
> 
> I think we can compormise on an informative log message suggesting to use
> interrupt mode. I'm not keen on reverting to previous buggy behavior :)
> 
Hi

I am not insisting to revert to previous behavior, I am just trying to
highlight some probable issues that many users might face as its an old
application.
Since many arm based soc might not be supporting frequency scaling, can
we add the following check as soon as the application starts, probe the
platform if it supports frequency scaling, if not automatically set the
mode to interrupt mode, something like:
if (access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
			F_OK))
	app_mode = APP_MODE_INTERRUPT;


Thanks
Harman

> > > -- 
> > > Thanks,
> > > Anatoly
> 
> 
> -- 
> Thanks,
> Anatoly
Harman Kalra June 2, 2020, 12:16 p.m. UTC | #6
On Tue, Jun 02, 2020 at 03:53:07PM +0530, Harman Kalra wrote:
> On Mon, Jun 01, 2020 at 01:50:26PM +0100, Burakov, Anatoly wrote:
> > On 30-May-20 11:02 AM, Harman Kalra wrote:
> > > On Fri, May 29, 2020 at 03:19:45PM +0100, Burakov, Anatoly wrote:
> > > > External Email
> > > > 
> > > > ----------------------------------------------------------------------
> > > > On 29-May-20 2:19 PM, Harman Kalra wrote:
> > > > 
> > > > > >    	if (ret < 0)
> > > > > >    		rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
> > > > > > -	if (app_mode != APP_MODE_TELEMETRY && init_power_library())
> > > > > > +	if (app_mode == APP_MODE_DEFAULT)
> > > > > > +		app_mode = APP_MODE_LEGACY;
> > > > > > +
> > > > > > +	/* only legacy and empty poll mode rely on power library */
> > > > > > +	if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
> > > > > > +			init_power_library())
> > > > > >    		rte_exit(EXIT_FAILURE, "init_power_library failed\n");
> > > > > Hi,
> > > > > 
> > > > > Rather than just exiting from here can we have a else condition to
> > > > > automatically enter into the "interrupt only" mode.
> > > > > Please correct me if I am missing something.
> > > > 
> > > > Hi,
> > > > 
> > > > Thanks for your review. I don't think silently proceeding is a good idea. If
> > > > the user wants interrupt-only mode, they should request it. Silently falling
> > > > back to interrupt-only mode will create an illusion of successful
> > > > initialization and set the wrong expectation for how the application will
> > > > behave.
> > > > 
> > > 
> > > Hi,
> > > 
> > > Thanks for the explanation which even I also believe is logically perfect.
> > > 
> > > But since l3fwd-power is an old application and has many users around
> > > which are currently using this app in interrupt only mode without giving
> > > an extra argument. But suddenly they will start getting failure messages with
> > > the new patchset.
> > > 
> > > My only intent with else condition was backward compatibility.
> > > Or may be we can have more descriptive failure message, something like
> > > "init_power_library failed, check manual for other modes".
> > > 
> > > Thanks
> > > Harman
> > > 
> > > 
> > 
> > I think we can compormise on an informative log message suggesting to use
> > interrupt mode. I'm not keen on reverting to previous buggy behavior :)
> > 
> Hi
> 
> I am not insisting to revert to previous behavior, I am just trying to
> highlight some probable issues that many users might face as its an old
> application.
> Since many arm based soc might not be supporting frequency scaling, can
> we add the following check as soon as the application starts, probe the
> platform if it supports frequency scaling, if not automatically set the
> mode to interrupt mode, something like:
> if (access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
> 			F_OK))
> 	app_mode = APP_MODE_INTERRUPT;

Sorry, no direct check in application but we can introduce a new API in
power library:
   bool rte_is_freq_scaling() {
	   return  access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
			   F_OK);
   }

and in the application we can use "rte_is_freq_scaling()" at the start.

> 
> 
> Thanks
> Harman
> 
> > > > -- 
> > > > Thanks,
> > > > Anatoly
> > 
> > 
> > -- 
> > Thanks,
> > Anatoly
Anatoly Burakov June 15, 2020, 11:31 a.m. UTC | #7
On 02-Jun-20 1:16 PM, Harman Kalra wrote:
> On Tue, Jun 02, 2020 at 03:53:07PM +0530, Harman Kalra wrote:
>> On Mon, Jun 01, 2020 at 01:50:26PM +0100, Burakov, Anatoly wrote:
>>> On 30-May-20 11:02 AM, Harman Kalra wrote:
>>>> On Fri, May 29, 2020 at 03:19:45PM +0100, Burakov, Anatoly wrote:
>>>>> External Email
>>>>>
>>>>> ----------------------------------------------------------------------
>>>>> On 29-May-20 2:19 PM, Harman Kalra wrote:
>>>>>
>>>>>>>     	if (ret < 0)
>>>>>>>     		rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
>>>>>>> -	if (app_mode != APP_MODE_TELEMETRY && init_power_library())
>>>>>>> +	if (app_mode == APP_MODE_DEFAULT)
>>>>>>> +		app_mode = APP_MODE_LEGACY;
>>>>>>> +
>>>>>>> +	/* only legacy and empty poll mode rely on power library */
>>>>>>> +	if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
>>>>>>> +			init_power_library())
>>>>>>>     		rte_exit(EXIT_FAILURE, "init_power_library failed\n");
>>>>>> Hi,
>>>>>>
>>>>>> Rather than just exiting from here can we have a else condition to
>>>>>> automatically enter into the "interrupt only" mode.
>>>>>> Please correct me if I am missing something.
>>>>>
>>>>> Hi,
>>>>>
>>>>> Thanks for your review. I don't think silently proceeding is a good idea. If
>>>>> the user wants interrupt-only mode, they should request it. Silently falling
>>>>> back to interrupt-only mode will create an illusion of successful
>>>>> initialization and set the wrong expectation for how the application will
>>>>> behave.
>>>>>
>>>>
>>>> Hi,
>>>>
>>>> Thanks for the explanation which even I also believe is logically perfect.
>>>>
>>>> But since l3fwd-power is an old application and has many users around
>>>> which are currently using this app in interrupt only mode without giving
>>>> an extra argument. But suddenly they will start getting failure messages with
>>>> the new patchset.
>>>>
>>>> My only intent with else condition was backward compatibility.
>>>> Or may be we can have more descriptive failure message, something like
>>>> "init_power_library failed, check manual for other modes".
>>>>
>>>> Thanks
>>>> Harman
>>>>
>>>>
>>>
>>> I think we can compormise on an informative log message suggesting to use
>>> interrupt mode. I'm not keen on reverting to previous buggy behavior :)
>>>
>> Hi
>>
>> I am not insisting to revert to previous behavior, I am just trying to
>> highlight some probable issues that many users might face as its an old
>> application.
>> Since many arm based soc might not be supporting frequency scaling, can
>> we add the following check as soon as the application starts, probe the
>> platform if it supports frequency scaling, if not automatically set the
>> mode to interrupt mode, something like:
>> if (access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
>> 			F_OK))
>> 	app_mode = APP_MODE_INTERRUPT;
> 
> Sorry, no direct check in application but we can introduce a new API in
> power library:
>     bool rte_is_freq_scaling() {
> 	   return  access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
> 			   F_OK);
>     }
> 
> and in the application we can use "rte_is_freq_scaling()" at the start.
> 

What you're suggesting here is effectively what you have already 
suggested: silently fall back to interrupt-only mode if power lib init 
failed. I already outlined why i don't think it's a good approach.

>>
>>
>> Thanks
>> Harman
>>
>>>>> -- 
>>>>> Thanks,
>>>>> Anatoly
>>>
>>>
>>> -- 
>>> Thanks,
>>> Anatoly
Jerin Jacob June 15, 2020, 11:43 a.m. UTC | #8
On Mon, Jun 15, 2020 at 5:02 PM Burakov, Anatoly
<anatoly.burakov@intel.com> wrote:
>
> On 02-Jun-20 1:16 PM, Harman Kalra wrote:
> > On Tue, Jun 02, 2020 at 03:53:07PM +0530, Harman Kalra wrote:
> >> On Mon, Jun 01, 2020 at 01:50:26PM +0100, Burakov, Anatoly wrote:
> >>> On 30-May-20 11:02 AM, Harman Kalra wrote:
> >>>> On Fri, May 29, 2020 at 03:19:45PM +0100, Burakov, Anatoly wrote:
> >>>>> External Email
> >>>>>
> >>>>> ----------------------------------------------------------------------
> >>>>> On 29-May-20 2:19 PM, Harman Kalra wrote:
> >>>>>
> >>>>>>>         if (ret < 0)
> >>>>>>>                 rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
> >>>>>>> -       if (app_mode != APP_MODE_TELEMETRY && init_power_library())
> >>>>>>> +       if (app_mode == APP_MODE_DEFAULT)
> >>>>>>> +               app_mode = APP_MODE_LEGACY;
> >>>>>>> +
> >>>>>>> +       /* only legacy and empty poll mode rely on power library */
> >>>>>>> +       if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
> >>>>>>> +                       init_power_library())
> >>>>>>>                 rte_exit(EXIT_FAILURE, "init_power_library failed\n");
> >>>>>> Hi,
> >>>>>>
> >>>>>> Rather than just exiting from here can we have a else condition to
> >>>>>> automatically enter into the "interrupt only" mode.
> >>>>>> Please correct me if I am missing something.
> >>>>>
> >>>>> Hi,
> >>>>>
> >>>>> Thanks for your review. I don't think silently proceeding is a good idea. If
> >>>>> the user wants interrupt-only mode, they should request it. Silently falling
> >>>>> back to interrupt-only mode will create an illusion of successful
> >>>>> initialization and set the wrong expectation for how the application will
> >>>>> behave.
> >>>>>
> >>>>
> >>>> Hi,
> >>>>
> >>>> Thanks for the explanation which even I also believe is logically perfect.
> >>>>
> >>>> But since l3fwd-power is an old application and has many users around
> >>>> which are currently using this app in interrupt only mode without giving
> >>>> an extra argument. But suddenly they will start getting failure messages with
> >>>> the new patchset.
> >>>>
> >>>> My only intent with else condition was backward compatibility.
> >>>> Or may be we can have more descriptive failure message, something like
> >>>> "init_power_library failed, check manual for other modes".
> >>>>
> >>>> Thanks
> >>>> Harman
> >>>>
> >>>>
> >>>
> >>> I think we can compormise on an informative log message suggesting to use
> >>> interrupt mode. I'm not keen on reverting to previous buggy behavior :)
> >>>
> >> Hi
> >>
> >> I am not insisting to revert to previous behavior, I am just trying to
> >> highlight some probable issues that many users might face as its an old
> >> application.
> >> Since many arm based soc might not be supporting frequency scaling, can
> >> we add the following check as soon as the application starts, probe the
> >> platform if it supports frequency scaling, if not automatically set the
> >> mode to interrupt mode, something like:
> >> if (access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
> >>                      F_OK))
> >>      app_mode = APP_MODE_INTERRUPT;
> >
> > Sorry, no direct check in application but we can introduce a new API in
> > power library:
> >     bool rte_is_freq_scaling() {
> >          return  access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
> >                          F_OK);
> >     }
> >
> > and in the application we can use "rte_is_freq_scaling()" at the start.
> >
>
> What you're suggesting here is effectively what you have already
> suggested: silently fall back to interrupt-only mode if power lib init
> failed. I already outlined why i don't think it's a good approach.

Is probing "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"
file presence,
detects the power lib availability . Right?  Not the failure. Right?
IMO, it make sense to have following case:
# first check, Is the system is capable of power lib if so use power lib
# if the system is not capable of using power lib use interrupt mode.

I think, there is difference between system capable of using power lib
vs power lib not available or power lib failure.




>
> >>
> >>
> >> Thanks
> >> Harman
> >>
> >>>>> --
> >>>>> Thanks,
> >>>>> Anatoly
> >>>
> >>>
> >>> --
> >>> Thanks,
> >>> Anatoly
>
>
> --
> Thanks,
> Anatoly
Anatoly Burakov June 15, 2020, 3:05 p.m. UTC | #9
On 15-Jun-20 12:43 PM, Jerin Jacob wrote:
> On Mon, Jun 15, 2020 at 5:02 PM Burakov, Anatoly
> <anatoly.burakov@intel.com> wrote:
>>
>> On 02-Jun-20 1:16 PM, Harman Kalra wrote:
>>> On Tue, Jun 02, 2020 at 03:53:07PM +0530, Harman Kalra wrote:
>>>> On Mon, Jun 01, 2020 at 01:50:26PM +0100, Burakov, Anatoly wrote:
>>>>> On 30-May-20 11:02 AM, Harman Kalra wrote:
>>>>>> On Fri, May 29, 2020 at 03:19:45PM +0100, Burakov, Anatoly wrote:
>>>>>>> External Email
>>>>>>>
>>>>>>> ----------------------------------------------------------------------
>>>>>>> On 29-May-20 2:19 PM, Harman Kalra wrote:
>>>>>>>
>>>>>>>>>          if (ret < 0)
>>>>>>>>>                  rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
>>>>>>>>> -       if (app_mode != APP_MODE_TELEMETRY && init_power_library())
>>>>>>>>> +       if (app_mode == APP_MODE_DEFAULT)
>>>>>>>>> +               app_mode = APP_MODE_LEGACY;
>>>>>>>>> +
>>>>>>>>> +       /* only legacy and empty poll mode rely on power library */
>>>>>>>>> +       if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
>>>>>>>>> +                       init_power_library())
>>>>>>>>>                  rte_exit(EXIT_FAILURE, "init_power_library failed\n");
>>>>>>>> Hi,
>>>>>>>>
>>>>>>>> Rather than just exiting from here can we have a else condition to
>>>>>>>> automatically enter into the "interrupt only" mode.
>>>>>>>> Please correct me if I am missing something.
>>>>>>>
>>>>>>> Hi,
>>>>>>>
>>>>>>> Thanks for your review. I don't think silently proceeding is a good idea. If
>>>>>>> the user wants interrupt-only mode, they should request it. Silently falling
>>>>>>> back to interrupt-only mode will create an illusion of successful
>>>>>>> initialization and set the wrong expectation for how the application will
>>>>>>> behave.
>>>>>>>
>>>>>>
>>>>>> Hi,
>>>>>>
>>>>>> Thanks for the explanation which even I also believe is logically perfect.
>>>>>>
>>>>>> But since l3fwd-power is an old application and has many users around
>>>>>> which are currently using this app in interrupt only mode without giving
>>>>>> an extra argument. But suddenly they will start getting failure messages with
>>>>>> the new patchset.
>>>>>>
>>>>>> My only intent with else condition was backward compatibility.
>>>>>> Or may be we can have more descriptive failure message, something like
>>>>>> "init_power_library failed, check manual for other modes".
>>>>>>
>>>>>> Thanks
>>>>>> Harman
>>>>>>
>>>>>>
>>>>>
>>>>> I think we can compormise on an informative log message suggesting to use
>>>>> interrupt mode. I'm not keen on reverting to previous buggy behavior :)
>>>>>
>>>> Hi
>>>>
>>>> I am not insisting to revert to previous behavior, I am just trying to
>>>> highlight some probable issues that many users might face as its an old
>>>> application.
>>>> Since many arm based soc might not be supporting frequency scaling, can
>>>> we add the following check as soon as the application starts, probe the
>>>> platform if it supports frequency scaling, if not automatically set the
>>>> mode to interrupt mode, something like:
>>>> if (access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
>>>>                       F_OK))
>>>>       app_mode = APP_MODE_INTERRUPT;
>>>
>>> Sorry, no direct check in application but we can introduce a new API in
>>> power library:
>>>      bool rte_is_freq_scaling() {
>>>           return  access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
>>>                           F_OK);
>>>      }
>>>
>>> and in the application we can use "rte_is_freq_scaling()" at the start.
>>>
>>
>> What you're suggesting here is effectively what you have already
>> suggested: silently fall back to interrupt-only mode if power lib init
>> failed. I already outlined why i don't think it's a good approach.
> 
> Is probing "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"
> file presence,
> detects the power lib availability . Right?  Not the failure. Right?
> IMO, it make sense to have following case:
> # first check, Is the system is capable of power lib if so use power lib
> # if the system is not capable of using power lib use interrupt mode.
> 
> I think, there is difference between system capable of using power lib
> vs power lib not available or power lib failure.

I am of the opinion that if a test sets up an unrealistic expectation of 
how an application should behave, it's a problem with the test, not with 
the application.

If the system is not capable of running with power lib - the application 
shouldn't be requested to run in such mode.

"The application behaved that way before" - yes, it did. It was a bug in 
the application, that it allowed users to effectively misuse the 
application and use it despite the fact that it was in a half-working 
state. This problem has been addressed by 1) not allowing the 
application to run in half-working state, and 2) adding a new mode where 
the old "expected" behavior is *actually* expected and is "full working 
state" now.

Therefore, all users who were previously misusing the application to do 
something it was not designed to do because of a bug in the 
implementation, should now fix their usage and use the correct mode - 
and such breakage is IMO necessary to call attention to earlier misuse 
in the tools, and to correct this usage.

What bothers me about your suggestion is that it is impossible to fail 
the test if the wrong mode was requested (as in, if we request the 
power-lib mode on a system that doesn't have freq scaling) - it instead 
silently falls back to a mode that is almost guaranteed to work.
Jerin Jacob June 15, 2020, 3:21 p.m. UTC | #10
On Mon, Jun 15, 2020 at 8:35 PM Burakov, Anatoly
<anatoly.burakov@intel.com> wrote:
>
> On 15-Jun-20 12:43 PM, Jerin Jacob wrote:
> > On Mon, Jun 15, 2020 at 5:02 PM Burakov, Anatoly
> > <anatoly.burakov@intel.com> wrote:
> >>
> >> On 02-Jun-20 1:16 PM, Harman Kalra wrote:
> >>> On Tue, Jun 02, 2020 at 03:53:07PM +0530, Harman Kalra wrote:
> >>>> On Mon, Jun 01, 2020 at 01:50:26PM +0100, Burakov, Anatoly wrote:
> >>>>> On 30-May-20 11:02 AM, Harman Kalra wrote:
> >>>>>> On Fri, May 29, 2020 at 03:19:45PM +0100, Burakov, Anatoly wrote:
> >>>>>>> External Email
> >>>>>>>
> >>>>>>> ----------------------------------------------------------------------
> >>>>>>> On 29-May-20 2:19 PM, Harman Kalra wrote:
> >>>>>>>
> >>>>>>>>>          if (ret < 0)
> >>>>>>>>>                  rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
> >>>>>>>>> -       if (app_mode != APP_MODE_TELEMETRY && init_power_library())
> >>>>>>>>> +       if (app_mode == APP_MODE_DEFAULT)
> >>>>>>>>> +               app_mode = APP_MODE_LEGACY;
> >>>>>>>>> +
> >>>>>>>>> +       /* only legacy and empty poll mode rely on power library */
> >>>>>>>>> +       if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
> >>>>>>>>> +                       init_power_library())
> >>>>>>>>>                  rte_exit(EXIT_FAILURE, "init_power_library failed\n");
> >>>>>>>> Hi,
> >>>>>>>>
> >>>>>>>> Rather than just exiting from here can we have a else condition to
> >>>>>>>> automatically enter into the "interrupt only" mode.
> >>>>>>>> Please correct me if I am missing something.
> >>>>>>>
> >>>>>>> Hi,
> >>>>>>>
> >>>>>>> Thanks for your review. I don't think silently proceeding is a good idea. If
> >>>>>>> the user wants interrupt-only mode, they should request it. Silently falling
> >>>>>>> back to interrupt-only mode will create an illusion of successful
> >>>>>>> initialization and set the wrong expectation for how the application will
> >>>>>>> behave.
> >>>>>>>
> >>>>>>
> >>>>>> Hi,
> >>>>>>
> >>>>>> Thanks for the explanation which even I also believe is logically perfect.
> >>>>>>
> >>>>>> But since l3fwd-power is an old application and has many users around
> >>>>>> which are currently using this app in interrupt only mode without giving
> >>>>>> an extra argument. But suddenly they will start getting failure messages with
> >>>>>> the new patchset.
> >>>>>>
> >>>>>> My only intent with else condition was backward compatibility.
> >>>>>> Or may be we can have more descriptive failure message, something like
> >>>>>> "init_power_library failed, check manual for other modes".
> >>>>>>
> >>>>>> Thanks
> >>>>>> Harman
> >>>>>>
> >>>>>>
> >>>>>
> >>>>> I think we can compormise on an informative log message suggesting to use
> >>>>> interrupt mode. I'm not keen on reverting to previous buggy behavior :)
> >>>>>
> >>>> Hi
> >>>>
> >>>> I am not insisting to revert to previous behavior, I am just trying to
> >>>> highlight some probable issues that many users might face as its an old
> >>>> application.
> >>>> Since many arm based soc might not be supporting frequency scaling, can
> >>>> we add the following check as soon as the application starts, probe the
> >>>> platform if it supports frequency scaling, if not automatically set the
> >>>> mode to interrupt mode, something like:
> >>>> if (access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
> >>>>                       F_OK))
> >>>>       app_mode = APP_MODE_INTERRUPT;
> >>>
> >>> Sorry, no direct check in application but we can introduce a new API in
> >>> power library:
> >>>      bool rte_is_freq_scaling() {
> >>>           return  access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
> >>>                           F_OK);
> >>>      }
> >>>
> >>> and in the application we can use "rte_is_freq_scaling()" at the start.
> >>>
> >>
> >> What you're suggesting here is effectively what you have already
> >> suggested: silently fall back to interrupt-only mode if power lib init
> >> failed. I already outlined why i don't think it's a good approach.
> >
> > Is probing "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"
> > file presence,
> > detects the power lib availability . Right?  Not the failure. Right?
> > IMO, it make sense to have following case:
> > # first check, Is the system is capable of power lib if so use power lib
> > # if the system is not capable of using power lib use interrupt mode.
> >
> > I think, there is difference between system capable of using power lib
> > vs power lib not available or power lib failure.
>
> I am of the opinion that if a test sets up an unrealistic expectation of
> how an application should behave, it's a problem with the test, not with
> the application.
>
> If the system is not capable of running with power lib - the application
> shouldn't be requested to run in such mode.

But with this patch, default proposed mode is power lib without any
explicit request.

>
> "The application behaved that way before" - yes, it did. It was a bug in
> the application, that it allowed users to effectively misuse the
> application and use it despite the fact that it was in a half-working
> state. This problem has been addressed by 1) not allowing the
> application to run in half-working state, and 2) adding a new mode where
> the old "expected" behavior is *actually* expected and is "full working
> state" now.
>
> Therefore, all users who were previously misusing the application to do
> something it was not designed to do because of a bug in the
> implementation, should now fix their usage and use the correct mode -
> and such breakage is IMO necessary to call attention to earlier misuse
> in the tools, and to correct this usage.
>
> What bothers me about your suggestion is that it is impossible to fail
> the test if the wrong mode was requested (as in, if we request the
> power-lib mode on a system that doesn't have freq scaling) - it instead
> silently falls back to a mode that is almost guaranteed to work.

I agree that it should fail, i.e someone explicitly request,
power-lib mode or any mode
and it should fail application if the platform we can not do that.

My suggestion is all about, what is the default, IMO, if no argument
is specified,
the application should _probe_ the capability of the platfrom and
choose the mode. One can override
the probe to select the desired one. In such mode, fail to configure
the mode should result in
an error.


>
> --
> Thanks,
> Anatoly
Anatoly Burakov June 15, 2020, 3:45 p.m. UTC | #11
On 15-Jun-20 4:21 PM, Jerin Jacob wrote:
> On Mon, Jun 15, 2020 at 8:35 PM Burakov, Anatoly
> <anatoly.burakov@intel.com> wrote:
>>
>> On 15-Jun-20 12:43 PM, Jerin Jacob wrote:
>>> On Mon, Jun 15, 2020 at 5:02 PM Burakov, Anatoly
>>> <anatoly.burakov@intel.com> wrote:
>>>>
>>>> On 02-Jun-20 1:16 PM, Harman Kalra wrote:
>>>>> On Tue, Jun 02, 2020 at 03:53:07PM +0530, Harman Kalra wrote:
>>>>>> On Mon, Jun 01, 2020 at 01:50:26PM +0100, Burakov, Anatoly wrote:
>>>>>>> On 30-May-20 11:02 AM, Harman Kalra wrote:
>>>>>>>> On Fri, May 29, 2020 at 03:19:45PM +0100, Burakov, Anatoly wrote:
>>>>>>>>> External Email
>>>>>>>>>
>>>>>>>>> ----------------------------------------------------------------------
>>>>>>>>> On 29-May-20 2:19 PM, Harman Kalra wrote:
>>>>>>>>>
>>>>>>>>>>>           if (ret < 0)
>>>>>>>>>>>                   rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
>>>>>>>>>>> -       if (app_mode != APP_MODE_TELEMETRY && init_power_library())
>>>>>>>>>>> +       if (app_mode == APP_MODE_DEFAULT)
>>>>>>>>>>> +               app_mode = APP_MODE_LEGACY;
>>>>>>>>>>> +
>>>>>>>>>>> +       /* only legacy and empty poll mode rely on power library */
>>>>>>>>>>> +       if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
>>>>>>>>>>> +                       init_power_library())
>>>>>>>>>>>                   rte_exit(EXIT_FAILURE, "init_power_library failed\n");
>>>>>>>>>> Hi,
>>>>>>>>>>
>>>>>>>>>> Rather than just exiting from here can we have a else condition to
>>>>>>>>>> automatically enter into the "interrupt only" mode.
>>>>>>>>>> Please correct me if I am missing something.
>>>>>>>>>
>>>>>>>>> Hi,
>>>>>>>>>
>>>>>>>>> Thanks for your review. I don't think silently proceeding is a good idea. If
>>>>>>>>> the user wants interrupt-only mode, they should request it. Silently falling
>>>>>>>>> back to interrupt-only mode will create an illusion of successful
>>>>>>>>> initialization and set the wrong expectation for how the application will
>>>>>>>>> behave.
>>>>>>>>>
>>>>>>>>
>>>>>>>> Hi,
>>>>>>>>
>>>>>>>> Thanks for the explanation which even I also believe is logically perfect.
>>>>>>>>
>>>>>>>> But since l3fwd-power is an old application and has many users around
>>>>>>>> which are currently using this app in interrupt only mode without giving
>>>>>>>> an extra argument. But suddenly they will start getting failure messages with
>>>>>>>> the new patchset.
>>>>>>>>
>>>>>>>> My only intent with else condition was backward compatibility.
>>>>>>>> Or may be we can have more descriptive failure message, something like
>>>>>>>> "init_power_library failed, check manual for other modes".
>>>>>>>>
>>>>>>>> Thanks
>>>>>>>> Harman
>>>>>>>>
>>>>>>>>
>>>>>>>
>>>>>>> I think we can compormise on an informative log message suggesting to use
>>>>>>> interrupt mode. I'm not keen on reverting to previous buggy behavior :)
>>>>>>>
>>>>>> Hi
>>>>>>
>>>>>> I am not insisting to revert to previous behavior, I am just trying to
>>>>>> highlight some probable issues that many users might face as its an old
>>>>>> application.
>>>>>> Since many arm based soc might not be supporting frequency scaling, can
>>>>>> we add the following check as soon as the application starts, probe the
>>>>>> platform if it supports frequency scaling, if not automatically set the
>>>>>> mode to interrupt mode, something like:
>>>>>> if (access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
>>>>>>                        F_OK))
>>>>>>        app_mode = APP_MODE_INTERRUPT;
>>>>>
>>>>> Sorry, no direct check in application but we can introduce a new API in
>>>>> power library:
>>>>>       bool rte_is_freq_scaling() {
>>>>>            return  access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
>>>>>                            F_OK);
>>>>>       }
>>>>>
>>>>> and in the application we can use "rte_is_freq_scaling()" at the start.
>>>>>
>>>>
>>>> What you're suggesting here is effectively what you have already
>>>> suggested: silently fall back to interrupt-only mode if power lib init
>>>> failed. I already outlined why i don't think it's a good approach.
>>>
>>> Is probing "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"
>>> file presence,
>>> detects the power lib availability . Right?  Not the failure. Right?
>>> IMO, it make sense to have following case:
>>> # first check, Is the system is capable of power lib if so use power lib
>>> # if the system is not capable of using power lib use interrupt mode.
>>>
>>> I think, there is difference between system capable of using power lib
>>> vs power lib not available or power lib failure.
>>
>> I am of the opinion that if a test sets up an unrealistic expectation of
>> how an application should behave, it's a problem with the test, not with
>> the application.
>>
>> If the system is not capable of running with power lib - the application
>> shouldn't be requested to run in such mode.
> 
> But with this patch, default proposed mode is power lib without any
> explicit request.

The default has always been "use the power library". That was always the 
expected behavior, i believe since the very first version of this 
application. In other words, even if the "requested" behavior is not 
requested explicitly, it has always been that implicitly, and this patch 
is *keeping existing behavior*.

> 
>>
>> "The application behaved that way before" - yes, it did. It was a bug in
>> the application, that it allowed users to effectively misuse the
>> application and use it despite the fact that it was in a half-working
>> state. This problem has been addressed by 1) not allowing the
>> application to run in half-working state, and 2) adding a new mode where
>> the old "expected" behavior is *actually* expected and is "full working
>> state" now.
>>
>> Therefore, all users who were previously misusing the application to do
>> something it was not designed to do because of a bug in the
>> implementation, should now fix their usage and use the correct mode -
>> and such breakage is IMO necessary to call attention to earlier misuse
>> in the tools, and to correct this usage.
>>
>> What bothers me about your suggestion is that it is impossible to fail
>> the test if the wrong mode was requested (as in, if we request the
>> power-lib mode on a system that doesn't have freq scaling) - it instead
>> silently falls back to a mode that is almost guaranteed to work.
> 
> I agree that it should fail, i.e someone explicitly request,
> power-lib mode or any mode
> and it should fail application if the platform we can not do that.
> 
> My suggestion is all about, what is the default, IMO, if no argument
> is specified,
> the application should _probe_ the capability of the platfrom and
> choose the mode. One can override
> the probe to select the desired one. In such mode, fail to configure
> the mode should result in
> an error.

This would change the default behavior that has always existed with this 
application, and would still be subject to silent failure issue 
*because* older tests may not account for this implied assumption of 
"the application will run no matter what", leading to a possible false 
positive test result.

Now, if the default was "not to run and ask explicitly what mode should 
the user use" - i can see how we could both agree on that. It's not 
unprecedented - l3fwd itself won't run unless you explicitly specify 
core config, so we *could* request additional parameters by default. I 
would've also agreed with the "probe" thing *if it was a new application 
without any pre-existing usages* - but it isn't, and in this case IMO 
this is doubly important because there may be tests out there that *rely 
on a bug* and thus should be explicitly broken and addressed (like the 
internal test we had that uncovered the issue in the first place).

In other words, in the perfect world, i would agree with you :) As it 
stands though, i think that intentionally breaking tests that are 
themselves broken and rely on wrong assumptions is more important than 
keeping them working.
Jerin Jacob June 15, 2020, 4:29 p.m. UTC | #12
On Mon, Jun 15, 2020 at 9:15 PM Burakov, Anatoly
<anatoly.burakov@intel.com> wrote:
>
> On 15-Jun-20 4:21 PM, Jerin Jacob wrote:
> > On Mon, Jun 15, 2020 at 8:35 PM Burakov, Anatoly
> > <anatoly.burakov@intel.com> wrote:
> >>
> >> On 15-Jun-20 12:43 PM, Jerin Jacob wrote:
> >>> On Mon, Jun 15, 2020 at 5:02 PM Burakov, Anatoly
> >>> <anatoly.burakov@intel.com> wrote:
> >>>>
> >>>> On 02-Jun-20 1:16 PM, Harman Kalra wrote:
> >>>>> On Tue, Jun 02, 2020 at 03:53:07PM +0530, Harman Kalra wrote:
> >>>>>> On Mon, Jun 01, 2020 at 01:50:26PM +0100, Burakov, Anatoly wrote:
> >>>>>>> On 30-May-20 11:02 AM, Harman Kalra wrote:
> >>>>>>>> On Fri, May 29, 2020 at 03:19:45PM +0100, Burakov, Anatoly wrote:
> >>>>>>>>> External Email
> >>>>>>>>>
> >>>>>>>>> ----------------------------------------------------------------------
> >>>>>>>>> On 29-May-20 2:19 PM, Harman Kalra wrote:
> >>>>>>>>>
> >>>>>>>>>>>           if (ret < 0)
> >>>>>>>>>>>                   rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
> >>>>>>>>>>> -       if (app_mode != APP_MODE_TELEMETRY && init_power_library())
> >>>>>>>>>>> +       if (app_mode == APP_MODE_DEFAULT)
> >>>>>>>>>>> +               app_mode = APP_MODE_LEGACY;
> >>>>>>>>>>> +
> >>>>>>>>>>> +       /* only legacy and empty poll mode rely on power library */
> >>>>>>>>>>> +       if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
> >>>>>>>>>>> +                       init_power_library())
> >>>>>>>>>>>                   rte_exit(EXIT_FAILURE, "init_power_library failed\n");
> >>>>>>>>>> Hi,
> >>>>>>>>>>
> >>>>>>>>>> Rather than just exiting from here can we have a else condition to
> >>>>>>>>>> automatically enter into the "interrupt only" mode.
> >>>>>>>>>> Please correct me if I am missing something.
> >>>>>>>>>
> >>>>>>>>> Hi,
> >>>>>>>>>
> >>>>>>>>> Thanks for your review. I don't think silently proceeding is a good idea. If
> >>>>>>>>> the user wants interrupt-only mode, they should request it. Silently falling
> >>>>>>>>> back to interrupt-only mode will create an illusion of successful
> >>>>>>>>> initialization and set the wrong expectation for how the application will
> >>>>>>>>> behave.
> >>>>>>>>>
> >>>>>>>>
> >>>>>>>> Hi,
> >>>>>>>>
> >>>>>>>> Thanks for the explanation which even I also believe is logically perfect.
> >>>>>>>>
> >>>>>>>> But since l3fwd-power is an old application and has many users around
> >>>>>>>> which are currently using this app in interrupt only mode without giving
> >>>>>>>> an extra argument. But suddenly they will start getting failure messages with
> >>>>>>>> the new patchset.
> >>>>>>>>
> >>>>>>>> My only intent with else condition was backward compatibility.
> >>>>>>>> Or may be we can have more descriptive failure message, something like
> >>>>>>>> "init_power_library failed, check manual for other modes".
> >>>>>>>>
> >>>>>>>> Thanks
> >>>>>>>> Harman
> >>>>>>>>
> >>>>>>>>
> >>>>>>>
> >>>>>>> I think we can compormise on an informative log message suggesting to use
> >>>>>>> interrupt mode. I'm not keen on reverting to previous buggy behavior :)
> >>>>>>>
> >>>>>> Hi
> >>>>>>
> >>>>>> I am not insisting to revert to previous behavior, I am just trying to
> >>>>>> highlight some probable issues that many users might face as its an old
> >>>>>> application.
> >>>>>> Since many arm based soc might not be supporting frequency scaling, can
> >>>>>> we add the following check as soon as the application starts, probe the
> >>>>>> platform if it supports frequency scaling, if not automatically set the
> >>>>>> mode to interrupt mode, something like:
> >>>>>> if (access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
> >>>>>>                        F_OK))
> >>>>>>        app_mode = APP_MODE_INTERRUPT;
> >>>>>
> >>>>> Sorry, no direct check in application but we can introduce a new API in
> >>>>> power library:
> >>>>>       bool rte_is_freq_scaling() {
> >>>>>            return  access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
> >>>>>                            F_OK);
> >>>>>       }
> >>>>>
> >>>>> and in the application we can use "rte_is_freq_scaling()" at the start.
> >>>>>
> >>>>
> >>>> What you're suggesting here is effectively what you have already
> >>>> suggested: silently fall back to interrupt-only mode if power lib init
> >>>> failed. I already outlined why i don't think it's a good approach.
> >>>
> >>> Is probing "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"
> >>> file presence,
> >>> detects the power lib availability . Right?  Not the failure. Right?
> >>> IMO, it make sense to have following case:
> >>> # first check, Is the system is capable of power lib if so use power lib
> >>> # if the system is not capable of using power lib use interrupt mode.
> >>>
> >>> I think, there is difference between system capable of using power lib
> >>> vs power lib not available or power lib failure.
> >>
> >> I am of the opinion that if a test sets up an unrealistic expectation of
> >> how an application should behave, it's a problem with the test, not with
> >> the application.
> >>
> >> If the system is not capable of running with power lib - the application
> >> shouldn't be requested to run in such mode.
> >
> > But with this patch, default proposed mode is power lib without any
> > explicit request.
>
> The default has always been "use the power library". That was always the
> expected behavior, i believe since the very first version of this
> application. In other words, even if the "requested" behavior is not
> requested explicitly, it has always been that implicitly, and this patch
> is *keeping existing behavior*.

See below,
>
> >
> >>
> >> "The application behaved that way before" - yes, it did. It was a bug in
> >> the application, that it allowed users to effectively misuse the
> >> application and use it despite the fact that it was in a half-working
> >> state. This problem has been addressed by 1) not allowing the
> >> application to run in half-working state, and 2) adding a new mode where
> >> the old "expected" behavior is *actually* expected and is "full working
> >> state" now.
> >>
> >> Therefore, all users who were previously misusing the application to do
> >> something it was not designed to do because of a bug in the
> >> implementation, should now fix their usage and use the correct mode -
> >> and such breakage is IMO necessary to call attention to earlier misuse
> >> in the tools, and to correct this usage.
> >>
> >> What bothers me about your suggestion is that it is impossible to fail
> >> the test if the wrong mode was requested (as in, if we request the
> >> power-lib mode on a system that doesn't have freq scaling) - it instead
> >> silently falls back to a mode that is almost guaranteed to work.
> >
> > I agree that it should fail, i.e someone explicitly request,
> > power-lib mode or any mode
> > and it should fail application if the platform we can not do that.
> >
> > My suggestion is all about, what is the default, IMO, if no argument
> > is specified,
> > the application should _probe_ the capability of the platfrom and
> > choose the mode. One can override
> > the probe to select the desired one. In such mode, fail to configure
> > the mode should result in
> > an error.
>
> This would change the default behavior that has always existed with this
> application, and would still be subject to silent failure issue
> *because* older tests may not account for this implied assumption of
> "the application will run no matter what", leading to a possible false
> positive test result.
>
> Now, if the default was "not to run and ask explicitly what mode should
> the user use" - i can see how we could both agree on that. It's not
> unprecedented - l3fwd itself won't run unless you explicitly specify
> core config, so we *could* request additional parameters by default. I
> would've also agreed with the "probe" thing *if it was a new application
> without any pre-existing usages* - but it isn't, and in this case IMO
> this is doubly important because there may be tests out there that *rely
> on a bug* and thus should be explicitly broken and addressed (like the
> internal test we had that uncovered the issue in the first place).
>
> In other words, in the perfect world, i would agree with you :) As it
> stands though, i think that intentionally breaking tests that are
> themselves broken and rely on wrong assumptions is more important than
> keeping them working.

OK. Let's enumerate the pros and cons of both approaches?
Approach a: Auto probe
Approach b: Current patch

Approach a:
+ Application picks up the mode based on platform capability
+ No change in behavior wrt existing l3fwd-power where the platform
has only interrupt support.
(otherwise, It will fail to boot up, the CI etc we need to patch based
on the DPDK version)

I am not sure approach b has pros wrt approach a.

I.e On the x86 platform where freq scaling is present then SHOULD NOT
have any difference in the approach a vs
approach b. ie. Auto probe finds the system is capable of freq scaling
and picks the powerlib. its is win-win case,
I am not sure, What I am missing?








>
> --
> Thanks,
> Anatoly
Anatoly Burakov June 16, 2020, 9:31 a.m. UTC | #13
On 15-Jun-20 5:29 PM, Jerin Jacob wrote:
> On Mon, Jun 15, 2020 at 9:15 PM Burakov, Anatoly
> <anatoly.burakov@intel.com> wrote:
>>
>> On 15-Jun-20 4:21 PM, Jerin Jacob wrote:
>>> On Mon, Jun 15, 2020 at 8:35 PM Burakov, Anatoly
>>> <anatoly.burakov@intel.com> wrote:
>>>>
>>>> On 15-Jun-20 12:43 PM, Jerin Jacob wrote:
>>>>> On Mon, Jun 15, 2020 at 5:02 PM Burakov, Anatoly
>>>>> <anatoly.burakov@intel.com> wrote:
>>>>>>
>>>>>> On 02-Jun-20 1:16 PM, Harman Kalra wrote:
>>>>>>> On Tue, Jun 02, 2020 at 03:53:07PM +0530, Harman Kalra wrote:
>>>>>>>> On Mon, Jun 01, 2020 at 01:50:26PM +0100, Burakov, Anatoly wrote:
>>>>>>>>> On 30-May-20 11:02 AM, Harman Kalra wrote:
>>>>>>>>>> On Fri, May 29, 2020 at 03:19:45PM +0100, Burakov, Anatoly wrote:
>>>>>>>>>>> External Email
>>>>>>>>>>>
>>>>>>>>>>> ----------------------------------------------------------------------
>>>>>>>>>>> On 29-May-20 2:19 PM, Harman Kalra wrote:
>>>>>>>>>>>
>>>>>>>>>>>>>            if (ret < 0)
>>>>>>>>>>>>>                    rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
>>>>>>>>>>>>> -       if (app_mode != APP_MODE_TELEMETRY && init_power_library())
>>>>>>>>>>>>> +       if (app_mode == APP_MODE_DEFAULT)
>>>>>>>>>>>>> +               app_mode = APP_MODE_LEGACY;
>>>>>>>>>>>>> +
>>>>>>>>>>>>> +       /* only legacy and empty poll mode rely on power library */
>>>>>>>>>>>>> +       if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
>>>>>>>>>>>>> +                       init_power_library())
>>>>>>>>>>>>>                    rte_exit(EXIT_FAILURE, "init_power_library failed\n");
>>>>>>>>>>>> Hi,
>>>>>>>>>>>>
>>>>>>>>>>>> Rather than just exiting from here can we have a else condition to
>>>>>>>>>>>> automatically enter into the "interrupt only" mode.
>>>>>>>>>>>> Please correct me if I am missing something.
>>>>>>>>>>>
>>>>>>>>>>> Hi,
>>>>>>>>>>>
>>>>>>>>>>> Thanks for your review. I don't think silently proceeding is a good idea. If
>>>>>>>>>>> the user wants interrupt-only mode, they should request it. Silently falling
>>>>>>>>>>> back to interrupt-only mode will create an illusion of successful
>>>>>>>>>>> initialization and set the wrong expectation for how the application will
>>>>>>>>>>> behave.
>>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>> Hi,
>>>>>>>>>>
>>>>>>>>>> Thanks for the explanation which even I also believe is logically perfect.
>>>>>>>>>>
>>>>>>>>>> But since l3fwd-power is an old application and has many users around
>>>>>>>>>> which are currently using this app in interrupt only mode without giving
>>>>>>>>>> an extra argument. But suddenly they will start getting failure messages with
>>>>>>>>>> the new patchset.
>>>>>>>>>>
>>>>>>>>>> My only intent with else condition was backward compatibility.
>>>>>>>>>> Or may be we can have more descriptive failure message, something like
>>>>>>>>>> "init_power_library failed, check manual for other modes".
>>>>>>>>>>
>>>>>>>>>> Thanks
>>>>>>>>>> Harman
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>
>>>>>>>>> I think we can compormise on an informative log message suggesting to use
>>>>>>>>> interrupt mode. I'm not keen on reverting to previous buggy behavior :)
>>>>>>>>>
>>>>>>>> Hi
>>>>>>>>
>>>>>>>> I am not insisting to revert to previous behavior, I am just trying to
>>>>>>>> highlight some probable issues that many users might face as its an old
>>>>>>>> application.
>>>>>>>> Since many arm based soc might not be supporting frequency scaling, can
>>>>>>>> we add the following check as soon as the application starts, probe the
>>>>>>>> platform if it supports frequency scaling, if not automatically set the
>>>>>>>> mode to interrupt mode, something like:
>>>>>>>> if (access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
>>>>>>>>                         F_OK))
>>>>>>>>         app_mode = APP_MODE_INTERRUPT;
>>>>>>>
>>>>>>> Sorry, no direct check in application but we can introduce a new API in
>>>>>>> power library:
>>>>>>>        bool rte_is_freq_scaling() {
>>>>>>>             return  access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
>>>>>>>                             F_OK);
>>>>>>>        }
>>>>>>>
>>>>>>> and in the application we can use "rte_is_freq_scaling()" at the start.
>>>>>>>
>>>>>>
>>>>>> What you're suggesting here is effectively what you have already
>>>>>> suggested: silently fall back to interrupt-only mode if power lib init
>>>>>> failed. I already outlined why i don't think it's a good approach.
>>>>>
>>>>> Is probing "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"
>>>>> file presence,
>>>>> detects the power lib availability . Right?  Not the failure. Right?
>>>>> IMO, it make sense to have following case:
>>>>> # first check, Is the system is capable of power lib if so use power lib
>>>>> # if the system is not capable of using power lib use interrupt mode.
>>>>>
>>>>> I think, there is difference between system capable of using power lib
>>>>> vs power lib not available or power lib failure.
>>>>
>>>> I am of the opinion that if a test sets up an unrealistic expectation of
>>>> how an application should behave, it's a problem with the test, not with
>>>> the application.
>>>>
>>>> If the system is not capable of running with power lib - the application
>>>> shouldn't be requested to run in such mode.
>>>
>>> But with this patch, default proposed mode is power lib without any
>>> explicit request.
>>
>> The default has always been "use the power library". That was always the
>> expected behavior, i believe since the very first version of this
>> application. In other words, even if the "requested" behavior is not
>> requested explicitly, it has always been that implicitly, and this patch
>> is *keeping existing behavior*.
> 
> See below,
>>
>>>
>>>>
>>>> "The application behaved that way before" - yes, it did. It was a bug in
>>>> the application, that it allowed users to effectively misuse the
>>>> application and use it despite the fact that it was in a half-working
>>>> state. This problem has been addressed by 1) not allowing the
>>>> application to run in half-working state, and 2) adding a new mode where
>>>> the old "expected" behavior is *actually* expected and is "full working
>>>> state" now.
>>>>
>>>> Therefore, all users who were previously misusing the application to do
>>>> something it was not designed to do because of a bug in the
>>>> implementation, should now fix their usage and use the correct mode -
>>>> and such breakage is IMO necessary to call attention to earlier misuse
>>>> in the tools, and to correct this usage.
>>>>
>>>> What bothers me about your suggestion is that it is impossible to fail
>>>> the test if the wrong mode was requested (as in, if we request the
>>>> power-lib mode on a system that doesn't have freq scaling) - it instead
>>>> silently falls back to a mode that is almost guaranteed to work.
>>>
>>> I agree that it should fail, i.e someone explicitly request,
>>> power-lib mode or any mode
>>> and it should fail application if the platform we can not do that.
>>>
>>> My suggestion is all about, what is the default, IMO, if no argument
>>> is specified,
>>> the application should _probe_ the capability of the platfrom and
>>> choose the mode. One can override
>>> the probe to select the desired one. In such mode, fail to configure
>>> the mode should result in
>>> an error.
>>
>> This would change the default behavior that has always existed with this
>> application, and would still be subject to silent failure issue
>> *because* older tests may not account for this implied assumption of
>> "the application will run no matter what", leading to a possible false
>> positive test result.
>>
>> Now, if the default was "not to run and ask explicitly what mode should
>> the user use" - i can see how we could both agree on that. It's not
>> unprecedented - l3fwd itself won't run unless you explicitly specify
>> core config, so we *could* request additional parameters by default. I
>> would've also agreed with the "probe" thing *if it was a new application
>> without any pre-existing usages* - but it isn't, and in this case IMO
>> this is doubly important because there may be tests out there that *rely
>> on a bug* and thus should be explicitly broken and addressed (like the
>> internal test we had that uncovered the issue in the first place).
>>
>> In other words, in the perfect world, i would agree with you :) As it
>> stands though, i think that intentionally breaking tests that are
>> themselves broken and rely on wrong assumptions is more important than
>> keeping them working.
> 
> OK. Let's enumerate the pros and cons of both approaches?
> Approach a: Auto probe
> Approach b: Current patch
> 
> Approach a:
> + Application picks up the mode based on platform capability
> + No change in behavior wrt existing l3fwd-power where the platform
> has only interrupt support.
> (otherwise, It will fail to boot up, the CI etc we need to patch based
> on the DPDK version)
> 
> I am not sure approach b has pros wrt approach a.

The power library has other ways of freq scaling, not just through the 
pstate/ACPI. Now, granted, the third way (KVM) isn't supposed to work on 
l3fwd-power, and is in fact explicitly stopped from working in this 
patch, but still - relying on the scaling governor alone is not good enough.

Perhaps we should add a "check available" API that would probe each of 
the supported modes and return 1 for (potentially) supported and 0 for 
(definitely) unsupported? As far as i know, all three are reliant on 
file access at the end of the day, so we can check if they exist before 
trying to access them. That would effectively be "autodetect" without 
being too specific to ACPI/pstate scaling checks.

> 
> I.e On the x86 platform where freq scaling is present then SHOULD NOT
> have any difference in the approach a vs
> approach b. ie. Auto probe finds the system is capable of freq scaling
> and picks the powerlib. its is win-win case,
> I am not sure, What I am missing?
> 
> 

I was worried that an x86 VM would still have this file without being 
capable of frequency scaling, but i just checked and it seems that VM's 
indeed don't expose the cpufreq filesystem.
Jerin Jacob June 16, 2020, 5:09 p.m. UTC | #14
On Tue, Jun 16, 2020 at 3:01 PM Burakov, Anatoly
<anatoly.burakov@intel.com> wrote:
>
> On 15-Jun-20 5:29 PM, Jerin Jacob wrote:
> > On Mon, Jun 15, 2020 at 9:15 PM Burakov, Anatoly
> > <anatoly.burakov@intel.com> wrote:
> >>
> >> On 15-Jun-20 4:21 PM, Jerin Jacob wrote:
> >>> On Mon, Jun 15, 2020 at 8:35 PM Burakov, Anatoly
> >>> <anatoly.burakov@intel.com> wrote:
> >>>>
> >>>> On 15-Jun-20 12:43 PM, Jerin Jacob wrote:
> >>>>> On Mon, Jun 15, 2020 at 5:02 PM Burakov, Anatoly
> >>>>> <anatoly.burakov@intel.com> wrote:
> >>>>>>
> >>>>>> On 02-Jun-20 1:16 PM, Harman Kalra wrote:
> >>>>>>> On Tue, Jun 02, 2020 at 03:53:07PM +0530, Harman Kalra wrote:
> >>>>>>>> On Mon, Jun 01, 2020 at 01:50:26PM +0100, Burakov, Anatoly wrote:
> >>>>>>>>> On 30-May-20 11:02 AM, Harman Kalra wrote:
> >>>>>>>>>> On Fri, May 29, 2020 at 03:19:45PM +0100, Burakov, Anatoly wrote:
> >>>>>>>>>>> External Email
> >>>>>>>>>>>
> >>>>>>>>>>> ----------------------------------------------------------------------
> >>>>>>>>>>> On 29-May-20 2:19 PM, Harman Kalra wrote:
> >>>>>>>>>>>
> >>>>>>>>>>>>>            if (ret < 0)
> >>>>>>>>>>>>>                    rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
> >>>>>>>>>>>>> -       if (app_mode != APP_MODE_TELEMETRY && init_power_library())
> >>>>>>>>>>>>> +       if (app_mode == APP_MODE_DEFAULT)
> >>>>>>>>>>>>> +               app_mode = APP_MODE_LEGACY;
> >>>>>>>>>>>>> +
> >>>>>>>>>>>>> +       /* only legacy and empty poll mode rely on power library */
> >>>>>>>>>>>>> +       if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
> >>>>>>>>>>>>> +                       init_power_library())
> >>>>>>>>>>>>>                    rte_exit(EXIT_FAILURE, "init_power_library failed\n");
> >>>>>>>>>>>> Hi,
> >>>>>>>>>>>>
> >>>>>>>>>>>> Rather than just exiting from here can we have a else condition to
> >>>>>>>>>>>> automatically enter into the "interrupt only" mode.
> >>>>>>>>>>>> Please correct me if I am missing something.
> >>>>>>>>>>>
> >>>>>>>>>>> Hi,
> >>>>>>>>>>>
> >>>>>>>>>>> Thanks for your review. I don't think silently proceeding is a good idea. If
> >>>>>>>>>>> the user wants interrupt-only mode, they should request it. Silently falling
> >>>>>>>>>>> back to interrupt-only mode will create an illusion of successful
> >>>>>>>>>>> initialization and set the wrong expectation for how the application will
> >>>>>>>>>>> behave.
> >>>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>> Hi,
> >>>>>>>>>>
> >>>>>>>>>> Thanks for the explanation which even I also believe is logically perfect.
> >>>>>>>>>>
> >>>>>>>>>> But since l3fwd-power is an old application and has many users around
> >>>>>>>>>> which are currently using this app in interrupt only mode without giving
> >>>>>>>>>> an extra argument. But suddenly they will start getting failure messages with
> >>>>>>>>>> the new patchset.
> >>>>>>>>>>
> >>>>>>>>>> My only intent with else condition was backward compatibility.
> >>>>>>>>>> Or may be we can have more descriptive failure message, something like
> >>>>>>>>>> "init_power_library failed, check manual for other modes".
> >>>>>>>>>>
> >>>>>>>>>> Thanks
> >>>>>>>>>> Harman
> >>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>
> >>>>>>>>> I think we can compormise on an informative log message suggesting to use
> >>>>>>>>> interrupt mode. I'm not keen on reverting to previous buggy behavior :)
> >>>>>>>>>
> >>>>>>>> Hi
> >>>>>>>>
> >>>>>>>> I am not insisting to revert to previous behavior, I am just trying to
> >>>>>>>> highlight some probable issues that many users might face as its an old
> >>>>>>>> application.
> >>>>>>>> Since many arm based soc might not be supporting frequency scaling, can
> >>>>>>>> we add the following check as soon as the application starts, probe the
> >>>>>>>> platform if it supports frequency scaling, if not automatically set the
> >>>>>>>> mode to interrupt mode, something like:
> >>>>>>>> if (access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
> >>>>>>>>                         F_OK))
> >>>>>>>>         app_mode = APP_MODE_INTERRUPT;
> >>>>>>>
> >>>>>>> Sorry, no direct check in application but we can introduce a new API in
> >>>>>>> power library:
> >>>>>>>        bool rte_is_freq_scaling() {
> >>>>>>>             return  access("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor",
> >>>>>>>                             F_OK);
> >>>>>>>        }
> >>>>>>>
> >>>>>>> and in the application we can use "rte_is_freq_scaling()" at the start.
> >>>>>>>
> >>>>>>
> >>>>>> What you're suggesting here is effectively what you have already
> >>>>>> suggested: silently fall back to interrupt-only mode if power lib init
> >>>>>> failed. I already outlined why i don't think it's a good approach.
> >>>>>
> >>>>> Is probing "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"
> >>>>> file presence,
> >>>>> detects the power lib availability . Right?  Not the failure. Right?
> >>>>> IMO, it make sense to have following case:
> >>>>> # first check, Is the system is capable of power lib if so use power lib
> >>>>> # if the system is not capable of using power lib use interrupt mode.
> >>>>>
> >>>>> I think, there is difference between system capable of using power lib
> >>>>> vs power lib not available or power lib failure.
> >>>>
> >>>> I am of the opinion that if a test sets up an unrealistic expectation of
> >>>> how an application should behave, it's a problem with the test, not with
> >>>> the application.
> >>>>
> >>>> If the system is not capable of running with power lib - the application
> >>>> shouldn't be requested to run in such mode.
> >>>
> >>> But with this patch, default proposed mode is power lib without any
> >>> explicit request.
> >>
> >> The default has always been "use the power library". That was always the
> >> expected behavior, i believe since the very first version of this
> >> application. In other words, even if the "requested" behavior is not
> >> requested explicitly, it has always been that implicitly, and this patch
> >> is *keeping existing behavior*.
> >
> > See below,
> >>
> >>>
> >>>>
> >>>> "The application behaved that way before" - yes, it did. It was a bug in
> >>>> the application, that it allowed users to effectively misuse the
> >>>> application and use it despite the fact that it was in a half-working
> >>>> state. This problem has been addressed by 1) not allowing the
> >>>> application to run in half-working state, and 2) adding a new mode where
> >>>> the old "expected" behavior is *actually* expected and is "full working
> >>>> state" now.
> >>>>
> >>>> Therefore, all users who were previously misusing the application to do
> >>>> something it was not designed to do because of a bug in the
> >>>> implementation, should now fix their usage and use the correct mode -
> >>>> and such breakage is IMO necessary to call attention to earlier misuse
> >>>> in the tools, and to correct this usage.
> >>>>
> >>>> What bothers me about your suggestion is that it is impossible to fail
> >>>> the test if the wrong mode was requested (as in, if we request the
> >>>> power-lib mode on a system that doesn't have freq scaling) - it instead
> >>>> silently falls back to a mode that is almost guaranteed to work.
> >>>
> >>> I agree that it should fail, i.e someone explicitly request,
> >>> power-lib mode or any mode
> >>> and it should fail application if the platform we can not do that.
> >>>
> >>> My suggestion is all about, what is the default, IMO, if no argument
> >>> is specified,
> >>> the application should _probe_ the capability of the platfrom and
> >>> choose the mode. One can override
> >>> the probe to select the desired one. In such mode, fail to configure
> >>> the mode should result in
> >>> an error.
> >>
> >> This would change the default behavior that has always existed with this
> >> application, and would still be subject to silent failure issue
> >> *because* older tests may not account for this implied assumption of
> >> "the application will run no matter what", leading to a possible false
> >> positive test result.
> >>
> >> Now, if the default was "not to run and ask explicitly what mode should
> >> the user use" - i can see how we could both agree on that. It's not
> >> unprecedented - l3fwd itself won't run unless you explicitly specify
> >> core config, so we *could* request additional parameters by default. I
> >> would've also agreed with the "probe" thing *if it was a new application
> >> without any pre-existing usages* - but it isn't, and in this case IMO
> >> this is doubly important because there may be tests out there that *rely
> >> on a bug* and thus should be explicitly broken and addressed (like the
> >> internal test we had that uncovered the issue in the first place).
> >>
> >> In other words, in the perfect world, i would agree with you :) As it
> >> stands though, i think that intentionally breaking tests that are
> >> themselves broken and rely on wrong assumptions is more important than
> >> keeping them working.
> >
> > OK. Let's enumerate the pros and cons of both approaches?
> > Approach a: Auto probe
> > Approach b: Current patch
> >
> > Approach a:
> > + Application picks up the mode based on platform capability
> > + No change in behavior wrt existing l3fwd-power where the platform
> > has only interrupt support.
> > (otherwise, It will fail to boot up, the CI etc we need to patch based
> > on the DPDK version)
> >
> > I am not sure approach b has pros wrt approach a.
>
> The power library has other ways of freq scaling, not just through the
> pstate/ACPI. Now, granted, the third way (KVM) isn't supposed to work on
> l3fwd-power, and is in fact explicitly stopped from working in this
> patch, but still - relying on the scaling governor alone is not good enough.
>
> Perhaps we should add a "check available" API that would probe each of
> the supported modes and return 1 for (potentially) supported and 0 for
> (definitely) unsupported?

+1 for adding "check available" API for the specific mode.


>  As far as i know, all three are reliant on
> file access at the end of the day, so we can check if they exist before
> trying to access them. That would effectively be "autodetect" without
> being too specific to ACPI/pstate scaling checks.

+1

>
> >
> > I.e On the x86 platform where freq scaling is present then SHOULD NOT
> > have any difference in the approach a vs
> > approach b. ie. Auto probe finds the system is capable of freq scaling
> > and picks the powerlib. its is win-win case,
> > I am not sure, What I am missing?
> >
> >
>
> I was worried that an x86 VM would still have this file without being
> capable of frequency scaling, but i just checked and it seems that VM's
> indeed don't expose the cpufreq filesystem.

Looks good then!!

>
> --
> Thanks,
> Anatoly

Patch
diff mbox series

diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index 5cee9d5387..4161e01974 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -195,9 +195,11 @@  static int parse_ptype; /**< Parse packet type using rx callback, and */
 			/**< disabled by default */
 
 enum appmode {
-	APP_MODE_LEGACY = 0,
+	APP_MODE_DEFAULT = 0,
+	APP_MODE_LEGACY,
 	APP_MODE_EMPTY_POLL,
-	APP_MODE_TELEMETRY
+	APP_MODE_TELEMETRY,
+	APP_MODE_INTERRUPT
 };
 
 enum appmode app_mode;
@@ -900,6 +902,170 @@  static int event_register(struct lcore_conf *qconf)
 
 	return 0;
 }
+
+/* main processing loop */
+static int main_intr_loop(__rte_unused void *dummy)
+{
+	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
+	unsigned int lcore_id;
+	uint64_t prev_tsc, diff_tsc, cur_tsc;
+	int i, j, nb_rx;
+	uint8_t queueid;
+	uint16_t portid;
+	struct lcore_conf *qconf;
+	struct lcore_rx_queue *rx_queue;
+	uint32_t lcore_rx_idle_count = 0;
+	uint32_t lcore_idle_hint = 0;
+	int intr_en = 0;
+
+	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
+				   US_PER_S * BURST_TX_DRAIN_US;
+
+	prev_tsc = 0;
+
+	lcore_id = rte_lcore_id();
+	qconf = &lcore_conf[lcore_id];
+
+	if (qconf->n_rx_queue == 0) {
+		RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n",
+				lcore_id);
+		return 0;
+	}
+
+	RTE_LOG(INFO, L3FWD_POWER, "entering main interrupt loop on lcore %u\n",
+			lcore_id);
+
+	for (i = 0; i < qconf->n_rx_queue; i++) {
+		portid = qconf->rx_queue_list[i].port_id;
+		queueid = qconf->rx_queue_list[i].queue_id;
+		RTE_LOG(INFO, L3FWD_POWER,
+				" -- lcoreid=%u portid=%u rxqueueid=%hhu\n",
+				lcore_id, portid, queueid);
+	}
+
+	/* add into event wait list */
+	if (event_register(qconf) == 0)
+		intr_en = 1;
+	else
+		RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n");
+
+	while (!is_done()) {
+		stats[lcore_id].nb_iteration_looped++;
+
+		cur_tsc = rte_rdtsc();
+
+		/*
+		 * TX burst queue drain
+		 */
+		diff_tsc = cur_tsc - prev_tsc;
+		if (unlikely(diff_tsc > drain_tsc)) {
+			for (i = 0; i < qconf->n_tx_port; ++i) {
+				portid = qconf->tx_port_id[i];
+				rte_eth_tx_buffer_flush(portid,
+						qconf->tx_queue_id[portid],
+						qconf->tx_buffer[portid]);
+			}
+			prev_tsc = cur_tsc;
+		}
+
+start_rx:
+		/*
+		 * Read packet from RX queues
+		 */
+		lcore_rx_idle_count = 0;
+		for (i = 0; i < qconf->n_rx_queue; ++i) {
+			rx_queue = &(qconf->rx_queue_list[i]);
+			rx_queue->idle_hint = 0;
+			portid = rx_queue->port_id;
+			queueid = rx_queue->queue_id;
+
+			nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst,
+					MAX_PKT_BURST);
+
+			stats[lcore_id].nb_rx_processed += nb_rx;
+			if (unlikely(nb_rx == 0)) {
+				/**
+				 * no packet received from rx queue, try to
+				 * sleep for a while forcing CPU enter deeper
+				 * C states.
+				 */
+				rx_queue->zero_rx_packet_count++;
+
+				if (rx_queue->zero_rx_packet_count <=
+						MIN_ZERO_POLL_COUNT)
+					continue;
+
+				rx_queue->idle_hint = power_idle_heuristic(
+						rx_queue->zero_rx_packet_count);
+				lcore_rx_idle_count++;
+			} else {
+				rx_queue->zero_rx_packet_count = 0;
+			}
+
+			/* Prefetch first packets */
+			for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
+				rte_prefetch0(rte_pktmbuf_mtod(
+						pkts_burst[j], void *));
+			}
+
+			/* Prefetch and forward already prefetched packets */
+			for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
+				rte_prefetch0(rte_pktmbuf_mtod(
+						pkts_burst[j + PREFETCH_OFFSET],
+						void *));
+				l3fwd_simple_forward(
+						pkts_burst[j], portid, qconf);
+			}
+
+			/* Forward remaining prefetched packets */
+			for (; j < nb_rx; j++) {
+				l3fwd_simple_forward(
+						pkts_burst[j], portid, qconf);
+			}
+		}
+
+		if (unlikely(lcore_rx_idle_count == qconf->n_rx_queue)) {
+			/**
+			 * All Rx queues empty in recent consecutive polls,
+			 * sleep in a conservative manner, meaning sleep as
+			 * less as possible.
+			 */
+			for (i = 1,
+			    lcore_idle_hint = qconf->rx_queue_list[0].idle_hint;
+					i < qconf->n_rx_queue; ++i) {
+				rx_queue = &(qconf->rx_queue_list[i]);
+				if (rx_queue->idle_hint < lcore_idle_hint)
+					lcore_idle_hint = rx_queue->idle_hint;
+			}
+
+			if (lcore_idle_hint < SUSPEND_THRESHOLD)
+				/**
+				 * execute "pause" instruction to avoid context
+				 * switch which generally take hundred of
+				 * microseconds for short sleep.
+				 */
+				rte_delay_us(lcore_idle_hint);
+			else {
+				/* suspend until rx interrupt triggers */
+				if (intr_en) {
+					turn_on_off_intr(qconf, 1);
+					sleep_until_rx_interrupt(
+							qconf->n_rx_queue);
+					turn_on_off_intr(qconf, 0);
+					/**
+					 * start receiving packets immediately
+					 */
+					if (likely(!is_done()))
+						goto start_rx;
+				}
+			}
+			stats[lcore_id].sleep_time += lcore_idle_hint;
+		}
+	}
+
+	return 0;
+}
+
 /* main processing loop */
 static int
 main_telemetry_loop(__rte_unused void *dummy)
@@ -1126,7 +1292,7 @@  main_empty_poll_loop(__rte_unused void *dummy)
 }
 /* main processing loop */
 static int
-main_loop(__rte_unused void *dummy)
+main_legacy_loop(__rte_unused void *dummy)
 {
 	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 	unsigned lcore_id;
@@ -1438,7 +1604,8 @@  print_usage(const char *prgname)
 		"  --empty-poll: enable empty poll detection"
 		" follow (training_flag, high_threshold, med_threshold)\n"
 		" --telemetry: enable telemetry mode, to update"
-		" empty polls, full polls, and core busyness to telemetry\n",
+		" empty polls, full polls, and core busyness to telemetry\n"
+		" --interrupt-only: enable interrupt-only mode\n",
 		prgname);
 }
 
@@ -1582,6 +1749,7 @@  parse_ep_config(const char *q_arg)
 }
 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype"
 #define CMD_LINE_OPT_TELEMETRY "telemetry"
+#define CMD_LINE_OPT_INTERRUPT_ONLY "interrupt-only"
 
 /* Parse the argument given in the command line of the application */
 static int
@@ -1601,6 +1769,7 @@  parse_args(int argc, char **argv)
 		{"empty-poll", 1, 0, 0},
 		{CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0},
 		{CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
+		{CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0},
 		{NULL, 0, 0, 0}
 	};
 
@@ -1674,8 +1843,8 @@  parse_args(int argc, char **argv)
 
 			if (!strncmp(lgopts[option_index].name,
 						"empty-poll", 10)) {
-				if (app_mode == APP_MODE_TELEMETRY) {
-					printf(" empty-poll cannot be enabled as telemetry mode is enabled\n");
+				if (app_mode != APP_MODE_DEFAULT) {
+					printf(" empty-poll mode is mutually exclusive with other modes\n");
 					return -1;
 				}
 				app_mode = APP_MODE_EMPTY_POLL;
@@ -1692,14 +1861,25 @@  parse_args(int argc, char **argv)
 			if (!strncmp(lgopts[option_index].name,
 					CMD_LINE_OPT_TELEMETRY,
 					sizeof(CMD_LINE_OPT_TELEMETRY))) {
-				if (app_mode == APP_MODE_EMPTY_POLL) {
-					printf("telemetry mode cannot be enabled as empty poll mode is enabled\n");
+				if (app_mode != APP_MODE_DEFAULT) {
+					printf(" telemetry mode is mutually exclusive with other modes\n");
 					return -1;
 				}
 				app_mode = APP_MODE_TELEMETRY;
 				printf("telemetry mode is enabled\n");
 			}
 
+			if (!strncmp(lgopts[option_index].name,
+					CMD_LINE_OPT_INTERRUPT_ONLY,
+					sizeof(CMD_LINE_OPT_INTERRUPT_ONLY))) {
+				if (app_mode != APP_MODE_DEFAULT) {
+					printf(" interrupt-only mode is mutually exclusive with other modes\n");
+					return -1;
+				}
+				app_mode = APP_MODE_INTERRUPT;
+				printf("interrupt-only mode is enabled\n");
+			}
+
 			if (!strncmp(lgopts[option_index].name,
 					"enable-jumbo", 12)) {
 				struct option lenopts =
@@ -2253,7 +2433,12 @@  main(int argc, char **argv)
 	if (ret < 0)
 		rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
 
-	if (app_mode != APP_MODE_TELEMETRY && init_power_library())
+	if (app_mode == APP_MODE_DEFAULT)
+		app_mode = APP_MODE_LEGACY;
+
+	/* only legacy and empty poll mode rely on power library */
+	if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
+			init_power_library())
 		rte_exit(EXIT_FAILURE, "init_power_library failed\n");
 
 	if (update_lcore_params() < 0)
@@ -2277,7 +2462,8 @@  main(int argc, char **argv)
 	RTE_ETH_FOREACH_DEV(portid) {
 		struct rte_eth_conf local_port_conf = port_conf;
 		/* not all app modes need interrupts */
-		bool need_intr = app_mode == APP_MODE_LEGACY;
+		bool need_intr = app_mode == APP_MODE_LEGACY ||
+				app_mode == APP_MODE_INTERRUPT;
 
 		/* skip ports that are not enabled */
 		if ((enabled_port_mask & (1 << portid)) == 0) {
@@ -2526,12 +2712,12 @@  main(int argc, char **argv)
 
 	/* launch per-lcore init on every lcore */
 	if (app_mode == APP_MODE_LEGACY) {
-		rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
+		rte_eal_mp_remote_launch(main_legacy_loop, NULL, CALL_MASTER);
 	} else if (app_mode == APP_MODE_EMPTY_POLL) {
 		empty_poll_stop = false;
 		rte_eal_mp_remote_launch(main_empty_poll_loop, NULL,
 				SKIP_MASTER);
-	} else {
+	} else if (app_mode == APP_MODE_TELEMETRY) {
 		unsigned int i;
 
 		/* Init metrics library */
@@ -2555,6 +2741,8 @@  main(int argc, char **argv)
 				"Returns global power stats. Parameters: None");
 		rte_eal_mp_remote_launch(main_telemetry_loop, NULL,
 						SKIP_MASTER);
+	} else if (app_mode == APP_MODE_INTERRUPT) {
+		rte_eal_mp_remote_launch(main_intr_loop, NULL, CALL_MASTER);
 	}
 
 	if (app_mode == APP_MODE_EMPTY_POLL || app_mode == APP_MODE_TELEMETRY)
@@ -2577,7 +2765,8 @@  main(int argc, char **argv)
 	if (app_mode == APP_MODE_EMPTY_POLL)
 		rte_power_empty_poll_stat_free();
 
-	if (app_mode != APP_MODE_TELEMETRY && deinit_power_library())
+	if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
+			deinit_power_library())
 		rte_exit(EXIT_FAILURE, "deinit_power_library failed\n");
 
 	if (rte_eal_cleanup() < 0)