event/dlb2: add missing delayed token pop logic

  The code contained in this commit was inadvertently omitted
when dissecting the dlb2 code base into discrete patches for
upstream.

Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
---
 drivers/event/dlb2/dlb2.c          | 314 +++++++++++++++++++++++--------------
 drivers/event/dlb2/dlb2_selftest.c |   4 +-
 2 files changed, 201 insertions(+), 117 deletions(-)
  

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Timothy McDaniel
> Sent: Wednesday, November 11, 2020 3:27 PM
> Cc: dev@dpdk.org; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads, Gage
> <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>;
> jerinj@marvell.com; thomas@monjalon.net; david.marchand@redhat.com
> Subject: [dpdk-dev] [PATCH] event/dlb2: add missing delayed token pop logic
> 
> The code contained in this commit was inadvertently omitted when dissecting
> the dlb2 code base into discrete patches for upstream.
> 
> Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> ---
>  drivers/event/dlb2/dlb2.c          | 314 +++++++++++++++++++++++--------------
>  drivers/event/dlb2/dlb2_selftest.c |   4 +-
>  2 files changed, 201 insertions(+), 117 deletions(-)
> 
> diff --git a/drivers/event/dlb2/dlb2.c b/drivers/event/dlb2/dlb2.c index
> d42e48b..8672486 100644
> --- a/drivers/event/dlb2/dlb2.c
> +++ b/drivers/event/dlb2/dlb2.c
> @@ -1082,6 +1082,25 @@ dlb2_init_qe_mem(struct dlb2_port *qm_port, char
> *mz_name)
>  	return ret;
>  }
> 
> +static inline uint16_t
> +dlb2_event_enqueue_delayed(void *event_port,
> +			   const struct rte_event events[]);
> +
> +static inline uint16_t
> +dlb2_event_enqueue_burst_delayed(void *event_port,
> +				 const struct rte_event events[],
> +				 uint16_t num);
> +
> +static inline uint16_t
> +dlb2_event_enqueue_new_burst_delayed(void *event_port,
> +				     const struct rte_event events[],
> +				     uint16_t num);
> +
> +static inline uint16_t
> +dlb2_event_enqueue_forward_burst_delayed(void *event_port,
> +					 const struct rte_event events[],
> +					 uint16_t num);
> +
>  static int
>  dlb2_hw_create_ldb_port(struct dlb2_eventdev *dlb2,
>  			struct dlb2_eventdev_port *ev_port,
> @@ -1198,6 +1217,20 @@ dlb2_hw_create_ldb_port(struct dlb2_eventdev
> *dlb2,
> 
>  	qm_port->dequeue_depth = dequeue_depth;
>  	qm_port->token_pop_thresh = dequeue_depth;
> +
> +	/* The default enqueue functions do not include delayed-pop support
> for
> +	 * performance reasons.
> +	 */
> +	if (qm_port->token_pop_mode == DELAYED_POP) {
> +		dlb2->event_dev->enqueue = dlb2_event_enqueue_delayed;
> +		dlb2->event_dev->enqueue_burst =
> +			dlb2_event_enqueue_burst_delayed;
> +		dlb2->event_dev->enqueue_new_burst =
> +			dlb2_event_enqueue_new_burst_delayed;
> +		dlb2->event_dev->enqueue_forward_burst =
> +			dlb2_event_enqueue_forward_burst_delayed;
> +	}
> +
>  	qm_port->owed_tokens = 0;
>  	qm_port->issued_releases = 0;
> 
> @@ -2427,11 +2460,6 @@ dlb2_event_build_hcws(struct dlb2_port *qm_port,
>  	case 3:
>  	case 2:
>  	case 1:
> -		/* At least one QE will be valid, so only zero out three */
> -		qe[1].cmd_byte = 0;
> -		qe[2].cmd_byte = 0;
> -		qe[3].cmd_byte = 0;
> -
>  		for (i = 0; i < num; i++) {
>  			qe[i].cmd_byte =
>  				cmd_byte_map[qm_port-
> >is_directed][ev[i].op];
> @@ -2452,6 +2480,8 @@ dlb2_event_build_hcws(struct dlb2_port *qm_port,
>  			qe[i].u.event_type.sub = ev[i].sub_event_type;
>  		}
>  		break;
> +	case 0:
> +		break;
>  	}
>  }
> 
> @@ -2578,29 +2608,57 @@ dlb2_event_enqueue_prep(struct
> dlb2_eventdev_port *ev_port,  }
> 
>  static inline uint16_t
> -dlb2_event_enqueue_burst(void *event_port,
> -			 const struct rte_event events[],
> -			 uint16_t num)
> +__dlb2_event_enqueue_burst(void *event_port,
> +			   const struct rte_event events[],
> +			   uint16_t num,
> +			   bool use_delayed)
>  {
>  	struct dlb2_eventdev_port *ev_port = event_port;
>  	struct dlb2_port *qm_port = &ev_port->qm_port;
>  	struct process_local_port_data *port_data;
> -	int i, cnt;
> +	int i;
> 
>  	RTE_ASSERT(ev_port->enq_configured);
>  	RTE_ASSERT(events != NULL);
> 
> -	cnt = 0;
> +	i = 0;
> 
>  	port_data = &dlb2_port[qm_port->id][PORT_TYPE(qm_port)];
> 
> -	for (i = 0; i < num; i += DLB2_NUM_QES_PER_CACHE_LINE) {
> +	while (i < num) {
>  		uint8_t sched_types[DLB2_NUM_QES_PER_CACHE_LINE];
>  		uint8_t queue_ids[DLB2_NUM_QES_PER_CACHE_LINE];
> +		int pop_offs = 0;
>  		int j = 0;
> 
> +		memset(qm_port->qe4,
> +		       0,
> +		       DLB2_NUM_QES_PER_CACHE_LINE *
> +		       sizeof(struct dlb2_enqueue_qe));
> +
>  		for (; j < DLB2_NUM_QES_PER_CACHE_LINE && (i + j) < num;
> j++) {
>  			const struct rte_event *ev = &events[i + j];
> +			int16_t thresh = qm_port->token_pop_thresh;
> +
> +			if (use_delayed &&
> +			    qm_port->token_pop_mode == DELAYED_POP &&
> +			    (ev->op == RTE_EVENT_OP_FORWARD ||
> +			     ev->op == RTE_EVENT_OP_RELEASE) &&
> +			    qm_port->issued_releases >= thresh - 1) {
> +				/* Insert the token pop QE and break out. This
> +				 * may result in a partial HCW, but that is
> +				 * simpler than supporting arbitrary QE
> +				 * insertion.
> +				 */
> +				dlb2_construct_token_pop_qe(qm_port, j);
> +
> +				/* Reset the releases for the next QE batch */
> +				qm_port->issued_releases -= thresh;
> +
> +				pop_offs = 1;
> +				j++;
> +				break;
> +			}
> 
>  			if (dlb2_event_enqueue_prep(ev_port, qm_port, ev,
>  						    &sched_types[j],
> @@ -2611,38 +2669,52 @@ dlb2_event_enqueue_burst(void *event_port,
>  		if (j == 0)
>  			break;
> 
> -		dlb2_event_build_hcws(qm_port, &events[i], j,
> +		dlb2_event_build_hcws(qm_port, &events[i], j - pop_offs,
>  				      sched_types, queue_ids);
> 
> -		if (qm_port->token_pop_mode == DELAYED_POP && j < 4 &&
> -		    qm_port->issued_releases >= qm_port->token_pop_thresh -
> 1) {
> -			dlb2_construct_token_pop_qe(qm_port, j);
> -
> -			/* Reset the releases counter for the next QE batch */
> -			qm_port->issued_releases -= qm_port-
> >token_pop_thresh;
> -		}
> -
>  		dlb2_hw_do_enqueue(qm_port, i == 0, port_data);
> 
> -		cnt += j;
> +		/* Don't include the token pop QE in the enqueue count */
> +		i += j - pop_offs;
> 
> -		if (j < DLB2_NUM_QES_PER_CACHE_LINE)
> +		/* Don't interpret j < DLB2_NUM_... as out-of-credits if
> +		 * pop_offs != 0
> +		 */
> +		if (j < DLB2_NUM_QES_PER_CACHE_LINE && pop_offs == 0)
>  			break;
>  	}
> 
> -	if (qm_port->token_pop_mode == DELAYED_POP &&
> -	    qm_port->issued_releases >= qm_port->token_pop_thresh - 1) {
> -		dlb2_consume_qe_immediate(qm_port, qm_port-
> >owed_tokens);
> -		qm_port->issued_releases -= qm_port->token_pop_thresh;
> -	}
> -	return cnt;
> +	return i;
> +}
> +
> +static uint16_t
> +dlb2_event_enqueue_burst(void *event_port,
> +			     const struct rte_event events[],
> +			     uint16_t num)
> +{
> +	return __dlb2_event_enqueue_burst(event_port, events, num, false); }
> +
> +static uint16_t
> +dlb2_event_enqueue_burst_delayed(void *event_port,
> +				     const struct rte_event events[],
> +				     uint16_t num)
> +{
> +	return __dlb2_event_enqueue_burst(event_port, events, num, true);
>  }
> 
>  static inline uint16_t
>  dlb2_event_enqueue(void *event_port,
>  		   const struct rte_event events[])
>  {
> -	return dlb2_event_enqueue_burst(event_port, events, 1);
> +	return __dlb2_event_enqueue_burst(event_port, events, 1, false); }
> +
> +static inline uint16_t
> +dlb2_event_enqueue_delayed(void *event_port,
> +			   const struct rte_event events[])
> +{
> +	return __dlb2_event_enqueue_burst(event_port, events, 1, true);
>  }
> 
>  static uint16_t
> @@ -2650,7 +2722,15 @@ dlb2_event_enqueue_new_burst(void *event_port,
>  			     const struct rte_event events[],
>  			     uint16_t num)
>  {
> -	return dlb2_event_enqueue_burst(event_port, events, num);
> +	return __dlb2_event_enqueue_burst(event_port, events, num, false); }
> +
> +static uint16_t
> +dlb2_event_enqueue_new_burst_delayed(void *event_port,
> +				     const struct rte_event events[],
> +				     uint16_t num)
> +{
> +	return __dlb2_event_enqueue_burst(event_port, events, num, true);
>  }
> 
>  static uint16_t
> @@ -2658,7 +2738,93 @@ dlb2_event_enqueue_forward_burst(void
> *event_port,
>  				 const struct rte_event events[],
>  				 uint16_t num)
>  {
> -	return dlb2_event_enqueue_burst(event_port, events, num);
> +	return __dlb2_event_enqueue_burst(event_port, events, num, false); }
> +
> +static uint16_t
> +dlb2_event_enqueue_forward_burst_delayed(void *event_port,
> +					 const struct rte_event events[],
> +					 uint16_t num)
> +{
> +	return __dlb2_event_enqueue_burst(event_port, events, num, true); }
> +
> +static void
> +dlb2_event_release(struct dlb2_eventdev *dlb2,
> +		   uint8_t port_id,
> +		   int n)
> +{
> +	struct process_local_port_data *port_data;
> +	struct dlb2_eventdev_port *ev_port;
> +	struct dlb2_port *qm_port;
> +	int i;
> +
> +	if (port_id > dlb2->num_ports) {
> +		DLB2_LOG_ERR("Invalid port id %d in dlb2-event_release\n",
> +			     port_id);
> +		rte_errno = -EINVAL;
> +		return;
> +	}
> +
> +	ev_port = &dlb2->ev_ports[port_id];
> +	qm_port = &ev_port->qm_port;
> +	port_data = &dlb2_port[qm_port->id][PORT_TYPE(qm_port)];
> +
> +	i = 0;
> +
> +	if (qm_port->is_directed) {
> +		i = n;
> +		goto sw_credit_update;
> +	}
> +
> +	while (i < n) {
> +		int pop_offs = 0;
> +		int j = 0;
> +
> +		/* Zero-out QEs */
> +		qm_port->qe4[0].cmd_byte = 0;
> +		qm_port->qe4[1].cmd_byte = 0;
> +		qm_port->qe4[2].cmd_byte = 0;
> +		qm_port->qe4[3].cmd_byte = 0;
> +
> +		for (; j < DLB2_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) {
> +			int16_t thresh = qm_port->token_pop_thresh;
> +
> +			if (qm_port->token_pop_mode == DELAYED_POP &&
> +			    qm_port->issued_releases >= thresh - 1) {
> +				/* Insert the token pop QE */
> +				dlb2_construct_token_pop_qe(qm_port, j);
> +
> +				/* Reset the releases for the next QE batch */
> +				qm_port->issued_releases -= thresh;
> +
> +				pop_offs = 1;
> +				j++;
> +				break;
> +			}
> +
> +			qm_port->qe4[j].cmd_byte = DLB2_COMP_CMD_BYTE;
> +			qm_port->issued_releases++;
> +		}
> +
> +		dlb2_hw_do_enqueue(qm_port, i == 0, port_data);
> +
> +		/* Don't include the token pop QE in the release count */
> +		i += j - pop_offs;
> +	}
> +
> +sw_credit_update:
> +	/* each release returns one credit */
> +	if (!ev_port->outstanding_releases) {
> +		DLB2_LOG_ERR("%s: Outstanding releases underflowed.\n",
> +			     __func__);
> +		return;
> +	}
> +	ev_port->outstanding_releases -= i;
> +	ev_port->inflight_credits += i;
> +
> +	/* Replenish s/w credits if enough releases are performed */
> +	dlb2_replenish_sw_credits(dlb2, ev_port);
>  }
> 
>  static inline void
> @@ -3067,86 +3233,6 @@ dlb2_inc_cq_idx(struct dlb2_port *qm_port, int cnt)
>  	qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) & 0x1;  }
> 
> -static int
> -dlb2_event_release(struct dlb2_eventdev *dlb2,
> -		   uint8_t port_id,
> -		   int n)
> -{
> -	struct process_local_port_data *port_data;
> -	struct dlb2_eventdev_port *ev_port;
> -	struct dlb2_port *qm_port;
> -	int i, cnt;
> -
> -	if (port_id > dlb2->num_ports) {
> -		DLB2_LOG_ERR("Invalid port id %d in dlb2-event_release\n",
> -			     port_id);
> -		rte_errno = -EINVAL;
> -		return rte_errno;
> -	}
> -
> -	ev_port = &dlb2->ev_ports[port_id];
> -	qm_port = &ev_port->qm_port;
> -	port_data = &dlb2_port[qm_port->id][PORT_TYPE(qm_port)];
> -
> -	cnt = 0;
> -
> -	if (qm_port->is_directed) {
> -		cnt = n;
> -		goto sw_credit_update;
> -	}
> -
> -	for (i = 0; i < n; i += DLB2_NUM_QES_PER_CACHE_LINE) {
> -		int j;
> -
> -		/* Zero-out QEs */
> -		qm_port->qe4[0].cmd_byte = 0;
> -		qm_port->qe4[1].cmd_byte = 0;
> -		qm_port->qe4[2].cmd_byte = 0;
> -		qm_port->qe4[3].cmd_byte = 0;
> -
> -		for (j = 0; j < DLB2_NUM_QES_PER_CACHE_LINE && (i + j) < n;
> j++)
> -			qm_port->qe4[j].cmd_byte = DLB2_COMP_CMD_BYTE;
> -
> -		qm_port->issued_releases += j;
> -
> -		if (j == 0)
> -			break;
> -
> -		if (qm_port->token_pop_mode == DELAYED_POP && j < 4 &&
> -		    qm_port->issued_releases >= qm_port->token_pop_thresh -
> 1) {
> -			dlb2_construct_token_pop_qe(qm_port, j);
> -
> -			/* Reset the releases counter for the next QE batch */
> -			qm_port->issued_releases -= qm_port-
> >token_pop_thresh;
> -		}
> -
> -		dlb2_hw_do_enqueue(qm_port, i == 0, port_data);
> -
> -		cnt += j;
> -	}
> -
> -	if (qm_port->token_pop_mode == DELAYED_POP &&
> -	    qm_port->issued_releases >= qm_port->token_pop_thresh - 1) {
> -		dlb2_consume_qe_immediate(qm_port, qm_port-
> >owed_tokens);
> -		qm_port->issued_releases -= qm_port->token_pop_thresh;
> -	}
> -
> -sw_credit_update:
> -	/* each release returns one credit */
> -	if (!ev_port->outstanding_releases) {
> -		DLB2_LOG_ERR("Unrecoverable application error. Outstanding
> releases underflowed.\n");
> -		rte_errno = -ENOTRECOVERABLE;
> -		return rte_errno;
> -	}
> -
> -	ev_port->outstanding_releases -= cnt;
> -	ev_port->inflight_credits += cnt;
> -
> -	/* Replenish s/w credits if enough releases are performed */
> -	dlb2_replenish_sw_credits(dlb2, ev_port);
> -	return 0;
> -}
> -
>  static inline int16_t
>  dlb2_hw_dequeue_sparse(struct dlb2_eventdev *dlb2,
>  		       struct dlb2_eventdev_port *ev_port, @@ -3367,8 +3453,7
> @@ dlb2_event_dequeue_burst(void *event_port, struct rte_event *ev,
> uint16_t num,
>  	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
>  		uint16_t out_rels = ev_port->outstanding_releases;
> 
> -		if (dlb2_event_release(dlb2, ev_port->id, out_rels))
> -			return 0; /* rte_errno is set */
> +		dlb2_event_release(dlb2, ev_port->id, out_rels);
> 
>  		DLB2_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
>  	}
> @@ -3405,8 +3490,7 @@ dlb2_event_dequeue_burst_sparse(void *event_port,
> struct rte_event *ev,
>  	if (ev_port->implicit_release && ev_port->outstanding_releases > 0) {
>  		uint16_t out_rels = ev_port->outstanding_releases;
> 
> -		if (dlb2_event_release(dlb2, ev_port->id, out_rels))
> -			return 0; /* rte_errno is set */
> +		dlb2_event_release(dlb2, ev_port->id, out_rels);
> 
>  		DLB2_INC_STAT(ev_port->stats.tx_implicit_rel, out_rels);
>  	}
> diff --git a/drivers/event/dlb2/dlb2_selftest.c
> b/drivers/event/dlb2/dlb2_selftest.c
> index f433654..5cf66c5 100644
> --- a/drivers/event/dlb2/dlb2_selftest.c
> +++ b/drivers/event/dlb2/dlb2_selftest.c
> @@ -1320,7 +1320,7 @@ test_delayed_pop(void)
>  		}
>  	}
> 
> -	/* Dequeue dequeue_depth events but only release dequeue_depth - 2.
> +	/* Dequeue dequeue_depth events but only release dequeue_depth - 1.
>  	 * Delayed pop won't perform the pop and no more events will be
>  	 * scheduled.
>  	 */
> @@ -1336,7 +1336,7 @@ test_delayed_pop(void)
> 
>  	ev.op = RTE_EVENT_OP_RELEASE;
> 
> -	for (i = 0; i < port_conf.dequeue_depth - 2; i++) {
> +	for (i = 0; i < port_conf.dequeue_depth - 1; i++) {
>  		if (rte_event_enqueue_burst(evdev, 0, &ev, 1) != 1) {
>  			printf("%d: RELEASE enqueue expected to succeed\n",
>  			       __LINE__);
> --
> 2.6.4

Looks good, and it fixed the delayed token pop issue in QA tests.

Reviewed-by: Mike Ximing Chen <mike.ximing.chen@intel.com>
  

On Thu, Nov 12, 2020 at 2:43 AM Chen, Mike Ximing
<mike.ximing.chen@intel.com> wrote:
>
>
>
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of Timothy McDaniel
> > Sent: Wednesday, November 11, 2020 3:27 PM
> > Cc: dev@dpdk.org; Carrillo, Erik G <erik.g.carrillo@intel.com>; Eads, Gage
> > <gage.eads@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>;
> > jerinj@marvell.com; thomas@monjalon.net; david.marchand@redhat.com
> > Subject: [dpdk-dev] [PATCH] event/dlb2: add missing delayed token pop logic
> >
> > The code contained in this commit was inadvertently omitted when dissecting
> > the dlb2 code base into discrete patches for upstream.
> >
> > Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
> > ---
> >  drivers/event/dlb2/dlb2.c          | 314 +++++++++++++++++++++++--------------
> >  drivers/event/dlb2/dlb2_selftest.c |   4 +-
> >  2 files changed, 201 insertions(+), 117 deletions(-)

>
> Looks good, and it fixed the delayed token pop issue in QA tests.
>
> Reviewed-by: Mike Ximing Chen <mike.ximing.chen@intel.com>

Series applied to dpdk-next-eventdev/for-main. Thanks.