net/mlx5: support hairpin between different ports
diff mbox series

Message ID 1575192909-105146-1-git-send-email-xiangxia.m.yue@gmail.com
State Rejected
Delegated to: Raslan Darawsheh
Headers show
Series
  • net/mlx5: support hairpin between different ports
Related show

Checks

Context Check Description
ci/travis-robot success Travis build: passed
ci/iol-mellanox-Performance success Performance Testing PASS
ci/Intel-compilation success Compilation OK
ci/iol-compilation success Compile Testing PASS
ci/iol-intel-Performance success Performance Testing PASS
ci/checkpatch warning coding style issues

Commit Message

Tonghao Zhang Dec. 1, 2019, 9:35 a.m. UTC
From: Tonghao Zhang <xiangxia.m.yue@gmail.com>

In the dpdk upstream, each hairpin Rxq can be connected
Txq which can belong to a same port. This patch allows
Rxq connected to different port Txq.

rte_eth_hairpin_conf_check will check the hairpin_conf valid
in high level.

Signed-off-by: Tonghao Zhang <xiangxia.m.yue@gmail.com>
---
 drivers/net/mlx5/mlx5_rxq.c     |  5 ++-
 drivers/net/mlx5/mlx5_trigger.c | 41 ++++++++++++++++-------
 drivers/net/mlx5/mlx5_txq.c     |  4 +--
 lib/librte_ethdev/rte_ethdev.c  | 73 ++++++++++++++++++++++++++++-------------
 4 files changed, 83 insertions(+), 40 deletions(-)

Comments

Ori Kam Jan. 16, 2020, 11:40 a.m. UTC | #1
Sorry for the late response,

Thank you very much for this patch.

We are planning to add such support, but this patch is not the best
way.

I will keep you updated about the progress, and will be very to hear 
your inputs.

Best,
Ori


> -----Original Message-----
> From: xiangxia.m.yue@gmail.com <xiangxia.m.yue@gmail.com>
> Sent: Sunday, December 1, 2019 11:35 AM
> To: Ori Kam <orika@mellanox.com>
> Cc: dev@dpdk.org; Tonghao Zhang <xiangxia.m.yue@gmail.com>
> Subject: [PATCH dpdk-dev] net/mlx5: support hairpin between different
> ports
> 
> From: Tonghao Zhang <xiangxia.m.yue@gmail.com>
> 
> In the dpdk upstream, each hairpin Rxq can be connected
> Txq which can belong to a same port. This patch allows
> Rxq connected to different port Txq.
> 
> rte_eth_hairpin_conf_check will check the hairpin_conf valid
> in high level.
> 
> Signed-off-by: Tonghao Zhang <xiangxia.m.yue@gmail.com>
> ---
>  drivers/net/mlx5/mlx5_rxq.c     |  5 ++-
>  drivers/net/mlx5/mlx5_trigger.c | 41 ++++++++++++++++-------
>  drivers/net/mlx5/mlx5_txq.c     |  4 +--
>  lib/librte_ethdev/rte_ethdev.c  | 73 ++++++++++++++++++++++++++++----
> ---------
>  4 files changed, 83 insertions(+), 40 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
> index 986ec01..c2daebd 100644
> --- a/drivers/net/mlx5/mlx5_rxq.c
> +++ b/drivers/net/mlx5/mlx5_rxq.c
> @@ -547,9 +547,8 @@
>  	res = mlx5_rx_queue_pre_setup(dev, idx, desc);
>  	if (res)
>  		return res;
> -	if (hairpin_conf->peer_count != 1 ||
> -	    hairpin_conf->peers[0].port != dev->data->port_id ||
> -	    hairpin_conf->peers[0].queue >= priv->txqs_n) {
> +
> +	if (hairpin_conf->peer_count != 1) {
>  		DRV_LOG(ERR, "port %u unable to setup hairpin queue index
> %u "
>  			" invalid hairpind configuration", dev->data->port_id,
>  			idx);
> diff --git a/drivers/net/mlx5/mlx5_trigger.c
> b/drivers/net/mlx5/mlx5_trigger.c
> index ab6937a..3eb2984 100644
> --- a/drivers/net/mlx5/mlx5_trigger.c
> +++ b/drivers/net/mlx5/mlx5_trigger.c
> @@ -185,6 +185,10 @@
>  	struct mlx5_rxq_ctrl *rxq_ctrl;
>  	struct mlx5_devx_obj *sq;
>  	struct mlx5_devx_obj *rq;
> +	struct rte_eth_dev *rxq_dev;
> +	struct mlx5_priv *rxq_priv;
> +	uint16_t peer_queue_id;
> +	uint16_t peer_port_id;
>  	unsigned int i;
>  	int ret = 0;
> 
> @@ -203,37 +207,50 @@
>  			mlx5_txq_release(dev, i);
>  			return -rte_errno;
>  		}
> -		sq = txq_ctrl->obj->sq;
> -		rxq_ctrl = mlx5_rxq_get(dev,
> -					txq_ctrl-
> >hairpin_conf.peers[0].queue);
> +		peer_port_id = txq_ctrl->hairpin_conf.peers[0].port;
> +		peer_queue_id = txq_ctrl->hairpin_conf.peers[0].queue;
> +		rxq_dev = &rte_eth_devices[peer_port_id];
> +		rxq_priv = rxq_dev->data->dev_private;
> +
> +		rxq_ctrl = mlx5_rxq_get(rxq_dev, peer_queue_id);
>  		if (!rxq_ctrl) {
>  			mlx5_txq_release(dev, i);
>  			rte_errno = EINVAL;
>  			DRV_LOG(ERR, "port %u no rxq object found: %d",
> -				dev->data->port_id,
> -				txq_ctrl->hairpin_conf.peers[0].queue);
> +				peer_port_id, peer_queue_id);
>  			return -rte_errno;
>  		}
>  		if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN ||
>  		    rxq_ctrl->hairpin_conf.peers[0].queue != i) {
>  			rte_errno = ENOMEM;
>  			DRV_LOG(ERR, "port %u Tx queue %d can't be
> binded to "
> -				"Rx queue %d", dev->data->port_id,
> -				i, txq_ctrl->hairpin_conf.peers[0].queue);
> +				"port %u Rx queue %d",
> +				dev->data->port_id, i,
> +				peer_port_id, peer_queue_id);
>  			goto error;
>  		}
> +		if (!rxq_ctrl->obj) {
> +			DRV_LOG(ERR, "port %u rxq obj not created, "
> +				"you may start it firstly.",
> +				peer_port_id);
> +			goto error;
> +		}
> +
> +		sq = txq_ctrl->obj->sq;
>  		rq = rxq_ctrl->obj->rq;
>  		if (!rq) {
>  			rte_errno = ENOMEM;
> -			DRV_LOG(ERR, "port %u hairpin no matching rxq:
> %d",
> +			DRV_LOG(ERR, "port %u hairpin no matching port"
> +				" %u rxq %d",
>  				dev->data->port_id,
> -				txq_ctrl->hairpin_conf.peers[0].queue);
> +				peer_port_id,
> +				peer_queue_id);
>  			goto error;
>  		}
>  		sq_attr.state = MLX5_SQC_STATE_RDY;
>  		sq_attr.sq_state = MLX5_SQC_STATE_RST;
>  		sq_attr.hairpin_peer_rq = rq->id;
> -		sq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id;
> +		sq_attr.hairpin_peer_vhca = rxq_priv-
> >config.hca_attr.vhca_id;
>  		ret = mlx5_devx_cmd_modify_sq(sq, &sq_attr);
>  		if (ret)
>  			goto error;
> @@ -245,12 +262,12 @@
>  		if (ret)
>  			goto error;
>  		mlx5_txq_release(dev, i);
> -		mlx5_rxq_release(dev, txq_ctrl-
> >hairpin_conf.peers[0].queue);
> +		mlx5_rxq_release(rxq_dev, peer_queue_id);
>  	}
>  	return 0;
>  error:
>  	mlx5_txq_release(dev, i);
> -	mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue);
> +	mlx5_rxq_release(dev, peer_queue_id);
>  	return -rte_errno;
>  }
> 
> diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
> index bac4f71..98ee35e 100644
> --- a/drivers/net/mlx5/mlx5_txq.c
> +++ b/drivers/net/mlx5/mlx5_txq.c
> @@ -254,9 +254,7 @@
>  	res = mlx5_tx_queue_pre_setup(dev, idx, desc);
>  	if (res)
>  		return res;
> -	if (hairpin_conf->peer_count != 1 ||
> -	    hairpin_conf->peers[0].port != dev->data->port_id ||
> -	    hairpin_conf->peers[0].queue >= priv->rxqs_n) {
> +	if (hairpin_conf->peer_count != 1) {
>  		DRV_LOG(ERR, "port %u unable to setup hairpin queue index
> %u "
>  			" invalid hairpind configuration", dev->data->port_id,
>  			idx);
> diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
> index 8d2ce31..0bd6d87 100644
> --- a/lib/librte_ethdev/rte_ethdev.c
> +++ b/lib/librte_ethdev/rte_ethdev.c
> @@ -1923,6 +1923,53 @@ struct rte_eth_dev *
>  	return eth_err(port_id, ret);
>  }
> 
> +static int
> +rte_eth_hairpin_conf_check(const struct rte_eth_hairpin_conf *conf,
> +			      const struct rte_eth_hairpin_cap *cap,
> +			      bool is_rx_hairpin)
> +{
> +	uint16_t max_queues_mapping = is_rx_hairpin ?
> +				      cap->max_rx_2_tx : cap->max_tx_2_rx;
> +	struct rte_eth_dev *dev;
> +	uint16_t nb_queues;
> +	uint16_t port_id;
> +	int i;
> +
> +	if (conf->peer_count == 0) {
> +		RTE_ETHDEV_LOG(ERR, "Invalid value for number of
> peers(=%hu),"
> +				" should be: > 0",
> +			       conf->peer_count);
> +		return -EINVAL;
> +	}
> +	if (conf->peer_count > max_queues_mapping) {
> +		RTE_ETHDEV_LOG(ERR, "Invalid value for number of
> peers(=%hu),"
> +			       " should be: <= %hu",
> +			       conf->peer_count, max_queues_mapping);
> +		return -EINVAL;
> +	}
> +	for (i = 0; i < conf->peer_count; i++) {
> +		port_id = conf->peers[i].port;
> +
> +		if (!rte_eth_dev_is_valid_port(port_id)) {
> +			RTE_ETHDEV_LOG(ERR, "Invalid port_id(=%hu) for"
> +				       " hairpin peers", conf->peer_count);
> +			return -EINVAL;
> +		}
> +
> +		dev = &rte_eth_devices[port_id];
> +		nb_queues = is_rx_hairpin ?
> +			    dev->data->nb_tx_queues : dev->data-
> >nb_rx_queues;
> +		if (conf->peers[i].queue >= nb_queues) {
> +			RTE_ETHDEV_LOG(ERR, "Invalid queue_id(=%hu)
> for"
> +				       " hairpin peers, shoud be < %hu",
> +				       conf->peer_count, nb_queues);
> +			return -EINVAL;
> +		}
> +	}
> +
> +	return 0;
> +}
> +
>  int
>  rte_eth_rx_hairpin_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
>  			       uint16_t nb_rx_desc,
> @@ -1956,18 +2003,9 @@ struct rte_eth_dev *
>  			nb_rx_desc, cap.max_nb_desc);
>  		return -EINVAL;
>  	}
> -	if (conf->peer_count > cap.max_rx_2_tx) {
> -		RTE_ETHDEV_LOG(ERR,
> -			"Invalid value for number of peers for Rx
> queue(=%hu), should be: <= %hu",
> -			conf->peer_count, cap.max_rx_2_tx);
> -		return -EINVAL;
> -	}
> -	if (conf->peer_count == 0) {
> -		RTE_ETHDEV_LOG(ERR,
> -			"Invalid value for number of peers for Rx
> queue(=%hu), should be: > 0",
> -			conf->peer_count);
> +	if (rte_eth_hairpin_conf_check(conf, &cap, true))
>  		return -EINVAL;
> -	}
> +
>  	for (i = 0, count = 0; i < dev->data->nb_rx_queues &&
>  	     cap.max_nb_queues != UINT16_MAX; i++) {
>  		if (i == rx_queue_id ||
> rte_eth_dev_is_rx_hairpin_queue(dev, i))
> @@ -2126,18 +2164,9 @@ struct rte_eth_dev *
>  			nb_tx_desc, cap.max_nb_desc);
>  		return -EINVAL;
>  	}
> -	if (conf->peer_count > cap.max_tx_2_rx) {
> -		RTE_ETHDEV_LOG(ERR,
> -			"Invalid value for number of peers for Tx
> queue(=%hu), should be: <= %hu",
> -			conf->peer_count, cap.max_tx_2_rx);
> +	if (rte_eth_hairpin_conf_check(conf, &cap, false))
>  		return -EINVAL;
> -	}
> -	if (conf->peer_count == 0) {
> -		RTE_ETHDEV_LOG(ERR,
> -			"Invalid value for number of peers for Tx
> queue(=%hu), should be: > 0",
> -			conf->peer_count);
> -		return -EINVAL;
> -	}
> +
>  	for (i = 0, count = 0; i < dev->data->nb_tx_queues &&
>  	     cap.max_nb_queues != UINT16_MAX; i++) {
>  		if (i == tx_queue_id ||
> rte_eth_dev_is_tx_hairpin_queue(dev, i))
> --
> 1.8.3.1

Patch
diff mbox series

diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 986ec01..c2daebd 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -547,9 +547,8 @@ 
 	res = mlx5_rx_queue_pre_setup(dev, idx, desc);
 	if (res)
 		return res;
-	if (hairpin_conf->peer_count != 1 ||
-	    hairpin_conf->peers[0].port != dev->data->port_id ||
-	    hairpin_conf->peers[0].queue >= priv->txqs_n) {
+
+	if (hairpin_conf->peer_count != 1) {
 		DRV_LOG(ERR, "port %u unable to setup hairpin queue index %u "
 			" invalid hairpind configuration", dev->data->port_id,
 			idx);
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index ab6937a..3eb2984 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -185,6 +185,10 @@ 
 	struct mlx5_rxq_ctrl *rxq_ctrl;
 	struct mlx5_devx_obj *sq;
 	struct mlx5_devx_obj *rq;
+	struct rte_eth_dev *rxq_dev;
+	struct mlx5_priv *rxq_priv;
+	uint16_t peer_queue_id;
+	uint16_t peer_port_id;
 	unsigned int i;
 	int ret = 0;
 
@@ -203,37 +207,50 @@ 
 			mlx5_txq_release(dev, i);
 			return -rte_errno;
 		}
-		sq = txq_ctrl->obj->sq;
-		rxq_ctrl = mlx5_rxq_get(dev,
-					txq_ctrl->hairpin_conf.peers[0].queue);
+		peer_port_id = txq_ctrl->hairpin_conf.peers[0].port;
+		peer_queue_id = txq_ctrl->hairpin_conf.peers[0].queue;
+		rxq_dev = &rte_eth_devices[peer_port_id];
+		rxq_priv = rxq_dev->data->dev_private;
+
+		rxq_ctrl = mlx5_rxq_get(rxq_dev, peer_queue_id);
 		if (!rxq_ctrl) {
 			mlx5_txq_release(dev, i);
 			rte_errno = EINVAL;
 			DRV_LOG(ERR, "port %u no rxq object found: %d",
-				dev->data->port_id,
-				txq_ctrl->hairpin_conf.peers[0].queue);
+				peer_port_id, peer_queue_id);
 			return -rte_errno;
 		}
 		if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN ||
 		    rxq_ctrl->hairpin_conf.peers[0].queue != i) {
 			rte_errno = ENOMEM;
 			DRV_LOG(ERR, "port %u Tx queue %d can't be binded to "
-				"Rx queue %d", dev->data->port_id,
-				i, txq_ctrl->hairpin_conf.peers[0].queue);
+				"port %u Rx queue %d",
+				dev->data->port_id, i,
+				peer_port_id, peer_queue_id);
 			goto error;
 		}
+		if (!rxq_ctrl->obj) {
+			DRV_LOG(ERR, "port %u rxq obj not created, "
+				"you may start it firstly.",
+				peer_port_id);
+			goto error;
+		}
+
+		sq = txq_ctrl->obj->sq;
 		rq = rxq_ctrl->obj->rq;
 		if (!rq) {
 			rte_errno = ENOMEM;
-			DRV_LOG(ERR, "port %u hairpin no matching rxq: %d",
+			DRV_LOG(ERR, "port %u hairpin no matching port"
+				" %u rxq %d",
 				dev->data->port_id,
-				txq_ctrl->hairpin_conf.peers[0].queue);
+				peer_port_id,
+				peer_queue_id);
 			goto error;
 		}
 		sq_attr.state = MLX5_SQC_STATE_RDY;
 		sq_attr.sq_state = MLX5_SQC_STATE_RST;
 		sq_attr.hairpin_peer_rq = rq->id;
-		sq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id;
+		sq_attr.hairpin_peer_vhca = rxq_priv->config.hca_attr.vhca_id;
 		ret = mlx5_devx_cmd_modify_sq(sq, &sq_attr);
 		if (ret)
 			goto error;
@@ -245,12 +262,12 @@ 
 		if (ret)
 			goto error;
 		mlx5_txq_release(dev, i);
-		mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue);
+		mlx5_rxq_release(rxq_dev, peer_queue_id);
 	}
 	return 0;
 error:
 	mlx5_txq_release(dev, i);
-	mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue);
+	mlx5_rxq_release(dev, peer_queue_id);
 	return -rte_errno;
 }
 
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index bac4f71..98ee35e 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -254,9 +254,7 @@ 
 	res = mlx5_tx_queue_pre_setup(dev, idx, desc);
 	if (res)
 		return res;
-	if (hairpin_conf->peer_count != 1 ||
-	    hairpin_conf->peers[0].port != dev->data->port_id ||
-	    hairpin_conf->peers[0].queue >= priv->rxqs_n) {
+	if (hairpin_conf->peer_count != 1) {
 		DRV_LOG(ERR, "port %u unable to setup hairpin queue index %u "
 			" invalid hairpind configuration", dev->data->port_id,
 			idx);
diff --git a/lib/librte_ethdev/rte_ethdev.c b/lib/librte_ethdev/rte_ethdev.c
index 8d2ce31..0bd6d87 100644
--- a/lib/librte_ethdev/rte_ethdev.c
+++ b/lib/librte_ethdev/rte_ethdev.c
@@ -1923,6 +1923,53 @@  struct rte_eth_dev *
 	return eth_err(port_id, ret);
 }
 
+static int
+rte_eth_hairpin_conf_check(const struct rte_eth_hairpin_conf *conf,
+			      const struct rte_eth_hairpin_cap *cap,
+			      bool is_rx_hairpin)
+{
+	uint16_t max_queues_mapping = is_rx_hairpin ?
+				      cap->max_rx_2_tx : cap->max_tx_2_rx;
+	struct rte_eth_dev *dev;
+	uint16_t nb_queues;
+	uint16_t port_id;
+	int i;
+
+	if (conf->peer_count == 0) {
+		RTE_ETHDEV_LOG(ERR, "Invalid value for number of peers(=%hu),"
+				" should be: > 0",
+			       conf->peer_count);
+		return -EINVAL;
+	}
+	if (conf->peer_count > max_queues_mapping) {
+		RTE_ETHDEV_LOG(ERR, "Invalid value for number of peers(=%hu),"
+			       " should be: <= %hu",
+			       conf->peer_count, max_queues_mapping);
+		return -EINVAL;
+	}
+	for (i = 0; i < conf->peer_count; i++) {
+		port_id = conf->peers[i].port;
+
+		if (!rte_eth_dev_is_valid_port(port_id)) {
+			RTE_ETHDEV_LOG(ERR, "Invalid port_id(=%hu) for"
+				       " hairpin peers", conf->peer_count);
+			return -EINVAL;
+		}
+
+		dev = &rte_eth_devices[port_id];
+		nb_queues = is_rx_hairpin ?
+			    dev->data->nb_tx_queues : dev->data->nb_rx_queues;
+		if (conf->peers[i].queue >= nb_queues) {
+			RTE_ETHDEV_LOG(ERR, "Invalid queue_id(=%hu) for"
+				       " hairpin peers, shoud be < %hu",
+				       conf->peer_count, nb_queues);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
 int
 rte_eth_rx_hairpin_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
 			       uint16_t nb_rx_desc,
@@ -1956,18 +2003,9 @@  struct rte_eth_dev *
 			nb_rx_desc, cap.max_nb_desc);
 		return -EINVAL;
 	}
-	if (conf->peer_count > cap.max_rx_2_tx) {
-		RTE_ETHDEV_LOG(ERR,
-			"Invalid value for number of peers for Rx queue(=%hu), should be: <= %hu",
-			conf->peer_count, cap.max_rx_2_tx);
-		return -EINVAL;
-	}
-	if (conf->peer_count == 0) {
-		RTE_ETHDEV_LOG(ERR,
-			"Invalid value for number of peers for Rx queue(=%hu), should be: > 0",
-			conf->peer_count);
+	if (rte_eth_hairpin_conf_check(conf, &cap, true))
 		return -EINVAL;
-	}
+
 	for (i = 0, count = 0; i < dev->data->nb_rx_queues &&
 	     cap.max_nb_queues != UINT16_MAX; i++) {
 		if (i == rx_queue_id || rte_eth_dev_is_rx_hairpin_queue(dev, i))
@@ -2126,18 +2164,9 @@  struct rte_eth_dev *
 			nb_tx_desc, cap.max_nb_desc);
 		return -EINVAL;
 	}
-	if (conf->peer_count > cap.max_tx_2_rx) {
-		RTE_ETHDEV_LOG(ERR,
-			"Invalid value for number of peers for Tx queue(=%hu), should be: <= %hu",
-			conf->peer_count, cap.max_tx_2_rx);
+	if (rte_eth_hairpin_conf_check(conf, &cap, false))
 		return -EINVAL;
-	}
-	if (conf->peer_count == 0) {
-		RTE_ETHDEV_LOG(ERR,
-			"Invalid value for number of peers for Tx queue(=%hu), should be: > 0",
-			conf->peer_count);
-		return -EINVAL;
-	}
+
 	for (i = 0, count = 0; i < dev->data->nb_tx_queues &&
 	     cap.max_nb_queues != UINT16_MAX; i++) {
 		if (i == tx_queue_id || rte_eth_dev_is_tx_hairpin_queue(dev, i))