[dpdk-dev,v6,10/13] ixgbe: support VMDq RSS in non-SRIOV environment
Commit Message
From: Changchun Ouyang <changchun.ouyang@intel.com>
In non-SRIOV environment, VMDq RSS could be enabled by MRQC register.
In theory, the queue number per pool could be 2 or 4, but only 2 queues
are available due to HW limitation, the same limit also exists in Linux
ixgbe driver.
Signed-off-by: Changchun Ouyang <changchun.ouyang@intel.com>
Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
---
drivers/net/ixgbe/ixgbe_rxtx.c | 86 +++++++++++++++++++++++++++++++++++-------
lib/librte_ether/rte_ethdev.c | 11 ++++++
2 files changed, 84 insertions(+), 13 deletions(-)
Comments
Helin, Konstantin,
could you review please?
2015-10-09 13:46, Yuanhan Liu:
> From: Changchun Ouyang <changchun.ouyang@intel.com>
>
> In non-SRIOV environment, VMDq RSS could be enabled by MRQC register.
> In theory, the queue number per pool could be 2 or 4, but only 2 queues
> are available due to HW limitation, the same limit also exists in Linux
> ixgbe driver.
>
> Signed-off-by: Changchun Ouyang <changchun.ouyang@intel.com>
> Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
Hi
> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Yuanhan Liu
> Sent: Friday, October 09, 2015 6:46 AM
> To: dev@dpdk.org
> Cc: Michael S. Tsirkin; marcel@redhat.com
> Subject: [dpdk-dev] [PATCH v6 10/13] ixgbe: support VMDq RSS in non-SRIOV environment
>
> From: Changchun Ouyang <changchun.ouyang@intel.com>
>
> In non-SRIOV environment, VMDq RSS could be enabled by MRQC register.
> In theory, the queue number per pool could be 2 or 4, but only 2 queues
> are available due to HW limitation, the same limit also exists in Linux
> ixgbe driver.
>
> Signed-off-by: Changchun Ouyang <changchun.ouyang@intel.com>
> Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
> ---
> drivers/net/ixgbe/ixgbe_rxtx.c | 86 +++++++++++++++++++++++++++++++++++-------
> lib/librte_ether/rte_ethdev.c | 11 ++++++
> 2 files changed, 84 insertions(+), 13 deletions(-)
>
> diff --git a/drivers/net/ixgbe/ixgbe_rxtx.c b/drivers/net/ixgbe/ixgbe_rxtx.c
> index a598a72..e502fe8 100644
> --- a/drivers/net/ixgbe/ixgbe_rxtx.c
> +++ b/drivers/net/ixgbe/ixgbe_rxtx.c
> @@ -3445,16 +3445,16 @@ void ixgbe_configure_dcb(struct rte_eth_dev *dev)
> return;
> }
>
> -/*
> - * VMDq only support for 10 GbE NIC.
> +/**
> + * Config pool for VMDq on 10 GbE NIC.
> */
> static void
> -ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
> +ixgbe_vmdq_pool_configure(struct rte_eth_dev *dev)
> {
> struct rte_eth_vmdq_rx_conf *cfg;
> struct ixgbe_hw *hw;
> enum rte_eth_nb_pools num_pools;
> - uint32_t mrqc, vt_ctl, vlanctrl;
> + uint32_t vt_ctl, vlanctrl;
> uint32_t vmolr = 0;
> int i;
>
> @@ -3463,12 +3463,6 @@ ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
> cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
> num_pools = cfg->nb_queue_pools;
>
> - ixgbe_rss_disable(dev);
> -
> - /* MRQC: enable vmdq */
> - mrqc = IXGBE_MRQC_VMDQEN;
> - IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
> -
> /* PFVTCTL: turn on virtualisation and set the default pool */
> vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
> if (cfg->enable_default_pool)
> @@ -3534,7 +3528,29 @@ ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
> IXGBE_WRITE_FLUSH(hw);
> }
>
> -/*
> +/**
> + * VMDq only support for 10 GbE NIC.
> + */
> +static void
> +ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
> +{
> + struct ixgbe_hw *hw;
> + uint32_t mrqc;
> +
> + PMD_INIT_FUNC_TRACE();
> + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> +
> + ixgbe_rss_disable(dev);
> +
> + /* MRQC: enable vmdq */
> + mrqc = IXGBE_MRQC_VMDQEN;
> + IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
> + IXGBE_WRITE_FLUSH(hw);
> +
> + ixgbe_vmdq_pool_configure(dev);
> +}
> +
> +/**
> * ixgbe_dcb_config_tx_hw_config - Configure general VMDq TX parameters
> * @hw: pointer to hardware structure
> */
> @@ -3639,6 +3655,41 @@ ixgbe_config_vf_rss(struct rte_eth_dev *dev)
> }
>
> static int
> +ixgbe_config_vmdq_rss(struct rte_eth_dev *dev)
> +{
> + struct ixgbe_hw *hw;
> + uint32_t mrqc;
> +
> + ixgbe_rss_configure(dev);
> +
> + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> +
> + /* MRQC: enable VMDQ RSS */
> + mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
> + mrqc &= ~IXGBE_MRQC_MRQE_MASK;
> +
> + switch (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) {
> + case 2:
> + mrqc |= IXGBE_MRQC_VMDQRSS64EN;
> + break;
> +
> + case 4:
> + mrqc |= IXGBE_MRQC_VMDQRSS32EN;
> + break;
> +
> + default:
> + PMD_INIT_LOG(ERR, "Invalid pool number in non-IOV mode with VMDQ RSS");
> + return -EINVAL;
> + }
> +
> + IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
> +
> + ixgbe_vmdq_pool_configure(dev);
> +
> + return 0;
> +}
So ixgbe_config_vmdq_rss() checks nb_q_per_pool value, and might return an error if the value is invalid.
Though this return value seems just ignored by ixgbe_dev_mq_rx_configure() below.
Probably, it is better to move nb_q_per_pool value checking into rte_eth_dev_check_mq_mode(),
as is done for other modes?
I know it is no ideal, as it probably should be HW specific check,
but seems anyway better than just ignoring the error.
Konstantin
> +
> +static int
> ixgbe_config_vf_default(struct rte_eth_dev *dev)
> {
> struct ixgbe_hw *hw =
> @@ -3694,6 +3745,10 @@ ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
> ixgbe_vmdq_rx_hw_configure(dev);
> break;
>
> + case ETH_MQ_RX_VMDQ_RSS:
> + ixgbe_config_vmdq_rss(dev);
> + break;
> +
> case ETH_MQ_RX_NONE:
> /* if mq_mode is none, disable rss mode.*/
> default: ixgbe_rss_disable(dev);
> @@ -4186,6 +4241,8 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
>
> /* Setup RX queues */
> for (i = 0; i < dev->data->nb_rx_queues; i++) {
> + uint32_t psrtype = 0;
> +
> rxq = dev->data->rx_queues[i];
>
> /*
> @@ -4213,12 +4270,10 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
> if (rx_conf->header_split) {
> if (hw->mac.type == ixgbe_mac_82599EB) {
> /* Must setup the PSRTYPE register */
> - uint32_t psrtype;
> psrtype = IXGBE_PSRTYPE_TCPHDR |
> IXGBE_PSRTYPE_UDPHDR |
> IXGBE_PSRTYPE_IPV4HDR |
> IXGBE_PSRTYPE_IPV6HDR;
> - IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype);
> }
> srrctl = ((rx_conf->split_hdr_size <<
> IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
> @@ -4228,6 +4283,11 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
> #endif
> srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
>
> + /* Set RQPL for VMDQ RSS according to max Rx queue */
> + psrtype |= (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool >> 1) <<
> + IXGBE_PSRTYPE_RQPL_SHIFT;
> + IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype);
> +
> /* Set if packets are dropped when no descriptors available */
> if (rxq->drop_en)
> srrctl |= IXGBE_SRRCTL_DROP_EN;
> diff --git a/lib/librte_ether/rte_ethdev.c b/lib/librte_ether/rte_ethdev.c
> index f593f6e..fe9dc5c 100644
> --- a/lib/librte_ether/rte_ethdev.c
> +++ b/lib/librte_ether/rte_ethdev.c
> @@ -1067,6 +1067,17 @@ rte_eth_dev_check_mq_mode(uint8_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
> return -EINVAL;
> }
> }
> +
> + if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_RSS) {
> + uint32_t nb_queue_pools =
> + dev_conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools;
> + struct rte_eth_dev_info dev_info;
> +
> + rte_eth_dev_info_get(port_id, &dev_info);
> + dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS;
> + RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool =
> + dev_info.vmdq_queue_num / nb_queue_pools;
> + }
> }
> return 0;
> }
> --
> 1.9.0
> > static int
> > +ixgbe_config_vmdq_rss(struct rte_eth_dev *dev)
> > +{
> > + struct ixgbe_hw *hw;
> > + uint32_t mrqc;
> > +
> > + ixgbe_rss_configure(dev);
> > +
> > + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> > +
> > + /* MRQC: enable VMDQ RSS */
> > + mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
> > + mrqc &= ~IXGBE_MRQC_MRQE_MASK;
> > +
> > + switch (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) {
> > + case 2:
> > + mrqc |= IXGBE_MRQC_VMDQRSS64EN;
> > + break;
> > +
> > + case 4:
> > + mrqc |= IXGBE_MRQC_VMDQRSS32EN;
> > + break;
> > +
> > + default:
> > + PMD_INIT_LOG(ERR, "Invalid pool number in non-IOV mode with VMDQ RSS");
> > + return -EINVAL;
> > + }
> > +
> > + IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
> > +
> > + ixgbe_vmdq_pool_configure(dev);
> > +
> > + return 0;
> > +}
>
> So ixgbe_config_vmdq_rss() checks nb_q_per_pool value, and might return an error if the value is invalid.
> Though this return value seems just ignored by ixgbe_dev_mq_rx_configure() below.
> Probably, it is better to move nb_q_per_pool value checking into rte_eth_dev_check_mq_mode(),
> as is done for other modes?
> I know it is no ideal, as it probably should be HW specific check,
> but seems anyway better than just ignoring the error.
>
Hi,
Thanks for the review. However, as you might have seen, I dropped this
patch in the lastest version. This patch is for using the NIC VMDq
feature to link with VM multiple queue to demonstrate the multiple
queue feature. However, this introduces too much limitation. So,
I dropped it.
However, despite of above usage, if you think it's necessary to add
such ability (support VMDq RSS in non-SRIOV), I could update this patch
based on your comments, and sent it out as a standalone patch. Otherwise,
I'll simply drop it.
--yliu
Hi Yliu,
> -----Original Message-----
> From: Yuanhan Liu [mailto:yuanhan.liu@linux.intel.com]
> Sent: Wednesday, October 21, 2015 2:01 PM
> To: Ananyev, Konstantin
> Cc: dev@dpdk.org; Michael S. Tsirkin; marcel@redhat.com
> Subject: Re: [dpdk-dev] [PATCH v6 10/13] ixgbe: support VMDq RSS in non-SRIOV environment
>
> > > static int
> > > +ixgbe_config_vmdq_rss(struct rte_eth_dev *dev)
> > > +{
> > > + struct ixgbe_hw *hw;
> > > + uint32_t mrqc;
> > > +
> > > + ixgbe_rss_configure(dev);
> > > +
> > > + hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
> > > +
> > > + /* MRQC: enable VMDQ RSS */
> > > + mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
> > > + mrqc &= ~IXGBE_MRQC_MRQE_MASK;
> > > +
> > > + switch (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) {
> > > + case 2:
> > > + mrqc |= IXGBE_MRQC_VMDQRSS64EN;
> > > + break;
> > > +
> > > + case 4:
> > > + mrqc |= IXGBE_MRQC_VMDQRSS32EN;
> > > + break;
> > > +
> > > + default:
> > > + PMD_INIT_LOG(ERR, "Invalid pool number in non-IOV mode with VMDQ RSS");
> > > + return -EINVAL;
> > > + }
> > > +
> > > + IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
> > > +
> > > + ixgbe_vmdq_pool_configure(dev);
> > > +
> > > + return 0;
> > > +}
> >
> > So ixgbe_config_vmdq_rss() checks nb_q_per_pool value, and might return an error if the value is invalid.
> > Though this return value seems just ignored by ixgbe_dev_mq_rx_configure() below.
> > Probably, it is better to move nb_q_per_pool value checking into rte_eth_dev_check_mq_mode(),
> > as is done for other modes?
> > I know it is no ideal, as it probably should be HW specific check,
> > but seems anyway better than just ignoring the error.
> >
>
> Hi,
>
> Thanks for the review. However, as you might have seen, I dropped this
> patch in the lastest version. This patch is for using the NIC VMDq
> feature to link with VM multiple queue to demonstrate the multiple
> queue feature. However, this introduces too much limitation. So,
> I dropped it.
Ah sorry, didn't notice it - too many patches are flying around these days.
>
> However, despite of above usage, if you think it's necessary to add
> such ability (support VMDq RSS in non-SRIOV), I could update this patch
> based on your comments, and sent it out as a standalone patch. Otherwise,
> I'll simply drop it.
That's ok by me.
Konstantin
>
> --yliu
> >
> > Hi,
> >
> > Thanks for the review. However, as you might have seen, I dropped this
> > patch in the lastest version. This patch is for using the NIC VMDq
> > feature to link with VM multiple queue to demonstrate the multiple
> > queue feature. However, this introduces too much limitation. So,
> > I dropped it.
>
> Ah sorry, didn't notice it - too many patches are flying around these days.
That's true :)
--yliu
>
> >
> > However, despite of above usage, if you think it's necessary to add
> > such ability (support VMDq RSS in non-SRIOV), I could update this patch
> > based on your comments, and sent it out as a standalone patch. Otherwise,
> > I'll simply drop it.
>
> That's ok by me.
> Konstantin
>
> >
> > --yliu
@@ -3445,16 +3445,16 @@ void ixgbe_configure_dcb(struct rte_eth_dev *dev)
return;
}
-/*
- * VMDq only support for 10 GbE NIC.
+/**
+ * Config pool for VMDq on 10 GbE NIC.
*/
static void
-ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
+ixgbe_vmdq_pool_configure(struct rte_eth_dev *dev)
{
struct rte_eth_vmdq_rx_conf *cfg;
struct ixgbe_hw *hw;
enum rte_eth_nb_pools num_pools;
- uint32_t mrqc, vt_ctl, vlanctrl;
+ uint32_t vt_ctl, vlanctrl;
uint32_t vmolr = 0;
int i;
@@ -3463,12 +3463,6 @@ ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
num_pools = cfg->nb_queue_pools;
- ixgbe_rss_disable(dev);
-
- /* MRQC: enable vmdq */
- mrqc = IXGBE_MRQC_VMDQEN;
- IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
-
/* PFVTCTL: turn on virtualisation and set the default pool */
vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
if (cfg->enable_default_pool)
@@ -3534,7 +3528,29 @@ ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
IXGBE_WRITE_FLUSH(hw);
}
-/*
+/**
+ * VMDq only support for 10 GbE NIC.
+ */
+static void
+ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
+{
+ struct ixgbe_hw *hw;
+ uint32_t mrqc;
+
+ PMD_INIT_FUNC_TRACE();
+ hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ ixgbe_rss_disable(dev);
+
+ /* MRQC: enable vmdq */
+ mrqc = IXGBE_MRQC_VMDQEN;
+ IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
+ IXGBE_WRITE_FLUSH(hw);
+
+ ixgbe_vmdq_pool_configure(dev);
+}
+
+/**
* ixgbe_dcb_config_tx_hw_config - Configure general VMDq TX parameters
* @hw: pointer to hardware structure
*/
@@ -3639,6 +3655,41 @@ ixgbe_config_vf_rss(struct rte_eth_dev *dev)
}
static int
+ixgbe_config_vmdq_rss(struct rte_eth_dev *dev)
+{
+ struct ixgbe_hw *hw;
+ uint32_t mrqc;
+
+ ixgbe_rss_configure(dev);
+
+ hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ /* MRQC: enable VMDQ RSS */
+ mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
+ mrqc &= ~IXGBE_MRQC_MRQE_MASK;
+
+ switch (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) {
+ case 2:
+ mrqc |= IXGBE_MRQC_VMDQRSS64EN;
+ break;
+
+ case 4:
+ mrqc |= IXGBE_MRQC_VMDQRSS32EN;
+ break;
+
+ default:
+ PMD_INIT_LOG(ERR, "Invalid pool number in non-IOV mode with VMDQ RSS");
+ return -EINVAL;
+ }
+
+ IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
+
+ ixgbe_vmdq_pool_configure(dev);
+
+ return 0;
+}
+
+static int
ixgbe_config_vf_default(struct rte_eth_dev *dev)
{
struct ixgbe_hw *hw =
@@ -3694,6 +3745,10 @@ ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
ixgbe_vmdq_rx_hw_configure(dev);
break;
+ case ETH_MQ_RX_VMDQ_RSS:
+ ixgbe_config_vmdq_rss(dev);
+ break;
+
case ETH_MQ_RX_NONE:
/* if mq_mode is none, disable rss mode.*/
default: ixgbe_rss_disable(dev);
@@ -4186,6 +4241,8 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
/* Setup RX queues */
for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ uint32_t psrtype = 0;
+
rxq = dev->data->rx_queues[i];
/*
@@ -4213,12 +4270,10 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
if (rx_conf->header_split) {
if (hw->mac.type == ixgbe_mac_82599EB) {
/* Must setup the PSRTYPE register */
- uint32_t psrtype;
psrtype = IXGBE_PSRTYPE_TCPHDR |
IXGBE_PSRTYPE_UDPHDR |
IXGBE_PSRTYPE_IPV4HDR |
IXGBE_PSRTYPE_IPV6HDR;
- IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype);
}
srrctl = ((rx_conf->split_hdr_size <<
IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
@@ -4228,6 +4283,11 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
#endif
srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
+ /* Set RQPL for VMDQ RSS according to max Rx queue */
+ psrtype |= (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool >> 1) <<
+ IXGBE_PSRTYPE_RQPL_SHIFT;
+ IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype);
+
/* Set if packets are dropped when no descriptors available */
if (rxq->drop_en)
srrctl |= IXGBE_SRRCTL_DROP_EN;
@@ -1067,6 +1067,17 @@ rte_eth_dev_check_mq_mode(uint8_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
return -EINVAL;
}
}
+
+ if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_RSS) {
+ uint32_t nb_queue_pools =
+ dev_conf->rx_adv_conf.vmdq_rx_conf.nb_queue_pools;
+ struct rte_eth_dev_info dev_info;
+
+ rte_eth_dev_info_get(port_id, &dev_info);
+ dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS;
+ RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool =
+ dev_info.vmdq_queue_num / nb_queue_pools;
+ }
}
return 0;
}