[dpdk-dev,v2,3/3] net/sfc: support multi-process
Checks
Commit Message
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
Reviewed-by: Andy Moreton <amoreton@solarflare.com>
---
v2:
- no changes
doc/guides/nics/features/sfc_efx.ini | 1 +
drivers/net/sfc/sfc.h | 11 +++
drivers/net/sfc/sfc_dp_rx.h | 1 +
drivers/net/sfc/sfc_dp_tx.h | 1 +
drivers/net/sfc/sfc_ef10_rx.c | 2 +-
drivers/net/sfc/sfc_ef10_tx.c | 5 +-
drivers/net/sfc/sfc_ethdev.c | 133 ++++++++++++++++++++++++++++++++++-
7 files changed, 148 insertions(+), 6 deletions(-)
Comments
On 5/18/2017 3:00 PM, Andrew Rybchenko wrote:
> Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
> Reviewed-by: Andy Moreton <amoreton@solarflare.com>
<...>
> Linux VFIO = Y
> diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h
> index 772a713..007ed24 100644
> --- a/drivers/net/sfc/sfc.h
> +++ b/drivers/net/sfc/sfc.h
> @@ -225,7 +225,18 @@ struct sfc_adapter {
> uint8_t rss_key[SFC_RSS_KEY_SIZE];
> #endif
>
> + /*
> + * Shared memory copy of the Rx datapath name to be used by
> + * the secondary process to find Rx datapath to be used.
> + */
> + char *dp_rx_name;
Why not use sa->dp_rx->dp.name to find the dp_rx? That variable should
be shared between processes already?
<...>
> diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
> index 1484bab..60812cb 100644
> --- a/drivers/net/sfc/sfc_ef10_rx.c
> +++ b/drivers/net/sfc/sfc_ef10_rx.c
> @@ -699,7 +699,7 @@ struct sfc_dp_rx sfc_ef10_rx = {
> .type = SFC_DP_RX,
> .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10,
> },
> - .features = 0,
> + .features = SFC_DP_RX_FEAT_MULTI_PROCESS,
Why this flag is needed, I mean why multi process support is not always
enabled by default?
<...>
On 05/22/2017 02:29 PM, Ferruh Yigit wrote:
> On 5/18/2017 3:00 PM, Andrew Rybchenko wrote:
>> Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
>> Reviewed-by: Andy Moreton <amoreton@solarflare.com>
> <...>
>
>> Linux VFIO = Y
>> diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h
>> index 772a713..007ed24 100644
>> --- a/drivers/net/sfc/sfc.h
>> +++ b/drivers/net/sfc/sfc.h
>> @@ -225,7 +225,18 @@ struct sfc_adapter {
>> uint8_t rss_key[SFC_RSS_KEY_SIZE];
>> #endif
>>
>> + /*
>> + * Shared memory copy of the Rx datapath name to be used by
>> + * the secondary process to find Rx datapath to be used.
>> + */
>> + char *dp_rx_name;
> Why not use sa->dp_rx->dp.name to find the dp_rx? That variable should
> be shared between processes already?
sa->dp_rx is a pointer to .data section (sfc_efx_rx or sfc_ef10_rx)
which is (may be) different in primary and secondary processes.
> <...>
>
>> diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
>> index 1484bab..60812cb 100644
>> --- a/drivers/net/sfc/sfc_ef10_rx.c
>> +++ b/drivers/net/sfc/sfc_ef10_rx.c
>> @@ -699,7 +699,7 @@ struct sfc_dp_rx sfc_ef10_rx = {
>> .type = SFC_DP_RX,
>> .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10,
>> },
>> - .features = 0,
>> + .features = SFC_DP_RX_FEAT_MULTI_PROCESS,
> Why this flag is needed, I mean why multi process support is not always
> enabled by default?
libefx-based datapath intensively uses function pointers (primary
process function pointers stored in data structures). So, it does not
work in multi process.
> <...>
On 5/22/2017 1:07 PM, Andrew Rybchenko wrote:
> On 05/22/2017 02:29 PM, Ferruh Yigit wrote:
>> On 5/18/2017 3:00 PM, Andrew Rybchenko wrote:
>>> Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
>>> Reviewed-by: Andy Moreton <amoreton@solarflare.com>
>> <...>
>>
>>> Linux VFIO = Y
>>> diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h
>>> index 772a713..007ed24 100644
>>> --- a/drivers/net/sfc/sfc.h
>>> +++ b/drivers/net/sfc/sfc.h
>>> @@ -225,7 +225,18 @@ struct sfc_adapter {
>>> uint8_t rss_key[SFC_RSS_KEY_SIZE];
>>> #endif
>>>
>>> + /*
>>> + * Shared memory copy of the Rx datapath name to be used by
>>> + * the secondary process to find Rx datapath to be used.
>>> + */
>>> + char *dp_rx_name;
>> Why not use sa->dp_rx->dp.name to find the dp_rx? That variable should
>> be shared between processes already?
>
> sa->dp_rx is a pointer to .data section (sfc_efx_rx or sfc_ef10_rx)
> which is (may be) different in primary and secondary processes.
OK, thanks.
Does it make sense to implement strdup as rte_strdup, so others can
re-use it? @sergio, what do you think?
>
>> <...>
>>
>>> diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
>>> index 1484bab..60812cb 100644
>>> --- a/drivers/net/sfc/sfc_ef10_rx.c
>>> +++ b/drivers/net/sfc/sfc_ef10_rx.c
>>> @@ -699,7 +699,7 @@ struct sfc_dp_rx sfc_ef10_rx = {
>>> .type = SFC_DP_RX,
>>> .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10,
>>> },
>>> - .features = 0,
>>> + .features = SFC_DP_RX_FEAT_MULTI_PROCESS,
>> Why this flag is needed, I mean why multi process support is not always
>> enabled by default?
>
> libefx-based datapath intensively uses function pointers (primary
> process function pointers stored in data structures). So, it does not
> work in multi process.
But this currently added always, if I don't miss anything. And only
checked once in secondary path and error returned if not set.
Is there any code path that behaves different based on this flag? Or any
case that this flags shouldn't be set?
What happens if this flag removed, and assumed it is always set? (this
and tx version of this flag)
>
>> <...>
>
>
On 05/22/2017 03:36 PM, Ferruh Yigit wrote:
> On 5/22/2017 1:07 PM, Andrew Rybchenko wrote:
>> On 05/22/2017 02:29 PM, Ferruh Yigit wrote:
>>> On 5/18/2017 3:00 PM, Andrew Rybchenko wrote:
>>>> Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
>>>> Reviewed-by: Andy Moreton <amoreton@solarflare.com>
>>> <...>
>>>
>>>> diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
>>>> index 1484bab..60812cb 100644
>>>> --- a/drivers/net/sfc/sfc_ef10_rx.c
>>>> +++ b/drivers/net/sfc/sfc_ef10_rx.c
>>>> @@ -699,7 +699,7 @@ struct sfc_dp_rx sfc_ef10_rx = {
>>>> .type = SFC_DP_RX,
>>>> .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10,
>>>> },
>>>> - .features = 0,
>>>> + .features = SFC_DP_RX_FEAT_MULTI_PROCESS,
>>> Why this flag is needed, I mean why multi process support is not always
>>> enabled by default?
>> libefx-based datapath intensively uses function pointers (primary
>> process function pointers stored in data structures). So, it does not
>> work in multi process.
> But this currently added always, if I don't miss anything. And only
> checked once in secondary path and error returned if not set.
>
> Is there any code path that behaves different based on this flag? Or any
> case that this flags shouldn't be set?
sfc_efx_rx (and sfc_efx_tx) does not have the flag.
> What happens if this flag removed, and assumed it is always set? (this
> and tx version of this flag)
Init in the secondary process fails if datapath chosen by the primary
process does not have the flag.
>>> <...>
On 5/22/2017 1:44 PM, Andrew Rybchenko wrote:
> On 05/22/2017 03:36 PM, Ferruh Yigit wrote:
>> On 5/22/2017 1:07 PM, Andrew Rybchenko wrote:
>>> On 05/22/2017 02:29 PM, Ferruh Yigit wrote:
>>>> On 5/18/2017 3:00 PM, Andrew Rybchenko wrote:
>>>>> Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
>>>>> Reviewed-by: Andy Moreton <amoreton@solarflare.com>
>>>> <...>
>>>>
>>>>> diff --git a/drivers/net/sfc/sfc_ef10_rx.c b/drivers/net/sfc/sfc_ef10_rx.c
>>>>> index 1484bab..60812cb 100644
>>>>> --- a/drivers/net/sfc/sfc_ef10_rx.c
>>>>> +++ b/drivers/net/sfc/sfc_ef10_rx.c
>>>>> @@ -699,7 +699,7 @@ struct sfc_dp_rx sfc_ef10_rx = {
>>>>> .type = SFC_DP_RX,
>>>>> .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10,
>>>>> },
>>>>> - .features = 0,
>>>>> + .features = SFC_DP_RX_FEAT_MULTI_PROCESS,
>>>> Why this flag is needed, I mean why multi process support is not always
>>>> enabled by default?
>>> libefx-based datapath intensively uses function pointers (primary
>>> process function pointers stored in data structures). So, it does not
>>> work in multi process.
>> But this currently added always, if I don't miss anything. And only
>> checked once in secondary path and error returned if not set.
>>
>> Is there any code path that behaves different based on this flag? Or any
>> case that this flags shouldn't be set?
>
> sfc_efx_rx (and sfc_efx_tx) does not have the flag.
Got it, thanks.
>
>> What happens if this flag removed, and assumed it is always set? (this
>> and tx version of this flag)
>
> Init in the secondary process fails if datapath chosen by the primary
> process does not have the flag.
>
>>>> <...>
>
>
On 22/05/2017 13:36, Ferruh Yigit wrote:
> On 5/22/2017 1:07 PM, Andrew Rybchenko wrote:
>> On 05/22/2017 02:29 PM, Ferruh Yigit wrote:
>>> On 5/18/2017 3:00 PM, Andrew Rybchenko wrote:
>>>> Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>
>>>> Reviewed-by: Andy Moreton <amoreton@solarflare.com>
>>> <...>
>>>
>>>> Linux VFIO = Y
>>>> diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h
>>>> index 772a713..007ed24 100644
>>>> --- a/drivers/net/sfc/sfc.h
>>>> +++ b/drivers/net/sfc/sfc.h
>>>> @@ -225,7 +225,18 @@ struct sfc_adapter {
>>>> uint8_t rss_key[SFC_RSS_KEY_SIZE];
>>>> #endif
>>>>
>>>> + /*
>>>> + * Shared memory copy of the Rx datapath name to be used by
>>>> + * the secondary process to find Rx datapath to be used.
>>>> + */
>>>> + char *dp_rx_name;
>>> Why not use sa->dp_rx->dp.name to find the dp_rx? That variable should
>>> be shared between processes already?
>> sa->dp_rx is a pointer to .data section (sfc_efx_rx or sfc_ef10_rx)
>> which is (may be) different in primary and secondary processes.
> OK, thanks.
> Does it make sense to implement strdup as rte_strdup, so others can
> re-use it? @sergio, what do you think?
IMHO I would hold until there are more cases using it.
Sergio
@@ -28,6 +28,7 @@ Packet type parsing = Y
Basic stats = Y
Extended stats = Y
FW version = Y
+Multiprocess aware = Y
BSD nic_uio = Y
Linux UIO = Y
Linux VFIO = Y
@@ -225,7 +225,18 @@ struct sfc_adapter {
uint8_t rss_key[SFC_RSS_KEY_SIZE];
#endif
+ /*
+ * Shared memory copy of the Rx datapath name to be used by
+ * the secondary process to find Rx datapath to be used.
+ */
+ char *dp_rx_name;
const struct sfc_dp_rx *dp_rx;
+
+ /*
+ * Shared memory copy of the Tx datapath name to be used by
+ * the secondary process to find Rx datapath to be used.
+ */
+ char *dp_tx_name;
const struct sfc_dp_tx *dp_tx;
};
@@ -161,6 +161,7 @@ struct sfc_dp_rx {
unsigned int features;
#define SFC_DP_RX_FEAT_SCATTER 0x1
+#define SFC_DP_RX_FEAT_MULTI_PROCESS 0x2
sfc_dp_rx_qcreate_t *qcreate;
sfc_dp_rx_qdestroy_t *qdestroy;
sfc_dp_rx_qstart_t *qstart;
@@ -135,6 +135,7 @@ struct sfc_dp_tx {
#define SFC_DP_TX_FEAT_VLAN_INSERT 0x1
#define SFC_DP_TX_FEAT_TSO 0x2
#define SFC_DP_TX_FEAT_MULTI_SEG 0x4
+#define SFC_DP_TX_FEAT_MULTI_PROCESS 0x8
sfc_dp_tx_qcreate_t *qcreate;
sfc_dp_tx_qdestroy_t *qdestroy;
sfc_dp_tx_qstart_t *qstart;
@@ -699,7 +699,7 @@ struct sfc_dp_rx sfc_ef10_rx = {
.type = SFC_DP_RX,
.hw_fw_caps = SFC_DP_HW_FW_CAP_EF10,
},
- .features = 0,
+ .features = SFC_DP_RX_FEAT_MULTI_PROCESS,
.qcreate = sfc_ef10_rx_qcreate,
.qdestroy = sfc_ef10_rx_qdestroy,
.qstart = sfc_ef10_rx_qstart,
@@ -534,7 +534,8 @@ struct sfc_dp_tx sfc_ef10_tx = {
.type = SFC_DP_TX,
.hw_fw_caps = SFC_DP_HW_FW_CAP_EF10,
},
- .features = SFC_DP_TX_FEAT_MULTI_SEG,
+ .features = SFC_DP_TX_FEAT_MULTI_SEG |
+ SFC_DP_TX_FEAT_MULTI_PROCESS,
.qcreate = sfc_ef10_tx_qcreate,
.qdestroy = sfc_ef10_tx_qdestroy,
.qstart = sfc_ef10_tx_qstart,
@@ -549,7 +550,7 @@ struct sfc_dp_tx sfc_ef10_simple_tx = {
.name = SFC_KVARG_DATAPATH_EF10_SIMPLE,
.type = SFC_DP_TX,
},
- .features = 0,
+ .features = SFC_DP_TX_FEAT_MULTI_PROCESS,
.qcreate = sfc_ef10_tx_qcreate,
.qdestroy = sfc_ef10_tx_qdestroy,
.qstart = sfc_ef10_tx_qstart,
@@ -913,6 +913,10 @@ sfc_set_mc_addr_list(struct rte_eth_dev *dev, struct ether_addr *mc_addr_set,
return -rc;
}
+/*
+ * The function is used by the secondary process as well. It must not
+ * use any process-local pointers from the adapter data.
+ */
static void
sfc_rx_queue_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
struct rte_eth_rxq_info *qinfo)
@@ -939,6 +943,10 @@ sfc_rx_queue_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
sfc_adapter_unlock(sa);
}
+/*
+ * The function is used by the secondary process as well. It must not
+ * use any process-local pointers from the adapter data.
+ */
static void
sfc_tx_queue_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
struct rte_eth_txq_info *qinfo)
@@ -1372,6 +1380,29 @@ static const struct eth_dev_ops sfc_eth_dev_ops = {
.fw_version_get = sfc_fw_version_get,
};
+/**
+ * Duplicate a string in potentially shared memory required for
+ * multi-process support.
+ *
+ * strdup() allocates from process-local heap/memory.
+ */
+static char *
+sfc_strdup(const char *str)
+{
+ size_t size;
+ char *copy;
+
+ if (str == NULL)
+ return NULL;
+
+ size = strlen(str) + 1;
+ copy = rte_malloc(__func__, size, 0);
+ if (copy != NULL)
+ rte_memcpy(copy, str, size);
+
+ return copy;
+}
+
static int
sfc_eth_dev_set_ops(struct rte_eth_dev *dev)
{
@@ -1419,7 +1450,13 @@ sfc_eth_dev_set_ops(struct rte_eth_dev *dev)
}
}
- sfc_info(sa, "use %s Rx datapath", sa->dp_rx->dp.name);
+ sa->dp_rx_name = sfc_strdup(sa->dp_rx->dp.name);
+ if (sa->dp_rx_name == NULL) {
+ rc = ENOMEM;
+ goto fail_dp_rx_name;
+ }
+
+ sfc_info(sa, "use %s Rx datapath", sa->dp_rx_name);
dev->rx_pkt_burst = sa->dp_rx->pkt_burst;
@@ -1452,7 +1489,13 @@ sfc_eth_dev_set_ops(struct rte_eth_dev *dev)
}
}
- sfc_info(sa, "use %s Tx datapath", sa->dp_tx->dp.name);
+ sa->dp_tx_name = sfc_strdup(sa->dp_tx->dp.name);
+ if (sa->dp_tx_name == NULL) {
+ rc = ENOMEM;
+ goto fail_dp_tx_name;
+ }
+
+ sfc_info(sa, "use %s Tx datapath", sa->dp_tx_name);
dev->tx_pkt_burst = sa->dp_tx->pkt_burst;
@@ -1460,11 +1503,16 @@ sfc_eth_dev_set_ops(struct rte_eth_dev *dev)
return 0;
+fail_dp_tx_name:
fail_dp_tx_caps:
sa->dp_tx = NULL;
fail_dp_tx:
fail_kvarg_tx_datapath:
+ rte_free(sa->dp_rx_name);
+ sa->dp_rx_name = NULL;
+
+fail_dp_rx_name:
fail_dp_rx_caps:
sa->dp_rx = NULL;
@@ -1482,10 +1530,80 @@ sfc_eth_dev_clear_ops(struct rte_eth_dev *dev)
dev->rx_pkt_burst = NULL;
dev->tx_pkt_burst = NULL;
+ rte_free(sa->dp_tx_name);
+ sa->dp_tx_name = NULL;
sa->dp_tx = NULL;
+
+ rte_free(sa->dp_rx_name);
+ sa->dp_rx_name = NULL;
sa->dp_rx = NULL;
}
+static const struct eth_dev_ops sfc_eth_dev_secondary_ops = {
+ .rxq_info_get = sfc_rx_queue_info_get,
+ .txq_info_get = sfc_tx_queue_info_get,
+};
+
+static int
+sfc_eth_dev_secondary_set_ops(struct rte_eth_dev *dev)
+{
+ /*
+ * Device private data has really many process-local pointers.
+ * Below code should be extremely careful to use data located
+ * in shared memory only.
+ */
+ struct sfc_adapter *sa = dev->data->dev_private;
+ const struct sfc_dp_rx *dp_rx;
+ const struct sfc_dp_tx *dp_tx;
+ int rc;
+
+ dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, sa->dp_rx_name);
+ if (dp_rx == NULL) {
+ sfc_err(sa, "cannot find %s Rx datapath", sa->dp_tx_name);
+ rc = ENOENT;
+ goto fail_dp_rx;
+ }
+ if (~dp_rx->features & SFC_DP_RX_FEAT_MULTI_PROCESS) {
+ sfc_err(sa, "%s Rx datapath does not support multi-process",
+ sa->dp_tx_name);
+ rc = EINVAL;
+ goto fail_dp_rx_multi_process;
+ }
+
+ dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, sa->dp_tx_name);
+ if (dp_tx == NULL) {
+ sfc_err(sa, "cannot find %s Tx datapath", sa->dp_tx_name);
+ rc = ENOENT;
+ goto fail_dp_tx;
+ }
+ if (~dp_tx->features & SFC_DP_TX_FEAT_MULTI_PROCESS) {
+ sfc_err(sa, "%s Tx datapath does not support multi-process",
+ sa->dp_tx_name);
+ rc = EINVAL;
+ goto fail_dp_tx_multi_process;
+ }
+
+ dev->rx_pkt_burst = dp_rx->pkt_burst;
+ dev->tx_pkt_burst = dp_tx->pkt_burst;
+ dev->dev_ops = &sfc_eth_dev_secondary_ops;
+
+ return 0;
+
+fail_dp_tx_multi_process:
+fail_dp_tx:
+fail_dp_rx_multi_process:
+fail_dp_rx:
+ return rc;
+}
+
+static void
+sfc_eth_dev_secondary_clear_ops(struct rte_eth_dev *dev)
+{
+ dev->dev_ops = NULL;
+ dev->tx_pkt_burst = NULL;
+ dev->rx_pkt_burst = NULL;
+}
+
static void
sfc_register_dp(void)
{
@@ -1512,6 +1630,9 @@ sfc_eth_dev_init(struct rte_eth_dev *dev)
sfc_register_dp();
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return -sfc_eth_dev_secondary_set_ops(dev);
+
/* Required for logging */
sa->pci_addr = pci_dev->addr;
sa->port_id = dev->data->port_id;
@@ -1595,8 +1716,14 @@ sfc_eth_dev_init(struct rte_eth_dev *dev)
static int
sfc_eth_dev_uninit(struct rte_eth_dev *dev)
{
- struct sfc_adapter *sa = dev->data->dev_private;
+ struct sfc_adapter *sa;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ sfc_eth_dev_secondary_clear_ops(dev);
+ return 0;
+ }
+ sa = dev->data->dev_private;
sfc_log_init(sa, "entry");
sfc_adapter_lock(sa);