Hi Qi,
I wanted to push this old series, but I still have some questions
and comments. Please let's fix them quickly.
Note: I did not review the IPC mechanism and rollback. I trust you :)
28/09/2018 06:23, Qi Zhang:
> --- a/doc/guides/rel_notes/release_18_11.rst
> +++ b/doc/guides/rel_notes/release_18_11.rst
> @@ -67,6 +67,12 @@ New Features
> SR-IOV option in Hyper-V and Azure. This is an alternative to the previous
> vdev_netvsc, tap, and failsafe drivers combination.
>
> +* **Support device multi-process hotplug.**
> +
> + Hotplug and hot-unplug for devices will now be supported in multiprocessing
> + scenario. Any ethdev devices created in the primary process will be regarded
> + as shared and will be available for all DPDK processes. Synchronization
> + between processes will be done using DPDK IPC.
>
A blank line is missing here.
> API Changes
> -----------
> @@ -91,6 +97,11 @@ API Changes
> flag the MAC can be properly configured in any case. This is particularly
> important for bonding.
>
> +* eal: scope of rte_eal_hotplug_add and rte_eal_hotplug_remove is extended.
> +
> + In primary-secondary process model, ``rte_eal_hotplug_add`` will guarantee
> + that device be attached on all processes, while ``rte_eal_hotplug_remove``
> + will guarantee device be detached on all processes.
>
Here too, double blank line before next heading.
> --- a/lib/librte_eal/common/eal_common_dev.c
> +++ b/lib/librte_eal/common/eal_common_dev.c
> -int
> -rte_eal_hotplug_add(const char *busname, const char *devname,
> - const char *drvargs)
> +/* help funciton to build devargs, caller should free the memory */
"help funciton" -> "helper function"
> +static char *
> +build_devargs(const char *busname, const char *devname,
> + const char *drvargs)
> {
> char *devargs = NULL;
> int size, length = -1;
> @@ -140,19 +143,33 @@ rte_eal_hotplug_add(const char *busname, const char *devname,
> if (length >= size)
> devargs = malloc(length + 1);
> if (devargs == NULL)
> - return -ENOMEM;
> + break;
> } while (size == 0);
It is an old code, please rebase on master.
> -int __rte_experimental
> -rte_dev_remove(struct rte_device *dev)
> +/* remove device at local process. */
> +int
> +local_dev_remove(struct rte_device *dev)
> {
> struct rte_bus *bus;
> int ret;
> @@ -248,7 +268,194 @@ rte_dev_remove(struct rte_device *dev)
> if (ret)
> RTE_LOG(ERR, EAL, "Driver cannot detach the device (%s)\n",
> dev->name);
> - rte_devargs_remove(dev->devargs);
> + else
> + rte_devargs_remove(dev->devargs);
It looks you are fixing a bug here. Good catch!
> +int __rte_experimental
> +rte_dev_probe(const char *devargs)
> +{
> + struct eal_dev_mp_req req;
> + struct rte_device *dev;
> + int ret;
> +
> + memset(&req, 0, sizeof(req));
> + req.t = EAL_DEV_REQ_TYPE_ATTACH;
> + strlcpy(req.devargs, devargs, EAL_DEV_MP_DEV_ARGS_MAX_LEN);
> +
> + if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
> + /**
> + * If in secondary process, just send IPC request to
> + * primary process.
> + */
> + ret = eal_dev_hotplug_request_to_primary(&req);
> + if (ret) {
> + RTE_LOG(ERR, EAL,
> + "Failed to send hotplug request to primary\n");
> + return -ENOMSG;
> + }
> + if (req.result)
> + RTE_LOG(ERR, EAL,
> + "Failed to hotplug add device\n");
> + return req.result;
> + }
> +
> + /* attach a shared device from primary start from here: */
> +
> + /* primary attach the new device itself. */
> + ret = local_dev_probe(devargs, &dev);
> +
> + if (ret) {
> + RTE_LOG(ERR, EAL,
> + "Failed to attach device on primary process\n");
> +
> + /**
> + * it is possible that secondary process failed to attached a
> + * device that primary process have during initialization,
> + * so for -EEXIST case, we still need to sync with secondary
> + * process.
> + */
> + if (ret != -EEXIST)
> + return ret;
> + }
> +
> + /* primary send attach sync request to secondary. */
> + ret = eal_dev_hotplug_request_to_secondary(&req);
> +
> + /* if any commnunication error, we need to rollback. */
typo: communication
> + if (ret) {
> + RTE_LOG(ERR, EAL,
> + "Failed to send hotplug add request to secondary\n");
> + ret = -ENOMSG;
> + goto rollback;
> + }
> +
> + /**
> + * if any secondary failed to attach, we need to consider if rollback
> + * is necessary.
> + */
> + if (req.result) {
> + RTE_LOG(ERR, EAL,
> + "Failed to attach device on secondary process\n");
> + ret = req.result;
> +
> + /* for -EEXIST, we don't need to rollback. */
> + if (ret == -EEXIST)
> + return ret;
> + goto rollback;
> + }
> +
> + return 0;
> +
> +rollback:
> + req.t = EAL_DEV_REQ_TYPE_ATTACH_ROLLBACK;
> +
> + /* primary send rollback request to secondary. */
> + if (eal_dev_hotplug_request_to_secondary(&req))
For all occurences of "if (function())", the coding style is requesting
an explicit check of the return value: if (function() != 0)
> + RTE_LOG(WARNING, EAL,
> + "Failed to rollback device attach on secondary."
> + "Devices in secondary may not sync with primary\n");
> +
> + /* primary rollback itself. */
> + if (local_dev_remove(dev))
> + RTE_LOG(WARNING, EAL,
> + "Failed to rollback device attach on primary."
> + "Devices in secondary may not sync with primary\n");
> +
> + return ret;
> +}
> +
> +int __rte_experimental
> +rte_dev_remove(struct rte_device *dev)
> +{
> + struct eal_dev_mp_req req;
> + char *devargs;
> + int ret;
> +
> + devargs = build_devargs(dev->devargs->bus->name, dev->name, "");
> + if (devargs == NULL)
> + return -ENOMEM;
> +
> + memset(&req, 0, sizeof(req));
> + req.t = EAL_DEV_REQ_TYPE_DETACH;
> + strlcpy(req.devargs, devargs, EAL_DEV_MP_DEV_ARGS_MAX_LEN);
> + free(devargs);
> +
> + if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
> + /**
> + * If in secondary process, just send IPC request to
> + * primary process.
> + */
> + ret = eal_dev_hotplug_request_to_primary(&req);
> + if (ret) {
> + RTE_LOG(ERR, EAL,
> + "Failed to send hotplug request to primary\n");
> + return -ENOMSG;
> + }
> + if (req.result)
> + RTE_LOG(ERR, EAL,
> + "Failed to hotplug remove device\n");
> + return req.result;
> + }
> +
> + /* detach a device from primary start from here: */
> +
> + /* primary send detach sync request to secondary */
> + ret = eal_dev_hotplug_request_to_secondary(&req);
> +
> + /**
> + * if communication error, we need to rollback, because it is possible
> + * part of the secondary processes still detached it successfully.
> + */
> + if (ret) {
ret is not a boolean, please do explicit check != 0.
> + RTE_LOG(ERR, EAL,
> + "Failed to send device detach request to secondary\n");
> + ret = -ENOMSG;
> + goto rollback;
> + }
> +
> + /**
> + * if any secondary failed to detach, we need to consider if rollback
> + * is necessary.
> + */
> + if (req.result) {
result is not a boolean, please do explicit check != 0.
> + RTE_LOG(ERR, EAL,
> + "Failed to detach device on secondary process\n");
> + ret = req.result;
> + /**
> + * if -ENOENT, we don't need to rollback, since devices is
> + * already detached on secondary process.
> + */
> + if (ret != -ENOENT)
> + goto rollback;
> + }
> +
> + /* primary detach the device itself. */
> + ret = local_dev_remove(dev);
> +
> + /* if primary failed, still need to consider if rollback is necessary */
> + if (ret) {
> + RTE_LOG(ERR, EAL,
> + "Failed to detach device on primary process\n");
> + /* if -ENOENT, we don't need to rollback */
> + if (ret == -ENOENT)
> + return ret;
> + goto rollback;
> + }
> +
> + return 0;
> +
> +rollback:
> + req.t = EAL_DEV_REQ_TYPE_DETACH_ROLLBACK;
> +
> + /* primary send rollback request to secondary. */
> + if (eal_dev_hotplug_request_to_secondary(&req))
> + RTE_LOG(WARNING, EAL,
> + "Failed to rollback device detach on secondary."
> + "Devices in secondary may not sync with primary\n");
> +
> return ret;
> }
> --- a/lib/librte_eal/common/eal_private.h
> +++ b/lib/librte_eal/common/eal_private.h
> +/**
> + * Register all mp action callbacks for hotplug.
> + *
> + * @return
> + * 0 on success, negative on error.
> + */
> +int rte_dev_hotplug_mp_init(void);
This function is called by the init, so it should not be private.
The app should be free to build its own init routine.
> --- /dev/null
> +++ b/lib/librte_eal/common/hotplug_mp.h
> +#include <rte_dev.h>
> +#include <rte_bus.h>
I think EAL headers should be included with quotes.
> +/**
> + * this is a synchronous wrapper for secondary process send
Missing uppercase at the beggining of sentence.
> + * request to primary process, this is invoked when an attach
> + * or detach request issued from primary process.
Missing "is" before "issued": request is issued.
> --- a/lib/librte_eal/common/include/rte_dev.h
> +++ b/lib/librte_eal/common/include/rte_dev.h
> @@ -190,6 +190,9 @@ int rte_eal_dev_detach(struct rte_device *dev);
>
> /**
> * Hotplug add a given device to a specific bus.
> + * In multi-process, this function will inform all other processes
> + * to hotplug add the same device. Any failure on other process rollback
> + * the action.
Better to leave a blank line before this comment.
Small reword:
* Hotplug add a given device to a specific bus.
*
* In multi-process, it will request other processes to add the same device.
* A failure, in any process, will rollback the action.
You should add the same comment for rte_dev_probe().
> @@ -219,6 +222,9 @@ int __rte_experimental rte_dev_probe(const char *devargs);
>
> /**
> * Hotplug remove a given device from a specific bus.
> + * In multi-process, this function will inform all other processes
> + * to hotplug remove the same device. Any failure on other process
> + * will rollback the action.
Same reword:
* Hotplug remove a given device from a specific bus.
*
* In multi-process, it will request other processes to remove the same device.
* A failure, in any process, will rollback the action.
[...]
> --- a/lib/librte_eal/linuxapp/eal/eal.c
> +++ b/lib/librte_eal/linuxapp/eal/eal.c
> + /* register mp action callbacks for hotplug */
> + if (rte_dev_hotplug_mp_init() < 0) {
In the comment, better to say "multi-process" instead of the cryptic "mp".
@@ -67,6 +67,12 @@ New Features
SR-IOV option in Hyper-V and Azure. This is an alternative to the previous
vdev_netvsc, tap, and failsafe drivers combination.
+* **Support device multi-process hotplug.**
+
+ Hotplug and hot-unplug for devices will now be supported in multiprocessing
+ scenario. Any ethdev devices created in the primary process will be regarded
+ as shared and will be available for all DPDK processes. Synchronization
+ between processes will be done using DPDK IPC.
API Changes
-----------
@@ -91,6 +97,11 @@ API Changes
flag the MAC can be properly configured in any case. This is particularly
important for bonding.
+* eal: scope of rte_eal_hotplug_add and rte_eal_hotplug_remove is extended.
+
+ In primary-secondary process model, ``rte_eal_hotplug_add`` will guarantee
+ that device be attached on all processes, while ``rte_eal_hotplug_remove``
+ will guarantee device be detached on all processes.
ABI Changes
-----------
@@ -62,6 +62,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_proc.c
SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_fbarray.c
SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += eal_common_uuid.c
SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_malloc.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += hotplug_mp.c
SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_elem.c
SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_heap.c
SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_mp.c
@@ -19,8 +19,10 @@
#include <rte_log.h>
#include <rte_spinlock.h>
#include <rte_malloc.h>
+#include <rte_string_fns.h>
#include "eal_private.h"
+#include "hotplug_mp.h"
/**
* The device event callback description.
@@ -127,9 +129,10 @@ int rte_eal_dev_detach(struct rte_device *dev)
return ret;
}
-int
-rte_eal_hotplug_add(const char *busname, const char *devname,
- const char *drvargs)
+/* help funciton to build devargs, caller should free the memory */
+static char *
+build_devargs(const char *busname, const char *devname,
+ const char *drvargs)
{
char *devargs = NULL;
int size, length = -1;
@@ -140,19 +143,33 @@ rte_eal_hotplug_add(const char *busname, const char *devname,
if (length >= size)
devargs = malloc(length + 1);
if (devargs == NULL)
- return -ENOMEM;
+ break;
} while (size == 0);
+ return devargs;
+}
+
+int
+rte_eal_hotplug_add(const char *busname, const char *devname,
+ const char *drvargs)
+{
+ char *devargs = build_devargs(busname, devname, drvargs);
+
+ if (devargs == NULL)
+ return -ENOMEM;
+
return rte_dev_probe(devargs);
}
-int __rte_experimental
-rte_dev_probe(const char *devargs)
+/* probe device at local process. */
+int
+local_dev_probe(const char *devargs, struct rte_device **new_dev)
{
struct rte_device *dev;
struct rte_devargs *da;
int ret;
+ *new_dev = NULL;
da = calloc(1, sizeof(*da));
if (da == NULL)
return -ENOMEM;
@@ -195,6 +212,8 @@ rte_dev_probe(const char *devargs)
dev->name);
goto err_devarg;
}
+
+ *new_dev = dev;
return 0;
err_devarg:
@@ -226,8 +245,9 @@ rte_eal_hotplug_remove(const char *busname, const char *devname)
return rte_dev_remove(dev);
}
-int __rte_experimental
-rte_dev_remove(struct rte_device *dev)
+/* remove device at local process. */
+int
+local_dev_remove(struct rte_device *dev)
{
struct rte_bus *bus;
int ret;
@@ -248,7 +268,194 @@ rte_dev_remove(struct rte_device *dev)
if (ret)
RTE_LOG(ERR, EAL, "Driver cannot detach the device (%s)\n",
dev->name);
- rte_devargs_remove(dev->devargs);
+ else
+ rte_devargs_remove(dev->devargs);
+
+ return ret;
+}
+
+int __rte_experimental
+rte_dev_probe(const char *devargs)
+{
+ struct eal_dev_mp_req req;
+ struct rte_device *dev;
+ int ret;
+
+ memset(&req, 0, sizeof(req));
+ req.t = EAL_DEV_REQ_TYPE_ATTACH;
+ strlcpy(req.devargs, devargs, EAL_DEV_MP_DEV_ARGS_MAX_LEN);
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ /**
+ * If in secondary process, just send IPC request to
+ * primary process.
+ */
+ ret = eal_dev_hotplug_request_to_primary(&req);
+ if (ret) {
+ RTE_LOG(ERR, EAL,
+ "Failed to send hotplug request to primary\n");
+ return -ENOMSG;
+ }
+ if (req.result)
+ RTE_LOG(ERR, EAL,
+ "Failed to hotplug add device\n");
+ return req.result;
+ }
+
+ /* attach a shared device from primary start from here: */
+
+ /* primary attach the new device itself. */
+ ret = local_dev_probe(devargs, &dev);
+
+ if (ret) {
+ RTE_LOG(ERR, EAL,
+ "Failed to attach device on primary process\n");
+
+ /**
+ * it is possible that secondary process failed to attached a
+ * device that primary process have during initialization,
+ * so for -EEXIST case, we still need to sync with secondary
+ * process.
+ */
+ if (ret != -EEXIST)
+ return ret;
+ }
+
+ /* primary send attach sync request to secondary. */
+ ret = eal_dev_hotplug_request_to_secondary(&req);
+
+ /* if any commnunication error, we need to rollback. */
+ if (ret) {
+ RTE_LOG(ERR, EAL,
+ "Failed to send hotplug add request to secondary\n");
+ ret = -ENOMSG;
+ goto rollback;
+ }
+
+ /**
+ * if any secondary failed to attach, we need to consider if rollback
+ * is necessary.
+ */
+ if (req.result) {
+ RTE_LOG(ERR, EAL,
+ "Failed to attach device on secondary process\n");
+ ret = req.result;
+
+ /* for -EEXIST, we don't need to rollback. */
+ if (ret == -EEXIST)
+ return ret;
+ goto rollback;
+ }
+
+ return 0;
+
+rollback:
+ req.t = EAL_DEV_REQ_TYPE_ATTACH_ROLLBACK;
+
+ /* primary send rollback request to secondary. */
+ if (eal_dev_hotplug_request_to_secondary(&req))
+ RTE_LOG(WARNING, EAL,
+ "Failed to rollback device attach on secondary."
+ "Devices in secondary may not sync with primary\n");
+
+ /* primary rollback itself. */
+ if (local_dev_remove(dev))
+ RTE_LOG(WARNING, EAL,
+ "Failed to rollback device attach on primary."
+ "Devices in secondary may not sync with primary\n");
+
+ return ret;
+}
+
+int __rte_experimental
+rte_dev_remove(struct rte_device *dev)
+{
+ struct eal_dev_mp_req req;
+ char *devargs;
+ int ret;
+
+ devargs = build_devargs(dev->devargs->bus->name, dev->name, "");
+ if (devargs == NULL)
+ return -ENOMEM;
+
+ memset(&req, 0, sizeof(req));
+ req.t = EAL_DEV_REQ_TYPE_DETACH;
+ strlcpy(req.devargs, devargs, EAL_DEV_MP_DEV_ARGS_MAX_LEN);
+ free(devargs);
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ /**
+ * If in secondary process, just send IPC request to
+ * primary process.
+ */
+ ret = eal_dev_hotplug_request_to_primary(&req);
+ if (ret) {
+ RTE_LOG(ERR, EAL,
+ "Failed to send hotplug request to primary\n");
+ return -ENOMSG;
+ }
+ if (req.result)
+ RTE_LOG(ERR, EAL,
+ "Failed to hotplug remove device\n");
+ return req.result;
+ }
+
+ /* detach a device from primary start from here: */
+
+ /* primary send detach sync request to secondary */
+ ret = eal_dev_hotplug_request_to_secondary(&req);
+
+ /**
+ * if communication error, we need to rollback, because it is possible
+ * part of the secondary processes still detached it successfully.
+ */
+ if (ret) {
+ RTE_LOG(ERR, EAL,
+ "Failed to send device detach request to secondary\n");
+ ret = -ENOMSG;
+ goto rollback;
+ }
+
+ /**
+ * if any secondary failed to detach, we need to consider if rollback
+ * is necessary.
+ */
+ if (req.result) {
+ RTE_LOG(ERR, EAL,
+ "Failed to detach device on secondary process\n");
+ ret = req.result;
+ /**
+ * if -ENOENT, we don't need to rollback, since devices is
+ * already detached on secondary process.
+ */
+ if (ret != -ENOENT)
+ goto rollback;
+ }
+
+ /* primary detach the device itself. */
+ ret = local_dev_remove(dev);
+
+ /* if primary failed, still need to consider if rollback is necessary */
+ if (ret) {
+ RTE_LOG(ERR, EAL,
+ "Failed to detach device on primary process\n");
+ /* if -ENOENT, we don't need to rollback */
+ if (ret == -ENOENT)
+ return ret;
+ goto rollback;
+ }
+
+ return 0;
+
+rollback:
+ req.t = EAL_DEV_REQ_TYPE_DETACH_ROLLBACK;
+
+ /* primary send rollback request to secondary. */
+ if (eal_dev_hotplug_request_to_secondary(&req))
+ RTE_LOG(WARNING, EAL,
+ "Failed to rollback device detach on secondary."
+ "Devices in secondary may not sync with primary\n");
+
return ret;
}
@@ -304,4 +304,34 @@ int
rte_devargs_layers_parse(struct rte_devargs *devargs,
const char *devstr);
+/*
+ * probe a device at local process.
+ *
+ * @param devargs
+ * Device arguments including bus, class and driver properties.
+ * @param new_dev
+ * new device be probed as output.
+ * @return
+ * 0 on success, negative on error.
+ */
+int local_dev_probe(const char *devargs, struct rte_device **new_dev);
+
+/**
+ * Hotplug remove a given device from a specific bus at local process.
+ *
+ * @param dev
+ * Data structure of the device to remove.
+ * @return
+ * 0 on success, negative on error.
+ */
+int local_dev_remove(struct rte_device *dev);
+
+/**
+ * Register all mp action callbacks for hotplug.
+ *
+ * @return
+ * 0 on success, negative on error.
+ */
+int rte_dev_hotplug_mp_init(void);
+
#endif /* _EAL_PRIVATE_H_ */
new file mode 100644
@@ -0,0 +1,221 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+#include <string.h>
+
+#include <rte_eal.h>
+#include <rte_alarm.h>
+#include <rte_string_fns.h>
+#include <rte_devargs.h>
+
+#include "hotplug_mp.h"
+#include "eal_private.h"
+
+#define MP_TIMEOUT_S 5 /**< 5 seconds timeouts */
+
+static int cmp_dev_name(const struct rte_device *dev, const void *_name)
+{
+ const char *name = _name;
+
+ return strcmp(dev->name, name);
+}
+
+struct mp_reply_bundle {
+ struct rte_mp_msg msg;
+ void *peer;
+};
+
+static int
+handle_secondary_request(const struct rte_mp_msg *msg, const void *peer)
+{
+ RTE_SET_USED(msg);
+ RTE_SET_USED(peer);
+ return -ENOTSUP;
+}
+
+static void __handle_primary_request(void *param)
+{
+ struct mp_reply_bundle *bundle = param;
+ struct rte_mp_msg *msg = &bundle->msg;
+ const struct eal_dev_mp_req *req =
+ (const struct eal_dev_mp_req *)msg->param;
+ struct rte_mp_msg mp_resp;
+ struct eal_dev_mp_req *resp =
+ (struct eal_dev_mp_req *)mp_resp.param;
+ struct rte_devargs *da;
+ struct rte_device *dev;
+ struct rte_bus *bus;
+ int ret = 0;
+
+ memset(&mp_resp, 0, sizeof(mp_resp));
+
+ switch (req->t) {
+ case EAL_DEV_REQ_TYPE_ATTACH:
+ case EAL_DEV_REQ_TYPE_DETACH_ROLLBACK:
+ ret = local_dev_probe(req->devargs, &dev);
+ break;
+ case EAL_DEV_REQ_TYPE_DETACH:
+ case EAL_DEV_REQ_TYPE_ATTACH_ROLLBACK:
+ da = calloc(1, sizeof(*da));
+ if (da == NULL) {
+ ret = -ENOMEM;
+ goto quit;
+ }
+
+ ret = rte_devargs_parse(da, req->devargs);
+ if (ret)
+ goto quit;
+
+ bus = rte_bus_find_by_name(da->bus->name);
+ if (bus == NULL) {
+ RTE_LOG(ERR, EAL, "Cannot find bus (%s)\n", da->bus->name);
+ ret = -ENOENT;
+ goto quit;
+ }
+
+ dev = bus->find_device(NULL, cmp_dev_name, da->name);
+ if (dev == NULL) {
+ RTE_LOG(ERR, EAL, "Cannot find plugged device (%s)\n", da->name);
+ ret = -ENOENT;
+ goto quit;
+ }
+
+ ret = local_dev_remove(dev);
+quit:
+ break;
+ default:
+ ret = -EINVAL;
+ }
+
+ strlcpy(mp_resp.name, EAL_DEV_MP_ACTION_REQUEST, sizeof(mp_resp.name));
+ mp_resp.len_param = sizeof(*req);
+ memcpy(resp, req, sizeof(*resp));
+ resp->result = ret;
+ if (rte_mp_reply(&mp_resp, bundle->peer) < 0)
+ RTE_LOG(ERR, EAL, "failed to send reply to primary request\n");
+
+ free(bundle->peer);
+ free(bundle);
+}
+
+static int
+handle_primary_request(const struct rte_mp_msg *msg, const void *peer)
+{
+ struct rte_mp_msg mp_resp;
+ const struct eal_dev_mp_req *req =
+ (const struct eal_dev_mp_req *)msg->param;
+ struct eal_dev_mp_req *resp =
+ (struct eal_dev_mp_req *)mp_resp.param;
+ struct mp_reply_bundle *bundle;
+ int ret = 0;
+
+ memset(&mp_resp, 0, sizeof(mp_resp));
+ strlcpy(mp_resp.name, EAL_DEV_MP_ACTION_REQUEST, sizeof(mp_resp.name));
+ mp_resp.len_param = sizeof(*req);
+ memcpy(resp, req, sizeof(*resp));
+
+ bundle = calloc(1, sizeof(*bundle));
+ if (bundle == NULL) {
+ resp->result = -ENOMEM;
+ ret = rte_mp_reply(&mp_resp, peer);
+ if (ret)
+ RTE_LOG(ERR, EAL, "failed to send reply to primary request\n");
+ return ret;
+ }
+
+ bundle->msg = *msg;
+ /**
+ * We need to send reply on interrupt thread, but peer can't be
+ * parsed directly, so this is a temporal hack, need to be fixed
+ * when it is ready.
+ */
+ bundle->peer = (void *)strdup(peer);
+
+ /**
+ * We are at IPC callback thread, sync IPC is not allowed due to
+ * dead lock, so we delegate the task to interrupt thread.
+ */
+ ret = rte_eal_alarm_set(1, __handle_primary_request, bundle);
+ if (ret) {
+ resp->result = ret;
+ ret = rte_mp_reply(&mp_resp, peer);
+ if (ret) {
+ RTE_LOG(ERR, EAL, "failed to send reply to primary request\n");
+ return ret;
+ }
+ }
+ return 0;
+}
+
+int eal_dev_hotplug_request_to_primary(struct eal_dev_mp_req *req)
+{
+ RTE_SET_USED(req);
+ return -ENOTSUP;
+}
+
+int eal_dev_hotplug_request_to_secondary(struct eal_dev_mp_req *req)
+{
+ struct rte_mp_msg mp_req;
+ struct rte_mp_reply mp_reply;
+ struct timespec ts = {.tv_sec = MP_TIMEOUT_S, .tv_nsec = 0};
+ int ret;
+ int i;
+
+ memset(&mp_req, 0, sizeof(mp_req));
+ memcpy(mp_req.param, req, sizeof(*req));
+ mp_req.len_param = sizeof(*req);
+ strlcpy(mp_req.name, EAL_DEV_MP_ACTION_REQUEST, sizeof(mp_req.name));
+
+ ret = rte_mp_request_sync(&mp_req, &mp_reply, &ts);
+ if (ret) {
+ RTE_LOG(ERR, EAL, "rte_mp_request_sync failed\n");
+ return ret;
+ }
+
+ if (mp_reply.nb_sent != mp_reply.nb_received) {
+ RTE_LOG(ERR, EAL, "not all secondary reply\n");
+ return -1;
+ }
+
+ req->result = 0;
+ for (i = 0; i < mp_reply.nb_received; i++) {
+ struct eal_dev_mp_req *resp =
+ (struct eal_dev_mp_req *)mp_reply.msgs[i].param;
+ if (resp->result) {
+ req->result = resp->result;
+ if (req->t == EAL_DEV_REQ_TYPE_ATTACH &&
+ req->result != -EEXIST)
+ break;
+ if (req->t == EAL_DEV_REQ_TYPE_DETACH &&
+ req->result != -ENOENT)
+ break;
+ }
+ }
+
+ return 0;
+}
+
+int rte_dev_hotplug_mp_init(void)
+{
+ int ret;
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ ret = rte_mp_action_register(EAL_DEV_MP_ACTION_REQUEST,
+ handle_secondary_request);
+ if (ret) {
+ RTE_LOG(ERR, EAL, "Couldn't register '%s' action\n",
+ EAL_DEV_MP_ACTION_REQUEST);
+ return ret;
+ }
+ } else {
+ ret = rte_mp_action_register(EAL_DEV_MP_ACTION_REQUEST,
+ handle_primary_request);
+ if (ret) {
+ RTE_LOG(ERR, EAL, "Couldn't register '%s' action\n",
+ EAL_DEV_MP_ACTION_REQUEST);
+ return ret;
+ }
+ }
+
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef _HOTPLUG_MP_H_
+#define _HOTPLUG_MP_H_
+
+#include <rte_dev.h>
+#include <rte_bus.h>
+
+#define EAL_DEV_MP_ACTION_REQUEST "eal_dev_mp_request"
+#define EAL_DEV_MP_ACTION_RESPONSE "eal_dev_mp_response"
+
+#define EAL_DEV_MP_DEV_NAME_MAX_LEN RTE_DEV_NAME_MAX_LEN
+#define EAL_DEV_MP_BUS_NAME_MAX_LEN 32
+#define EAL_DEV_MP_DEV_ARGS_MAX_LEN 128
+
+enum eal_dev_req_type {
+ EAL_DEV_REQ_TYPE_ATTACH,
+ EAL_DEV_REQ_TYPE_DETACH,
+ EAL_DEV_REQ_TYPE_ATTACH_ROLLBACK,
+ EAL_DEV_REQ_TYPE_DETACH_ROLLBACK,
+};
+
+struct eal_dev_mp_req {
+ enum eal_dev_req_type t;
+ char devargs[EAL_DEV_MP_DEV_ARGS_MAX_LEN];
+ int result;
+};
+
+/**
+ * this is a synchronous wrapper for secondary process send
+ * request to primary process, this is invoked when an attach
+ * or detach request issued from primary process.
+ */
+int eal_dev_hotplug_request_to_primary(struct eal_dev_mp_req *req);
+
+/**
+ * this is a synchronous wrapper for primary process send
+ * request to secondary process, this is invoked when an attach
+ * or detach request issued from secondary process.
+ */
+int eal_dev_hotplug_request_to_secondary(struct eal_dev_mp_req *req);
+
+
+#endif /* _HOTPLUG_MP_H_ */
@@ -190,6 +190,9 @@ int rte_eal_dev_detach(struct rte_device *dev);
/**
* Hotplug add a given device to a specific bus.
+ * In multi-process, this function will inform all other processes
+ * to hotplug add the same device. Any failure on other process rollback
+ * the action.
*
* @param busname
* The bus name the device is added to.
@@ -219,6 +222,9 @@ int __rte_experimental rte_dev_probe(const char *devargs);
/**
* Hotplug remove a given device from a specific bus.
+ * In multi-process, this function will inform all other processes
+ * to hotplug remove the same device. Any failure on other process
+ * will rollback the action.
*
* @param busname
* The bus name the device is removed from.
@@ -234,6 +240,9 @@ int rte_eal_hotplug_remove(const char *busname, const char *devname);
* @b EXPERIMENTAL: this API may change without prior notice
*
* Remove one device.
+ * In multi-process, this function will inform all other processes
+ * to hotplug remove the same device. Any failure on other process
+ * will rollback the action.
*
* @param dev
* Data structure of the device to remove.
@@ -28,6 +28,7 @@ common_sources = files(
'eal_common_thread.c',
'eal_common_timer.c',
'eal_common_uuid.c',
+ 'hotplug_mp.c',
'malloc_elem.c',
'malloc_heap.c',
'malloc_mp.c',
@@ -70,6 +70,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_proc.c
SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_fbarray.c
SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += eal_common_uuid.c
SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_malloc.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += hotplug_mp.c
SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_elem.c
SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_heap.c
SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_mp.c
@@ -865,6 +865,12 @@ rte_eal_init(int argc, char **argv)
}
}
+ /* register mp action callbacks for hotplug */
+ if (rte_dev_hotplug_mp_init() < 0) {
+ rte_eal_init_alert("failed to register mp callback for hotplug\n");
+ return -1;
+ }
+
if (rte_bus_scan()) {
rte_eal_init_alert("Cannot scan the buses for devices\n");
rte_errno = ENODEV;