[V5,6/7] eal: add failure handle mechanism for hotplug
diff mbox series

Message ID 1530776333-30318-7-git-send-email-jia.guo@intel.com
State Superseded, archived
Delegated to: Thomas Monjalon
Headers show
Series
  • hot plug failure handle mechanism
Related show

Checks

Context Check Description
ci/checkpatch warning coding style issues
ci/Intel-compilation success Compilation OK

Commit Message

Jeff Guo July 5, 2018, 7:38 a.m. UTC
This patch introduces a failure handler mechanism to handle device
hot plug removal event.

First register sigbus handler, once sigbus error be captured, will
check the failure address and accordingly remap the invalid memory
for the corresponding device. Bese on this mechanism, it could
guaranty the application not to be crash when hotplug out device.

Signed-off-by: Jeff Guo <jia.guo@intel.com>
---
v5->v4:
add sigbus old handler recover.
---
 lib/librte_eal/linuxapp/eal/eal_dev.c | 111 +++++++++++++++++++++++++++++++++-
 1 file changed, 110 insertions(+), 1 deletion(-)

Comments

He, Shaopeng July 6, 2018, 3:22 p.m. UTC | #1
> -----Original Message-----
> From: Guo, Jia
> Sent: Thursday, July 5, 2018 3:39 PM
> 
> This patch introduces a failure handler mechanism to handle device
> hot plug removal event.
> 
> First register sigbus handler, once sigbus error be captured, will
> check the failure address and accordingly remap the invalid memory
> for the corresponding device. Bese on this mechanism, it could

" Besed on this mechanism "?

> guaranty the application not to be crash when hotplug out device.
> 
> Signed-off-by: Jeff Guo <jia.guo@intel.com>

Acked-by: Shaopeng He <shaopeng.he@intel.com>
Andrew Rybchenko July 8, 2018, 1:46 p.m. UTC | #2
On 05.07.2018 10:38, Jeff Guo wrote:
> This patch introduces a failure handler mechanism to handle device
> hot plug removal event.
>
> First register sigbus handler, once sigbus error be captured, will
> check the failure address and accordingly remap the invalid memory
> for the corresponding device. Bese on this mechanism, it could
> guaranty the application not to be crash when hotplug out device.
>
> Signed-off-by: Jeff Guo <jia.guo@intel.com>
> ---
> v5->v4:
> add sigbus old handler recover.
> ---
>   lib/librte_eal/linuxapp/eal/eal_dev.c | 111 +++++++++++++++++++++++++++++++++-
>   1 file changed, 110 insertions(+), 1 deletion(-)
>
> diff --git a/lib/librte_eal/linuxapp/eal/eal_dev.c b/lib/librte_eal/linuxapp/eal/eal_dev.c
> index 1cf6aeb..a22cb9a 100644
> --- a/lib/librte_eal/linuxapp/eal/eal_dev.c
> +++ b/lib/librte_eal/linuxapp/eal/eal_dev.c
> @@ -4,6 +4,8 @@
>   
>   #include <string.h>
>   #include <unistd.h>
> +#include <fcntl.h>
> +#include <signal.h>
>   #include <sys/socket.h>
>   #include <linux/netlink.h>
>   
> @@ -14,15 +16,28 @@
>   #include <rte_malloc.h>
>   #include <rte_interrupts.h>
>   #include <rte_alarm.h>
> +#include <rte_bus.h>
> +#include <rte_eal.h>
> +#include <rte_spinlock.h>
> +#include <rte_errno.h>
>   
>   #include "eal_private.h"
>   
>   static struct rte_intr_handle intr_handle = {.fd = -1 };
>   static bool monitor_started;
>   
> +extern struct rte_bus_list rte_bus_list;
> +

Shouldn't rte_bus.h provide it?

>   #define EAL_UEV_MSG_LEN 4096
>   #define EAL_UEV_MSG_ELEM_LEN 128
>   
> +/* spinlock for device failure process */
> +static rte_spinlock_t dev_failure_lock = RTE_SPINLOCK_INITIALIZER;

It would be useful to explain why the lock is needed and when
it should be obtained/released. Which resources are protected
by the lock?

> +
> +static struct sigaction sigbus_action_old;
> +
> +static int sigbus_need_recover;
> +
>   static void dev_uev_handler(__rte_unused void *param);
>   
>   /* identify the system layer which reports this event. */
> @@ -33,6 +48,49 @@ enum eal_dev_event_subsystem {
>   	EAL_DEV_EVENT_SUBSYSTEM_MAX
>   };
>   
> +static void
> +sigbus_action_recover(void)
> +{
> +	if (sigbus_need_recover) {
> +		sigaction(SIGBUS, &sigbus_action_old, NULL);
> +		sigbus_need_recover = 0;
> +	}
> +}
> +
> +static void sigbus_handler(int signum, siginfo_t *info,
> +				void *ctx __rte_unused)
> +{
> +	int ret;
> +
> +	RTE_LOG(INFO, EAL, "Thread[%d] catch SIGBUS, fault address:%p\n",
> +		(int)pthread_self(), info->si_addr);
> +
> +	rte_spinlock_lock(&dev_failure_lock);
> +	ret = rte_bus_sigbus_handler(info->si_addr);
> +	rte_spinlock_unlock(&dev_failure_lock);
> +	if (ret == -1) {
> +		rte_exit(EXIT_FAILURE,
> +			 "Failed to handle SIGBUS for hotplug, "
> +			 "(rte_errno: %s)!", strerror(rte_errno));
> +	} else if (ret == 1) {
> +		if (sigbus_action_old.sa_handler)
> +			(*(sigbus_action_old.sa_handler))(signum);
> +		else
> +			rte_exit(EXIT_FAILURE,
> +				 "Failed to handle generic SIGBUS!");
> +	}
> +
> +	RTE_LOG(INFO, EAL, "Success to handle SIGBUS for hotplug!\n");
> +}
> +
> +static int cmp_dev_name(const struct rte_device *dev,
> +	const void *_name)
> +{
> +	const char *name = _name;
> +
> +	return strcmp(dev->name, name);
> +}
> +
>   static int
>   dev_uev_socket_fd_create(void)
>   {
> @@ -147,6 +205,9 @@ dev_uev_handler(__rte_unused void *param)
>   	struct rte_dev_event uevent;
>   	int ret;
>   	char buf[EAL_UEV_MSG_LEN];
> +	struct rte_bus *bus;
> +	struct rte_device *dev;
> +	const char *busname;
>   
>   	memset(&uevent, 0, sizeof(struct rte_dev_event));
>   	memset(buf, 0, EAL_UEV_MSG_LEN);
> @@ -171,13 +232,50 @@ dev_uev_handler(__rte_unused void *param)
>   	RTE_LOG(DEBUG, EAL, "receive uevent(name:%s, type:%d, subsystem:%d)\n",
>   		uevent.devname, uevent.type, uevent.subsystem);
>   
> -	if (uevent.devname)
> +	switch (uevent.subsystem) {
> +	case EAL_DEV_EVENT_SUBSYSTEM_PCI:
> +	case EAL_DEV_EVENT_SUBSYSTEM_UIO:
> +		busname = "pci";
> +		break;
> +	default:
> +		break;
> +	}
> +
> +	if (uevent.devname) {
> +		if (uevent.type == RTE_DEV_EVENT_REMOVE) {
> +			rte_spinlock_lock(&dev_failure_lock);
> +			bus = rte_bus_find_by_name(busname);

It looks like busname could be uninitialized here.

> +			if (bus == NULL) {
> +				RTE_LOG(ERR, EAL, "Cannot find bus (%s)\n",
> +					busname);
> +				return;
> +			}
> +
> +			dev = bus->find_device(NULL, cmp_dev_name,
> +					       uevent.devname);
> +			if (dev == NULL) {
> +				RTE_LOG(ERR, EAL, "Cannot find device (%s) on "
> +					"bus (%s)\n", uevent.devname, busname);
> +				return;
> +			}
> +
> +			ret = bus->hotplug_failure_handler(dev);
> +			rte_spinlock_unlock(&dev_failure_lock);
> +			if (ret) {
> +				RTE_LOG(ERR, EAL, "Can not handle hotplug for "
> +					"device (%s)\n", dev->name);
> +				return;
> +			}
> +		}
>   		dev_callback_process(uevent.devname, uevent.type);
> +	}
>   }
>   
>   int __rte_experimental
>   rte_dev_event_monitor_start(void)
>   {
> +	sigset_t mask;
> +	struct sigaction action;
>   	int ret;
>   
>   	if (monitor_started)
> @@ -197,6 +295,14 @@ rte_dev_event_monitor_start(void)
>   		return -1;
>   	}
>   
> +	/* register sigbus handler */
> +	sigemptyset(&mask);
> +	sigaddset(&mask, SIGBUS);
> +	action.sa_flags = SA_SIGINFO;
> +	action.sa_mask = mask;
> +	action.sa_sigaction = sigbus_handler;
> +	sigbus_need_recover = !sigaction(SIGBUS, &action, &sigbus_action_old);
> +
>   	monitor_started = true;
>   
>   	return 0;
> @@ -217,8 +323,11 @@ rte_dev_event_monitor_stop(void)
>   		return ret;
>   	}
>   
> +	sigbus_action_recover();
> +
>   	close(intr_handle.fd);
>   	intr_handle.fd = -1;
>   	monitor_started = false;
> +
>   	return 0;
>   }
Jeff Guo July 9, 2018, 5:40 a.m. UTC | #3
On 7/8/2018 9:46 PM, Andrew Rybchenko wrote:
> On 05.07.2018 10:38, Jeff Guo wrote:
>> This patch introduces a failure handler mechanism to handle device
>> hot plug removal event.
>>
>> First register sigbus handler, once sigbus error be captured, will
>> check the failure address and accordingly remap the invalid memory
>> for the corresponding device. Bese on this mechanism, it could
>> guaranty the application not to be crash when hotplug out device.
>>
>> Signed-off-by: Jeff Guo <jia.guo@intel.com>
>> ---
>> v5->v4:
>> add sigbus old handler recover.
>> ---
>>   lib/librte_eal/linuxapp/eal/eal_dev.c | 111 
>> +++++++++++++++++++++++++++++++++-
>>   1 file changed, 110 insertions(+), 1 deletion(-)
>>
>> diff --git a/lib/librte_eal/linuxapp/eal/eal_dev.c 
>> b/lib/librte_eal/linuxapp/eal/eal_dev.c
>> index 1cf6aeb..a22cb9a 100644
>> --- a/lib/librte_eal/linuxapp/eal/eal_dev.c
>> +++ b/lib/librte_eal/linuxapp/eal/eal_dev.c
>> @@ -4,6 +4,8 @@
>>     #include <string.h>
>>   #include <unistd.h>
>> +#include <fcntl.h>
>> +#include <signal.h>
>>   #include <sys/socket.h>
>>   #include <linux/netlink.h>
>>   @@ -14,15 +16,28 @@
>>   #include <rte_malloc.h>
>>   #include <rte_interrupts.h>
>>   #include <rte_alarm.h>
>> +#include <rte_bus.h>
>> +#include <rte_eal.h>
>> +#include <rte_spinlock.h>
>> +#include <rte_errno.h>
>>     #include "eal_private.h"
>>     static struct rte_intr_handle intr_handle = {.fd = -1 };
>>   static bool monitor_started;
>>   +extern struct rte_bus_list rte_bus_list;
>> +
>
> Shouldn't rte_bus.h provide it?
>

I think rte_bus.h provide the rte_bus_list structure,  and then 
announcement a variable in eal_common_bus.c, then i use it by extern in 
eal_dev.c.

>>   #define EAL_UEV_MSG_LEN 4096
>>   #define EAL_UEV_MSG_ELEM_LEN 128
>>   +/* spinlock for device failure process */
>> +static rte_spinlock_t dev_failure_lock = RTE_SPINLOCK_INITIALIZER;
>
> It would be useful to explain why the lock is needed and when
> it should be obtained/released. Which resources are protected
> by the lock?
>

make sense, this locker should be use both bus and device access 
protection. Will explicit to let it to be more readable.

>> +
>> +static struct sigaction sigbus_action_old;
>> +
>> +static int sigbus_need_recover;
>> +
>>   static void dev_uev_handler(__rte_unused void *param);
>>     /* identify the system layer which reports this event. */
>> @@ -33,6 +48,49 @@ enum eal_dev_event_subsystem {
>>       EAL_DEV_EVENT_SUBSYSTEM_MAX
>>   };
>>   +static void
>> +sigbus_action_recover(void)
>> +{
>> +    if (sigbus_need_recover) {
>> +        sigaction(SIGBUS, &sigbus_action_old, NULL);
>> +        sigbus_need_recover = 0;
>> +    }
>> +}
>> +
>> +static void sigbus_handler(int signum, siginfo_t *info,
>> +                void *ctx __rte_unused)
>> +{
>> +    int ret;
>> +
>> +    RTE_LOG(INFO, EAL, "Thread[%d] catch SIGBUS, fault address:%p\n",
>> +        (int)pthread_self(), info->si_addr);
>> +
>> +    rte_spinlock_lock(&dev_failure_lock);
>> +    ret = rte_bus_sigbus_handler(info->si_addr);
>> +    rte_spinlock_unlock(&dev_failure_lock);
>> +    if (ret == -1) {
>> +        rte_exit(EXIT_FAILURE,
>> +             "Failed to handle SIGBUS for hotplug, "
>> +             "(rte_errno: %s)!", strerror(rte_errno));
>> +    } else if (ret == 1) {
>> +        if (sigbus_action_old.sa_handler)
>> +            (*(sigbus_action_old.sa_handler))(signum);
>> +        else
>> +            rte_exit(EXIT_FAILURE,
>> +                 "Failed to handle generic SIGBUS!");
>> +    }
>> +
>> +    RTE_LOG(INFO, EAL, "Success to handle SIGBUS for hotplug!\n");
>> +}
>> +
>> +static int cmp_dev_name(const struct rte_device *dev,
>> +    const void *_name)
>> +{
>> +    const char *name = _name;
>> +
>> +    return strcmp(dev->name, name);
>> +}
>> +
>>   static int
>>   dev_uev_socket_fd_create(void)
>>   {
>> @@ -147,6 +205,9 @@ dev_uev_handler(__rte_unused void *param)
>>       struct rte_dev_event uevent;
>>       int ret;
>>       char buf[EAL_UEV_MSG_LEN];
>> +    struct rte_bus *bus;
>> +    struct rte_device *dev;
>> +    const char *busname;
>>         memset(&uevent, 0, sizeof(struct rte_dev_event));
>>       memset(buf, 0, EAL_UEV_MSG_LEN);
>> @@ -171,13 +232,50 @@ dev_uev_handler(__rte_unused void *param)
>>       RTE_LOG(DEBUG, EAL, "receive uevent(name:%s, type:%d, 
>> subsystem:%d)\n",
>>           uevent.devname, uevent.type, uevent.subsystem);
>>   -    if (uevent.devname)
>> +    switch (uevent.subsystem) {
>> +    case EAL_DEV_EVENT_SUBSYSTEM_PCI:
>> +    case EAL_DEV_EVENT_SUBSYSTEM_UIO:
>> +        busname = "pci";
>> +        break;
>> +    default:
>> +        break;
>> +    }
>> +
>> +    if (uevent.devname) {
>> +        if (uevent.type == RTE_DEV_EVENT_REMOVE) {
>> +            rte_spinlock_lock(&dev_failure_lock);
>> +            bus = rte_bus_find_by_name(busname);
>
> It looks like busname could be uninitialized here.
>

you are correct i think.

>> +            if (bus == NULL) {
>> +                RTE_LOG(ERR, EAL, "Cannot find bus (%s)\n",
>> +                    busname);
>> +                return;
>> +            }
>> +
>> +            dev = bus->find_device(NULL, cmp_dev_name,
>> +                           uevent.devname);
>> +            if (dev == NULL) {
>> +                RTE_LOG(ERR, EAL, "Cannot find device (%s) on "
>> +                    "bus (%s)\n", uevent.devname, busname);
>> +                return;
>> +            }
>> +
>> +            ret = bus->hotplug_failure_handler(dev);
>> +            rte_spinlock_unlock(&dev_failure_lock);
>> +            if (ret) {
>> +                RTE_LOG(ERR, EAL, "Can not handle hotplug for "
>> +                    "device (%s)\n", dev->name);
>> +                return;
>> +            }
>> +        }
>>           dev_callback_process(uevent.devname, uevent.type);
>> +    }
>>   }
>>     int __rte_experimental
>>   rte_dev_event_monitor_start(void)
>>   {
>> +    sigset_t mask;
>> +    struct sigaction action;
>>       int ret;
>>         if (monitor_started)
>> @@ -197,6 +295,14 @@ rte_dev_event_monitor_start(void)
>>           return -1;
>>       }
>>   +    /* register sigbus handler */
>> +    sigemptyset(&mask);
>> +    sigaddset(&mask, SIGBUS);
>> +    action.sa_flags = SA_SIGINFO;
>> +    action.sa_mask = mask;
>> +    action.sa_sigaction = sigbus_handler;
>> +    sigbus_need_recover = !sigaction(SIGBUS, &action, 
>> &sigbus_action_old);
>> +
>>       monitor_started = true;
>>         return 0;
>> @@ -217,8 +323,11 @@ rte_dev_event_monitor_stop(void)
>>           return ret;
>>       }
>>   +    sigbus_action_recover();
>> +
>>       close(intr_handle.fd);
>>       intr_handle.fd = -1;
>>       monitor_started = false;
>> +
>>       return 0;
>>   }
>

Patch
diff mbox series

diff --git a/lib/librte_eal/linuxapp/eal/eal_dev.c b/lib/librte_eal/linuxapp/eal/eal_dev.c
index 1cf6aeb..a22cb9a 100644
--- a/lib/librte_eal/linuxapp/eal/eal_dev.c
+++ b/lib/librte_eal/linuxapp/eal/eal_dev.c
@@ -4,6 +4,8 @@ 
 
 #include <string.h>
 #include <unistd.h>
+#include <fcntl.h>
+#include <signal.h>
 #include <sys/socket.h>
 #include <linux/netlink.h>
 
@@ -14,15 +16,28 @@ 
 #include <rte_malloc.h>
 #include <rte_interrupts.h>
 #include <rte_alarm.h>
+#include <rte_bus.h>
+#include <rte_eal.h>
+#include <rte_spinlock.h>
+#include <rte_errno.h>
 
 #include "eal_private.h"
 
 static struct rte_intr_handle intr_handle = {.fd = -1 };
 static bool monitor_started;
 
+extern struct rte_bus_list rte_bus_list;
+
 #define EAL_UEV_MSG_LEN 4096
 #define EAL_UEV_MSG_ELEM_LEN 128
 
+/* spinlock for device failure process */
+static rte_spinlock_t dev_failure_lock = RTE_SPINLOCK_INITIALIZER;
+
+static struct sigaction sigbus_action_old;
+
+static int sigbus_need_recover;
+
 static void dev_uev_handler(__rte_unused void *param);
 
 /* identify the system layer which reports this event. */
@@ -33,6 +48,49 @@  enum eal_dev_event_subsystem {
 	EAL_DEV_EVENT_SUBSYSTEM_MAX
 };
 
+static void
+sigbus_action_recover(void)
+{
+	if (sigbus_need_recover) {
+		sigaction(SIGBUS, &sigbus_action_old, NULL);
+		sigbus_need_recover = 0;
+	}
+}
+
+static void sigbus_handler(int signum, siginfo_t *info,
+				void *ctx __rte_unused)
+{
+	int ret;
+
+	RTE_LOG(INFO, EAL, "Thread[%d] catch SIGBUS, fault address:%p\n",
+		(int)pthread_self(), info->si_addr);
+
+	rte_spinlock_lock(&dev_failure_lock);
+	ret = rte_bus_sigbus_handler(info->si_addr);
+	rte_spinlock_unlock(&dev_failure_lock);
+	if (ret == -1) {
+		rte_exit(EXIT_FAILURE,
+			 "Failed to handle SIGBUS for hotplug, "
+			 "(rte_errno: %s)!", strerror(rte_errno));
+	} else if (ret == 1) {
+		if (sigbus_action_old.sa_handler)
+			(*(sigbus_action_old.sa_handler))(signum);
+		else
+			rte_exit(EXIT_FAILURE,
+				 "Failed to handle generic SIGBUS!");
+	}
+
+	RTE_LOG(INFO, EAL, "Success to handle SIGBUS for hotplug!\n");
+}
+
+static int cmp_dev_name(const struct rte_device *dev,
+	const void *_name)
+{
+	const char *name = _name;
+
+	return strcmp(dev->name, name);
+}
+
 static int
 dev_uev_socket_fd_create(void)
 {
@@ -147,6 +205,9 @@  dev_uev_handler(__rte_unused void *param)
 	struct rte_dev_event uevent;
 	int ret;
 	char buf[EAL_UEV_MSG_LEN];
+	struct rte_bus *bus;
+	struct rte_device *dev;
+	const char *busname;
 
 	memset(&uevent, 0, sizeof(struct rte_dev_event));
 	memset(buf, 0, EAL_UEV_MSG_LEN);
@@ -171,13 +232,50 @@  dev_uev_handler(__rte_unused void *param)
 	RTE_LOG(DEBUG, EAL, "receive uevent(name:%s, type:%d, subsystem:%d)\n",
 		uevent.devname, uevent.type, uevent.subsystem);
 
-	if (uevent.devname)
+	switch (uevent.subsystem) {
+	case EAL_DEV_EVENT_SUBSYSTEM_PCI:
+	case EAL_DEV_EVENT_SUBSYSTEM_UIO:
+		busname = "pci";
+		break;
+	default:
+		break;
+	}
+
+	if (uevent.devname) {
+		if (uevent.type == RTE_DEV_EVENT_REMOVE) {
+			rte_spinlock_lock(&dev_failure_lock);
+			bus = rte_bus_find_by_name(busname);
+			if (bus == NULL) {
+				RTE_LOG(ERR, EAL, "Cannot find bus (%s)\n",
+					busname);
+				return;
+			}
+
+			dev = bus->find_device(NULL, cmp_dev_name,
+					       uevent.devname);
+			if (dev == NULL) {
+				RTE_LOG(ERR, EAL, "Cannot find device (%s) on "
+					"bus (%s)\n", uevent.devname, busname);
+				return;
+			}
+
+			ret = bus->hotplug_failure_handler(dev);
+			rte_spinlock_unlock(&dev_failure_lock);
+			if (ret) {
+				RTE_LOG(ERR, EAL, "Can not handle hotplug for "
+					"device (%s)\n", dev->name);
+				return;
+			}
+		}
 		dev_callback_process(uevent.devname, uevent.type);
+	}
 }
 
 int __rte_experimental
 rte_dev_event_monitor_start(void)
 {
+	sigset_t mask;
+	struct sigaction action;
 	int ret;
 
 	if (monitor_started)
@@ -197,6 +295,14 @@  rte_dev_event_monitor_start(void)
 		return -1;
 	}
 
+	/* register sigbus handler */
+	sigemptyset(&mask);
+	sigaddset(&mask, SIGBUS);
+	action.sa_flags = SA_SIGINFO;
+	action.sa_mask = mask;
+	action.sa_sigaction = sigbus_handler;
+	sigbus_need_recover = !sigaction(SIGBUS, &action, &sigbus_action_old);
+
 	monitor_started = true;
 
 	return 0;
@@ -217,8 +323,11 @@  rte_dev_event_monitor_stop(void)
 		return ret;
 	}
 
+	sigbus_action_recover();
+
 	close(intr_handle.fd);
 	intr_handle.fd = -1;
 	monitor_started = false;
+
 	return 0;
 }