Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/24869/?format=api
{ "id": 24869, "url": "https://patches.dpdk.org/api/patches/24869/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/4c08830ecc812443bada2dade0cebb95e7fa52c4.1496065002.git.gaetan.rivet@6wind.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<4c08830ecc812443bada2dade0cebb95e7fa52c4.1496065002.git.gaetan.rivet@6wind.com>", "list_archive_url": "https://inbox.dpdk.org/dev/4c08830ecc812443bada2dade0cebb95e7fa52c4.1496065002.git.gaetan.rivet@6wind.com", "date": "2017-05-29T13:42:22", "name": "[dpdk-dev,v4,10/12] net/failsafe: support device removal", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "28da85d417e711ca177fb81a620f3a84ac35abe6", "submitter": { "id": 269, "url": "https://patches.dpdk.org/api/people/269/?format=api", "name": "Gaëtan Rivet", "email": "gaetan.rivet@6wind.com" }, "delegate": { "id": 319, "url": "https://patches.dpdk.org/api/users/319/?format=api", "username": "fyigit", "first_name": "Ferruh", "last_name": "Yigit", "email": "ferruh.yigit@amd.com" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/4c08830ecc812443bada2dade0cebb95e7fa52c4.1496065002.git.gaetan.rivet@6wind.com/mbox/", "series": [], "comments": "https://patches.dpdk.org/api/patches/24869/comments/", "check": "warning", "checks": "https://patches.dpdk.org/api/patches/24869/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [IPv6:::1])\n\tby dpdk.org (Postfix) with ESMTP id 6FABC7D93;\n\tMon, 29 May 2017 15:43:29 +0200 (CEST)", "from mail-wm0-f51.google.com (mail-wm0-f51.google.com\n\t[74.125.82.51]) by dpdk.org (Postfix) with ESMTP id BC0187D3B\n\tfor <dev@dpdk.org>; Mon, 29 May 2017 15:43:18 +0200 (CEST)", "by mail-wm0-f51.google.com with SMTP id e127so61304098wmg.1\n\tfor <dev@dpdk.org>; Mon, 29 May 2017 06:43:18 -0700 (PDT)", "from bidouze.dev.6wind.com (host.78.145.23.62.rev.coltfrance.com.\n\t[62.23.145.78]) by smtp.gmail.com with ESMTPSA id\n\tl8sm12906671wmd.8.2017.05.29.06.43.16\n\t(version=TLS1_2 cipher=ECDHE-RSA-AES128-SHA bits=128/128);\n\tMon, 29 May 2017 06:43:17 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=6wind-com.20150623.gappssmtp.com; s=20150623;\n\th=from:to:cc:subject:date:message-id:in-reply-to:references\n\t:in-reply-to:references;\n\tbh=e2CAeSPpy2SSmNWSHk7d2yW6whz/E65XRnzZd2wnGHo=;\n\tb=EQku8gnBCvOJMtEStigQJXOAguXDWiTiv6mEA03fqkj3P83JFK9oHsQRwBZmSxSQ1P\n\tqHRaGS/7rM5lxW+gu+sqpyFN4E+22G3KwKVhW0pY99BhZGgcxlcUAmOQkVa+8KD8nXFB\n\ty9GVdmKQ5fa0GpFq14rCVLIdDdpF+rRYdAXQ5basDl8BecKmF+xcMzRQZjJ/k5trX0Ni\n\tLunUcf5vMWW94LNtd4GRUym+B7uPi2SNG7eaezflRYNSsCW1k4TYpD2967ojDs/9VodF\n\te0el7kkCkgY7+OgW7vbY416+P1iVz7gKGe57e9oEANjonnW/RA2scLI1O34c3HTbyh8b\n\tqh6g==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to\n\t:references:in-reply-to:references;\n\tbh=e2CAeSPpy2SSmNWSHk7d2yW6whz/E65XRnzZd2wnGHo=;\n\tb=i8For+751liMjtUXSvDpneV9d1t95kxNBkjABaf01sZUz72i1HxqMDe9oaBm0/uVyt\n\tlpoMm0r/pXaAiwE259DDjaOQSsAZph+nvwsfx61MALlCWh/T/C5aVg8Jn6qDM9sVr+dN\n\tjsZI0We0FHunWbz6WWVbnQDojwSKokV9dI6x46gJfkG5fng8CN1sezEqMtPpOd+RbpSQ\n\tISI8WECmU/4M0Fr9UkFi31uF3TZAJ+weH4USpqneypj0zfG67HpEd3IvoviIlropunaH\n\tPJd9D8k68xdXIaTtGGv5/7NmWlL316sw7Y6xBvMCRoiCRwTVAxfX/ex8MM9t/+hoJ2ik\n\t2CqQ==", "X-Gm-Message-State": "AODbwcAQ+fwxG/+Y0dUN7Sh/ojvHZacoOYuiRV8xbxs8kOSKU9s8A/jL\n\tB5gHx7dJrGHcVEFMr48=", "X-Received": "by 10.223.139.25 with SMTP id n25mr10370372wra.17.1496065397933; \n\tMon, 29 May 2017 06:43:17 -0700 (PDT)", "From": "Gaetan Rivet <gaetan.rivet@6wind.com>", "To": "dev@dpdk.org", "Cc": "Gaetan Rivet <gaetan.rivet@6wind.com>", "Date": "Mon, 29 May 2017 15:42:22 +0200", "Message-Id": "<4c08830ecc812443bada2dade0cebb95e7fa52c4.1496065002.git.gaetan.rivet@6wind.com>", "X-Mailer": "git-send-email 2.1.4", "In-Reply-To": [ "<cover.1496065002.git.gaetan.rivet@6wind.com>", "<cover.1496065002.git.gaetan.rivet@6wind.com>" ], "References": [ "<cover.1495634974.git.gaetan.rivet@6wind.com>\n\t<cover.1496065002.git.gaetan.rivet@6wind.com>", "<cover.1496065002.git.gaetan.rivet@6wind.com>" ], "Subject": "[dpdk-dev] [PATCH v4 10/12] net/failsafe: support device removal", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<http://dpdk.org/ml/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://dpdk.org/ml/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<http://dpdk.org/ml/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Listen to INTR_RMV events issued by slaves.\nAdd atomic flags on slave queues to detect use of slave bursts function.\nIf a removal is detected, set the recollection flag on this slave.\n\nDuring a slave upkeep round, if its recollection flag is set and its\nburst functions are not in use by any thread, remove that slave.\n\nSigned-off-by: Gaetan Rivet <gaetan.rivet@6wind.com>\nAcked-by: Olga Shern <olgas@mellanox.com>\n---\n doc/guides/nics/fail_safe.rst | 14 +++++\n drivers/net/failsafe/failsafe.c | 1 +\n drivers/net/failsafe/failsafe_args.c | 13 +++++\n drivers/net/failsafe/failsafe_eal.c | 3 +-\n drivers/net/failsafe/failsafe_ether.c | 93 ++++++++++++++++++++++++++++++---\n drivers/net/failsafe/failsafe_ops.c | 38 ++++++++++++--\n drivers/net/failsafe/failsafe_private.h | 73 +++++++++++++++++++++++---\n drivers/net/failsafe/failsafe_rxtx.c | 17 +++++-\n 8 files changed, 232 insertions(+), 20 deletions(-)", "diff": "diff --git a/doc/guides/nics/fail_safe.rst b/doc/guides/nics/fail_safe.rst\nindex 1b6e110..4154f0a 100644\n--- a/doc/guides/nics/fail_safe.rst\n+++ b/doc/guides/nics/fail_safe.rst\n@@ -51,6 +51,12 @@ The Fail-safe PMD only supports a limited set of features. If you plan to use a\n device underneath the Fail-safe PMD with a specific feature, this feature must\n be supported by the Fail-safe PMD to avoid throwing any error.\n \n+A notable exception is the device removal feature. The fail-safe PMD being a\n+virtual device, it cannot currently be removed in the sense of a specific bus\n+hotplug, like for PCI for example. It will however enable this feature for its\n+sub-device automatically, detecting those that are capable and register the\n+relevant callback for such event.\n+\n Check the feature matrix for the complete set of supported features.\n \n Compilation options\n@@ -170,3 +176,11 @@ emit and receive packets. It will store any applied configuration, and try to\n apply it upon the probing of its missing sub-device. After this configuration\n pass, the new sub-device will be synchronized with other sub-devices, i.e. be\n started if the fail-safe PMD has been started by the user before.\n+\n+Plug-out feature\n+----------------\n+\n+A sub-device supporting the device removal event can be removed from its bus at\n+any time. The fail-safe PMD will register a callback for such event and react\n+accordingly. It will try to safely stop, close and uninit the sub-device having\n+emitted this event, allowing it to free its eventual resources.\ndiff --git a/drivers/net/failsafe/failsafe.c b/drivers/net/failsafe/failsafe.c\nindex 6557255..4d35860 100644\n--- a/drivers/net/failsafe/failsafe.c\n+++ b/drivers/net/failsafe/failsafe.c\n@@ -132,6 +132,7 @@ fs_hotplug_alarm(void *arg)\n \tif (!PRIV(dev)->pending_alarm)\n \t\treturn;\n \tPRIV(dev)->pending_alarm = 0;\n+\tfailsafe_dev_remove(dev);\n \tFOREACH_SUBDEV(sdev, i, dev)\n \t\tif (sdev->state != PRIV(dev)->state)\n \t\t\tbreak;\ndiff --git a/drivers/net/failsafe/failsafe_args.c b/drivers/net/failsafe/failsafe_args.c\nindex c723ca3..dd55aaf 100644\n--- a/drivers/net/failsafe/failsafe_args.c\n+++ b/drivers/net/failsafe/failsafe_args.c\n@@ -443,6 +443,17 @@ failsafe_args_count_subdevice(struct rte_eth_dev *dev,\n \t\t\t\t dev, params);\n }\n \n+static int\n+fs_parse_sub_device(struct sub_device *sdev)\n+{\n+\tstruct rte_devargs *da;\n+\tchar devstr[DEVARGS_MAXLEN] = \"\";\n+\n+\tda = &sdev->devargs;\n+\tsnprintf(devstr, sizeof(devstr), \"%s,%s\", da->name, da->args);\n+\treturn fs_parse_device(sdev, devstr);\n+}\n+\n int\n failsafe_args_parse_subs(struct rte_eth_dev *dev)\n {\n@@ -455,6 +466,8 @@ failsafe_args_parse_subs(struct rte_eth_dev *dev)\n \t\t\tcontinue;\n \t\tif (sdev->cmdline)\n \t\t\tret = fs_execute_cmd(sdev, sdev->cmdline);\n+\t\telse\n+\t\t\tret = fs_parse_sub_device(sdev);\n \t\tif (ret == 0)\n \t\t\tsdev->state = DEV_PARSED;\n \t}\ndiff --git a/drivers/net/failsafe/failsafe_eal.c b/drivers/net/failsafe/failsafe_eal.c\nindex 1d9ddab..4e5d70e 100644\n--- a/drivers/net/failsafe/failsafe_eal.c\n+++ b/drivers/net/failsafe/failsafe_eal.c\n@@ -81,6 +81,7 @@ fs_bus_init(struct rte_eth_dev *dev)\n \t\t\treturn -ENODEV;\n \t\t}\n \t\tSUB_ID(sdev) = i;\n+\t\tsdev->fs_dev = dev;\n \t\tsdev->dev = ETH(sdev)->device;\n \t\tETH(sdev)->state = RTE_ETH_DEV_DEFERRED;\n \t\tsdev->state = DEV_PROBED;\n@@ -98,7 +99,7 @@ failsafe_eal_init(struct rte_eth_dev *dev)\n \t\treturn ret;\n \tif (PRIV(dev)->state < DEV_PROBED)\n \t\tPRIV(dev)->state = DEV_PROBED;\n-\tfs_switch_dev(dev);\n+\tfs_switch_dev(dev, NULL);\n \treturn 0;\n }\n \ndiff --git a/drivers/net/failsafe/failsafe_ether.c b/drivers/net/failsafe/failsafe_ether.c\nindex 2958207..bb6fcff 100644\n--- a/drivers/net/failsafe/failsafe_ether.c\n+++ b/drivers/net/failsafe/failsafe_ether.c\n@@ -250,6 +250,64 @@ fs_eth_dev_conf_apply(struct rte_eth_dev *dev,\n \treturn 0;\n }\n \n+static void\n+fs_dev_remove(struct sub_device *sdev)\n+{\n+\tif (sdev == NULL)\n+\t\treturn;\n+\tswitch (sdev->state) {\n+\tcase DEV_STARTED:\n+\t\trte_eth_dev_stop(PORT_ID(sdev));\n+\t\tsdev->state = DEV_ACTIVE;\n+\t\t/* fallthrough */\n+\tcase DEV_ACTIVE:\n+\t\trte_eth_dev_close(PORT_ID(sdev));\n+\t\tsdev->state = DEV_PROBED;\n+\t\t/* fallthrough */\n+\tcase DEV_PROBED:\n+\t\tif (sdev->bus->detach == NULL ||\n+\t\t sdev->bus->detach(sdev->dev)) {\n+\t\t\tERROR(\"Bus detach failed for sub_device %u\",\n+\t\t\t SUB_ID(sdev));\n+\t\t} else {\n+\t\t\tETH(sdev)->state = RTE_ETH_DEV_UNUSED;\n+\t\t}\n+\t\tsdev->state = DEV_PARSED;\n+\t\t/* fallthrough */\n+\tcase DEV_PARSED:\n+\tcase DEV_UNDEFINED:\n+\t\tsdev->state = DEV_UNDEFINED;\n+\t\t/* the end */\n+\t\tbreak;\n+\t}\n+\tfailsafe_hotplug_alarm_install(sdev->fs_dev);\n+}\n+\n+static inline int\n+fs_rxtx_clean(struct sub_device *sdev)\n+{\n+\tuint16_t i;\n+\n+\tfor (i = 0; i < ETH(sdev)->data->nb_rx_queues; i++)\n+\t\tif (FS_ATOMIC_RX(sdev, i))\n+\t\t\treturn 0;\n+\tfor (i = 0; i < ETH(sdev)->data->nb_tx_queues; i++)\n+\t\tif (FS_ATOMIC_TX(sdev, i))\n+\t\t\treturn 0;\n+\treturn 1;\n+}\n+\n+void\n+failsafe_dev_remove(struct rte_eth_dev *dev)\n+{\n+\tstruct sub_device *sdev;\n+\tuint8_t i;\n+\n+\tFOREACH_SUBDEV(sdev, i, dev)\n+\t\tif (sdev->remove && fs_rxtx_clean(sdev))\n+\t\t\tfs_dev_remove(sdev);\n+}\n+\n int\n failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)\n {\n@@ -263,13 +321,13 @@ failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)\n \n \tret = failsafe_args_parse_subs(dev);\n \tif (ret)\n-\t\treturn ret;\n+\t\tgoto err_remove;\n \n \tif (PRIV(dev)->state < DEV_PROBED)\n \t\treturn 0;\n \tret = failsafe_eal_init(dev);\n \tif (ret)\n-\t\treturn ret;\n+\t\tgoto err_remove;\n \tif (PRIV(dev)->state < DEV_ACTIVE)\n \t\treturn 0;\n \tinactive = 0;\n@@ -278,15 +336,14 @@ failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)\n \t\t\tinactive |= UINT32_C(1) << i;\n \tret = dev->dev_ops->dev_configure(dev);\n \tif (ret)\n-\t\treturn ret;\n+\t\tgoto err_remove;\n \tFOREACH_SUBDEV(sdev, i, dev) {\n \t\tif (inactive & (UINT32_C(1) << i)) {\n \t\t\tret = fs_eth_dev_conf_apply(dev, sdev);\n \t\t\tif (ret) {\n \t\t\t\tERROR(\"Could not apply configuration to sub_device %d\",\n \t\t\t\t i);\n-\t\t\t\t/* TODO: disable device */\n-\t\t\t\treturn ret;\n+\t\t\t\tgoto err_remove;\n \t\t\t}\n \t\t}\n \t}\n@@ -300,6 +357,30 @@ failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)\n \t\treturn 0;\n \tret = dev->dev_ops->dev_start(dev);\n \tif (ret)\n-\t\treturn ret;\n+\t\tgoto err_remove;\n \treturn 0;\n+err_remove:\n+\tFOREACH_SUBDEV(sdev, i, dev)\n+\t\tif (sdev->state != PRIV(dev)->state)\n+\t\t\tsdev->remove = 1;\n+\treturn ret;\n+}\n+\n+void\n+failsafe_eth_rmv_event_callback(uint8_t port_id __rte_unused,\n+\t\t\t\tenum rte_eth_event_type event __rte_unused,\n+\t\t\t\tvoid *cb_arg)\n+{\n+\tstruct sub_device *sdev = cb_arg;\n+\n+\t/* Switch as soon as possible tx_dev. */\n+\tfs_switch_dev(sdev->fs_dev, sdev);\n+\t/* Use safe bursts in any case. */\n+\tset_burst_fn(sdev->fs_dev, 1);\n+\tusleep(FAILSAFE_PLUGOUT_ASYNC_RESCHED_US);\n+\t/*\n+\t * Async removal, the sub-PMD will try to unregister\n+\t * the callback at the source of the current thread context.\n+\t */\n+\tsdev->remove = 1;\n }\ndiff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c\nindex 5fb0135..2e1c798 100644\n--- a/drivers/net/failsafe/failsafe_ops.c\n+++ b/drivers/net/failsafe/failsafe_ops.c\n@@ -33,6 +33,8 @@\n \n #include <assert.h>\n #include <stdint.h>\n+\n+#include <rte_atomic.h>\n #include <rte_ethdev.h>\n #include <rte_malloc.h>\n #include <rte_flow.h>\n@@ -204,9 +206,21 @@ fs_dev_configure(struct rte_eth_dev *dev)\n \t\t}\n \t}\n \tFOREACH_SUBDEV(sdev, i, dev) {\n+\t\tint rmv_interrupt = 0;\n+\n \t\tif (sdev->state != DEV_PROBED)\n \t\t\tcontinue;\n+\n+\t\trmv_interrupt = ETH(sdev)->data->dev_flags &\n+\t\t\t\tRTE_ETH_DEV_INTR_RMV;\n+\t\tif (rmv_interrupt) {\n+\t\t\tDEBUG(\"Enabling RMV interrupts for sub_device %d\", i);\n+\t\t\tdev->data->dev_conf.intr_conf.rmv = 1;\n+\t\t} else {\n+\t\t\tDEBUG(\"sub_device %d does not support RMV event\", i);\n+\t\t}\n \t\tDEBUG(\"Configuring sub-device %d\", i);\n+\t\tsdev->remove = 0;\n \t\tret = rte_eth_dev_configure(PORT_ID(sdev),\n \t\t\t\t\tdev->data->nb_rx_queues,\n \t\t\t\t\tdev->data->nb_tx_queues,\n@@ -215,6 +229,16 @@ fs_dev_configure(struct rte_eth_dev *dev)\n \t\t\tERROR(\"Could not configure sub_device %d\", i);\n \t\t\treturn ret;\n \t\t}\n+\t\tif (rmv_interrupt) {\n+\t\t\tret = rte_eth_dev_callback_register(PORT_ID(sdev),\n+\t\t\t\t\tRTE_ETH_EVENT_INTR_RMV,\n+\t\t\t\t\tfailsafe_eth_rmv_event_callback,\n+\t\t\t\t\tsdev);\n+\t\t\tif (ret)\n+\t\t\t\tWARN(\"Failed to register RMV callback for sub_device %d\",\n+\t\t\t\t SUB_ID(sdev));\n+\t\t}\n+\t\tdev->data->dev_conf.intr_conf.rmv = 0;\n \t\tsdev->state = DEV_ACTIVE;\n \t}\n \tif (PRIV(dev)->state < DEV_ACTIVE)\n@@ -240,7 +264,7 @@ fs_dev_start(struct rte_eth_dev *dev)\n \t}\n \tif (PRIV(dev)->state < DEV_STARTED)\n \t\tPRIV(dev)->state = DEV_STARTED;\n-\tfs_switch_dev(dev);\n+\tfs_switch_dev(dev, NULL);\n \treturn 0;\n }\n \n@@ -351,10 +375,14 @@ fs_rx_queue_setup(struct rte_eth_dev *dev,\n \t\tfs_rx_queue_release(rxq);\n \t\tdev->data->rx_queues[rx_queue_id] = NULL;\n \t}\n-\trxq = rte_zmalloc(NULL, sizeof(*rxq),\n+\trxq = rte_zmalloc(NULL,\n+\t\t\t sizeof(*rxq) +\n+\t\t\t sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,\n \t\t\t RTE_CACHE_LINE_SIZE);\n \tif (rxq == NULL)\n \t\treturn -ENOMEM;\n+\tFOREACH_SUBDEV(sdev, i, dev)\n+\t\trte_atomic64_init(&rxq->refcnt[i]);\n \trxq->qid = rx_queue_id;\n \trxq->socket_id = socket_id;\n \trxq->info.mp = mb_pool;\n@@ -414,10 +442,14 @@ fs_tx_queue_setup(struct rte_eth_dev *dev,\n \t\tfs_tx_queue_release(txq);\n \t\tdev->data->tx_queues[tx_queue_id] = NULL;\n \t}\n-\ttxq = rte_zmalloc(\"ethdev TX queue\", sizeof(*txq),\n+\ttxq = rte_zmalloc(\"ethdev TX queue\",\n+\t\t\t sizeof(*txq) +\n+\t\t\t sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,\n \t\t\t RTE_CACHE_LINE_SIZE);\n \tif (txq == NULL)\n \t\treturn -ENOMEM;\n+\tFOREACH_SUBDEV(sdev, i, dev)\n+\t\trte_atomic64_init(&txq->refcnt[i]);\n \ttxq->qid = tx_queue_id;\n \ttxq->socket_id = socket_id;\n \ttxq->info.conf = *tx_conf;\ndiff --git a/drivers/net/failsafe/failsafe_private.h b/drivers/net/failsafe/failsafe_private.h\nindex 25a4dac..ca20109 100644\n--- a/drivers/net/failsafe/failsafe_private.h\n+++ b/drivers/net/failsafe/failsafe_private.h\n@@ -36,6 +36,7 @@\n \n #include <sys/queue.h>\n \n+#include <rte_atomic.h>\n #include <rte_dev.h>\n #include <rte_ethdev.h>\n #include <rte_devargs.h>\n@@ -52,6 +53,7 @@\n \t\"\"\n \n #define FAILSAFE_HOTPLUG_DEFAULT_TIMEOUT_MS 2000\n+#define FAILSAFE_PLUGOUT_ASYNC_RESCHED_US 100\n \n #define FAILSAFE_MAX_ETHPORTS 2\n #define FAILSAFE_MAX_ETHADDR 128\n@@ -65,6 +67,7 @@ struct rxq {\n \tuint8_t last_polled;\n \tunsigned int socket_id;\n \tstruct rte_eth_rxq_info info;\n+\trte_atomic64_t refcnt[];\n };\n \n struct txq {\n@@ -72,6 +75,7 @@ struct txq {\n \tuint16_t qid;\n \tunsigned int socket_id;\n \tstruct rte_eth_txq_info info;\n+\trte_atomic64_t refcnt[];\n };\n \n struct rte_flow {\n@@ -101,6 +105,10 @@ struct sub_device {\n \tenum dev_state state;\n \t/* Some device are defined as a command line */\n \tchar *cmdline;\n+\t/* fail-safe device backreference */\n+\tstruct rte_eth_dev *fs_dev;\n+\t/* flag calling for recollection */\n+\tvolatile unsigned int remove:1;\n };\n \n struct fs_priv {\n@@ -168,6 +176,10 @@ int failsafe_eal_uninit(struct rte_eth_dev *dev);\n /* ETH_DEV */\n \n int failsafe_eth_dev_state_sync(struct rte_eth_dev *dev);\n+void failsafe_dev_remove(struct rte_eth_dev *dev);\n+void failsafe_eth_rmv_event_callback(uint8_t port_id,\n+\t\t\t\t enum rte_eth_event_type type,\n+\t\t\t\t void *arg);\n \n /* GLOBALS */\n \n@@ -233,6 +245,39 @@ extern int mac_from_arg;\n #define SUBOPS(s, ops) \\\n \t(ETH(s)->dev_ops->ops)\n \n+/**\n+ * Atomic guard\n+ */\n+\n+/**\n+ * a: (rte_atomic64_t *)\n+ */\n+#define FS_ATOMIC_P(a) \\\n+\trte_atomic64_add(&(a), 1)\n+\n+/**\n+ * a: (rte_atomic64_t *)\n+ */\n+#define FS_ATOMIC_V(a) \\\n+\trte_atomic64_sub(&(a), 1)\n+\n+/**\n+ * s: (struct sub_device *)\n+ * i: uint16_t qid\n+ */\n+#define FS_ATOMIC_RX(s, i) \\\n+\trte_atomic64_read( \\\n+\t &((struct rxq *)((s)->fs_dev->data->rx_queues[i]))->refcnt[(s)->sid] \\\n+\t)\n+/**\n+ * s: (struct sub_device *)\n+ * i: uint16_t qid\n+ */\n+#define FS_ATOMIC_TX(s, i) \\\n+\trte_atomic64_read( \\\n+\t &((struct txq *)((s)->fs_dev->data->tx_queues[i]))->refcnt[(s)->sid] \\\n+\t)\n+\n #ifndef NDEBUG\n #include <stdio.h>\n #define DEBUG__(m, ...)\t\t\t\t\t\t\\\n@@ -274,33 +319,45 @@ fs_find_next(struct rte_eth_dev *dev, uint8_t sid,\n \treturn sid;\n }\n \n+/*\n+ * Switch emitting device.\n+ * If banned is set, banned must not be considered for\n+ * the role of emitting device.\n+ */\n static inline void\n-fs_switch_dev(struct rte_eth_dev *dev)\n+fs_switch_dev(struct rte_eth_dev *dev,\n+\t struct sub_device *banned)\n {\n+\tstruct sub_device *txd;\n \tenum dev_state req_state;\n \n \treq_state = PRIV(dev)->state;\n-\tif (PREFERRED_SUBDEV(dev)->state >= req_state) {\n-\t\tif (TX_SUBDEV(dev) != PREFERRED_SUBDEV(dev) &&\n-\t\t (TX_SUBDEV(dev) == NULL ||\n+\ttxd = TX_SUBDEV(dev);\n+\tif (PREFERRED_SUBDEV(dev)->state >= req_state &&\n+\t PREFERRED_SUBDEV(dev) != banned) {\n+\t\tif (txd != PREFERRED_SUBDEV(dev) &&\n+\t\t (txd == NULL ||\n \t\t (req_state == DEV_STARTED) ||\n-\t\t (TX_SUBDEV(dev) && TX_SUBDEV(dev)->state < DEV_STARTED))) {\n+\t\t (txd && txd->state < DEV_STARTED))) {\n \t\t\tDEBUG(\"Switching tx_dev to preferred sub_device\");\n \t\t\tPRIV(dev)->subs_tx = 0;\n \t\t}\n-\t} else if ((TX_SUBDEV(dev) && TX_SUBDEV(dev)->state < req_state) ||\n-\t\t TX_SUBDEV(dev) == NULL) {\n+\t} else if ((txd && txd->state < req_state) ||\n+\t\t txd == NULL ||\n+\t\t txd == banned) {\n \t\tstruct sub_device *sdev;\n \t\tuint8_t i;\n \n \t\t/* Using acceptable device */\n \t\tFOREACH_SUBDEV_ST(sdev, i, dev, req_state) {\n+\t\t\tif (sdev == banned)\n+\t\t\t\tcontinue;\n \t\t\tDEBUG(\"Switching tx_dev to sub_device %d\",\n \t\t\t i);\n \t\t\tPRIV(dev)->subs_tx = i;\n \t\t\tbreak;\n \t\t}\n-\t} else if (TX_SUBDEV(dev) && TX_SUBDEV(dev)->state < req_state) {\n+\t} else if (txd && txd->state < req_state) {\n \t\tDEBUG(\"No device ready, deactivating tx_dev\");\n \t\tPRIV(dev)->subs_tx = PRIV(dev)->subs_tail;\n \t} else {\ndiff --git a/drivers/net/failsafe/failsafe_rxtx.c b/drivers/net/failsafe/failsafe_rxtx.c\nindex c15025f..82a8c4e 100644\n--- a/drivers/net/failsafe/failsafe_rxtx.c\n+++ b/drivers/net/failsafe/failsafe_rxtx.c\n@@ -33,6 +33,7 @@\n \n #include <assert.h>\n \n+#include <rte_atomic.h>\n #include <rte_mbuf.h>\n #include <rte_ethdev.h>\n \n@@ -113,8 +114,10 @@ failsafe_rx_burst(void *queue,\n \t\tif (unlikely(fs_rx_unsafe(sdev)))\n \t\t\tcontinue;\n \t\tsub_rxq = ETH(sdev)->data->rx_queues[rxq->qid];\n+\t\tFS_ATOMIC_P(rxq->refcnt[sdev->sid]);\n \t\tnb_rx = ETH(sdev)->\n \t\t\trx_pkt_burst(sub_rxq, rx_pkts, nb_pkts);\n+\t\tFS_ATOMIC_V(rxq->refcnt[sdev->sid]);\n \t\tif (nb_rx) {\n \t\t\trxq->last_polled = i;\n \t\t\treturn nb_rx;\n@@ -147,8 +150,10 @@ failsafe_rx_burst_fast(void *queue,\n \t\tsdev = &priv->subs[i];\n \t\tassert(!fs_rx_unsafe(sdev));\n \t\tsub_rxq = ETH(sdev)->data->rx_queues[rxq->qid];\n+\t\tFS_ATOMIC_P(rxq->refcnt[sdev->sid]);\n \t\tnb_rx = ETH(sdev)->\n \t\t\trx_pkt_burst(sub_rxq, rx_pkts, nb_pkts);\n+\t\tFS_ATOMIC_V(rxq->refcnt[sdev->sid]);\n \t\tif (nb_rx) {\n \t\t\trxq->last_polled = i;\n \t\t\treturn nb_rx;\n@@ -165,13 +170,17 @@ failsafe_tx_burst(void *queue,\n \tstruct sub_device *sdev;\n \tstruct txq *txq;\n \tvoid *sub_txq;\n+\tuint16_t nb_tx;\n \n \ttxq = queue;\n \tsdev = TX_SUBDEV(txq->priv->dev);\n \tif (unlikely(fs_tx_unsafe(sdev)))\n \t\treturn 0;\n \tsub_txq = ETH(sdev)->data->tx_queues[txq->qid];\n-\treturn ETH(sdev)->tx_pkt_burst(sub_txq, tx_pkts, nb_pkts);\n+\tFS_ATOMIC_P(txq->refcnt[sdev->sid]);\n+\tnb_tx = ETH(sdev)->tx_pkt_burst(sub_txq, tx_pkts, nb_pkts);\n+\tFS_ATOMIC_V(txq->refcnt[sdev->sid]);\n+\treturn nb_tx;\n }\n \n uint16_t\n@@ -182,10 +191,14 @@ failsafe_tx_burst_fast(void *queue,\n \tstruct sub_device *sdev;\n \tstruct txq *txq;\n \tvoid *sub_txq;\n+\tuint16_t nb_tx;\n \n \ttxq = queue;\n \tsdev = TX_SUBDEV(txq->priv->dev);\n \tassert(!fs_tx_unsafe(sdev));\n \tsub_txq = ETH(sdev)->data->tx_queues[txq->qid];\n-\treturn ETH(sdev)->tx_pkt_burst(sub_txq, tx_pkts, nb_pkts);\n+\tFS_ATOMIC_P(txq->refcnt[sdev->sid]);\n+\tnb_tx = ETH(sdev)->tx_pkt_burst(sub_txq, tx_pkts, nb_pkts);\n+\tFS_ATOMIC_V(txq->refcnt[sdev->sid]);\n+\treturn nb_tx;\n }\n", "prefixes": [ "dpdk-dev", "v4", "10/12" ] }