Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/51453/?format=api
{ "id": 51453, "url": "http://patches.dpdk.org/api/patches/51453/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1553155888-27498-6-git-send-email-viacheslavo@mellanox.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1553155888-27498-6-git-send-email-viacheslavo@mellanox.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1553155888-27498-6-git-send-email-viacheslavo@mellanox.com", "date": "2019-03-21T08:11:19", "name": "[05/14] net/mlx5: add multiport IB device support to probing", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "a350001d2c74b7ea6944389b196fe0939cf3b3ff", "submitter": { "id": 1102, "url": "http://patches.dpdk.org/api/people/1102/?format=api", "name": "Slava Ovsiienko", "email": "viacheslavo@mellanox.com" }, "delegate": { "id": 6624, "url": "http://patches.dpdk.org/api/users/6624/?format=api", "username": "shahafs", "first_name": "Shahaf", "last_name": "Shuler", "email": "shahafs@mellanox.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1553155888-27498-6-git-send-email-viacheslavo@mellanox.com/mbox/", "series": [ { "id": 3842, "url": "http://patches.dpdk.org/api/series/3842/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=3842", "date": "2019-03-21T08:11:18", "name": "net/mlx5: add support for multiport IB devices", "version": 1, "mbox": "http://patches.dpdk.org/series/3842/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/51453/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/51453/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 303D41B462;\n\tThu, 21 Mar 2019 09:12:01 +0100 (CET)", "from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129])\n\tby dpdk.org (Postfix) with ESMTP id 9B49F1B43E\n\tfor <dev@dpdk.org>; Thu, 21 Mar 2019 09:11:42 +0100 (CET)", "from Internal Mail-Server by MTLPINE1 (envelope-from\n\tviacheslavo@mellanox.com)\n\twith ESMTPS (AES256-SHA encrypted); 21 Mar 2019 10:11:37 +0200", "from pegasus12.mtr.labs.mlnx. (pegasus12.mtr.labs.mlnx\n\t[10.210.17.40])\n\tby labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2L8Bai3003643;\n\tThu, 21 Mar 2019 10:11:37 +0200" ], "From": "Viacheslav Ovsiienko <viacheslavo@mellanox.com>", "To": "dev@dpdk.org", "Cc": "shahafs@mellanox.com", "Date": "Thu, 21 Mar 2019 08:11:19 +0000", "Message-Id": "<1553155888-27498-6-git-send-email-viacheslavo@mellanox.com>", "X-Mailer": "git-send-email 1.8.3.1", "In-Reply-To": "<1553155888-27498-1-git-send-email-viacheslavo@mellanox.com>", "References": "<1551376985-11096-1-git-send-email-viacheslavo@mellanox.com>\n\t<1553155888-27498-1-git-send-email-viacheslavo@mellanox.com>", "Subject": "[dpdk-dev] [PATCH 05/14] net/mlx5: add multiport IB device support\n\tto probing", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "mlx5_pci_probe() routine is refactored to probe the ports\nof found Infiniband devices. All active ports (with attached\nnetwork interface), belonging to the same Infiniband device\nwill use the signle shared Infiniband context of that device.\n\nSigned-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>\n---\n drivers/net/mlx5/mlx5.c | 302 +++++++++++++++++++++++++++++++++---------------\n 1 file changed, 210 insertions(+), 92 deletions(-)", "diff": "diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c\nindex 89c30af..100e9f4 100644\n--- a/drivers/net/mlx5/mlx5.c\n+++ b/drivers/net/mlx5/mlx5.c\n@@ -130,6 +130,16 @@\n /** Driver-specific log messages type. */\n int mlx5_logtype;\n \n+/** Data associated with devices to spawn. */\n+struct mlx5_dev_spawn_data {\n+\tuint32_t ifindex; /**< Network interface index. */\n+\tuint32_t max_port; /**< IB device maximal port index. */\n+\tuint32_t ibv_port; /**< IB device physical port index. */\n+\tstruct mlx5_switch_info info; /**< Switch information. */\n+\tstruct ibv_device *ibv_dev; /**< Associated IB device. */\n+\tstruct rte_eth_dev *eth_dev; /**< Associated Ethernet device. */\n+};\n+\n /**\n * Prepare shared data between primary and secondary process.\n */\n@@ -716,12 +726,10 @@\n *\n * @param dpdk_dev\n * Backing DPDK device.\n- * @param ibv_dev\n- * Verbs device.\n+ * @param spawn\n+ * Verbs device parameters (name, port, switch_info) to spawn.\n * @param config\n * Device configuration parameters.\n- * @param[in] switch_info\n- * Switch properties of Ethernet device.\n *\n * @return\n * A valid Ethernet device object on success, NULL otherwise and rte_errno\n@@ -732,10 +740,11 @@\n */\n static struct rte_eth_dev *\n mlx5_dev_spawn(struct rte_device *dpdk_dev,\n-\t struct ibv_device *ibv_dev,\n-\t struct mlx5_dev_config config,\n-\t const struct mlx5_switch_info *switch_info)\n+\t struct mlx5_dev_spawn_data *spawn,\n+\t struct mlx5_dev_config config)\n {\n+\tconst struct mlx5_switch_info *switch_info = &spawn->info;\n+\tstruct ibv_device *ibv_dev = spawn->ibv_dev;\n \tstruct ibv_context *ctx = NULL;\n \tstruct ibv_device_attr_ex attr;\n \tstruct ibv_port_attr port_attr;\n@@ -952,7 +961,7 @@\n \t\treturn eth_dev;\n \t}\n \t/* Check port status. */\n-\terr = mlx5_glue->query_port(ctx, 1, &port_attr);\n+\terr = mlx5_glue->query_port(ctx, spawn->ibv_port, &port_attr);\n \tif (err) {\n \t\tDRV_LOG(ERR, \"port query failed: %s\", strerror(err));\n \t\tgoto error;\n@@ -1316,14 +1325,6 @@\n \treturn NULL;\n }\n \n-/** Data associated with devices to spawn. */\n-struct mlx5_dev_spawn_data {\n-\tunsigned int ifindex; /**< Network interface index. */\n-\tstruct mlx5_switch_info info; /**< Switch information. */\n-\tstruct ibv_device *ibv_dev; /**< Associated IB device. */\n-\tstruct rte_eth_dev *eth_dev; /**< Associated Ethernet device. */\n-};\n-\n /**\n * Comparison callback to sort device data.\n *\n@@ -1380,7 +1381,9 @@ struct mlx5_dev_spawn_data {\n \t struct rte_pci_device *pci_dev)\n {\n \tstruct ibv_device **ibv_list;\n-\tunsigned int n = 0;\n+\tunsigned int nd = 0;\n+\tunsigned int np = 0;\n+\tunsigned int ns = 0;\n \tstruct mlx5_dev_config dev_config;\n \tint ret;\n \n@@ -1392,8 +1395,14 @@ struct mlx5_dev_spawn_data {\n \t\tDRV_LOG(ERR, \"cannot list devices, is ib_uverbs loaded?\");\n \t\treturn -rte_errno;\n \t}\n-\n+\t/*\n+\t * First scan the list of all Infiniband devices to find\n+\t * matching ones, gathering into the list.\n+\t */\n \tstruct ibv_device *ibv_match[ret + 1];\n+\tint nl_route = -1;\n+\tint nl_rdma = -1;\n+\tunsigned int i;\n \n \twhile (ret-- > 0) {\n \t\tstruct rte_pci_addr pci_addr;\n@@ -1408,77 +1417,183 @@ struct mlx5_dev_spawn_data {\n \t\t\tcontinue;\n \t\tDRV_LOG(INFO, \"PCI information matches for device \\\"%s\\\"\",\n \t\t\tibv_list[ret]->name);\n-\t\tibv_match[n++] = ibv_list[ret];\n+\t\tibv_match[nd++] = ibv_list[ret];\n+\t}\n+\tibv_match[nd] = NULL;\n+\tif (!nd) {\n+\t\t/* No device macthes, just complain and bail out. */\n+\t\tmlx5_glue->free_device_list(ibv_list);\n+\t\tDRV_LOG(WARNING,\n+\t\t\t\"no Verbs device matches PCI device \" PCI_PRI_FMT \",\"\n+\t\t\t\" are kernel drivers loaded?\",\n+\t\t\tpci_dev->addr.domain, pci_dev->addr.bus,\n+\t\t\tpci_dev->addr.devid, pci_dev->addr.function);\n+\t\trte_errno = ENOENT;\n+\t\tret = -rte_errno;\n+\t\treturn ret;\n+\t}\n+\tnl_route = mlx5_nl_init(NETLINK_ROUTE);\n+\tnl_rdma = mlx5_nl_init(NETLINK_RDMA);\n+\tif (nd == 1) {\n+\t\t/*\n+\t\t * Found single matching device may have multiple ports.\n+\t\t * Each port may be representor, we have to check the port\n+\t\t * number and check the representors existence.\n+\t\t */\n+\t\tif (nl_rdma >= 0)\n+\t\t\tnp = mlx5_nl_portnum(nl_rdma, ibv_match[0]->name);\n+\t\tif (!np)\n+\t\t\tDRV_LOG(WARNING, \"can not get IB device \\\"%s\\\"\"\n+\t\t\t\t\t \" ports number\", ibv_match[0]->name);\n \t}\n-\tibv_match[n] = NULL;\n-\n-\tstruct mlx5_dev_spawn_data list[n];\n-\tint nl_route = n ? mlx5_nl_init(NETLINK_ROUTE) : -1;\n-\tint nl_rdma = n ? mlx5_nl_init(NETLINK_RDMA) : -1;\n-\tunsigned int i;\n-\tunsigned int u;\n-\n \t/*\n-\t * The existence of several matching entries (n > 1) means port\n-\t * representors have been instantiated. No existing Verbs call nor\n-\t * /sys entries can tell them apart, this can only be done through\n-\t * Netlink calls assuming kernel drivers are recent enough to\n-\t * support them.\n-\t *\n-\t * In the event of identification failure through Netlink, try again\n-\t * through sysfs, then either:\n-\t *\n-\t * 1. No device matches (n == 0), complain and bail out.\n-\t * 2. A single IB device matches (n == 1) and is not a representor,\n-\t * assume no switch support.\n-\t * 3. Otherwise no safe assumptions can be made; complain louder and\n-\t * bail out.\n+\t * Now we can determine the maximal\n+\t * amount of devices to be spawned.\n \t */\n-\tfor (i = 0; i != n; ++i) {\n-\t\tlist[i].ibv_dev = ibv_match[i];\n-\t\tlist[i].eth_dev = NULL;\n-\t\tif (nl_rdma < 0)\n-\t\t\tlist[i].ifindex = 0;\n-\t\telse\n-\t\t\tlist[i].ifindex = mlx5_nl_ifindex\n-\t\t\t\t(nl_rdma, list[i].ibv_dev->name, 1);\n-\t\tif (nl_route < 0 ||\n-\t\t !list[i].ifindex ||\n-\t\t mlx5_nl_switch_info(nl_route, list[i].ifindex,\n-\t\t\t\t\t&list[i].info) ||\n-\t\t ((!list[i].info.representor && !list[i].info.master) &&\n-\t\t mlx5_sysfs_switch_info(list[i].ifindex, &list[i].info))) {\n-\t\t\tlist[i].ifindex = 0;\n-\t\t\tmemset(&list[i].info, 0, sizeof(list[i].info));\n-\t\t\tcontinue;\n+\tstruct mlx5_dev_spawn_data list[np ? np : nd];\n+\n+\tif (np > 1) {\n+\t\t/*\n+\t\t * Signle IB device with multiple ports found,\n+\t\t * it may be E-Switch master device and representors.\n+\t\t * We have to perform identification trough the ports.\n+\t\t */\n+\t\tassert(nl_rdma >= 0);\n+\t\tassert(ns == 0);\n+\t\tassert(nd == 1);\n+\t\tfor (i = 1; i <= np; ++i) {\n+\t\t\tlist[ns].max_port = np;\n+\t\t\tlist[ns].ibv_port = i;\n+\t\t\tlist[ns].ibv_dev = ibv_match[0];\n+\t\t\tlist[ns].eth_dev = NULL;\n+\t\t\tlist[ns].ifindex = mlx5_nl_ifindex\n+\t\t\t\t\t(nl_rdma, list[ns].ibv_dev->name, i);\n+\t\t\tif (!list[ns].ifindex) {\n+\t\t\t\t/*\n+\t\t\t\t * No network interface index found for the\n+\t\t\t\t * specified port, it means there is no\n+\t\t\t\t * representor on this port. It's OK,\n+\t\t\t\t * there can be disabled ports, for example\n+\t\t\t\t * if sriov_numvfs < sriov_totalvfs.\n+\t\t\t\t */\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tret = -1;\n+\t\t\tif (nl_route >= 0)\n+\t\t\t\tret = mlx5_nl_switch_info\n+\t\t\t\t\t (nl_route,\n+\t\t\t\t\t\tlist[ns].ifindex,\n+\t\t\t\t\t\t&list[ns].info);\n+\t\t\tif (ret || (!list[ns].info.representor &&\n+\t\t\t\t !list[ns].info.master)) {\n+\t\t\t\t/*\n+\t\t\t\t * We failed to recognize representors with\n+\t\t\t\t * Netlink, let's try to perform the task\n+\t\t\t\t * with sysfs.\n+\t\t\t\t */\n+\t\t\t\tret = mlx5_sysfs_switch_info\n+\t\t\t\t\t\t(list[ns].ifindex,\n+\t\t\t\t\t\t &list[ns].info);\n+\t\t\t}\n+\t\t\tif (!ret && (list[ns].info.representor ^\n+\t\t\t\t list[ns].info.master))\n+\t\t\t\tns++;\n \t\t}\n-\t}\n-\tif (nl_rdma >= 0)\n-\t\tclose(nl_rdma);\n-\tif (nl_route >= 0)\n-\t\tclose(nl_route);\n-\t/* Count unidentified devices. */\n-\tfor (u = 0, i = 0; i != n; ++i)\n-\t\tif (!list[i].info.master && !list[i].info.representor)\n-\t\t\t++u;\n-\tif (u) {\n-\t\tif (n == 1 && u == 1) {\n-\t\t\t/* Case #2. */\n-\t\t\tDRV_LOG(INFO, \"no switch support detected\");\n-\t\t} else {\n-\t\t\t/* Case #3. */\n+\t\tif (!ns) {\n+\t\t\tDRV_LOG(ERR,\n+\t\t\t\t\"unable to recognize master/representors\"\n+\t\t\t\t\" on the IB device with multiple ports\");\n+\t\t\trte_errno = ENOENT;\n+\t\t\tret = -rte_errno;\n+\t\t\tgoto exit;\n+\t\t}\n+\t} else {\n+\t\t/*\n+\t\t * The existence of several matching entries (nd > 1) means\n+\t\t * port representors have been instantiated. No existing Verbs\n+\t\t * call nor sysfs entries can tell them apart, this can only\n+\t\t * be done through Netlink calls assuming kernel drivers are\n+\t\t * recent enough to support them.\n+\t\t *\n+\t\t * In the event of identification failure through Netlink,\n+\t\t * try again through sysfs, then:\n+\t\t *\n+\t\t * 1. A single IB device matches (nd == 1) with single\n+\t\t * port (np=0/1) and is not a representor, assume\n+\t\t * no switch support.\n+\t\t *\n+\t\t * 2. Otherwise no safe assumptions can be made;\n+\t\t * complain louder and bail out.\n+\t\t */\n+\t\tnp = 1;\n+\t\tfor (i = 0; i != nd; ++i) {\n+\t\t\tmemset(&list[ns].info, 0, sizeof(list[ns].info));\n+\t\t\tlist[ns].max_port = 1;\n+\t\t\tlist[ns].ibv_port = 1;\n+\t\t\tlist[ns].ibv_dev = ibv_match[i];\n+\t\t\tlist[ns].eth_dev = NULL;\n+\t\t\tlist[ns].ifindex = 0;\n+\t\t\tif (nl_rdma >= 0)\n+\t\t\t\tlist[ns].ifindex = mlx5_nl_ifindex\n+\t\t\t\t\t(nl_rdma, list[ns].ibv_dev->name, 1);\n+\t\t\tif (!list[ns].ifindex) {\n+\t\t\t\t/*\n+\t\t\t\t * No network interface index found for the\n+\t\t\t\t * specified device, it means there it is not\n+\t\t\t\t * a representor/master.\n+\t\t\t\t */\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tret = -1;\n+\t\t\tif (nl_route >= 0)\n+\t\t\t\tret = mlx5_nl_switch_info\n+\t\t\t\t\t (nl_route,\n+\t\t\t\t\t\tlist[ns].ifindex,\n+\t\t\t\t\t\t&list[ns].info);\n+\t\t\tif (ret || (!list[ns].info.representor &&\n+\t\t\t\t !list[ns].info.master)) {\n+\t\t\t\t/*\n+\t\t\t\t * We failed to recognize representors with\n+\t\t\t\t * Netlink, let's try to perform the task\n+\t\t\t\t * with sysfs.\n+\t\t\t\t */\n+\t\t\t\tret = mlx5_sysfs_switch_info\n+\t\t\t\t\t\t(list[ns].ifindex,\n+\t\t\t\t\t\t &list[ns].info);\n+\t\t\t}\n+\t\t\tif (!ret && (list[ns].info.representor ^\n+\t\t\t\t list[ns].info.master)) {\n+\t\t\t\tns++;\n+\t\t\t} else if ((nd == 1) &&\n+\t\t\t\t !list[ns].info.representor &&\n+\t\t\t\t !list[ns].info.master) {\n+\t\t\t\t/*\n+\t\t\t\t * Single IB device with\n+\t\t\t\t * one physical port and\n+\t\t\t\t * attached network device.\n+\t\t\t\t * May be SRIOV is not enabled\n+\t\t\t\t * or there is no representors.\n+\t\t\t\t */\n+\t\t\t\tDRV_LOG(INFO, \"no E-Switch support detected\");\n+\t\t\t\tns++;\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t}\n+\t\tif (!ns) {\n \t\t\tDRV_LOG(ERR,\n-\t\t\t\t\"unable to tell which of the matching devices\"\n-\t\t\t\t\" is the master (lack of kernel support?)\");\n-\t\t\tn = 0;\n+\t\t\t\t\"unable to recognize master/representors\"\n+\t\t\t\t\" on the multiple IB devices\");\n+\t\t\trte_errno = ENOENT;\n+\t\t\tret = -rte_errno;\n+\t\t\tgoto exit;\n \t\t}\n \t}\n+\tassert(ns);\n \t/*\n \t * Sort list to probe devices in natural order for users convenience\n \t * (i.e. master first, then representors from lowest to highest ID).\n \t */\n-\tif (n)\n-\t\tqsort(list, n, sizeof(*list), mlx5_dev_spawn_data_cmp);\n+\tqsort(list, ns, sizeof(*list), mlx5_dev_spawn_data_cmp);\n \t/* Default configuration. */\n \tdev_config = (struct mlx5_dev_config){\n \t\t.hw_padding = 0,\n@@ -1497,7 +1612,7 @@ struct mlx5_dev_spawn_data {\n \t\t\t.min_rxqs_num = MLX5_MPRQ_MIN_RXQS,\n \t\t},\n \t};\n-\t/* Device speicific configuration. */\n+\t/* Device specific configuration. */\n \tswitch (pci_dev->id.device_id) {\n \tcase PCI_DEVICE_ID_MELLANOX_CONNECTX5BF:\n \t\tdev_config.txqs_vec = MLX5_VPMD_MAX_TXQS_BLUEFIELD;\n@@ -1514,12 +1629,12 @@ struct mlx5_dev_spawn_data {\n \t/* Set architecture-dependent default value if unset. */\n \tif (dev_config.txqs_vec == MLX5_ARG_UNSET)\n \t\tdev_config.txqs_vec = MLX5_VPMD_MAX_TXQS;\n-\tfor (i = 0; i != n; ++i) {\n+\tfor (i = 0; i != ns; ++i) {\n \t\tuint32_t restore;\n \n \t\tlist[i].eth_dev = mlx5_dev_spawn(&pci_dev->device,\n-\t\t\t\t\t\t list[i].ibv_dev, dev_config,\n-\t\t\t\t\t\t &list[i].info);\n+\t\t\t\t\t\t &list[i],\n+\t\t\t\t\t\t dev_config);\n \t\tif (!list[i].eth_dev) {\n \t\t\tif (rte_errno != EBUSY && rte_errno != EEXIST)\n \t\t\t\tbreak;\n@@ -1532,16 +1647,7 @@ struct mlx5_dev_spawn_data {\n \t\tlist[i].eth_dev->data->dev_flags |= restore;\n \t\trte_eth_dev_probing_finish(list[i].eth_dev);\n \t}\n-\tmlx5_glue->free_device_list(ibv_list);\n-\tif (!n) {\n-\t\tDRV_LOG(WARNING,\n-\t\t\t\"no Verbs device matches PCI device \" PCI_PRI_FMT \",\"\n-\t\t\t\" are kernel drivers loaded?\",\n-\t\t\tpci_dev->addr.domain, pci_dev->addr.bus,\n-\t\t\tpci_dev->addr.devid, pci_dev->addr.function);\n-\t\trte_errno = ENOENT;\n-\t\tret = -rte_errno;\n-\t} else if (i != n) {\n+\tif (i != ns) {\n \t\tDRV_LOG(ERR,\n \t\t\t\"probe of PCI device \" PCI_PRI_FMT \" aborted after\"\n \t\t\t\" encountering an error: %s\",\n@@ -1563,6 +1669,18 @@ struct mlx5_dev_spawn_data {\n \t} else {\n \t\tret = 0;\n \t}\n+exit:\n+\t/*\n+\t * Do the routine cleanup:\n+\t * - close opened Netlink sockets\n+\t * - free the Infiniband device list\n+\t */\n+\tif (nl_rdma >= 0)\n+\t\tclose(nl_rdma);\n+\tif (nl_route >= 0)\n+\t\tclose(nl_route);\n+\tassert(ibv_list);\n+\tmlx5_glue->free_device_list(ibv_list);\n \treturn ret;\n }\n \n", "prefixes": [ "05/14" ] }