Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/54793/?format=api
{ "id": 54793, "url": "http://patches.dpdk.org/api/patches/54793/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20190614021940.78631-4-xiaoyun.li@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20190614021940.78631-4-xiaoyun.li@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20190614021940.78631-4-xiaoyun.li@intel.com", "date": "2019-06-14T02:19:37", "name": "[v5,3/6] raw/ntb: add handshake process", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "c89578294ca01b2aa3da573ad9bcd628125e66b3", "submitter": { "id": 798, "url": "http://patches.dpdk.org/api/people/798/?format=api", "name": "Li, Xiaoyun", "email": "xiaoyun.li@intel.com" }, "delegate": { "id": 1, "url": "http://patches.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20190614021940.78631-4-xiaoyun.li@intel.com/mbox/", "series": [ { "id": 5008, "url": "http://patches.dpdk.org/api/series/5008/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=5008", "date": "2019-06-14T02:19:34", "name": "rawdev driver for ntb", "version": 5, "mbox": "http://patches.dpdk.org/series/5008/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/54793/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/54793/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id BC4871D47D;\n\tFri, 14 Jun 2019 04:20:34 +0200 (CEST)", "from mga02.intel.com (mga02.intel.com [134.134.136.20])\n\tby dpdk.org (Postfix) with ESMTP id 567652BA5\n\tfor <dev@dpdk.org>; Fri, 14 Jun 2019 04:20:30 +0200 (CEST)", "from orsmga007.jf.intel.com ([10.7.209.58])\n\tby orsmga101.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t13 Jun 2019 19:20:30 -0700", "from dpdk-xiaoyun3.sh.intel.com ([10.67.119.132])\n\tby orsmga007.jf.intel.com with ESMTP; 13 Jun 2019 19:20:28 -0700" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "From": "Xiaoyun Li <xiaoyun.li@intel.com>", "To": "jingjing.wu@intel.com, keith.wiles@intel.com, cunming.liang@intel.com,\n\tomkar.maslekar@intel.com", "Cc": "dev@dpdk.org,\n\tXiaoyun Li <xiaoyun.li@intel.com>", "Date": "Fri, 14 Jun 2019 10:19:37 +0800", "Message-Id": "<20190614021940.78631-4-xiaoyun.li@intel.com>", "X-Mailer": "git-send-email 2.17.1", "In-Reply-To": "<20190614021940.78631-1-xiaoyun.li@intel.com>", "References": "<20190613075649.67472-1-xiaoyun.li@intel.com>\n\t<20190614021940.78631-1-xiaoyun.li@intel.com>", "Subject": "[dpdk-dev] [PATCH v5 3/6] raw/ntb: add handshake process", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Add handshake process using doorbell so that two hosts can\ncommunicate to start and stop.\n\nSigned-off-by: Xiaoyun Li <xiaoyun.li@intel.com>\n---\n drivers/raw/ntb_rawdev/ntb_rawdev.c | 336 +++++++++++++++++++++++++++-\n 1 file changed, 335 insertions(+), 1 deletion(-)", "diff": "diff --git a/drivers/raw/ntb_rawdev/ntb_rawdev.c b/drivers/raw/ntb_rawdev/ntb_rawdev.c\nindex a03decd55..d9088e825 100644\n--- a/drivers/raw/ntb_rawdev/ntb_rawdev.c\n+++ b/drivers/raw/ntb_rawdev/ntb_rawdev.c\n@@ -28,6 +28,183 @@ static const struct rte_pci_id pci_id_ntb_map[] = {\n \t{ .vendor_id = 0, /* sentinel */ },\n };\n \n+static int\n+ntb_set_mw(struct rte_rawdev *dev, int mw_idx, uint64_t mw_size)\n+{\n+\tstruct ntb_hw *hw = dev->dev_private;\n+\tchar mw_name[RTE_MEMZONE_NAMESIZE];\n+\tconst struct rte_memzone *mz;\n+\tint ret = 0;\n+\n+\tif (hw->ntb_ops->mw_set_trans == NULL) {\n+\t\tNTB_LOG(ERR, \"Not supported to set mw.\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\tsnprintf(mw_name, sizeof(mw_name), \"ntb_%d_mw_%d\",\n+\t\t dev->dev_id, mw_idx);\n+\n+\tmz = rte_memzone_lookup(mw_name);\n+\tif (mz)\n+\t\treturn 0;\n+\n+\t/**\n+\t * Hardware requires that mapped memory base address should be\n+\t * aligned with EMBARSZ and needs continuous memzone.\n+\t */\n+\tmz = rte_memzone_reserve_aligned(mw_name, mw_size, dev->socket_id,\n+\t\t\t\tRTE_MEMZONE_IOVA_CONTIG, hw->mw_size[mw_idx]);\n+\tif (!mz) {\n+\t\tNTB_LOG(ERR, \"Cannot allocate aligned memzone.\");\n+\t\treturn -EIO;\n+\t}\n+\thw->mz[mw_idx] = mz;\n+\n+\tret = (*hw->ntb_ops->mw_set_trans)(dev, mw_idx, mz->iova, mw_size);\n+\tif (ret) {\n+\t\tNTB_LOG(ERR, \"Cannot set mw translation.\");\n+\t\treturn ret;\n+\t}\n+\n+\treturn ret;\n+}\n+\n+static void\n+ntb_link_cleanup(struct rte_rawdev *dev)\n+{\n+\tstruct ntb_hw *hw = dev->dev_private;\n+\tint status, i;\n+\n+\tif (hw->ntb_ops->spad_write == NULL ||\n+\t hw->ntb_ops->mw_set_trans == NULL) {\n+\t\tNTB_LOG(ERR, \"Not supported to clean up link.\");\n+\t\treturn;\n+\t}\n+\n+\t/* Clean spad registers. */\n+\tfor (i = 0; i < hw->spad_cnt; i++) {\n+\t\tstatus = (*hw->ntb_ops->spad_write)(dev, i, 0, 0);\n+\t\tif (status)\n+\t\t\tNTB_LOG(ERR, \"Failed to clean local spad.\");\n+\t}\n+\n+\t/* Clear mw so that peer cannot access local memory.*/\n+\tfor (i = 0; i < hw->mw_cnt; i++) {\n+\t\tstatus = (*hw->ntb_ops->mw_set_trans)(dev, i, 0, 0);\n+\t\tif (status)\n+\t\t\tNTB_LOG(ERR, \"Failed to clean mw.\");\n+\t}\n+}\n+\n+static void\n+ntb_dev_intr_handler(void *param)\n+{\n+\tstruct rte_rawdev *dev = (struct rte_rawdev *)param;\n+\tstruct ntb_hw *hw = dev->dev_private;\n+\tuint32_t mw_size_h, mw_size_l;\n+\tuint64_t db_bits = 0;\n+\tint i = 0;\n+\n+\tif (hw->ntb_ops->db_read == NULL ||\n+\t hw->ntb_ops->db_clear == NULL ||\n+\t hw->ntb_ops->peer_db_set == NULL) {\n+\t\tNTB_LOG(ERR, \"Doorbell is not supported.\");\n+\t\treturn;\n+\t}\n+\n+\tdb_bits = (*hw->ntb_ops->db_read)(dev);\n+\tif (!db_bits)\n+\t\tNTB_LOG(ERR, \"No doorbells\");\n+\n+\t/* Doorbell 0 is for peer device ready. */\n+\tif (db_bits & 1) {\n+\t\tNTB_LOG(DEBUG, \"DB0: Peer device is up.\");\n+\t\t/* Clear received doorbell. */\n+\t\t(*hw->ntb_ops->db_clear)(dev, 1);\n+\n+\t\t/**\n+\t\t * Peer dev is already up. All mw settings are already done.\n+\t\t * Skip them.\n+\t\t */\n+\t\tif (hw->peer_dev_up)\n+\t\t\treturn;\n+\n+\t\tif (hw->ntb_ops->spad_read == NULL ||\n+\t\t hw->ntb_ops->spad_write == NULL) {\n+\t\t\tNTB_LOG(ERR, \"Scratchpad is not supported.\");\n+\t\t\treturn;\n+\t\t}\n+\n+\t\thw->peer_mw_cnt = (*hw->ntb_ops->spad_read)\n+\t\t\t\t (dev, SPAD_NUM_MWS, 0);\n+\t\thw->peer_mw_size = rte_zmalloc(\"uint64_t\",\n+\t\t\t\t hw->peer_mw_cnt * sizeof(uint64_t), 0);\n+\t\tfor (i = 0; i < hw->mw_cnt; i++) {\n+\t\t\tmw_size_h = (*hw->ntb_ops->spad_read)\n+\t\t\t\t (dev, SPAD_MW0_SZ_H + 2 * i, 0);\n+\t\t\tmw_size_l = (*hw->ntb_ops->spad_read)\n+\t\t\t\t (dev, SPAD_MW0_SZ_L + 2 * i, 0);\n+\t\t\thw->peer_mw_size[i] = ((uint64_t)mw_size_h << 32) |\n+\t\t\t\t\t mw_size_l;\n+\t\t\tNTB_LOG(DEBUG, \"Peer %u mw size: 0x%\"PRIx64\"\", i,\n+\t\t\t\t\thw->peer_mw_size[i]);\n+\t\t}\n+\n+\t\thw->peer_dev_up = 1;\n+\n+\t\t/**\n+\t\t * Handshake with peer. Spad_write only works when both\n+\t\t * devices are up. So write spad again when db is received.\n+\t\t * And set db again for the later device who may miss\n+\t\t * the 1st db.\n+\t\t */\n+\t\tfor (i = 0; i < hw->mw_cnt; i++) {\n+\t\t\t(*hw->ntb_ops->spad_write)(dev, SPAD_NUM_MWS,\n+\t\t\t\t\t\t 1, hw->mw_cnt);\n+\t\t\tmw_size_h = hw->mw_size[i] >> 32;\n+\t\t\t(*hw->ntb_ops->spad_write)(dev, SPAD_MW0_SZ_H + 2 * i,\n+\t\t\t\t\t\t 1, mw_size_h);\n+\n+\t\t\tmw_size_l = hw->mw_size[i];\n+\t\t\t(*hw->ntb_ops->spad_write)(dev, SPAD_MW0_SZ_L + 2 * i,\n+\t\t\t\t\t\t 1, mw_size_l);\n+\t\t}\n+\t\t(*hw->ntb_ops->peer_db_set)(dev, 0);\n+\n+\t\t/* To get the link info. */\n+\t\tif (hw->ntb_ops->get_link_status == NULL) {\n+\t\t\tNTB_LOG(ERR, \"Not supported to get link status.\");\n+\t\t\treturn;\n+\t\t}\n+\t\t(*hw->ntb_ops->get_link_status)(dev);\n+\t\tNTB_LOG(INFO, \"Link is up. Link speed: %u. Link width: %u\",\n+\t\t\thw->link_speed, hw->link_width);\n+\t\treturn;\n+\t}\n+\n+\tif (db_bits & (1 << 1)) {\n+\t\tNTB_LOG(DEBUG, \"DB1: Peer device is down.\");\n+\t\t/* Clear received doorbell. */\n+\t\t(*hw->ntb_ops->db_clear)(dev, 2);\n+\n+\t\t/* Peer device will be down, So clean local side too. */\n+\t\tntb_link_cleanup(dev);\n+\n+\t\thw->peer_dev_up = 0;\n+\t\t/* Response peer's dev_stop request. */\n+\t\t(*hw->ntb_ops->peer_db_set)(dev, 2);\n+\t\treturn;\n+\t}\n+\n+\tif (db_bits & (1 << 2)) {\n+\t\tNTB_LOG(DEBUG, \"DB2: Peer device agrees dev to be down.\");\n+\t\t/* Clear received doorbell. */\n+\t\t(*hw->ntb_ops->db_clear)(dev, (1 << 2));\n+\t\thw->peer_dev_up = 0;\n+\t\treturn;\n+\t}\n+}\n+\n static void\n ntb_queue_conf_get(struct rte_rawdev *dev __rte_unused,\n \t\t uint16_t queue_id __rte_unused,\n@@ -147,7 +324,22 @@ ntb_dev_configure(const struct rte_rawdev *dev __rte_unused,\n static int\n ntb_dev_start(struct rte_rawdev *dev)\n {\n+\tstruct ntb_hw *hw = dev->dev_private;\n+\tint ret, i;\n+\n \t/* TODO: init queues and start queues. */\n+\n+\t/* Map memory of bar_size to remote. */\n+\thw->mz = rte_zmalloc(\"struct rte_memzone *\",\n+\t\t\t hw->mw_cnt * sizeof(struct rte_memzone *), 0);\n+\tfor (i = 0; i < hw->mw_cnt; i++) {\n+\t\tret = ntb_set_mw(dev, i, hw->mw_size[i]);\n+\t\tif (ret) {\n+\t\t\tNTB_LOG(ERR, \"Fail to set mw.\");\n+\t\t\treturn ret;\n+\t\t}\n+\t}\n+\n \tdev->started = 1;\n \n \treturn 0;\n@@ -156,13 +348,59 @@ ntb_dev_start(struct rte_rawdev *dev)\n static void\n ntb_dev_stop(struct rte_rawdev *dev)\n {\n+\tstruct ntb_hw *hw = dev->dev_private;\n+\tuint32_t time_out;\n+\tint status;\n+\n \t/* TODO: stop rx/tx queues. */\n+\n+\tif (!hw->peer_dev_up)\n+\t\tgoto clean;\n+\n+\tntb_link_cleanup(dev);\n+\n+\t/* Notify the peer that device will be down. */\n+\tif (hw->ntb_ops->peer_db_set == NULL) {\n+\t\tNTB_LOG(ERR, \"Peer doorbell setting is not supported.\");\n+\t\treturn;\n+\t}\n+\tstatus = (*hw->ntb_ops->peer_db_set)(dev, 1);\n+\tif (status) {\n+\t\tNTB_LOG(ERR, \"Failed to tell peer device is down.\");\n+\t\treturn;\n+\t}\n+\n+\t/*\n+\t * Set time out as 1s in case that the peer is stopped accidently\n+\t * without any notification.\n+\t */\n+\ttime_out = 1000000;\n+\n+\t/* Wait for cleanup work down before db mask clear. */\n+\twhile (hw->peer_dev_up && time_out) {\n+\t\ttime_out -= 10;\n+\t\trte_delay_us(10);\n+\t}\n+\n+clean:\n+\t/* Clear doorbells mask. */\n+\tif (hw->ntb_ops->db_set_mask == NULL) {\n+\t\tNTB_LOG(ERR, \"Doorbell mask setting is not supported.\");\n+\t\treturn;\n+\t}\n+\tstatus = (*hw->ntb_ops->db_set_mask)(dev,\n+\t\t\t\t(((uint64_t)1 << hw->db_cnt) - 1));\n+\tif (status)\n+\t\tNTB_LOG(ERR, \"Failed to clear doorbells.\");\n+\n \tdev->started = 0;\n }\n \n static int\n ntb_dev_close(struct rte_rawdev *dev)\n {\n+\tstruct ntb_hw *hw = dev->dev_private;\n+\tstruct rte_intr_handle *intr_handle;\n \tint ret = 0;\n \n \tif (dev->started)\n@@ -170,6 +408,20 @@ ntb_dev_close(struct rte_rawdev *dev)\n \n \t/* TODO: free queues. */\n \n+\tintr_handle = &hw->pci_dev->intr_handle;\n+\t/* Clean datapath event and vec mapping */\n+\trte_intr_efd_disable(intr_handle);\n+\tif (intr_handle->intr_vec) {\n+\t\trte_free(intr_handle->intr_vec);\n+\t\tintr_handle->intr_vec = NULL;\n+\t}\n+\t/* Disable uio intr before callback unregister */\n+\trte_intr_disable(intr_handle);\n+\n+\t/* Unregister callback func to eal lib */\n+\trte_intr_callback_unregister(intr_handle,\n+\t\t\t\t ntb_dev_intr_handler, dev);\n+\n \treturn ret;\n }\n \n@@ -356,7 +608,9 @@ static int\n ntb_init_hw(struct rte_rawdev *dev, struct rte_pci_device *pci_dev)\n {\n \tstruct ntb_hw *hw = dev->dev_private;\n-\tint ret;\n+\tstruct rte_intr_handle *intr_handle;\n+\tuint32_t val;\n+\tint ret, i;\n \n \thw->pci_dev = pci_dev;\n \thw->peer_dev_up = 0;\n@@ -387,6 +641,86 @@ ntb_init_hw(struct rte_rawdev *dev, struct rte_pci_device *pci_dev)\n \tif (ret)\n \t\treturn ret;\n \n+\t/* Init doorbell. */\n+\thw->db_valid_mask = ((uint64_t)1 << hw->db_cnt) - 1;\n+\n+\tintr_handle = &pci_dev->intr_handle;\n+\t/* Register callback func to eal lib */\n+\trte_intr_callback_register(intr_handle,\n+\t\t\t\t ntb_dev_intr_handler, dev);\n+\n+\tret = rte_intr_efd_enable(intr_handle, hw->db_cnt);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\t/* To clarify, the interrupt for each doorbell is already mapped\n+\t * by default for intel gen3. They are mapped to msix vec 1-32,\n+\t * and hardware intr is mapped to 0. Map all to 0 for uio.\n+\t */\n+\tif (!rte_intr_cap_multiple(intr_handle)) {\n+\t\tfor (i = 0; i < hw->db_cnt; i++) {\n+\t\t\tif (hw->ntb_ops->vector_bind == NULL)\n+\t\t\t\treturn -ENOTSUP;\n+\t\t\tret = (*hw->ntb_ops->vector_bind)(dev, i, 0);\n+\t\t\tif (ret)\n+\t\t\t\treturn ret;\n+\t\t}\n+\t}\n+\n+\tif (hw->ntb_ops->db_set_mask == NULL ||\n+\t hw->ntb_ops->peer_db_set == NULL) {\n+\t\tNTB_LOG(ERR, \"Doorbell is not supported.\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\thw->db_mask = 0;\n+\tret = (*hw->ntb_ops->db_set_mask)(dev, hw->db_mask);\n+\tif (ret) {\n+\t\tNTB_LOG(ERR, \"Unanle to enable intr for all dbs.\");\n+\t\treturn ret;\n+\t}\n+\n+\t/* enable uio intr after callback register */\n+\trte_intr_enable(intr_handle);\n+\n+\tif (hw->ntb_ops->spad_write == NULL) {\n+\t\tNTB_LOG(ERR, \"Scratchpad is not supported.\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\t/* Tell peer the mw_cnt of local side. */\n+\tret = (*hw->ntb_ops->spad_write)(dev, SPAD_NUM_MWS, 1, hw->mw_cnt);\n+\tif (ret) {\n+\t\tNTB_LOG(ERR, \"Failed to tell peer mw count.\");\n+\t\treturn ret;\n+\t}\n+\n+\t/* Tell peer each mw size on local side. */\n+\tfor (i = 0; i < hw->mw_cnt; i++) {\n+\t\tNTB_LOG(DEBUG, \"Local %u mw size: 0x%\"PRIx64\"\", i,\n+\t\t\t\thw->mw_size[i]);\n+\t\tval = hw->mw_size[i] >> 32;\n+\t\tret = (*hw->ntb_ops->spad_write)\n+\t\t\t\t(dev, SPAD_MW0_SZ_H + 2 * i, 1, val);\n+\t\tif (ret) {\n+\t\t\tNTB_LOG(ERR, \"Failed to tell peer mw size.\");\n+\t\t\treturn ret;\n+\t\t}\n+\n+\t\tval = hw->mw_size[i];\n+\t\tret = (*hw->ntb_ops->spad_write)\n+\t\t\t\t(dev, SPAD_MW0_SZ_L + 2 * i, 1, val);\n+\t\tif (ret) {\n+\t\t\tNTB_LOG(ERR, \"Failed to tell peer mw size.\");\n+\t\t\treturn ret;\n+\t\t}\n+\t}\n+\n+\t/* Ring doorbell 0 to tell peer the device is ready. */\n+\tret = (*hw->ntb_ops->peer_db_set)(dev, 0);\n+\tif (ret) {\n+\t\tNTB_LOG(ERR, \"Failed to tell peer device is probed.\");\n+\t\treturn ret;\n+\t}\n+\n \treturn ret;\n }\n \n", "prefixes": [ "v5", "3/6" ] }