Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/97480/?format=api
{ "id": 97480, "url": "http://patches.dpdk.org/api/patches/97480/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20210827172048.558704-11-kevin.laatz@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20210827172048.558704-11-kevin.laatz@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20210827172048.558704-11-kevin.laatz@intel.com", "date": "2021-08-27T17:20:45", "name": "[10/13] dma/idxd: add data-path job submission functions", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "07ad165acacc63184b84bacb21e5fbd3650d2f9d", "submitter": { "id": 921, "url": "http://patches.dpdk.org/api/people/921/?format=api", "name": "Kevin Laatz", "email": "kevin.laatz@intel.com" }, "delegate": { "id": 1, "url": "http://patches.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20210827172048.558704-11-kevin.laatz@intel.com/mbox/", "series": [ { "id": 18500, "url": "http://patches.dpdk.org/api/series/18500/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=18500", "date": "2021-08-27T17:20:35", "name": "add dmadev driver for idxd devices", "version": 1, "mbox": "http://patches.dpdk.org/series/18500/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/97480/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/97480/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 2EE5DA0548;\n\tFri, 27 Aug 2021 19:22:17 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 23BCF4128E;\n\tFri, 27 Aug 2021 19:21:29 +0200 (CEST)", "from mga17.intel.com (mga17.intel.com [192.55.52.151])\n by mails.dpdk.org (Postfix) with ESMTP id E1FDF41286\n for <dev@dpdk.org>; Fri, 27 Aug 2021 19:21:26 +0200 (CEST)", "from orsmga002.jf.intel.com ([10.7.209.21])\n by fmsmga107.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 27 Aug 2021 10:21:26 -0700", "from silpixa00401122.ir.intel.com ([10.55.128.10])\n by orsmga002.jf.intel.com with ESMTP; 27 Aug 2021 10:21:24 -0700" ], "X-IronPort-AV": [ "E=McAfee;i=\"6200,9189,10089\"; a=\"198235946\"", "E=Sophos;i=\"5.84,357,1620716400\"; d=\"scan'208\";a=\"198235946\"", "E=Sophos;i=\"5.84,357,1620716400\"; d=\"scan'208\";a=\"445009573\"" ], "X-ExtLoop1": "1", "From": "Kevin Laatz <kevin.laatz@intel.com>", "To": "dev@dpdk.org", "Cc": "bruce.richardson@intel.com, fengchengwen@huawei.com, jerinj@marvell.com,\n conor.walsh@intel.com, Kevin Laatz <kevin.laatz@intel.com>", "Date": "Fri, 27 Aug 2021 17:20:45 +0000", "Message-Id": "<20210827172048.558704-11-kevin.laatz@intel.com>", "X-Mailer": "git-send-email 2.30.2", "In-Reply-To": "<20210827172048.558704-1-kevin.laatz@intel.com>", "References": "<20210827172048.558704-1-kevin.laatz@intel.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[dpdk-dev] [PATCH 10/13] dma/idxd: add data-path job submission\n functions", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Add data path functions for enqueuing and submitting operations to DSA\ndevices.\n\nSigned-off-by: Bruce Richardson <bruce.richardson@intel.com>\nSigned-off-by: Kevin Laatz <kevin.laatz@intel.com>\n---\n doc/guides/dmadevs/idxd.rst | 64 ++++++++++++++\n drivers/dma/idxd/idxd_common.c | 138 +++++++++++++++++++++++++++++++\n drivers/dma/idxd/idxd_internal.h | 5 ++\n drivers/dma/idxd/meson.build | 1 +\n 4 files changed, 208 insertions(+)", "diff": "diff --git a/doc/guides/dmadevs/idxd.rst b/doc/guides/dmadevs/idxd.rst\nindex 66bc9fe744..0c4c105e0f 100644\n--- a/doc/guides/dmadevs/idxd.rst\n+++ b/doc/guides/dmadevs/idxd.rst\n@@ -153,3 +153,67 @@ The following code shows how the device is configured in\n \n Once configured, the device can then be made ready for use by calling the\n ``rte_dmadev_start()`` API.\n+\n+Performing Data Copies\n+~~~~~~~~~~~~~~~~~~~~~~~\n+\n+To perform data copies using IDXD dmadev devices, descriptors should be enqueued\n+using the ``rte_dmadev_copy()`` API. The HW can be triggered to perform the copy\n+in two ways, either via a ``RTE_DMA_OP_FLAG_SUBMIT`` flag or by calling\n+``rte_dmadev_submit()``. Once copies have been completed, the completion will\n+be reported back when the application calls ``rte_dmadev_completed()`` or\n+``rte_dmadev_completed_status()``. The latter will also report the status of each\n+completed operation.\n+\n+The ``rte_dmadev_copy()`` function enqueues a single copy to the device ring for\n+copying at a later point. The parameters to that function include the IOVA addresses\n+of both the source and destination buffers, as well as the length of the copy.\n+\n+The ``rte_dmadev_copy()`` function enqueues a copy operation on the device ring.\n+If the ``RTE_DMA_OP_FLAG_SUBMIT`` flag is set when calling ``rte_dmadev_copy()``,\n+the device hardware will be informed of the elements. Alternatively, if the flag\n+is not set, the application need to call the ``rte_dmadev_submit()`` function to\n+notify the device hardware. Once the device hardware is informed of the elements\n+enqueued on the ring, and the device will begin to process them. It is expected\n+that, for efficiency reasons, a burst of operations will be enqueued to the\n+device via multiple enqueue calls between calls to the ``rte_dmadev_submit()``\n+function.\n+\n+The following code from demonstrates how to enqueue a burst of copies to the\n+device and start the hardware processing of them:\n+\n+.. code-block:: C\n+\n+ struct rte_mbuf *srcs[COMP_BURST_SZ], *dsts[COMP_BURST_SZ];\n+ unsigned int i;\n+\n+ for (i = 0; i < RTE_DIM(srcs); i++) {\n+ uint64_t *src_data;\n+\n+ srcs[i] = rte_pktmbuf_alloc(pool);\n+ dsts[i] = rte_pktmbuf_alloc(pool);\n+ src_data = rte_pktmbuf_mtod(srcs[i], uint64_t *);\n+ if (srcs[i] == NULL || dsts[i] == NULL) {\n+ PRINT_ERR(\"Error allocating buffers\\n\");\n+ return -1;\n+ }\n+\n+ for (j = 0; j < COPY_LEN/sizeof(uint64_t); j++)\n+ src_data[j] = rte_rand();\n+\n+ if (rte_dmadev_copy(dev_id, vchan, srcs[i]->buf_iova + srcs[i]->data_off,\n+ dsts[i]->buf_iova + dsts[i]->data_off, COPY_LEN, 0) < 0) {\n+ PRINT_ERR(\"Error with rte_dmadev_copy for buffer %u\\n\", i);\n+ return -1;\n+ }\n+ }\n+ rte_dmadev_submit(dev_id, vchan);\n+\n+Filling an Area of Memory\n+~~~~~~~~~~~~~~~~~~~~~~~~~~\n+\n+The IDXD driver also has support for the ``fill`` operation, where an area\n+of memory is overwritten, or filled, with a short pattern of data.\n+Fill operations can be performed in much the same was as copy operations\n+described above, just using the ``rte_dmadev_fill()`` function rather than the\n+``rte_dmadev_copy()`` function.\ndiff --git a/drivers/dma/idxd/idxd_common.c b/drivers/dma/idxd/idxd_common.c\nindex ea2c0b7f19..e2ef7b3b95 100644\n--- a/drivers/dma/idxd/idxd_common.c\n+++ b/drivers/dma/idxd/idxd_common.c\n@@ -2,14 +2,148 @@\n * Copyright 2021 Intel Corporation\n */\n \n+#include <x86intrin.h>\n+\n #include <rte_dmadev_pmd.h>\n #include <rte_malloc.h>\n #include <rte_common.h>\n+#include <rte_prefetch.h>\n \n #include \"idxd_internal.h\"\n \n #define IDXD_PMD_NAME_STR \"dmadev_idxd\"\n \n+static __rte_always_inline rte_iova_t\n+__desc_idx_to_iova(struct idxd_dmadev *idxd, uint16_t n)\n+{\n+\treturn idxd->desc_iova + (n * sizeof(struct idxd_hw_desc));\n+}\n+\n+static __rte_always_inline void\n+__idxd_movdir64b(volatile void *dst, const struct idxd_hw_desc *src)\n+{\n+\tasm volatile (\".byte 0x66, 0x0f, 0x38, 0xf8, 0x02\"\n+\t\t\t:\n+\t\t\t: \"a\" (dst), \"d\" (src)\n+\t\t\t: \"memory\");\n+}\n+\n+static __rte_always_inline void\n+__submit(struct idxd_dmadev *idxd)\n+{\n+\t/* TODO have flag setting indicating polling on same core as submission */\n+\trte_prefetch1(&idxd->batch_comp_ring[idxd->batch_idx_read]);\n+\n+\tif (idxd->batch_size == 0)\n+\t\treturn;\n+\n+\t/* write completion to batch comp ring */\n+\trte_iova_t comp_addr = idxd->batch_iova +\n+\t\t\t(idxd->batch_idx_write * sizeof(struct idxd_completion));\n+\n+\tif (idxd->batch_size == 1) {\n+\t\t/* submit batch directly */\n+\t\tstruct idxd_hw_desc desc =\n+\t\t\t\tidxd->desc_ring[idxd->batch_start & idxd->desc_ring_mask];\n+\t\tdesc.completion = comp_addr;\n+\t\tdesc.op_flags |= IDXD_FLAG_REQUEST_COMPLETION;\n+\t\t_mm_sfence(); /* fence before writing desc to device */\n+\t\t__idxd_movdir64b(idxd->portal, &desc);\n+\t} else {\n+\t\tconst struct idxd_hw_desc batch_desc = {\n+\t\t\t\t.op_flags = (idxd_op_batch << IDXD_CMD_OP_SHIFT) |\n+\t\t\t\tIDXD_FLAG_COMPLETION_ADDR_VALID |\n+\t\t\t\tIDXD_FLAG_REQUEST_COMPLETION,\n+\t\t\t\t.desc_addr = __desc_idx_to_iova(idxd,\n+\t\t\t\t\t\tidxd->batch_start & idxd->desc_ring_mask),\n+\t\t\t\t.completion = comp_addr,\n+\t\t\t\t.size = idxd->batch_size,\n+\t\t};\n+\t\t_mm_sfence(); /* fence before writing desc to device */\n+\t\t__idxd_movdir64b(idxd->portal, &batch_desc);\n+\t}\n+\n+\tif (++idxd->batch_idx_write > idxd->max_batches)\n+\t\tidxd->batch_idx_write = 0;\n+\n+\tidxd->batch_start += idxd->batch_size;\n+\tidxd->batch_size = 0;\n+\tidxd->batch_idx_ring[idxd->batch_idx_write] = idxd->batch_start;\n+\t_mm256_store_si256((void *)&idxd->batch_comp_ring[idxd->batch_idx_write],\n+\t\t\t_mm256_setzero_si256());\n+}\n+\n+static __rte_always_inline int\n+__idxd_write_desc(struct rte_dmadev *dev,\n+\t\tconst uint32_t op_flags,\n+\t\tconst rte_iova_t src,\n+\t\tconst rte_iova_t dst,\n+\t\tconst uint32_t size,\n+\t\tconst uint32_t flags)\n+{\n+\tstruct idxd_dmadev *idxd = dev->dev_private;\n+\tuint16_t mask = idxd->desc_ring_mask;\n+\tuint16_t job_id = idxd->batch_start + idxd->batch_size;\n+\t/* we never wrap batches, so we only mask the start and allow start+size to overflow */\n+\tuint16_t write_idx = (idxd->batch_start & mask) + idxd->batch_size;\n+\n+\t/* first check batch ring space then desc ring space */\n+\tif ((idxd->batch_idx_read == 0 && idxd->batch_idx_write == idxd->max_batches) ||\n+\t\t\tidxd->batch_idx_write + 1 == idxd->batch_idx_read)\n+\t\tgoto failed;\n+\tif (((write_idx + 1) & mask) == (idxd->ids_returned & mask))\n+\t\tgoto failed;\n+\n+\t/* write desc. Note: descriptors don't wrap, but the completion address does */\n+\tconst uint64_t op_flags64 = (uint64_t)(op_flags | IDXD_FLAG_COMPLETION_ADDR_VALID) << 32;\n+\tconst uint64_t comp_addr = __desc_idx_to_iova(idxd, write_idx & mask);\n+\t_mm256_store_si256((void *)&idxd->desc_ring[write_idx],\n+\t\t\t_mm256_set_epi64x(dst, src, comp_addr, op_flags64));\n+\t_mm256_store_si256((void *)&idxd->desc_ring[write_idx].size,\n+\t\t\t_mm256_set_epi64x(0, 0, 0, size));\n+\n+\tidxd->batch_size++;\n+\n+\trte_prefetch0_write(&idxd->desc_ring[write_idx + 1]);\n+\n+\tif (flags & RTE_DMA_OP_FLAG_SUBMIT)\n+\t\t__submit(idxd);\n+\n+\treturn job_id;\n+\n+failed:\n+\treturn -1;\n+}\n+\n+int\n+idxd_enqueue_copy(struct rte_dmadev *dev, uint16_t qid __rte_unused, rte_iova_t src,\n+\t\trte_iova_t dst, unsigned int length, uint64_t flags)\n+{\n+\t/* we can take advantage of the fact that the fence flag in dmadev and DSA are the same,\n+\t * but check it at compile time to be sure.\n+\t */\n+\tRTE_BUILD_BUG_ON(RTE_DMA_OP_FLAG_FENCE != IDXD_FLAG_FENCE);\n+\tuint32_t memmove = (idxd_op_memmove << IDXD_CMD_OP_SHIFT) |\n+\t\t\tIDXD_FLAG_CACHE_CONTROL | (flags & IDXD_FLAG_FENCE);\n+\treturn __idxd_write_desc(dev, memmove, src, dst, length, flags);\n+}\n+\n+int\n+idxd_enqueue_fill(struct rte_dmadev *dev, uint16_t qid __rte_unused, uint64_t pattern,\n+\t\trte_iova_t dst, unsigned int length, uint64_t flags)\n+{\n+\tuint32_t fill = (idxd_op_fill << IDXD_CMD_OP_SHIFT) |\n+\t\t\tIDXD_FLAG_CACHE_CONTROL | (flags & IDXD_FLAG_FENCE);\n+\treturn __idxd_write_desc(dev, fill, pattern, dst, length, flags);\n+}\n+\n+int\n+idxd_submit(struct rte_dmadev *dev, uint16_t qid __rte_unused)\n+{\n+\t__submit(dev->dev_private);\n+\treturn 0;\n+}\n+\n int\n idxd_dump(const struct rte_dmadev *dev, FILE *f)\n {\n@@ -135,6 +269,10 @@ idxd_dmadev_create(const char *name, struct rte_device *dev,\n \tdmadev->dev_ops = ops;\n \tdmadev->device = dev;\n \n+\tdmadev->copy = idxd_enqueue_copy;\n+\tdmadev->fill = idxd_enqueue_fill;\n+\tdmadev->submit = idxd_submit;\n+\n \tidxd = rte_malloc_socket(NULL, sizeof(struct idxd_dmadev), 0, dev->numa_node);\n \tif (idxd == NULL) {\n \t\tIDXD_PMD_ERR(\"Unable to allocate memory for device\");\ndiff --git a/drivers/dma/idxd/idxd_internal.h b/drivers/dma/idxd/idxd_internal.h\nindex 18fc65d00c..6a6c69fd61 100644\n--- a/drivers/dma/idxd/idxd_internal.h\n+++ b/drivers/dma/idxd/idxd_internal.h\n@@ -85,5 +85,10 @@ int idxd_vchan_setup(struct rte_dmadev *dev, uint16_t vchan,\n \t\tconst struct rte_dmadev_vchan_conf *qconf);\n int idxd_info_get(const struct rte_dmadev *dev, struct rte_dmadev_info *dev_info,\n \t\tuint32_t size);\n+int idxd_enqueue_copy(struct rte_dmadev *dev, uint16_t qid, rte_iova_t src,\n+\t\trte_iova_t dst, unsigned int length, uint64_t flags);\n+int idxd_enqueue_fill(struct rte_dmadev *dev, uint16_t qid, uint64_t pattern,\n+\t\trte_iova_t dst, unsigned int length, uint64_t flags);\n+int idxd_submit(struct rte_dmadev *dev, uint16_t qid);\n \n #endif /* _IDXD_INTERNAL_H_ */\ndiff --git a/drivers/dma/idxd/meson.build b/drivers/dma/idxd/meson.build\nindex 81150e6f25..2de5130fd2 100644\n--- a/drivers/dma/idxd/meson.build\n+++ b/drivers/dma/idxd/meson.build\n@@ -2,6 +2,7 @@\n # Copyright(c) 2021 Intel Corporation\n \n deps += ['bus_pci']\n+cflags += '-mavx2' # all platforms with idxd HW support AVX\n sources = files(\n 'idxd_bus.c',\n 'idxd_common.c',\n", "prefixes": [ "10/13" ] }