Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/53935/?format=api
{ "id": 53935, "url": "http://patches.dpdk.org/api/patches/53935/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20190530212525.40370-9-bruce.richardson@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20190530212525.40370-9-bruce.richardson@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20190530212525.40370-9-bruce.richardson@intel.com", "date": "2019-05-30T21:25:25", "name": "[8/8] raw/ioat: add local API to perform copies", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "a4c6690a54fbb97686e141397377bc3a7c46629e", "submitter": { "id": 20, "url": "http://patches.dpdk.org/api/people/20/?format=api", "name": "Bruce Richardson", "email": "bruce.richardson@intel.com" }, "delegate": { "id": 1, "url": "http://patches.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20190530212525.40370-9-bruce.richardson@intel.com/mbox/", "series": [ { "id": 4831, "url": "http://patches.dpdk.org/api/series/4831/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=4831", "date": "2019-05-30T21:25:17", "name": "raw/ioat: driver for Intel QuickData Technology", "version": 1, "mbox": "http://patches.dpdk.org/series/4831/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/53935/comments/", "check": "fail", "checks": "http://patches.dpdk.org/api/patches/53935/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 0DF9A1B99A;\n\tThu, 30 May 2019 23:26:08 +0200 (CEST)", "from mga11.intel.com (mga11.intel.com [192.55.52.93])\n\tby dpdk.org (Postfix) with ESMTP id B2C7B1B955\n\tfor <dev@dpdk.org>; Thu, 30 May 2019 23:25:57 +0200 (CEST)", "from orsmga001.jf.intel.com ([10.7.209.18])\n\tby fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t30 May 2019 14:25:56 -0700", "from silpixa00399126.ir.intel.com (HELO\n\tsilpixa00399126.ger.corp.intel.com) ([10.237.223.2])\n\tby orsmga001.jf.intel.com with ESMTP; 30 May 2019 14:25:56 -0700" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "From": "Bruce Richardson <bruce.richardson@intel.com>", "To": "dev@dpdk.org", "Cc": "Bruce Richardson <bruce.richardson@intel.com>", "Date": "Thu, 30 May 2019 22:25:25 +0100", "Message-Id": "<20190530212525.40370-9-bruce.richardson@intel.com>", "X-Mailer": "git-send-email 2.21.0", "In-Reply-To": "<20190530212525.40370-1-bruce.richardson@intel.com>", "References": "<20190530212525.40370-1-bruce.richardson@intel.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[dpdk-dev] [PATCH 8/8] raw/ioat: add local API to perform copies", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Add local APIs to trigger data copies, and retrieve handle values once\nthose copies are completed. Included are unit tests to validate the data\nis copies correctly.\n\nSigned-off-by: Bruce Richardson <bruce.richardson@intel.com>\n---\n app/test/test_ioat_rawdev.c | 159 ++++++++++++++++++++++++++++-\n doc/guides/rawdevs/ioat_rawdev.rst | 100 ++++++++++++++++++\n drivers/raw/ioat/rte_ioat_rawdev.h | 155 +++++++++++++++++++++++++++-\n 3 files changed, 410 insertions(+), 4 deletions(-)", "diff": "diff --git a/app/test/test_ioat_rawdev.c b/app/test/test_ioat_rawdev.c\nindex 7081f3365..f2240adec 100644\n--- a/app/test/test_ioat_rawdev.c\n+++ b/app/test/test_ioat_rawdev.c\n@@ -18,6 +18,131 @@ test_ioat_rawdev(void) { return TEST_SKIPPED; }\n #include <rte_rawdev.h>\n #include <rte_ioat_rawdev.h>\n \n+static struct rte_mempool *pool;\n+\n+static int\n+test_enqueue_copies(int dev_id)\n+{\n+\tconst unsigned int length = 1024;\n+\tunsigned int i;\n+\n+\tdo {\n+\t\tstruct rte_mbuf *src, *dst;\n+\t\tchar *src_data, *dst_data;\n+\t\tstruct rte_mbuf *completed[2] = {0};\n+\n+\t\t/* test doing a single copy */\n+\t\tsrc = rte_pktmbuf_alloc(pool);\n+\t\tdst = rte_pktmbuf_alloc(pool);\n+\t\tsrc->data_len = src->pkt_len = length;\n+\t\tdst->data_len = dst->pkt_len = length;\n+\t\tsrc_data = rte_pktmbuf_mtod(src, char *);\n+\t\tdst_data = rte_pktmbuf_mtod(dst, char *);\n+\n+\t\tfor (i = 0; i < length; i++)\n+\t\t\tsrc_data[i] = rand() & 0xFF;\n+\n+\t\tif (rte_ioat_enqueue_copy(dev_id,\n+\t\t\t\tsrc->buf_iova + src->data_off,\n+\t\t\t\tdst->buf_iova + dst->data_off,\n+\t\t\t\tlength,\n+\t\t\t\t(uintptr_t)src,\n+\t\t\t\t(uintptr_t)dst,\n+\t\t\t\t0 /* no fence */) != 1) {\n+\t\t\tprintf(\"Error with rte_ioat_enqueue_copy\\n\");\n+\t\t\treturn -1;\n+\t\t}\n+\t\trte_ioat_do_copies(dev_id);\n+\t\tusleep(10);\n+\n+\t\tif (rte_ioat_completed_copies(dev_id, 1, (void *)&completed[0],\n+\t\t\t\t(void *)&completed[1]) != 1) {\n+\t\t\tprintf(\"Error with rte_ioat_completed_copies\\n\");\n+\t\t\treturn -1;\n+\t\t}\n+\t\tif (completed[0] != src || completed[1] != dst) {\n+\t\t\tprintf(\"Error with completions: got (%p, %p), not (%p,%p)\\n\",\n+\t\t\t\t\tcompleted[0], completed[1], src, dst);\n+\t\t\treturn -1;\n+\t\t}\n+\n+\t\tfor (i = 0; i < length; i++)\n+\t\t\tif (dst_data[i] != src_data[i]) {\n+\t\t\t\tprintf(\"Data mismatch at char %u\\n\", i);\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\t\trte_pktmbuf_free(src);\n+\t\trte_pktmbuf_free(dst);\n+\t} while(0);\n+\n+\t/* test doing multiple copies */\n+\tdo {\n+\t\tstruct rte_mbuf *srcs[32], *dsts[32];\n+\t\tstruct rte_mbuf *completed_src[64];\n+\t\tstruct rte_mbuf *completed_dst[64];\n+\t\tunsigned int j;\n+\n+\t\tfor (i = 0; i < RTE_DIM(srcs); i++) {\n+\t\t\tchar *src_data;\n+\n+\t\t\tsrcs[i] = rte_pktmbuf_alloc(pool);\n+\t\t\tdsts[i] = rte_pktmbuf_alloc(pool);\n+\t\t\tsrcs[i]->data_len = srcs[i]->pkt_len = length;\n+\t\t\tdsts[i]->data_len = dsts[i]->pkt_len = length;\n+\t\t\tsrc_data = rte_pktmbuf_mtod(srcs[i], char *);\n+\n+\t\t\tfor (j = 0; j < length; j++)\n+\t\t\t\tsrc_data[j] = rand() & 0xFF;\n+\n+\t\t\tif (rte_ioat_enqueue_copy(dev_id,\n+\t\t\t\t\tsrcs[i]->buf_iova + srcs[i]->data_off,\n+\t\t\t\t\tdsts[i]->buf_iova + dsts[i]->data_off,\n+\t\t\t\t\tlength,\n+\t\t\t\t\t(uintptr_t)srcs[i],\n+\t\t\t\t\t(uintptr_t)dsts[i],\n+\t\t\t\t\t0 /* nofence */) != 1) {\n+\t\t\t\tprintf(\"Error with rte_ioat_enqueue_copy for buffer %u\\n\",\n+\t\t\t\t\t\ti);\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\t\t}\n+\t\trte_ioat_do_copies(dev_id);\n+\t\tusleep(100);\n+\n+\t\tif (rte_ioat_completed_copies(dev_id, 64, (void *)completed_src,\n+\t\t\t\t(void *)completed_dst) != RTE_DIM(srcs)) {\n+\t\t\tprintf(\"Error with rte_ioat_completed_copies\\n\");\n+\t\t\treturn -1;\n+\t\t}\n+\t\tfor (i = 0; i < RTE_DIM(srcs); i++) {\n+\t\t\tchar *src_data, *dst_data;\n+\n+\t\t\tif (completed_src[i] != srcs[i]) {\n+\t\t\t\tprintf(\"Error with source pointer %u\\n\", i);\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\t\t\tif (completed_dst[i] != dsts[i]) {\n+\t\t\t\tprintf(\"Error with dest pointer %u\\n\", i);\n+\t\t\t\treturn -1;\n+\t\t\t}\n+\n+\t\t\tsrc_data = rte_pktmbuf_mtod(srcs[i], char *);\n+\t\t\tdst_data = rte_pktmbuf_mtod(dsts[i], char *);\n+\t\t\tfor (j = 0; j < length; j++)\n+\t\t\t\tif (src_data[j] != dst_data[j]) {\n+\t\t\t\t\tprintf(\"Error with copy of packet %u, byte %u\\n\",\n+\t\t\t\t\t\t\ti, j);\n+\t\t\t\t\treturn -1;\n+\t\t\t\t}\n+\t\t\trte_pktmbuf_free(srcs[i]);\n+\t\t\trte_pktmbuf_free(dsts[i]);\n+\t\t}\n+\n+\t} while(0);\n+\n+\treturn 0;\n+}\n+\n static int\n run_ioat_tests(int dev_id)\n {\n@@ -54,6 +179,17 @@ run_ioat_tests(int dev_id)\n \t\treturn -1;\n \t}\n \n+\tpool = rte_pktmbuf_pool_create(\"TEST_IOAT_POOL\",\n+\t\t\t256, /* n == num elements */\n+\t\t\t32, /* cache size */\n+\t\t\t0, /* priv size */\n+\t\t\t2048, /* data room size */\n+\t\t\tinfo.socket_id);\n+\tif (pool == NULL) {\n+\t\tprintf(\"Error with mempool creation\\n\");\n+\t\treturn -1;\n+\t}\n+\n \t/* allocate memory for xstats names and values */\n \tnb_xstats = rte_rawdev_xstats_names_get(dev_id, NULL, 0);\n \n@@ -78,15 +214,32 @@ run_ioat_tests(int dev_id)\n \t\treturn -1;\n \t}\n \n-\trte_rawdev_xstats_get(dev_id, ids, stats, nb_xstats);\n-\tfor (i = 0; i < nb_xstats; i++)\n-\t\tprintf(\"%s: %\"PRIu64\" \", snames[i].name, stats[i]);\n+\t/* run the test cases */\n+\tfor (i = 0; i < 100; i++) {\n+\t\tunsigned int j;\n+\n+\t\tif (test_enqueue_copies(dev_id) != 0)\n+\t\t\tgoto err;\n+\n+\t\trte_rawdev_xstats_get(dev_id, ids, stats, nb_xstats);\n+\t\tfor (j = 0; j < nb_xstats; j++)\n+\t\t\tprintf(\"%s: %\"PRIu64\" \", snames[j].name, stats[j]);\n+\t\tprintf(\"\\r\");\n+\t}\n \tprintf(\"\\n\");\n \n+\trte_mempool_free(pool);\n \tfree(snames);\n \tfree(stats);\n \tfree(ids);\n \treturn 0;\n+\n+err:\n+\trte_mempool_free(pool);\n+\tfree(snames);\n+\tfree(stats);\n+\tfree(ids);\n+\treturn -1;\n }\n \n static int\ndiff --git a/doc/guides/rawdevs/ioat_rawdev.rst b/doc/guides/rawdevs/ioat_rawdev.rst\nindex 47f12e95c..3ed12c964 100644\n--- a/doc/guides/rawdevs/ioat_rawdev.rst\n+++ b/doc/guides/rawdevs/ioat_rawdev.rst\n@@ -112,6 +112,106 @@ The following code shows how the device is configured in\n Once configured, the device can then be made ready for use by calling the\n ``rte_rawdev_start()`` API.\n \n+Performing Data Copies\n+~~~~~~~~~~~~~~~~~~~~~~~\n+\n+To perform data copies using IOAT rawdev devices, the functions\n+``rte_ioat_enqueue_copy()`` and ``rte_ioat_do_copies()`` should be used.\n+Once copies have been completed, the completion will be reported back when\n+the application calls ``rte_ioat_completed_copies()``.\n+\n+The ``rte_ioat_enqueue_copy()`` function enqueues a single copy to the\n+device ring for copying at a later point. The parameters to that function\n+include the physical addresses of both the source and destination buffers,\n+as well as two \"handles\" to be returned to the user when the copy is\n+completed. These handles can be arbitrary values, but two are provided so\n+that the library can track handles for both source and destination on\n+behalf of the user, e.g. virtual addresses for the buffers, or mbuf\n+pointers if packet data is being copied.\n+\n+While the ``rte_ioat_enqueue_copy()`` function enqueues a copy operation on\n+the device ring, the copy will not actually be performed until after the\n+application calls the ``rte_ioat_do_copies()`` function. This function\n+informs the device hardware of the elements enqueued on the ring, and the\n+device will begin to process them. It is expected that, for efficiency\n+reasons, a burst of operations will be enqueued to the device via multiple\n+enqueue calls between calls to the ``rte_ioat_do_copies()`` function.\n+\n+The following code from ``test_ioat_rawdev.c`` demonstrates how to enqueue\n+a burst of copies to the device and start the hardware processing of them:\n+\n+.. code-block:: C\n+\n+ struct rte_mbuf *srcs[32], *dsts[32];\n+ unsigned int j;\n+\n+ for (i = 0; i < RTE_DIM(srcs); i++) {\n+ char *src_data;\n+\n+ srcs[i] = rte_pktmbuf_alloc(pool);\n+ dsts[i] = rte_pktmbuf_alloc(pool);\n+ srcs[i]->data_len = srcs[i]->pkt_len = length;\n+ dsts[i]->data_len = dsts[i]->pkt_len = length;\n+ src_data = rte_pktmbuf_mtod(srcs[i], char *);\n+\n+ for (j = 0; j < length; j++)\n+ src_data[j] = rand() & 0xFF;\n+\n+ if (rte_ioat_enqueue_copy(dev_id,\n+ srcs[i]->buf_iova + srcs[i]->data_off,\n+ dsts[i]->buf_iova + dsts[i]->data_off,\n+ length,\n+ (uintptr_t)srcs[i],\n+ (uintptr_t)dsts[i],\n+ 0 /* nofence */) != 1) {\n+ printf(\"Error with rte_ioat_enqueue_copy for buffer %u\\n\",\n+ i);\n+ return -1;\n+ }\n+ }\n+ rte_ioat_do_copies(dev_id);\n+\n+To retrieve information about completed copies, the API\n+``rte_ioat_completed_copies()`` should be used. This API will return to the\n+application a set of completion handles passed in when the relevant copies\n+were enqueued.\n+\n+The following code from ``test_ioat_rawdev.c`` shows the test code\n+retrieving information about the completed copies and validating the data\n+is correct before freeing the data buffers using the returned handles:\n+\n+.. code-block:: C\n+\n+ if (rte_ioat_completed_copies(dev_id, 64, (void *)completed_src,\n+ (void *)completed_dst) != RTE_DIM(srcs)) {\n+ printf(\"Error with rte_ioat_completed_copies\\n\");\n+ return -1;\n+ }\n+ for (i = 0; i < RTE_DIM(srcs); i++) {\n+ char *src_data, *dst_data;\n+\n+ if (completed_src[i] != srcs[i]) {\n+ printf(\"Error with source pointer %u\\n\", i);\n+ return -1;\n+ }\n+ if (completed_dst[i] != dsts[i]) {\n+ printf(\"Error with dest pointer %u\\n\", i);\n+ return -1;\n+ }\n+\n+ src_data = rte_pktmbuf_mtod(srcs[i], char *);\n+ dst_data = rte_pktmbuf_mtod(dsts[i], char *);\n+ for (j = 0; j < length; j++)\n+ if (src_data[j] != dst_data[j]) {\n+ printf(\"Error with copy of packet %u, byte %u\\n\",\n+ i, j);\n+ return -1;\n+ }\n+ rte_pktmbuf_free(srcs[i]);\n+ rte_pktmbuf_free(dsts[i]);\n+ }\n+\n+\n Querying Device Statistics\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~\n \ndiff --git a/drivers/raw/ioat/rte_ioat_rawdev.h b/drivers/raw/ioat/rte_ioat_rawdev.h\nindex abe5ba298..7750edba7 100644\n--- a/drivers/raw/ioat/rte_ioat_rawdev.h\n+++ b/drivers/raw/ioat/rte_ioat_rawdev.h\n@@ -14,7 +14,10 @@\n * @b EXPERIMENTAL: these structures and APIs may change without prior notice\n */\n \n+#include <x86intrin.h>\n+\n #include <rte_memory.h>\n+#include <rte_prefetch.h>\n #include <rte_ioat_spec.h>\n \n /** Name of the device driver */\n@@ -49,6 +52,10 @@ struct rte_ioat_rawdev {\n \tstruct rte_ioat_desc *desc_ring;\n \t__m128i *hdls; /* completion handles for returning to user */\n \n+\n+\tunsigned short next_read;\n+\tunsigned short next_write;\n+\n \t/* some statistics for tracking, if added/changed update xstats fns*/\n \tuint64_t enqueue_failed __rte_cache_aligned;\n \tuint64_t enqueued;\n@@ -72,4 +79,150 @@ struct rte_ioat_desc {\n \tuint64_t op_type_specific[4];\n };\n \n-#endif\n+/**\n+ * Enqueue a copy operation onto the ioat device\n+ *\n+ * This queues up a copy operation to be performed by hardware, but does not\n+ * trigger hardware to begin that operation.\n+ *\n+ * @param dev_id\n+ * The rawdev device id of the ioat instance\n+ * @param src\n+ * The physical address of the source buffer\n+ * @param dst\n+ * The physical address of the destination buffer\n+ * @param length\n+ * The length of the data to be copied\n+ * @param src_hdl\n+ * An opaque handle for the source data, to be returned when this operation\n+ * has been completed and the user polls for the completion details\n+ * @param dst_hdl\n+ * An opaque handle for the destination data, to be returned when this\n+ * operation has been completed and the user polls for the completion details\n+ * @param fence\n+ * A flag parameter indicating that hardware should not begin to perform any\n+ * subsequently enqueued copy operations until after this operation has\n+ * completed\n+ * @return\n+ * Number of operations enqueued, either 0 or 1\n+ */\n+static inline int\n+rte_ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst,\n+\t\tunsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl,\n+\t\tint fence)\n+{\n+\tstruct rte_ioat_rawdev *ioat = rte_rawdevs[dev_id].dev_private;\n+\tunsigned short read = ioat->next_read;\n+\tunsigned short write = ioat->next_write;\n+\tunsigned short mask = ioat->ring_size - 1;\n+\tunsigned short space = mask + read - write;\n+\n+\tif (space == 0) {\n+\t\tioat->enqueue_failed++;\n+\t\treturn 0;\n+\t}\n+\n+\tioat->next_write = write + 1;\n+\twrite &= mask;\n+\t/* write transfer size and the descriptor control bits */\n+\t*((uint64_t *)&ioat->desc_ring[write]) =\n+\t\t\tlength | (uint64_t)!!fence << 36 |\n+\t\t\t(uint64_t)(!(write & 0xF)) << 35;\n+\tioat->desc_ring[write].src_addr = src;\n+\tioat->desc_ring[write].dest_addr = dst;\n+\tioat->hdls[write] = _mm_set_epi64((__m64)dst_hdl, (__m64)src_hdl);\n+\trte_prefetch0(&ioat->desc_ring[ioat->next_write & mask]);\n+\n+\tioat->enqueued++;\n+\treturn 1;\n+}\n+\n+/**\n+ * Trigger hardware to begin performing enqueued copy operations\n+ *\n+ * This API is used to write the \"doorbell\" to the hardware to trigger it\n+ * to begin the copy operations previously enqueued by rte_ioat_enqueue_copy()\n+ *\n+ * @param dev_id\n+ * The rawdev device id of the ioat instance\n+ */\n+static inline void\n+rte_ioat_do_copies(int dev_id)\n+{\n+\tstruct rte_ioat_rawdev *ioat = rte_rawdevs[dev_id].dev_private;\n+\tioat->desc_ring[(ioat->next_write - 1) & (ioat->ring_size - 1)].desc_control = 8;\n+\trte_compiler_barrier();\n+\tioat->regs->dmacount = ioat->next_write;\n+\tioat->started = ioat->enqueued;\n+}\n+\n+/**\n+ * @internal\n+ * Returns the index of the last completed operation.\n+ */\n+static inline int\n+rte_ioat_get_last_completed(struct rte_ioat_rawdev *ioat)\n+{\n+\tuint64_t status = ioat->status;\n+\t/* lower 3 bits indicate \"transfer status\" : active, idle, halted.. */\n+\tif (status & 0x6)\n+\t\trte_panic(\"Error with cbdma: %u\\n\", (unsigned)status & 0x7);\n+\n+\treturn (status - ioat->ring_addr) >> 6;\n+}\n+\n+/**\n+ * Returns details of copy operations that have been completed\n+ *\n+ * Returns to the caller the user-provided \"handles\" for the copy operations\n+ * which have been completed by the hardware, and not already returned by\n+ * a previous call to this API.\n+ *\n+ * @param dev_id\n+ * The rawdev device id of the ioat instance\n+ * @param max_copies\n+ * The number of entries which can fit in the src_hdls and dst_hdls\n+ * arrays, i.e. max number of completed operations to report\n+ * @param src_hdls\n+ * Array to hold the source handle parameters of the completed copies\n+ * @param dst_hdls\n+ * Array to hold the destination handle parameters of the completed copies\n+ * @return\n+ * Number of completed operations i.e. number of entries written to the\n+ * src_hdls and dst_hdls array parameters.\n+ */\n+static inline int\n+rte_ioat_completed_copies(int dev_id, uint8_t max_copies,\n+\t\tuintptr_t *src_hdls, uintptr_t *dst_hdls)\n+{\n+\tstruct rte_ioat_rawdev *ioat = rte_rawdevs[dev_id].dev_private;\n+\tunsigned short mask = (ioat->ring_size - 1);\n+\tunsigned short read = ioat->next_read;\n+\tunsigned short end_read = (rte_ioat_get_last_completed(ioat) + 1) & mask;\n+\tunsigned short count = (end_read - (read & mask)) & mask;\n+\tint i = 0;\n+\n+\tif (count > max_copies)\n+\t\tcount = max_copies;\n+\n+\tfor (; i < count - 1; i += 2, read += 2) {\n+\t\t__m128i hdls0 = _mm_load_si128(&ioat->hdls[read & mask]);\n+\t\t__m128i hdls1 = _mm_load_si128(&ioat->hdls[(read + 1) & mask]);\n+\n+\t\t_mm_storeu_si128((void *)&src_hdls[i],\n+\t\t\t\t_mm_unpacklo_epi64(hdls0, hdls1));\n+\t\t_mm_storeu_si128((void *)&dst_hdls[i],\n+\t\t\t\t_mm_unpackhi_epi64(hdls0, hdls1));\n+\t}\n+\tfor (; i < count; i++, read++) {\n+\t\tuintptr_t *hdls = (void *)&ioat->hdls[read & mask];\n+\t\tsrc_hdls[i] = hdls[0];\n+\t\tdst_hdls[i] = hdls[1];\n+\t}\n+\n+\tioat->next_read = read;\n+\tioat->completed += count;\n+\treturn count;\n+}\n+\n+#endif /* _RTE_IOAT_RAWDEV_H_ */\n", "prefixes": [ "8/8" ] }