Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/101334/?format=api
{ "id": 101334, "url": "http://patches.dpdk.org/api/patches/101334/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20211013122500.51552-4-fengchengwen@huawei.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20211013122500.51552-4-fengchengwen@huawei.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20211013122500.51552-4-fengchengwen@huawei.com", "date": "2021-10-13T12:24:57", "name": "[v26,3/6] dmadev: add data plane API support", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "99c33bc56fcefbbc88b9485f56f86080fd6cea90", "submitter": { "id": 2146, "url": "http://patches.dpdk.org/api/people/2146/?format=api", "name": "fengchengwen", "email": "fengchengwen@huawei.com" }, "delegate": { "id": 1, "url": "http://patches.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20211013122500.51552-4-fengchengwen@huawei.com/mbox/", "series": [ { "id": 19594, "url": "http://patches.dpdk.org/api/series/19594/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=19594", "date": "2021-10-13T12:24:55", "name": "support dmadev", "version": 26, "mbox": "http://patches.dpdk.org/series/19594/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/101334/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/101334/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id D91A7A0C55;\n\tWed, 13 Oct 2021 14:29:51 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 81EFD411E7;\n\tWed, 13 Oct 2021 14:29:27 +0200 (CEST)", "from szxga01-in.huawei.com (szxga01-in.huawei.com [45.249.212.187])\n by mails.dpdk.org (Postfix) with ESMTP id 88F5441162\n for <dev@dpdk.org>; Wed, 13 Oct 2021 14:29:21 +0200 (CEST)", "from dggemv711-chm.china.huawei.com (unknown [172.30.72.56])\n by szxga01-in.huawei.com (SkyGuard) with ESMTP id 4HTsDN08Bgzbn4j;\n Wed, 13 Oct 2021 20:24:52 +0800 (CST)", "from dggpeml500024.china.huawei.com (7.185.36.10) by\n dggemv711-chm.china.huawei.com (10.1.198.66) with Microsoft SMTP Server\n (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id\n 15.1.2308.8; Wed, 13 Oct 2021 20:29:18 +0800", "from localhost.localdomain (10.67.165.24) by\n dggpeml500024.china.huawei.com (7.185.36.10) with Microsoft SMTP Server\n (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id\n 15.1.2308.8; Wed, 13 Oct 2021 20:29:18 +0800" ], "From": "Chengwen Feng <fengchengwen@huawei.com>", "To": "<thomas@monjalon.net>, <ferruh.yigit@intel.com>,\n <bruce.richardson@intel.com>, <jerinj@marvell.com>, <jerinjacobk@gmail.com>,\n <andrew.rybchenko@oktetlabs.ru>", "CC": "<dev@dpdk.org>, <mb@smartsharesystems.com>, <nipun.gupta@nxp.com>,\n <hemant.agrawal@nxp.com>, <maxime.coquelin@redhat.com>,\n <honnappa.nagarahalli@arm.com>, <david.marchand@redhat.com>,\n <sburla@marvell.com>, <pkapoor@marvell.com>, <konstantin.ananyev@intel.com>,\n <conor.walsh@intel.com>, <kevin.laatz@intel.com>", "Date": "Wed, 13 Oct 2021 20:24:57 +0800", "Message-ID": "<20211013122500.51552-4-fengchengwen@huawei.com>", "X-Mailer": "git-send-email 2.33.0", "In-Reply-To": "<20211013122500.51552-1-fengchengwen@huawei.com>", "References": "<1625231891-2963-1-git-send-email-fengchengwen@huawei.com>\n <20211013122500.51552-1-fengchengwen@huawei.com>", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=\"UTF-8\"", "Content-Transfer-Encoding": "8bit", "X-Originating-IP": "[10.67.165.24]", "X-ClientProxiedBy": "dggems704-chm.china.huawei.com (10.3.19.181) To\n dggpeml500024.china.huawei.com (7.185.36.10)", "X-CFilter-Loop": "Reflected", "Subject": "[dpdk-dev] [PATCH v26 3/6] dmadev: add data plane API support", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "This patch add data plane API for dmadev.\n\nSigned-off-by: Chengwen Feng <fengchengwen@huawei.com>\nAcked-by: Bruce Richardson <bruce.richardson@intel.com>\nAcked-by: Morten Brørup <mb@smartsharesystems.com>\nReviewed-by: Kevin Laatz <kevin.laatz@intel.com>\nReviewed-by: Conor Walsh <conor.walsh@intel.com>\n---\n doc/guides/prog_guide/dmadev.rst | 22 ++\n lib/dmadev/meson.build | 1 +\n lib/dmadev/rte_dmadev.c | 112 ++++++++\n lib/dmadev/rte_dmadev.h | 451 +++++++++++++++++++++++++++++++\n lib/dmadev/rte_dmadev_core.h | 76 ++++++\n lib/dmadev/rte_dmadev_pmd.h | 2 +\n lib/dmadev/version.map | 7 +\n 7 files changed, 671 insertions(+)\n create mode 100644 lib/dmadev/rte_dmadev_core.h", "diff": "diff --git a/doc/guides/prog_guide/dmadev.rst b/doc/guides/prog_guide/dmadev.rst\nindex b1b5bff639..32f7147862 100644\n--- a/doc/guides/prog_guide/dmadev.rst\n+++ b/doc/guides/prog_guide/dmadev.rst\n@@ -66,3 +66,25 @@ can be used to get the device info and supported features.\n \n Silent mode is a special device capability which does not require the\n application to invoke dequeue APIs.\n+\n+\n+Enqueue / Dequeue APIs\n+~~~~~~~~~~~~~~~~~~~~~~\n+\n+Enqueue APIs such as ``rte_dma_copy`` and ``rte_dma_fill`` can be used to\n+enqueue operations to hardware. If an enqueue is successful, a ``ring_idx`` is\n+returned. This ``ring_idx`` can be used by applications to track per operation\n+metadata in an application-defined circular ring.\n+\n+The ``rte_dma_submit`` API is used to issue doorbell to hardware.\n+Alternatively the ``RTE_DMA_OP_FLAG_SUBMIT`` flag can be passed to the enqueue\n+APIs to also issue the doorbell to hardware.\n+\n+There are two dequeue APIs ``rte_dma_completed`` and\n+``rte_dma_completed_status``, these are used to obtain the results of the\n+enqueue requests. ``rte_dma_completed`` will return the number of successfully\n+completed operations. ``rte_dma_completed_status`` will return the number of\n+completed operations along with the status of each operation (filled into the\n+``status`` array passed by user). These two APIs can also return the last\n+completed operation's ``ring_idx`` which could help user track operations within\n+their own application-defined rings.\ndiff --git a/lib/dmadev/meson.build b/lib/dmadev/meson.build\nindex f8d54c6e74..d2fc85e8c7 100644\n--- a/lib/dmadev/meson.build\n+++ b/lib/dmadev/meson.build\n@@ -3,4 +3,5 @@\n \n sources = files('rte_dmadev.c')\n headers = files('rte_dmadev.h')\n+indirect_headers += files('rte_dmadev_core.h')\n driver_sdk_headers += files('rte_dmadev_pmd.h')\ndiff --git a/lib/dmadev/rte_dmadev.c b/lib/dmadev/rte_dmadev.c\nindex 9223ae1c95..54e15817fa 100644\n--- a/lib/dmadev/rte_dmadev.c\n+++ b/lib/dmadev/rte_dmadev.c\n@@ -17,6 +17,7 @@\n \n static int16_t dma_devices_max;\n \n+struct rte_dma_fp_object *rte_dma_fp_objs;\n struct rte_dma_dev *rte_dma_devices;\n \n RTE_LOG_REGISTER_DEFAULT(rte_dma_logtype, INFO);\n@@ -97,6 +98,38 @@ dma_find_by_name(const char *name)\n \treturn NULL;\n }\n \n+static void dma_fp_object_dummy(struct rte_dma_fp_object *obj);\n+\n+static int\n+dma_fp_data_prepare(void)\n+{\n+\tsize_t size;\n+\tvoid *ptr;\n+\tint i;\n+\n+\tif (rte_dma_fp_objs != NULL)\n+\t\treturn 0;\n+\n+\t/* Fast-path object must align cacheline, but the return value of malloc\n+\t * may not be aligned to the cache line. Therefore, extra memory is\n+\t * applied for realignment.\n+\t * note: We do not call posix_memalign/aligned_alloc because it is\n+\t * version dependent on libc.\n+\t */\n+\tsize = dma_devices_max * sizeof(struct rte_dma_fp_object) +\n+\t\tRTE_CACHE_LINE_SIZE;\n+\tptr = malloc(size);\n+\tif (ptr == NULL)\n+\t\treturn -ENOMEM;\n+\tmemset(ptr, 0, size);\n+\n+\trte_dma_fp_objs = RTE_PTR_ALIGN(ptr, RTE_CACHE_LINE_SIZE);\n+\tfor (i = 0; i < dma_devices_max; i++)\n+\t\tdma_fp_object_dummy(&rte_dma_fp_objs[i]);\n+\n+\treturn 0;\n+}\n+\n static int\n dma_dev_data_prepare(void)\n {\n@@ -117,8 +150,15 @@ dma_dev_data_prepare(void)\n static int\n dma_data_prepare(void)\n {\n+\tint ret;\n+\n \tif (dma_devices_max == 0)\n \t\tdma_devices_max = RTE_DMADEV_DEFAULT_MAX;\n+\n+\tret = dma_fp_data_prepare();\n+\tif (ret)\n+\t\treturn ret;\n+\n \treturn dma_dev_data_prepare();\n }\n \n@@ -161,6 +201,8 @@ dma_allocate(const char *name, int numa_node, size_t private_data_size)\n \tdev->dev_id = dev_id;\n \tdev->numa_node = numa_node;\n \tdev->dev_private = dev_private;\n+\tdev->fp_obj = &rte_dma_fp_objs[dev_id];\n+\tdma_fp_object_dummy(dev->fp_obj);\n \n \treturn dev;\n }\n@@ -169,6 +211,7 @@ static void\n dma_release(struct rte_dma_dev *dev)\n {\n \trte_free(dev->dev_private);\n+\tdma_fp_object_dummy(dev->fp_obj);\n \tmemset(dev, 0, sizeof(struct rte_dma_dev));\n }\n \n@@ -605,3 +648,72 @@ rte_dma_dump(int16_t dev_id, FILE *f)\n \n \treturn 0;\n }\n+\n+static int\n+dummy_copy(__rte_unused void *dev_private, __rte_unused uint16_t vchan,\n+\t __rte_unused rte_iova_t src, __rte_unused rte_iova_t dst,\n+\t __rte_unused uint32_t length, __rte_unused uint64_t flags)\n+{\n+\tRTE_DMA_LOG(ERR, \"copy is not configured or not supported.\");\n+\treturn -EINVAL;\n+}\n+\n+static int\n+dummy_copy_sg(__rte_unused void *dev_private, __rte_unused uint16_t vchan,\n+\t __rte_unused const struct rte_dma_sge *src,\n+\t __rte_unused const struct rte_dma_sge *dst,\n+\t __rte_unused uint16_t nb_src, __rte_unused uint16_t nb_dst,\n+\t __rte_unused uint64_t flags)\n+{\n+\tRTE_DMA_LOG(ERR, \"copy_sg is not configured or not supported.\");\n+\treturn -EINVAL;\n+}\n+\n+static int\n+dummy_fill(__rte_unused void *dev_private, __rte_unused uint16_t vchan,\n+\t __rte_unused uint64_t pattern, __rte_unused rte_iova_t dst,\n+\t __rte_unused uint32_t length, __rte_unused uint64_t flags)\n+{\n+\tRTE_DMA_LOG(ERR, \"fill is not configured or not supported.\");\n+\treturn -EINVAL;\n+}\n+\n+static int\n+dummy_submit(__rte_unused void *dev_private, __rte_unused uint16_t vchan)\n+{\n+\tRTE_DMA_LOG(ERR, \"submit is not configured or not supported.\");\n+\treturn -EINVAL;\n+}\n+\n+static uint16_t\n+dummy_completed(__rte_unused void *dev_private,\t__rte_unused uint16_t vchan,\n+\t\t__rte_unused const uint16_t nb_cpls,\n+\t\t__rte_unused uint16_t *last_idx, __rte_unused bool *has_error)\n+{\n+\tRTE_DMA_LOG(ERR, \"completed is not configured or not supported.\");\n+\treturn 0;\n+}\n+\n+static uint16_t\n+dummy_completed_status(__rte_unused void *dev_private,\n+\t\t __rte_unused uint16_t vchan,\n+\t\t __rte_unused const uint16_t nb_cpls,\n+\t\t __rte_unused uint16_t *last_idx,\n+\t\t __rte_unused enum rte_dma_status_code *status)\n+{\n+\tRTE_DMA_LOG(ERR,\n+\t\t \"completed_status is not configured or not supported.\");\n+\treturn 0;\n+}\n+\n+static void\n+dma_fp_object_dummy(struct rte_dma_fp_object *obj)\n+{\n+\tobj->dev_private = NULL;\n+\tobj->copy = dummy_copy;\n+\tobj->copy_sg = dummy_copy_sg;\n+\tobj->fill = dummy_fill;\n+\tobj->submit = dummy_submit;\n+\tobj->completed = dummy_completed;\n+\tobj->completed_status = dummy_completed_status;\n+}\ndiff --git a/lib/dmadev/rte_dmadev.h b/lib/dmadev/rte_dmadev.h\nindex 03ed304709..e46c001404 100644\n--- a/lib/dmadev/rte_dmadev.h\n+++ b/lib/dmadev/rte_dmadev.h\n@@ -65,6 +65,77 @@\n * Finally, an application can close a dmadev by invoking the rte_dma_close()\n * function.\n *\n+ * The dataplane APIs include two parts:\n+ * The first part is the submission of operation requests:\n+ * - rte_dma_copy()\n+ * - rte_dma_copy_sg()\n+ * - rte_dma_fill()\n+ * - rte_dma_submit()\n+ *\n+ * These APIs could work with different virtual DMA channels which have\n+ * different contexts.\n+ *\n+ * The first three APIs are used to submit the operation request to the virtual\n+ * DMA channel, if the submission is successful, a positive\n+ * ring_idx <= UINT16_MAX is returned, otherwise a negative number is returned.\n+ *\n+ * The last API is used to issue doorbell to hardware, and also there are flags\n+ * (@see RTE_DMA_OP_FLAG_SUBMIT) parameter of the first three APIs could do the\n+ * same work.\n+ * @note When enqueuing a set of jobs to the device, having a separate submit\n+ * outside a loop makes for clearer code than having a check for the last\n+ * iteration inside the loop to set a special submit flag. However, for cases\n+ * where one item alone is to be submitted or there is a small set of jobs to\n+ * be submitted sequentially, having a submit flag provides a lower-overhead\n+ * way of doing the submission while still keeping the code clean.\n+ *\n+ * The second part is to obtain the result of requests:\n+ * - rte_dma_completed()\n+ * - return the number of operation requests completed successfully.\n+ * - rte_dma_completed_status()\n+ * - return the number of operation requests completed.\n+ *\n+ * @note If the dmadev works in silent mode (@see RTE_DMA_CAPA_SILENT),\n+ * application does not invoke the above two completed APIs.\n+ *\n+ * About the ring_idx which enqueue APIs (e.g. rte_dma_copy(), rte_dma_fill())\n+ * return, the rules are as follows:\n+ * - ring_idx for each virtual DMA channel are independent.\n+ * - For a virtual DMA channel, the ring_idx is monotonically incremented,\n+ * when it reach UINT16_MAX, it wraps back to zero.\n+ * - This ring_idx can be used by applications to track per-operation\n+ * metadata in an application-defined circular ring.\n+ * - The initial ring_idx of a virtual DMA channel is zero, after the\n+ * device is stopped, the ring_idx needs to be reset to zero.\n+ *\n+ * One example:\n+ * - step-1: start one dmadev\n+ * - step-2: enqueue a copy operation, the ring_idx return is 0\n+ * - step-3: enqueue a copy operation again, the ring_idx return is 1\n+ * - ...\n+ * - step-101: stop the dmadev\n+ * - step-102: start the dmadev\n+ * - step-103: enqueue a copy operation, the ring_idx return is 0\n+ * - ...\n+ * - step-x+0: enqueue a fill operation, the ring_idx return is 65535\n+ * - step-x+1: enqueue a copy operation, the ring_idx return is 0\n+ * - ...\n+ *\n+ * The DMA operation address used in enqueue APIs (i.e. rte_dma_copy(),\n+ * rte_dma_copy_sg(), rte_dma_fill()) is defined as rte_iova_t type.\n+ *\n+ * The dmadev supports two types of address: memory address and device address.\n+ *\n+ * - memory address: the source and destination address of the memory-to-memory\n+ * transfer type, or the source address of the memory-to-device transfer type,\n+ * or the destination address of the device-to-memory transfer type.\n+ * @note If the device support SVA (@see RTE_DMA_CAPA_SVA), the memory address\n+ * can be any VA address, otherwise it must be an IOVA address.\n+ *\n+ * - device address: the source and destination address of the device-to-device\n+ * transfer type, or the source address of the device-to-memory transfer type,\n+ * or the destination address of the memory-to-device transfer type.\n+ *\n * About MT-safe, all the functions of the dmadev API implemented by a PMD are\n * lock-free functions which assume to not be invoked in parallel on different\n * logical cores to work on the same target dmadev object.\n@@ -591,6 +662,386 @@ int rte_dma_stats_reset(int16_t dev_id, uint16_t vchan);\n __rte_experimental\n int rte_dma_dump(int16_t dev_id, FILE *f);\n \n+/**\n+ * DMA transfer result status code defines.\n+ *\n+ * @see rte_dma_completed_status\n+ */\n+enum rte_dma_status_code {\n+\t/** The operation completed successfully. */\n+\tRTE_DMA_STATUS_SUCCESSFUL,\n+\t/** The operation failed to complete due abort by user.\n+\t * This is mainly used when processing dev_stop, user could modidy the\n+\t * descriptors (e.g. change one bit to tell hardware abort this job),\n+\t * it allows outstanding requests to be complete as much as possible,\n+\t * so reduce the time to stop the device.\n+\t */\n+\tRTE_DMA_STATUS_USER_ABORT,\n+\t/** The operation failed to complete due to following scenarios:\n+\t * The jobs in a particular batch are not attempted because they\n+\t * appeared after a fence where a previous job failed. In some HW\n+\t * implementation it's possible for jobs from later batches would be\n+\t * completed, though, so report the status from the not attempted jobs\n+\t * before reporting those newer completed jobs.\n+\t */\n+\tRTE_DMA_STATUS_NOT_ATTEMPTED,\n+\t/** The operation failed to complete due invalid source address. */\n+\tRTE_DMA_STATUS_INVALID_SRC_ADDR,\n+\t/** The operation failed to complete due invalid destination address. */\n+\tRTE_DMA_STATUS_INVALID_DST_ADDR,\n+\t/** The operation failed to complete due invalid source or destination\n+\t * address, cover the case that only knows the address error, but not\n+\t * sure which address error.\n+\t */\n+\tRTE_DMA_STATUS_INVALID_ADDR,\n+\t/** The operation failed to complete due invalid length. */\n+\tRTE_DMA_STATUS_INVALID_LENGTH,\n+\t/** The operation failed to complete due invalid opcode.\n+\t * The DMA descriptor could have multiple format, which are\n+\t * distinguished by the opcode field.\n+\t */\n+\tRTE_DMA_STATUS_INVALID_OPCODE,\n+\t/** The operation failed to complete due bus read error. */\n+\tRTE_DMA_STATUS_BUS_READ_ERROR,\n+\t/** The operation failed to complete due bus write error. */\n+\tRTE_DMA_STATUS_BUS_WRITE_ERROR,\n+\t/** The operation failed to complete due bus error, cover the case that\n+\t * only knows the bus error, but not sure which direction error.\n+\t */\n+\tRTE_DMA_STATUS_BUS_ERROR,\n+\t/** The operation failed to complete due data poison. */\n+\tRTE_DMA_STATUS_DATA_POISION,\n+\t/** The operation failed to complete due descriptor read error. */\n+\tRTE_DMA_STATUS_DESCRIPTOR_READ_ERROR,\n+\t/** The operation failed to complete due device link error.\n+\t * Used to indicates that the link error in the memory-to-device/\n+\t * device-to-memory/device-to-device transfer scenario.\n+\t */\n+\tRTE_DMA_STATUS_DEV_LINK_ERROR,\n+\t/** The operation failed to complete due lookup page fault. */\n+\tRTE_DMA_STATUS_PAGE_FAULT,\n+\t/** The operation failed to complete due unknown reason.\n+\t * The initial value is 256, which reserves space for future errors.\n+\t */\n+\tRTE_DMA_STATUS_ERROR_UNKNOWN = 0x100,\n+};\n+\n+/**\n+ * A structure used to hold scatter-gather DMA operation request entry.\n+ *\n+ * @see rte_dma_copy_sg\n+ */\n+struct rte_dma_sge {\n+\trte_iova_t addr; /**< The DMA operation address. */\n+\tuint32_t length; /**< The DMA operation length. */\n+};\n+\n+#include \"rte_dmadev_core.h\"\n+\n+/**@{@name DMA operation flag\n+ * @see rte_dma_copy()\n+ * @see rte_dma_copy_sg()\n+ * @see rte_dma_fill()\n+ */\n+/** Fence flag.\n+ * It means the operation with this flag must be processed only after all\n+ * previous operations are completed.\n+ * If the specify DMA HW works in-order (it means it has default fence between\n+ * operations), this flag could be NOP.\n+ */\n+#define RTE_DMA_OP_FLAG_FENCE RTE_BIT64(0)\n+/** Submit flag.\n+ * It means the operation with this flag must issue doorbell to hardware after\n+ * enqueued jobs.\n+ */\n+#define RTE_DMA_OP_FLAG_SUBMIT RTE_BIT64(1)\n+/** Write data to low level cache hint.\n+ * Used for performance optimization, this is just a hint, and there is no\n+ * capability bit for this, driver should not return error if this flag was set.\n+ */\n+#define RTE_DMA_OP_FLAG_LLC RTE_BIT64(2)\n+/**@}*/\n+\n+/**\n+ * @warning\n+ * @b EXPERIMENTAL: this API may change without prior notice.\n+ *\n+ * Enqueue a copy operation onto the virtual DMA channel.\n+ *\n+ * This queues up a copy operation to be performed by hardware, if the 'flags'\n+ * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin\n+ * this operation, otherwise do not trigger doorbell.\n+ *\n+ * @param dev_id\n+ * The identifier of the device.\n+ * @param vchan\n+ * The identifier of virtual DMA channel.\n+ * @param src\n+ * The address of the source buffer.\n+ * @param dst\n+ * The address of the destination buffer.\n+ * @param length\n+ * The length of the data to be copied.\n+ * @param flags\n+ * An flags for this operation.\n+ * @see RTE_DMA_OP_FLAG_*\n+ *\n+ * @return\n+ * - 0..UINT16_MAX: index of enqueued job.\n+ * - -ENOSPC: if no space left to enqueue.\n+ * - other values < 0 on failure.\n+ */\n+__rte_experimental\n+static inline int\n+rte_dma_copy(int16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst,\n+\t uint32_t length, uint64_t flags)\n+{\n+\tstruct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];\n+\n+#ifdef RTE_DMADEV_DEBUG\n+\tif (!rte_dma_is_valid(dev_id) || length == 0)\n+\t\treturn -EINVAL;\n+\tRTE_FUNC_PTR_OR_ERR_RET(*obj->copy, -ENOTSUP);\n+#endif\n+\n+\treturn (*obj->copy)(obj->dev_private, vchan, src, dst, length, flags);\n+}\n+\n+/**\n+ * @warning\n+ * @b EXPERIMENTAL: this API may change without prior notice.\n+ *\n+ * Enqueue a scatter-gather list copy operation onto the virtual DMA channel.\n+ *\n+ * This queues up a scatter-gather list copy operation to be performed by\n+ * hardware, if the 'flags' parameter contains RTE_DMA_OP_FLAG_SUBMIT then\n+ * trigger doorbell to begin this operation, otherwise do not trigger doorbell.\n+ *\n+ * @param dev_id\n+ * The identifier of the device.\n+ * @param vchan\n+ * The identifier of virtual DMA channel.\n+ * @param src\n+ * The pointer of source scatter-gather entry array.\n+ * @param dst\n+ * The pointer of destination scatter-gather entry array.\n+ * @param nb_src\n+ * The number of source scatter-gather entry.\n+ * @see struct rte_dma_info::max_sges\n+ * @param nb_dst\n+ * The number of destination scatter-gather entry.\n+ * @see struct rte_dma_info::max_sges\n+ * @param flags\n+ * An flags for this operation.\n+ * @see RTE_DMA_OP_FLAG_*\n+ *\n+ * @return\n+ * - 0..UINT16_MAX: index of enqueued job.\n+ * - -ENOSPC: if no space left to enqueue.\n+ * - other values < 0 on failure.\n+ */\n+__rte_experimental\n+static inline int\n+rte_dma_copy_sg(int16_t dev_id, uint16_t vchan, struct rte_dma_sge *src,\n+\t\tstruct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst,\n+\t\tuint64_t flags)\n+{\n+\tstruct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];\n+\n+#ifdef RTE_DMADEV_DEBUG\n+\tif (!rte_dma_is_valid(dev_id) || src == NULL || dst == NULL ||\n+\t nb_src == 0 || nb_dst == 0)\n+\t\treturn -EINVAL;\n+\tRTE_FUNC_PTR_OR_ERR_RET(*obj->copy_sg, -ENOTSUP);\n+#endif\n+\n+\treturn (*obj->copy_sg)(obj->dev_private, vchan, src, dst, nb_src,\n+\t\t\t nb_dst, flags);\n+}\n+\n+/**\n+ * @warning\n+ * @b EXPERIMENTAL: this API may change without prior notice.\n+ *\n+ * Enqueue a fill operation onto the virtual DMA channel.\n+ *\n+ * This queues up a fill operation to be performed by hardware, if the 'flags'\n+ * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin\n+ * this operation, otherwise do not trigger doorbell.\n+ *\n+ * @param dev_id\n+ * The identifier of the device.\n+ * @param vchan\n+ * The identifier of virtual DMA channel.\n+ * @param pattern\n+ * The pattern to populate the destination buffer with.\n+ * @param dst\n+ * The address of the destination buffer.\n+ * @param length\n+ * The length of the destination buffer.\n+ * @param flags\n+ * An flags for this operation.\n+ * @see RTE_DMA_OP_FLAG_*\n+ *\n+ * @return\n+ * - 0..UINT16_MAX: index of enqueued job.\n+ * - -ENOSPC: if no space left to enqueue.\n+ * - other values < 0 on failure.\n+ */\n+__rte_experimental\n+static inline int\n+rte_dma_fill(int16_t dev_id, uint16_t vchan, uint64_t pattern,\n+\t rte_iova_t dst, uint32_t length, uint64_t flags)\n+{\n+\tstruct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];\n+\n+#ifdef RTE_DMADEV_DEBUG\n+\tif (!rte_dma_is_valid(dev_id) || length == 0)\n+\t\treturn -EINVAL;\n+\tRTE_FUNC_PTR_OR_ERR_RET(*obj->fill, -ENOTSUP);\n+#endif\n+\n+\treturn (*obj->fill)(obj->dev_private, vchan, pattern, dst, length,\n+\t\t\t flags);\n+}\n+\n+/**\n+ * @warning\n+ * @b EXPERIMENTAL: this API may change without prior notice.\n+ *\n+ * Trigger hardware to begin performing enqueued operations.\n+ *\n+ * This API is used to write the \"doorbell\" to the hardware to trigger it\n+ * to begin the operations previously enqueued by rte_dma_copy/fill().\n+ *\n+ * @param dev_id\n+ * The identifier of the device.\n+ * @param vchan\n+ * The identifier of virtual DMA channel.\n+ *\n+ * @return\n+ * 0 on success. Otherwise negative value is returned.\n+ */\n+__rte_experimental\n+static inline int\n+rte_dma_submit(int16_t dev_id, uint16_t vchan)\n+{\n+\tstruct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];\n+\n+#ifdef RTE_DMADEV_DEBUG\n+\tif (!rte_dma_is_valid(dev_id))\n+\t\treturn -EINVAL;\n+\tRTE_FUNC_PTR_OR_ERR_RET(*obj->submit, -ENOTSUP);\n+#endif\n+\n+\treturn (*obj->submit)(obj->dev_private, vchan);\n+}\n+\n+/**\n+ * @warning\n+ * @b EXPERIMENTAL: this API may change without prior notice.\n+ *\n+ * Return the number of operations that have been successfully completed.\n+ *\n+ * @param dev_id\n+ * The identifier of the device.\n+ * @param vchan\n+ * The identifier of virtual DMA channel.\n+ * @param nb_cpls\n+ * The maximum number of completed operations that can be processed.\n+ * @param[out] last_idx\n+ * The last completed operation's ring_idx.\n+ * If not required, NULL can be passed in.\n+ * @param[out] has_error\n+ * Indicates if there are transfer error.\n+ * If not required, NULL can be passed in.\n+ *\n+ * @return\n+ * The number of operations that successfully completed. This return value\n+ * must be less than or equal to the value of nb_cpls.\n+ */\n+__rte_experimental\n+static inline uint16_t\n+rte_dma_completed(int16_t dev_id, uint16_t vchan, const uint16_t nb_cpls,\n+\t\t uint16_t *last_idx, bool *has_error)\n+{\n+\tstruct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];\n+\tuint16_t idx;\n+\tbool err;\n+\n+#ifdef RTE_DMADEV_DEBUG\n+\tif (!rte_dma_is_valid(dev_id) || nb_cpls == 0)\n+\t\treturn 0;\n+\tRTE_FUNC_PTR_OR_ERR_RET(*obj->completed, 0);\n+#endif\n+\n+\t/* Ensure the pointer values are non-null to simplify drivers.\n+\t * In most cases these should be compile time evaluated, since this is\n+\t * an inline function.\n+\t * - If NULL is explicitly passed as parameter, then compiler knows the\n+\t * value is NULL\n+\t * - If address of local variable is passed as parameter, then compiler\n+\t * can know it's non-NULL.\n+\t */\n+\tif (last_idx == NULL)\n+\t\tlast_idx = &idx;\n+\tif (has_error == NULL)\n+\t\thas_error = &err;\n+\n+\t*has_error = false;\n+\treturn (*obj->completed)(obj->dev_private, vchan, nb_cpls, last_idx,\n+\t\t\t\t has_error);\n+}\n+\n+/**\n+ * @warning\n+ * @b EXPERIMENTAL: this API may change without prior notice.\n+ *\n+ * Return the number of operations that have been completed, and the operations\n+ * result may succeed or fail.\n+ *\n+ * @param dev_id\n+ * The identifier of the device.\n+ * @param vchan\n+ * The identifier of virtual DMA channel.\n+ * @param nb_cpls\n+ * Indicates the size of status array.\n+ * @param[out] last_idx\n+ * The last completed operation's ring_idx.\n+ * If not required, NULL can be passed in.\n+ * @param[out] status\n+ * This is a pointer to an array of length 'nb_cpls' that holds the completion\n+ * status code of each operation.\n+ * @see enum rte_dma_status_code\n+ *\n+ * @return\n+ * The number of operations that completed. This return value must be less\n+ * than or equal to the value of nb_cpls.\n+ * If this number is greater than zero (assuming n), then n values in the\n+ * status array are also set.\n+ */\n+__rte_experimental\n+static inline uint16_t\n+rte_dma_completed_status(int16_t dev_id, uint16_t vchan,\n+\t\t\t const uint16_t nb_cpls, uint16_t *last_idx,\n+\t\t\t enum rte_dma_status_code *status)\n+{\n+\tstruct rte_dma_fp_object *obj = &rte_dma_fp_objs[dev_id];\n+\tuint16_t idx;\n+\n+#ifdef RTE_DMADEV_DEBUG\n+\tif (!rte_dma_is_valid(dev_id) || nb_cpls == 0 || status == NULL)\n+\t\treturn 0;\n+\tRTE_FUNC_PTR_OR_ERR_RET(*obj->completed_status, 0);\n+#endif\n+\n+\tif (last_idx == NULL)\n+\t\tlast_idx = &idx;\n+\n+\treturn (*obj->completed_status)(obj->dev_private, vchan, nb_cpls,\n+\t\t\t\t\tlast_idx, status);\n+}\n+\n #ifdef __cplusplus\n }\n #endif\ndiff --git a/lib/dmadev/rte_dmadev_core.h b/lib/dmadev/rte_dmadev_core.h\nnew file mode 100644\nindex 0000000000..a6946052db\n--- /dev/null\n+++ b/lib/dmadev/rte_dmadev_core.h\n@@ -0,0 +1,76 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright(c) 2021 HiSilicon Limited\n+ * Copyright(c) 2021 Intel Corporation\n+ */\n+\n+#ifndef RTE_DMADEV_CORE_H\n+#define RTE_DMADEV_CORE_H\n+\n+/**\n+ * @file\n+ *\n+ * DMA Device internal header.\n+ *\n+ * This header contains internal data types which are used by dataplane inline\n+ * function.\n+ *\n+ * Applications should not use these functions directly.\n+ */\n+\n+/** @internal Used to enqueue a copy operation. */\n+typedef int (*rte_dma_copy_t)(void *dev_private, uint16_t vchan,\n+\t\t\t rte_iova_t src, rte_iova_t dst,\n+\t\t\t uint32_t length, uint64_t flags);\n+\n+/** @internal Used to enqueue a scatter-gather list copy operation. */\n+typedef int (*rte_dma_copy_sg_t)(void *dev_private, uint16_t vchan,\n+\t\t\t\t const struct rte_dma_sge *src,\n+\t\t\t\t const struct rte_dma_sge *dst,\n+\t\t\t\t uint16_t nb_src, uint16_t nb_dst,\n+\t\t\t\t uint64_t flags);\n+\n+/** @internal Used to enqueue a fill operation. */\n+typedef int (*rte_dma_fill_t)(void *dev_private, uint16_t vchan,\n+\t\t\t uint64_t pattern, rte_iova_t dst,\n+\t\t\t uint32_t length, uint64_t flags);\n+\n+/** @internal Used to trigger hardware to begin working. */\n+typedef int (*rte_dma_submit_t)(void *dev_private, uint16_t vchan);\n+\n+/** @internal Used to return number of successful completed operations. */\n+typedef uint16_t (*rte_dma_completed_t)(void *dev_private,\n+\t\t\t\tuint16_t vchan, const uint16_t nb_cpls,\n+\t\t\t\tuint16_t *last_idx, bool *has_error);\n+\n+/** @internal Used to return number of completed operations. */\n+typedef uint16_t (*rte_dma_completed_status_t)(void *dev_private,\n+\t\t\tuint16_t vchan, const uint16_t nb_cpls,\n+\t\t\tuint16_t *last_idx, enum rte_dma_status_code *status);\n+\n+/**\n+ * @internal\n+ * Fast-path dmadev functions and related data are hold in a flat array.\n+ * One entry per dmadev.\n+ *\n+ * This structure occupy exactly 128B which reserve space for future IO\n+ * functions.\n+ *\n+ * The 'dev_private' field was placed in the first cache line to optimize\n+ * performance because the PMD driver mainly depends on this field.\n+ */\n+struct rte_dma_fp_object {\n+\t/** PMD-specific private data. The driver should copy\n+\t * rte_dma_dev.dev_private to this field during initialization.\n+\t */\n+\tvoid *dev_private;\n+\trte_dma_copy_t copy;\n+\trte_dma_copy_sg_t copy_sg;\n+\trte_dma_fill_t fill;\n+\trte_dma_submit_t submit;\n+\trte_dma_completed_t completed;\n+\trte_dma_completed_status_t completed_status;\n+} __rte_aligned(128);\n+\n+extern struct rte_dma_fp_object *rte_dma_fp_objs;\n+\n+#endif /* RTE_DMADEV_CORE_H */\ndiff --git a/lib/dmadev/rte_dmadev_pmd.h b/lib/dmadev/rte_dmadev_pmd.h\nindex 5fcf0f60b8..d6d2161306 100644\n--- a/lib/dmadev/rte_dmadev_pmd.h\n+++ b/lib/dmadev/rte_dmadev_pmd.h\n@@ -100,6 +100,8 @@ struct rte_dma_dev {\n \tvoid *dev_private; /**< PMD-specific private data. */\n \t/** Device info which supplied during device initialization. */\n \tstruct rte_device *device;\n+\t/**< Fast-path functions and related data. */\n+\tstruct rte_dma_fp_object *fp_obj;\n \t/** Functions implemented by PMD. */\n \tconst struct rte_dma_dev_ops *dev_ops;\n \tstruct rte_dma_conf dev_conf; /**< DMA device configuration. */\ndiff --git a/lib/dmadev/version.map b/lib/dmadev/version.map\nindex e925dfcd6d..e17207b212 100644\n--- a/lib/dmadev/version.map\n+++ b/lib/dmadev/version.map\n@@ -2,10 +2,15 @@ EXPERIMENTAL {\n \tglobal:\n \n \trte_dma_close;\n+\trte_dma_completed;\n+\trte_dma_completed_status;\n \trte_dma_configure;\n+\trte_dma_copy;\n+\trte_dma_copy_sg;\n \trte_dma_count_avail;\n \trte_dma_dev_max;\n \trte_dma_dump;\n+\trte_dma_fill;\n \trte_dma_get_dev_id_by_name;\n \trte_dma_info_get;\n \trte_dma_is_valid;\n@@ -13,6 +18,7 @@ EXPERIMENTAL {\n \trte_dma_stats_get;\n \trte_dma_stats_reset;\n \trte_dma_stop;\n+\trte_dma_submit;\n \trte_dma_vchan_setup;\n \n \tlocal: *;\n@@ -22,6 +28,7 @@ INTERNAL {\n \tglobal:\n \n \trte_dma_devices;\n+\trte_dma_fp_objs;\n \trte_dma_pmd_allocate;\n \trte_dma_pmd_release;\n \n", "prefixes": [ "v26", "3/6" ] }