[v8] dmadev: introduce DMA device library
Checks
Commit Message
This patch introduce 'dmadevice' which is a generic type of DMA
device.
The APIs of dmadev library exposes some generic operations which can
enable configuration and I/O with the DMA devices.
Signed-off-by: Chengwen Feng <fengchengwen@huawei.com>
---
v8:
* fix pcie access port diagram doxygen problem.
* fix typo.
* fix compile warning when enable DMADEV_DEBUG.
v7:
* add rte_dmadev_get_dev_id API.
* fix typo.
* use the default macro assignment scheme.
* rename RTE_DMA_DEV_CAPA_* to RTE_DMADEV_CAPA_*.
* rename rte_dmadev_conf.silent_mode to enable_silent.
* add memset when get stats.
v6:
* delete fence capability.
* delete vchan_release ops.
* copy_sg direct use src/dst/nb_src/nb_dst as paramter.
* define rte_dma_direction, don't support multiple direction in the
same vchan.
* fix segment fault when allocate.
* fix typo.
* fix comments format.
v5:
* add doxy-api-* file modify.
* use RTE_LOG_REGISTER_DEFAULT.
* fix typo.
* resolve some incorrect comments.
* fix some doxgen problem.
* fix version.map still hold rte_dmadev_completed_fails.
v4:
* replace xxx_complete_fails with xxx_completed_status.
* add SILENT capability, also a silent_mode in rte_dmadev_conf.
* add op_flag_llc for performance.
* rename dmadev_xxx_t to rte_dmadev_xxx_t to avoid namespace conflict.
* delete filed 'enqueued_count' from rte_dmadev_stats.
* make rte_dmadev hold 'dev_private' filed.
* add RTE_DMA_STATUS_NOT_ATTEMPED status code.
* rename RTE_DMA_STATUS_ACTIVE_DROP to RTE_DMA_STATUS_USER_ABORT.
* rename rte_dma_sg(e) to rte_dmadev_sg(e) to make sure all struct
prefix with rte_dmadev.
* put the comment afterwards.
* fix some doxgen problem.
* delete macro RTE_DMADEV_VALID_DEV_ID_OR_RET and
RTE_DMADEV_PTR_OR_ERR_RET.
* replace strlcpy with rte_strscpy.
* other minor modifications from review comment.
v3:
* rm reset and fill_sg ops.
* rm MT-safe capabilities.
* add submit flag.
* redefine rte_dma_sg to implement asymmetric copy.
* delete some reserved field for future use.
* rearrangement rte_dmadev/rte_dmadev_data struct.
* refresh rte_dmadev.h copyright.
* update vchan setup parameter.
* modified some inappropriate descriptions.
* arrange version.map alphabetically.
* other minor modifications from review comment.
---
MAINTAINERS | 4 +
config/rte_config.h | 3 +
doc/api/doxy-api-index.md | 1 +
doc/api/doxy-api.conf.in | 1 +
lib/dmadev/meson.build | 7 +
lib/dmadev/rte_dmadev.c | 545 ++++++++++++++++++++++
lib/dmadev/rte_dmadev.h | 1041 ++++++++++++++++++++++++++++++++++++++++++
lib/dmadev/rte_dmadev_core.h | 182 ++++++++
lib/dmadev/rte_dmadev_pmd.h | 72 +++
lib/dmadev/version.map | 37 ++
lib/meson.build | 1 +
11 files changed, 1894 insertions(+)
create mode 100644 lib/dmadev/meson.build
create mode 100644 lib/dmadev/rte_dmadev.c
create mode 100644 lib/dmadev/rte_dmadev.h
create mode 100644 lib/dmadev/rte_dmadev_core.h
create mode 100644 lib/dmadev/rte_dmadev_pmd.h
create mode 100644 lib/dmadev/version.map
Comments
On Tue, Jul 20, 2021 at 6:48 AM Chengwen Feng <fengchengwen@huawei.com> wrote:
>
> This patch introduce 'dmadevice' which is a generic type of DMA
> device.
>
> The APIs of dmadev library exposes some generic operations which can
> enable configuration and I/O with the DMA devices.
>
> Signed-off-by: Chengwen Feng <fengchengwen@huawei.com>
> ---
> +DMA device API - EXPERIMENTAL
> +M: Chengwen Feng <fengchengwen@huawei.com>
> +F: lib/dmadev/
> +
>
> Memory Pool Drivers
> -------------------
> diff --git a/config/rte_config.h b/config/rte_config.h
> index 590903c..331a431 100644
> --- a/config/rte_config.h
> +++ b/config/rte_config.h
> @@ -81,6 +81,9 @@
> /* rawdev defines */
> #define RTE_RAWDEV_MAX_DEVS 64
>
> +/* dmadev defines */
> +#define RTE_DMADEV_MAX_DEVS 64
> +
> /* ip_fragmentation defines */
> #define RTE_LIBRTE_IP_FRAG_MAX_FRAG 4
> #undef RTE_LIBRTE_IP_FRAG_TBL_STAT
> diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
> index 1992107..ce08250 100644
> --- a/doc/api/doxy-api-index.md
> +++ b/doc/api/doxy-api-index.md
> @@ -27,6 +27,7 @@ The public API headers are grouped by topics:
> [event_timer_adapter] (@ref rte_event_timer_adapter.h),
> [event_crypto_adapter] (@ref rte_event_crypto_adapter.h),
> [rawdev] (@ref rte_rawdev.h),
> + [dmadev] (@ref rte_dmadev.h),
> [metrics] (@ref rte_metrics.h),
> [bitrate] (@ref rte_bitrate.h),
> [latency] (@ref rte_latencystats.h),
> diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
> index 325a019..109ec1f 100644
> --- a/doc/api/doxy-api.conf.in
> +++ b/doc/api/doxy-api.conf.in
> @@ -35,6 +35,7 @@ INPUT = @TOPDIR@/doc/api/doxy-api-index.md \
> @TOPDIR@/lib/compressdev \
> @TOPDIR@/lib/cryptodev \
> @TOPDIR@/lib/distributor \
> + @TOPDIR@/lib/dmadev \
> @TOPDIR@/lib/efd \
> @TOPDIR@/lib/ethdev \
> @TOPDIR@/lib/eventdev \
> diff --git a/lib/dmadev/meson.build b/lib/dmadev/meson.build
> new file mode 100644
> index 0000000..d2fc85e
> --- /dev/null
> +++ b/lib/dmadev/meson.build
> @@ -0,0 +1,7 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2021 HiSilicon Limited.
> +
> +sources = files('rte_dmadev.c')
> +headers = files('rte_dmadev.h')
> +indirect_headers += files('rte_dmadev_core.h')
> +driver_sdk_headers += files('rte_dmadev_pmd.h')
> diff --git a/lib/dmadev/rte_dmadev.c b/lib/dmadev/rte_dmadev.c
> new file mode 100644
> index 0000000..a6d8e3be
> --- /dev/null
> +++ b/lib/dmadev/rte_dmadev.c
> @@ -0,0 +1,545 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2021 HiSilicon Limited.
> + * Copyright(c) 2021 Intel Corporation.
> + */
> +
> +#include <ctype.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +
> +#include <rte_debug.h>
> +#include <rte_dev.h>
> +#include <rte_eal.h>
> +#include <rte_errno.h>
> +#include <rte_lcore.h>
> +#include <rte_log.h>
> +#include <rte_memory.h>
> +#include <rte_memzone.h>
> +#include <rte_malloc.h>
> +#include <rte_string_fns.h>
> +
> +#include "rte_dmadev.h"
> +#include "rte_dmadev_pmd.h"
> +
> +struct rte_dmadev rte_dmadevices[RTE_DMADEV_MAX_DEVS];
> +
> +static const char *MZ_RTE_DMADEV_DATA = "rte_dmadev_data";
You may change MZ_RTE_DMADEV_DATA as mz_rte_dmadev_data
> +/* Shared memory between primary and secondary processes. */
> +static struct {
> + struct rte_dmadev_data data[RTE_DMADEV_MAX_DEVS];
> +} *dmadev_shared_data;
> +
> +RTE_LOG_REGISTER_DEFAULT(rte_dmadev_logtype, INFO);
> +#define RTE_DMADEV_LOG(level, ...) \
> + rte_log(RTE_LOG_ ## level, rte_dmadev_logtype, "" __VA_ARGS__)
> +
> +/* Macros to check for valid device id */
> +#define RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, retval) do { \
> + if (!rte_dmadev_is_valid_dev(dev_id)) { \
> + RTE_DMADEV_LOG(ERR, "Invalid dev_id=%u\n", dev_id); \
> + return retval; \
> + } \
> +} while (0)
> +
> +static int
> +dmadev_check_name(const char *name)
> +{
> + size_t name_len;
> +
> + if (name == NULL) {
> + RTE_DMADEV_LOG(ERR, "Name can't be NULL\n");
> + return -EINVAL;
> + }
> +
> + name_len = strnlen(name, RTE_DMADEV_NAME_MAX_LEN);
> + if (name_len == 0) {
> + RTE_DMADEV_LOG(ERR, "Zero length DMA device name\n");
> + return -EINVAL;
> + }
> + if (name_len >= RTE_DMADEV_NAME_MAX_LEN) {
> + RTE_DMADEV_LOG(ERR, "DMA device name is too long\n");
> + return -EINVAL;
> + }
> +
> + return 0;
> +}
> +
> +static uint16_t
> +dmadev_find_free_dev(void)
> +{
> + uint16_t i;
> +
> + for (i = 0; i < RTE_DMADEV_MAX_DEVS; i++) {
> + if (dmadev_shared_data->data[i].dev_name[0] == '\0') {
Please check, are you writing \0 on freeing the device or not?
> + RTE_ASSERT(rte_dmadevices[i].state ==
> + RTE_DMADEV_UNUSED);
Please remove RTE_ASSERT from the library.
> + return i;
> + }
> + }
> +
> + return RTE_DMADEV_MAX_DEVS;
> +}
> +
> +static struct rte_dmadev*
> +dmadev_find(const char *name)
> +{
> + uint16_t i;
> +
> + for (i = 0; i < RTE_DMADEV_MAX_DEVS; i++) {
> + if ((rte_dmadevices[i].state == RTE_DMADEV_ATTACHED) &&
> + (!strcmp(name, rte_dmadevices[i].data->dev_name)))
> + return &rte_dmadevices[i];
> + }
> +
> + return NULL;
> +}
> +
> +static int
> +dmadev_shared_data_prepare(void)
> +{
> + const struct rte_memzone *mz;
> +
> + if (dmadev_shared_data == NULL) {
> + if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
> + /* Allocate port data and ownership shared memory. */
> + mz = rte_memzone_reserve(MZ_RTE_DMADEV_DATA,
> + sizeof(*dmadev_shared_data),
> + rte_socket_id(), 0);
> + } else
> + mz = rte_memzone_lookup(MZ_RTE_DMADEV_DATA);
> + if (mz == NULL)
> + return -ENOMEM;
> +
> + dmadev_shared_data = mz->addr;
> + if (rte_eal_process_type() == RTE_PROC_PRIMARY)
> + memset(dmadev_shared_data->data, 0,
> + sizeof(dmadev_shared_data->data));
> + }
> +
> + return 0;
> +}
> +
> +static struct rte_dmadev *
> +dmadev_allocate(const char *name)
> +{
> + struct rte_dmadev *dev;
> + uint16_t dev_id;
> +
> + dev = dmadev_find(name);
> + if (dev != NULL) {
> + RTE_DMADEV_LOG(ERR, "DMA device already allocated\n");
> + return NULL;
> + }
> +
> + if (dmadev_shared_data_prepare() != 0) {
> + RTE_DMADEV_LOG(ERR, "Cannot allocate DMA shared data\n");
> + return NULL;
> + }
> +
> + dev_id = dmadev_find_free_dev();
> + if (dev_id == RTE_DMADEV_MAX_DEVS) {
> + RTE_DMADEV_LOG(ERR, "Reached maximum number of DMA devices\n");
> + return NULL;
> + }
> +
> + dev = &rte_dmadevices[dev_id];
> + dev->data = &dmadev_shared_data->data[dev_id];
> + dev->data->dev_id = dev_id;
> + rte_strscpy(dev->data->dev_name, name, sizeof(dev->data->dev_name));
> +
> + return dev;
> +}
> +
> +static struct rte_dmadev *
> +dmadev_attach_secondary(const char *name)
> +{
> + struct rte_dmadev *dev;
> + uint16_t i;
> +
> + if (dmadev_shared_data_prepare() != 0) {
> + RTE_DMADEV_LOG(ERR, "Cannot allocate DMA shared data\n");
> + return NULL;
> + }
> +
> + for (i = 0; i < RTE_DMADEV_MAX_DEVS; i++) {
> + if (!strcmp(dmadev_shared_data->data[i].dev_name, name))
> + break;
> + }
> + if (i == RTE_DMADEV_MAX_DEVS) {
> + RTE_DMADEV_LOG(ERR,
> + "Device %s is not driven by the primary process\n",
> + name);
> + return NULL;
> + }
> +
> + dev = &rte_dmadevices[i];
> + dev->data = &dmadev_shared_data->data[i];
> + RTE_ASSERT(dev->data->dev_id == i);
Remove ASSERT from library.
> + dev->dev_private = dev->data->dev_private;
> +
> + return dev;
> +}
> +
> +struct rte_dmadev *
> +rte_dmadev_pmd_allocate(const char *name)
> +{
> + struct rte_dmadev *dev;
> +
> + if (dmadev_check_name(name) != 0)
> + return NULL;
> +
> + if (rte_eal_process_type() == RTE_PROC_PRIMARY)
> + dev = dmadev_allocate(name);
> + else
> + dev = dmadev_attach_secondary(name);
> +
> + if (dev == NULL)
> + return NULL;
> + dev->state = RTE_DMADEV_ATTACHED;
> +
> + return dev;
> +}
> +
> +int
> +rte_dmadev_pmd_release(struct rte_dmadev *dev)
> +{
> + void *dev_private_bak;
How about bak -> tmp
> +
> + if (dev == NULL)
> + return -EINVAL;
> +
> + if (dev->state == RTE_DMADEV_UNUSED)
> + return 0;
> +
> + if (rte_eal_process_type() == RTE_PROC_PRIMARY)
> + memset(dev->data, 0, sizeof(struct rte_dmadev_data));
> +
> + dev_private_bak = dev->dev_private;
> + memset(dev, 0, sizeof(struct rte_dmadev));
> + if (rte_eal_process_type() == RTE_PROC_PRIMARY)
> + dev->dev_private = dev_private_bak;
> + dev->state = RTE_DMADEV_UNUSED;
> +
> + return 0;
> +}
> +
> +struct rte_dmadev *
> +rte_dmadev_get_device_by_name(const char *name)
> +{
> + if (dmadev_check_name(name) != 0)
> + return NULL;
> + return dmadev_find(name);
> +}
> +
> +int
> +rte_dmadev_get_dev_id(const char *name)
> +{
> + struct rte_dmadev *dev = rte_dmadev_get_device_by_name(name);
> + if (dev != NULL)
> + return dev->data->dev_id;
> + return -EINVAL;
> +}
> +
> +bool
> +rte_dmadev_is_valid_dev(uint16_t dev_id)
> +{
> + return (dev_id < RTE_DMADEV_MAX_DEVS) &&
> + rte_dmadevices[dev_id].state == RTE_DMADEV_ATTACHED;
> +}
> +
> +uint16_t
> +rte_dmadev_count(void)
> +{
> + uint16_t count = 0;
> + uint16_t i;
> +
> + for (i = 0; i < RTE_DMADEV_MAX_DEVS; i++) {
> + if (rte_dmadevices[i].state == RTE_DMADEV_ATTACHED)
> + count++;
> + }
> +
> + return count;
> +}
> +
> +int
> +rte_dmadev_info_get(uint16_t dev_id, struct rte_dmadev_info *dev_info)
> +{
> + const struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> + int ret;
> +
> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> + if (dev_info == NULL)
> + return -EINVAL;
> +
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_info_get, -ENOTSUP);
> + memset(dev_info, 0, sizeof(struct rte_dmadev_info));
> + ret = (*dev->dev_ops->dev_info_get)(dev, dev_info,
> + sizeof(struct rte_dmadev_info));
> + if (ret != 0)
> + return ret;
> +
> + dev_info->device = dev->device;
> + dev_info->nb_vchans = dev->data->dev_conf.max_vchans;
This will be updated after configure stage.
> +
> + return 0;
> +}
> +
> +int
> +rte_dmadev_configure(uint16_t dev_id, const struct rte_dmadev_conf *dev_conf)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> + struct rte_dmadev_info info;
> + int ret;
> +
> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> + if (dev_conf == NULL)
> + return -EINVAL;
> +
> + ret = rte_dmadev_info_get(dev_id, &info);
> + if (ret != 0) {
> + RTE_DMADEV_LOG(ERR, "Device %u get device info fail\n", dev_id);
> + return -EINVAL;
> + }
> + if (dev_conf->max_vchans == 0) {
> + RTE_DMADEV_LOG(ERR,
> + "Device %u configure zero vchans\n", dev_id);
> + return -EINVAL;
> + }
> + if (dev_conf->max_vchans > info.max_vchans) {
> + RTE_DMADEV_LOG(ERR,
> + "Device %u configure too many vchans\n", dev_id);
> + return -EINVAL;
> + }
> + if (dev_conf->enable_silent &&
> + !(info.dev_capa & RTE_DMADEV_CAPA_SILENT)) {
> + RTE_DMADEV_LOG(ERR, "Device %u don't support silent\n", dev_id);
> + return -EINVAL;
> + }
> +
> + if (dev->data->dev_started != 0) {
> + RTE_DMADEV_LOG(ERR,
> + "Device %u must be stopped to allow configuration\n",
> + dev_id);
> + return -EBUSY;
> + }
ethdev and other device class common code handles the reconfigure case. i.e
the application configures N vchan first and reconfigures to N - M
then free the resources
attached to M - N. Please do the same here.
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
> + ret = (*dev->dev_ops->dev_configure)(dev, dev_conf);
> + if (ret == 0)
> + memcpy(&dev->data->dev_conf, dev_conf, sizeof(*dev_conf));
> +
> + return ret;
> +}
> +
> +int
> +rte_dmadev_start(uint16_t dev_id)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> + int ret;
> +
> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> +
> + if (dev->data->dev_started != 0) {
> + RTE_DMADEV_LOG(WARNING, "Device %u already started\n", dev_id);
> + return 0;
> + }
> +
> + if (dev->dev_ops->dev_start == NULL)
> + goto mark_started;
> +
> + ret = (*dev->dev_ops->dev_start)(dev);
> + if (ret != 0)
> + return ret;
> +
> +mark_started:
> + dev->data->dev_started = 1;
> + return 0;
> +}
> +
> +int
> +rte_dmadev_stop(uint16_t dev_id)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> + int ret;
> +
> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> +
> + if (dev->data->dev_started == 0) {
> + RTE_DMADEV_LOG(WARNING, "Device %u already stopped\n", dev_id);
> + return 0;
> + }
> +
> + if (dev->dev_ops->dev_stop == NULL)
> + goto mark_stopped;
> +
> + ret = (*dev->dev_ops->dev_stop)(dev);
> + if (ret != 0)
> + return ret;
> +
> +mark_stopped:
> + dev->data->dev_started = 0;
> + return 0;
> +}
> +
> +int
> +rte_dmadev_close(uint16_t dev_id)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> +
> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> +
> + /* Device must be stopped before it can be closed */
> + if (dev->data->dev_started == 1) {
> + RTE_DMADEV_LOG(ERR,
> + "Device %u must be stopped before closing\n", dev_id);
> + return -EBUSY;
> + }
> +
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
> + return (*dev->dev_ops->dev_close)(dev);
> +}
> +
> +int
> +rte_dmadev_vchan_setup(uint16_t dev_id,
> + const struct rte_dmadev_vchan_conf *conf)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> + struct rte_dmadev_info info;
> + int ret;
> +
> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> + if (conf == NULL)
> + return -EINVAL;
> +
> + dev = &rte_dmadevices[dev_id];
> +
> + ret = rte_dmadev_info_get(dev_id, &info);
> + if (ret != 0) {
> + RTE_DMADEV_LOG(ERR, "Device %u get device info fail\n", dev_id);
> + return -EINVAL;
> + }
> + if (conf->direction != RTE_DMA_DIR_MEM_TO_MEM &&
> + conf->direction != RTE_DMA_DIR_MEM_TO_DEV &&
> + conf->direction != RTE_DMA_DIR_DEV_TO_MEM &&
> + conf->direction != RTE_DMA_DIR_DEV_TO_DEV) {
> + RTE_DMADEV_LOG(ERR, "Device %u direction invalid!\n", dev_id);
> + return -EINVAL;
> + }
> + if (conf->direction == RTE_DMA_DIR_MEM_TO_MEM &&
> + !(info.dev_capa & RTE_DMADEV_CAPA_MEM_TO_MEM)) {
> + RTE_DMADEV_LOG(ERR,
> + "Device %u don't support mem2mem transfer\n", dev_id);
> + return -EINVAL;
> + }
> + if (conf->direction == RTE_DMA_DIR_MEM_TO_DEV &&
> + !(info.dev_capa & RTE_DMADEV_CAPA_MEM_TO_DEV)) {
> + RTE_DMADEV_LOG(ERR,
> + "Device %u don't support mem2dev transfer\n", dev_id);
> + return -EINVAL;
> + }
> + if (conf->direction == RTE_DMA_DIR_DEV_TO_MEM &&
> + !(info.dev_capa & RTE_DMADEV_CAPA_DEV_TO_MEM)) {
> + RTE_DMADEV_LOG(ERR,
> + "Device %u don't support dev2mem transfer\n", dev_id);
> + return -EINVAL;
> + }
> + if (conf->direction == RTE_DMA_DIR_DEV_TO_DEV &&
> + !(info.dev_capa & RTE_DMADEV_CAPA_DEV_TO_DEV)) {
> + RTE_DMADEV_LOG(ERR,
> + "Device %u don't support dev2dev transfer\n", dev_id);
> + return -EINVAL;
> + }
> + if (conf->nb_desc < info.min_desc || conf->nb_desc > info.max_desc) {
> + RTE_DMADEV_LOG(ERR,
> + "Device %u number of descriptors invalid\n", dev_id);
> + return -EINVAL;
> + }
Missing sanity check on src and dst port incase if the direction is
based on device.
> +
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vchan_setup, -ENOTSUP);
> + return (*dev->dev_ops->vchan_setup)(dev, conf);
> +}
> +
> +int
> +rte_dmadev_stats_get(uint16_t dev_id, uint16_t vchan,
> + struct rte_dmadev_stats *stats)
> +{
> + const struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> +
> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> + if (stats == NULL)
> + return -EINVAL;
> + if (vchan >= dev->data->dev_conf.max_vchans &&
> + vchan != RTE_DMADEV_ALL_VCHAN) {
> + RTE_DMADEV_LOG(ERR,
> + "Device %u vchan %u out of range\n", dev_id, vchan);
> + return -EINVAL;
> + }
> +
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
> + memset(stats, 0, sizeof(struct rte_dmadev_stats));
> + return (*dev->dev_ops->stats_get)(dev, vchan, stats,
> + sizeof(struct rte_dmadev_stats));
> +}
> +
> +int
> +rte_dmadev_stats_reset(uint16_t dev_id, uint16_t vchan)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> +
> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> + if (vchan >= dev->data->dev_conf.max_vchans &&
> + vchan != RTE_DMADEV_ALL_VCHAN) {
> + RTE_DMADEV_LOG(ERR,
> + "Device %u vchan %u out of range\n", dev_id, vchan);
> + return -EINVAL;
> + }
> +
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP);
> + return (*dev->dev_ops->stats_reset)(dev, vchan);
> +}
> +
> +int
> +rte_dmadev_dump(uint16_t dev_id, FILE *f)
> +{
> + const struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> + struct rte_dmadev_info info;
> + int ret;
> +
> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> + if (f == NULL)
> + return -EINVAL;
> +
> + ret = rte_dmadev_info_get(dev_id, &info);
> + if (ret != 0) {
> + RTE_DMADEV_LOG(ERR, "Device %u get device info fail\n", dev_id);
> + return -EINVAL;
> + }
> +
> + fprintf(f, "DMA Dev %u, '%s' [%s]\n",
> + dev->data->dev_id,
> + dev->data->dev_name,
> + dev->data->dev_started ? "started" : "stopped");
> + fprintf(f, " dev_capa: 0x%" PRIx64 "\n", info.dev_capa);
> + fprintf(f, " max_vchans_supported: %u\n", info.max_vchans);
> + fprintf(f, " max_vchans_configured: %u\n", info.nb_vchans);
> + fprintf(f, " silent_mode: %s\n",
> + dev->data->dev_conf.enable_silent ? "on" : "off");
Probably iterate over each vchan and dumping the at least direction
will be usefull.
> +
> + if (dev->dev_ops->dev_dump != NULL)
> + return (*dev->dev_ops->dev_dump)(dev, f);
> +
> + return 0;
> +}
> +
> +int
> +rte_dmadev_selftest(uint16_t dev_id)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> +
> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_selftest, -ENOTSUP);
> + return (*dev->dev_ops->dev_selftest)(dev_id);
> +}
> diff --git a/lib/dmadev/rte_dmadev.h b/lib/dmadev/rte_dmadev.h
> new file mode 100644
> index 0000000..53d85f1
> --- /dev/null
> +++ b/lib/dmadev/rte_dmadev.h
> @@ -0,0 +1,1041 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2021 HiSilicon Limited.
> + * Copyright(c) 2021 Intel Corporation.
> + * Copyright(c) 2021 Marvell International Ltd.
> + * Copyright(c) 2021 SmartShare Systems.
> + */
> +
> +#ifndef _RTE_DMADEV_H_
> +#define _RTE_DMADEV_H_
> +
> +/**
> + * @file rte_dmadev.h
> + *
> + * RTE DMA (Direct Memory Access) device APIs.
> + *
> + * The DMA framework is built on the following model:
> + *
> + * --------------- --------------- ---------------
> + * | virtual DMA | | virtual DMA | | virtual DMA |
> + * | channel | | channel | | channel |
> + * --------------- --------------- ---------------
> + * | | |
> + * ------------------ |
> + * | |
> + * ------------ ------------
> + * | dmadev | | dmadev |
> + * ------------ ------------
> + * | |
> + * ------------------ ------------------
> + * | HW-DMA-channel | | HW-DMA-channel |
> + * ------------------ ------------------
> + * | |
> + * --------------------------------
> + * |
> + * ---------------------
> + * | HW-DMA-Controller |
> + * ---------------------
> + *
> + * The DMA controller could have multiple HW-DMA-channels (aka. HW-DMA-queues),
> + * each HW-DMA-channel should be represented by a dmadev.
> + *
> + * The dmadev could create multiple virtual DMA channels, each virtual DMA
> + * channel represents a different transfer context. The DMA operation request
> + * must be submitted to the virtual DMA channel. e.g. Application could create
> + * virtual DMA channel 0 for memory-to-memory transfer scenario, and create
> + * virtual DMA channel 1 for memory-to-device transfer scenario.
> + *
> + * The dmadev are dynamically allocated by rte_dmadev_pmd_allocate() during the
> + * PCI/SoC device probing phase performed at EAL initialization time. And could
> + * be released by rte_dmadev_pmd_release() during the PCI/SoC device removing
> + * phase.
> + *
> + * This framework uses 'uint16_t dev_id' as the device identifier of a dmadev,
> + * and 'uint16_t vchan' as the virtual DMA channel identifier in one dmadev.
> + *
> + * The functions exported by the dmadev API to setup a device designated by its
> + * device identifier must be invoked in the following order:
> + * - rte_dmadev_configure()
> + * - rte_dmadev_vchan_setup()
> + * - rte_dmadev_start()
> + *
> + * Then, the application can invoke dataplane APIs to process jobs.
> + *
> + * If the application wants to change the configuration (i.e. invoke
> + * rte_dmadev_configure() or rte_dmadev_vchan_setup()), it must invoke
> + * rte_dmadev_stop() first to stop the device and then do the reconfiguration
> + * before invoking rte_dmadev_start() again. The dataplane APIs should not be
> + * invoked when the device is stopped.
> + *
> + * Finally, an application can close a dmadev by invoking the
> + * rte_dmadev_close() function.
> + *
> + * The dataplane APIs include two parts:
> + * The first part is the submission of operation requests:
> + * - rte_dmadev_copy()
> + * - rte_dmadev_copy_sg()
> + * - rte_dmadev_fill()
> + * - rte_dmadev_submit()
> + *
> + * These APIs could work with different virtual DMA channels which have
> + * different contexts.
> + *
> + * The first three APIs are used to submit the operation request to the virtual
> + * DMA channel, if the submission is successful, a uint16_t ring_idx is
> + * returned, otherwise a negative number is returned.
> + *
> + * The last API was used to issue doorbell to hardware, and also there are flags
> + * (@see RTE_DMA_OP_FLAG_SUBMIT) parameter of the first three APIs could do the
> + * same work.
> + *
> + * The second part is to obtain the result of requests:
> + * - rte_dmadev_completed()
> + * - return the number of operation requests completed successfully.
> + * - rte_dmadev_completed_status()
> + * - return the number of operation requests completed.
> + *
> + * @note If the dmadev works in silent mode (@see RTE_DMADEV_CAPA_SILENT),
> + * application does not invoke the above two completed APIs.
> + *
> + * About the ring_idx which enqueue APIs (e.g. rte_dmadev_copy()
> + * rte_dmadev_fill()) returned, the rules are as follows:
> + * - ring_idx for each virtual DMA channel are independent.
> + * - For a virtual DMA channel, the ring_idx is monotonically incremented,
> + * when it reach UINT16_MAX, it wraps back to zero.
> + * - This ring_idx can be used by applications to track per-operation
> + * metadata in an application-defined circular ring.
> + * - The initial ring_idx of a virtual DMA channel is zero, after the
> + * device is stopped, the ring_idx needs to be reset to zero.
> + *
> + * One example:
> + * - step-1: start one dmadev
> + * - step-2: enqueue a copy operation, the ring_idx return is 0
> + * - step-3: enqueue a copy operation again, the ring_idx return is 1
> + * - ...
> + * - step-101: stop the dmadev
> + * - step-102: start the dmadev
> + * - step-103: enqueue a copy operation, the cookie return is 0
> + * - ...
> + * - step-x+0: enqueue a fill operation, the ring_idx return is 65535
> + * - step-x+1: enqueue a copy operation, the ring_idx return is 0
> + * - ...
> + *
> + * The DMA operation address used in enqueue APIs (i.e. rte_dmadev_copy(),
> + * rte_dmadev_copy_sg(), rte_dmadev_fill()) defined as rte_iova_t type. The
> + * dmadev supports two types of address: memory address and device address.
> + *
> + * - memory address: the source and destination address of the memory-to-memory
> + * transfer type, or the source address of the memory-to-device transfer type,
> + * or the destination address of the device-to-memory transfer type.
> + * @note If the device support SVA (@see RTE_DMADEV_CAPA_SVA), the memory
> + * address can be any VA address, otherwise it must be an IOVA address.
> + *
> + * - device address: the source and destination address of the device-to-device
> + * transfer type, or the source address of the device-to-memory transfer type,
> + * or the destination address of the memory-to-device transfer type.
> + *
> + * By default, all the functions of the dmadev API exported by a PMD are
> + * lock-free functions which assume to not be invoked in parallel on different
> + * logical cores to work on the same target dmadev object.
> + * @note Different virtual DMA channels on the same dmadev *DO NOT* support
> + * parallel invocation because these virtual DMA channels share the same
> + * HW-DMA-channel.
> + *
> + */
> +
> +#include <rte_common.h>
> +#include <rte_compat.h>
> +#include <rte_dev.h>
> +#include <rte_errno.h>
> +#include <rte_memory.h>
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#define RTE_DMADEV_NAME_MAX_LEN RTE_DEV_NAME_MAX_LEN
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Get the device identifier for the named DMA device.
> + *
> + * @param name
> + * DMA device name.
> + *
> + * @return
> + * Returns DMA device identifier on success.
> + * - <0: Failure to find named DMA device.
> + */
> +__rte_experimental
> +int
> +rte_dmadev_get_dev_id(const char *name);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * @param dev_id
> + * DMA device index.
> + *
> + * @return
> + * - If the device index is valid (true) or not (false).
> + */
> +__rte_experimental
> +bool
> +rte_dmadev_is_valid_dev(uint16_t dev_id);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Get the total number of DMA devices that have been successfully
> + * initialised.
> + *
> + * @return
> + * The total number of usable DMA devices.
> + */
> +__rte_experimental
> +uint16_t
> +rte_dmadev_count(void);
> +
> +/* Enumerates DMA device capabilities. */
> +#define RTE_DMADEV_CAPA_MEM_TO_MEM (1ull << 0)
> +/**< DMA device support memory-to-memory transfer.
> + *
> + * @see struct rte_dmadev_info::dev_capa
> + */
> +
> +#define RTE_DMADEV_CAPA_MEM_TO_DEV (1ull << 1)
> +/**< DMA device support memory-to-device transfer.
> + *
> + * @see struct rte_dmadev_info::dev_capa
> + * @see struct rte_dmadev_port_param::port_type
> + */
> +
> +#define RTE_DMADEV_CAPA_DEV_TO_MEM (1ull << 2)
> +/**< DMA device support device-to-memory transfer.
> + *
> + * @see struct rte_dmadev_info::dev_capa
> + * @see struct rte_dmadev_port_param::port_type
> + */
> +
> +#define RTE_DMADEV_CAPA_DEV_TO_DEV (1ull << 3)
> +/**< DMA device support device-to-device transfer.
> + *
> + * @see struct rte_dmadev_info::dev_capa
> + * @see struct rte_dmadev_port_param::port_type
> + */
> +
> +#define RTE_DMADEV_CAPA_SVA (1ull << 4)
> +/**< DMA device support SVA which could use VA as DMA address.
> + * If device support SVA then application could pass any VA address like memory
> + * from rte_malloc(), rte_memzone(), malloc, stack memory.
> + * If device don't support SVA, then application should pass IOVA address which
> + * from rte_malloc(), rte_memzone().
> + *
> + * @see struct rte_dmadev_info::dev_capa
> + */
> +
> +#define RTE_DMADEV_CAPA_SILENT (1ull << 5)
> +/**< DMA device support work in silent mode.
> + * In this mode, application don't required to invoke rte_dmadev_completed*()
> + * API.
> + *
> + * @see struct rte_dmadev_conf::silent_mode
> + */
> +
> +#define RTE_DMADEV_CAPA_OPS_COPY (1ull << 32)
> +/**< DMA device support copy ops.
> + * This capability start with index of 32, so that it could leave gap between
> + * normal capability and ops capability.
> + *
> + * @see struct rte_dmadev_info::dev_capa
> + */
> +
> +#define RTE_DMADEV_CAPA_OPS_COPY_SG (1ull << 33)
> +/**< DMA device support scatter-list copy ops.
> + *
> + * @see struct rte_dmadev_info::dev_capa
> + */
> +
> +#define RTE_DMADEV_CAPA_OPS_FILL (1ull << 34)
> +/**< DMA device support fill ops.
> + *
> + * @see struct rte_dmadev_info::dev_capa
> + */
> +
> +/**
> + * A structure used to retrieve the information of a DMA device.
> + */
> +struct rte_dmadev_info {
> + struct rte_device *device; /**< Generic Device information. */
> + uint64_t dev_capa; /**< Device capabilities (RTE_DMADEV_CAPA_*). */
> + uint16_t max_vchans;
> + /**< Maximum number of virtual DMA channels supported. */
> + uint16_t max_desc;
> + /**< Maximum allowed number of virtual DMA channel descriptors. */
> + uint16_t min_desc;
> + /**< Minimum allowed number of virtual DMA channel descriptors. */
> + uint16_t nb_vchans; /**< Number of virtual DMA channel configured. */
> +};
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Retrieve information of a DMA device.
> + *
> + * @param dev_id
> + * The identifier of the device.
> + * @param[out] dev_info
> + * A pointer to a structure of type *rte_dmadev_info* to be filled with the
> + * information of the device.
> + *
> + * @return
> + * - =0: Success, driver updates the information of the DMA device.
> + * - <0: Error code returned by the driver info get function.
> + *
> + */
> +__rte_experimental
> +int
> +rte_dmadev_info_get(uint16_t dev_id, struct rte_dmadev_info *dev_info);
> +
> +/**
> + * A structure used to configure a DMA device.
> + */
> +struct rte_dmadev_conf {
> + uint16_t max_vchans;
> + /**< Maximum number of virtual DMA channel to use.
> + * This value cannot be greater than the field 'max_vchans' of struct
> + * rte_dmadev_info which get from rte_dmadev_info_get().
> + */
> + bool enable_silent;
> + /**< Indicates whether to enable silent mode.
> + * false-default mode, true-silent mode.
> + * This value can be set to true only when the SILENT capability is
> + * supported.
> + *
> + * @see RTE_DMADEV_CAPA_SILENT
> + */
> +};
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Configure a DMA device.
> + *
> + * This function must be invoked first before any other function in the
> + * API. This function can also be re-invoked when a device is in the
> + * stopped state.
> + *
> + * @param dev_id
> + * The identifier of the device to configure.
> + * @param dev_conf
> + * The DMA device configuration structure encapsulated into rte_dmadev_conf
> + * object.
> + *
> + * @return
> + * - =0: Success, device configured.
> + * - <0: Error code returned by the driver configuration function.
> + */
> +__rte_experimental
> +int
> +rte_dmadev_configure(uint16_t dev_id, const struct rte_dmadev_conf *dev_conf);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Start a DMA device.
> + *
> + * The device start step is the last one and consists of setting the DMA
> + * to start accepting jobs.
> + *
> + * @param dev_id
> + * The identifier of the device.
> + *
> + * @return
> + * - =0: Success, device started.
> + * - <0: Error code returned by the driver start function.
> + */
> +__rte_experimental
> +int
> +rte_dmadev_start(uint16_t dev_id);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Stop a DMA device.
> + *
> + * The device can be restarted with a call to rte_dmadev_start().
> + *
> + * @param dev_id
> + * The identifier of the device.
> + *
> + * @return
> + * - =0: Success, device stopped.
> + * - <0: Error code returned by the driver stop function.
> + */
> +__rte_experimental
> +int
> +rte_dmadev_stop(uint16_t dev_id);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Close a DMA device.
> + *
> + * The device cannot be restarted after this call.
> + *
> + * @param dev_id
> + * The identifier of the device.
> + *
> + * @return
> + * - =0: Successfully close device
> + * - <0: Failure to close device
> + */
> +__rte_experimental
> +int
> +rte_dmadev_close(uint16_t dev_id);
> +
> +/**
> + * rte_dma_direction - DMA transfer direction defines.
> + */
> +enum rte_dma_direction {
> + RTE_DMA_DIR_MEM_TO_MEM,
> + /**< DMA transfer direction - from memory to memory.
> + *
> + * @see struct rte_dmadev_vchan_conf::direction
> + */
> + RTE_DMA_DIR_MEM_TO_DEV,
> + /**< DMA transfer direction - from memory to device.
> + * In a typical scenario, the SoCs are installed on host servers as
> + * iNICs through the PCIe interface. In this case, the SoCs works in
> + * EP(endpoint) mode, it could initiate a DMA move request from memory
> + * (which is SoCs memory) to device (which is host memory).
> + *
> + * @see struct rte_dmadev_vchan_conf::direction
> + */
> + RTE_DMA_DIR_DEV_TO_MEM,
> + /**< DMA transfer direction - from device to memory.
> + * In a typical scenario, the SoCs are installed on host servers as
> + * iNICs through the PCIe interface. In this case, the SoCs works in
> + * EP(endpoint) mode, it could initiate a DMA move request from device
> + * (which is host memory) to memory (which is SoCs memory).
> + *
> + * @see struct rte_dmadev_vchan_conf::direction
> + */
> + RTE_DMA_DIR_DEV_TO_DEV,
> + /**< DMA transfer direction - from device to device.
> + * In a typical scenario, the SoCs are installed on host servers as
> + * iNICs through the PCIe interface. In this case, the SoCs works in
> + * EP(endpoint) mode, it could initiate a DMA move request from device
> + * (which is host memory) to the device (which is another host memory).
> + *
> + * @see struct rte_dmadev_vchan_conf::direction
> + */
> +};
> +
> +/**
> + * enum rte_dmadev_port_type - DMA access port type defines.
> + *
> + * @see struct rte_dmadev_port_param::port_type
> + */
> +enum rte_dmadev_port_type {
> + RTE_DMADEV_PORT_NONE,
> + RTE_DMADEV_PORT_PCIE, /**< The DMA access port is PCIe. */
> +};
> +
> +/**
> + * A structure used to descript DMA access port parameters.
> + *
> + * @see struct rte_dmadev_vchan_conf::src_port
> + * @see struct rte_dmadev_vchan_conf::dst_port
> + */
> +struct rte_dmadev_port_param {
> + enum rte_dmadev_port_type port_type;
> + /**< The device access port type.
> + * @see enum rte_dmadev_port_type
> + */
> + union {
> + /** PCIe access port parameters.
> + *
> + * The following model shows SoC's PCIe module connects to
> + * multiple PCIe hosts and multiple endpoints. The PCIe module
> + * has an integrated DMA controller.
> + *
> + * If the DMA wants to access the memory of host A, it can be
> + * initiated by PF1 in core0, or by VF0 of PF0 in core0.
> + *
> + * \code{.unparsed}
> + * System Bus
> + * | ----------PCIe module----------
> + * | Bus
> + * | Interface
> + * | ----- ------------------
> + * | | | | PCIe Core0 |
> + * | | | | | -----------
> + * | | | | PF-0 -- VF-0 | | Host A |
> + * | | |--------| |- VF-1 |--------| Root |
> + * | | | | PF-1 | | Complex |
> + * | | | | PF-2 | -----------
> + * | | | ------------------
> + * | | |
> + * | | | ------------------
> + * | | | | PCIe Core1 |
> + * | | | | | -----------
> + * | | | | PF-0 -- VF-0 | | Host B |
> + * |-----| |--------| PF-1 -- VF-0 |--------| Root |
> + * | | | | |- VF-1 | | Complex |
> + * | | | | PF-2 | -----------
> + * | | | ------------------
> + * | | |
> + * | | | ------------------
> + * | |DMA| | | ------
> + * | | | | |--------| EP |
> + * | | |--------| PCIe Core2 | ------
> + * | | | | | ------
> + * | | | | |--------| EP |
> + * | | | | | ------
> + * | ----- ------------------
> + *
> + * \endcode
> + *
> + * @note If some fields can not be supported by the
> + * hardware/driver, then the driver ignores those fields.
> + * Please check driver-specific documentation for limitations
> + * and capablites.
> + */
> + struct {
> + uint64_t coreid : 4; /**< PCIe core id used. */
> + uint64_t pfid : 8; /**< PF id used. */
> + uint64_t vfen : 1; /**< VF enable bit. */
> + uint64_t vfid : 16; /**< VF id used. */
> + uint64_t pasid : 20;
> + /**< The pasid filed in TLP packet. */
> + uint64_t attr : 3;
> + /**< The attributes filed in TLP packet. */
> + uint64_t ph : 2;
> + /**< The processing hint filed in TLP packet. */
> + uint64_t st : 16;
> + /**< The steering tag filed in TLP packet. */
> + } pcie;
> + };
> + uint64_t reserved[2]; /**< Reserved for future fields. */
> +};
> +
> +/**
> + * A structure used to configure a virtual DMA channel.
> + */
> +struct rte_dmadev_vchan_conf {
> + enum rte_dma_direction direction;
> + /**< Transfer direction
> + * @see enum rte_dma_direction
> + */
> + uint16_t nb_desc;
> + /**< Number of descriptor for the virtual DMA channel */
> + struct rte_dmadev_port_param src_port;
> + /**< 1) Used to describes the device access port parameter in the
> + * device-to-memory transfer scenario.
> + * 2) Used to describes the source device access port parameter in the
> + * device-to-device transfer scenario.
> + * @see struct rte_dmadev_port_param
> + */
> + struct rte_dmadev_port_param dst_port;
> + /**< 1) Used to describes the device access port parameter in the
> + * memory-to-device transfer scenario.
> + * 2) Used to describes the destination device access port parameter in
> + * the device-to-device transfer scenario.
> + * @see struct rte_dmadev_port_param
> + */
> +};
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Allocate and set up a virtual DMA channel.
> + *
> + * @param dev_id
> + * The identifier of the device.
> + * @param conf
> + * The virtual DMA channel configuration structure encapsulated into
> + * rte_dmadev_vchan_conf object.
> + *
> + * @return
> + * - >=0: Allocate success, it is the virtual DMA channel id. This value must
> + * be less than the field 'max_vchans' of struct rte_dmadev_conf
> + * which configured by rte_dmadev_configure().
> + * - <0: Error code returned by the driver virtual channel setup function.
> + */
> +__rte_experimental
> +int
> +rte_dmadev_vchan_setup(uint16_t dev_id,
> + const struct rte_dmadev_vchan_conf *conf);
> +
> +/**
> + * rte_dmadev_stats - running statistics.
> + */
> +struct rte_dmadev_stats {
> + uint64_t submitted_count;
> + /**< Count of operations which were submitted to hardware. */
> + uint64_t completed_fail_count;
> + /**< Count of operations which failed to complete. */
> + uint64_t completed_count;
> + /**< Count of operations which successfully complete. */
> +};
> +
> +#define RTE_DMADEV_ALL_VCHAN 0xFFFFu
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Retrieve basic statistics of a or all virtual DMA channel(s).
> + *
> + * @param dev_id
> + * The identifier of the device.
> + * @param vchan
> + * The identifier of virtual DMA channel.
> + * If equal RTE_DMADEV_ALL_VCHAN means all channels.
> + * @param[out] stats
> + * The basic statistics structure encapsulated into rte_dmadev_stats
> + * object.
> + *
> + * @return
> + * - =0: Successfully retrieve stats.
> + * - <0: Failure to retrieve stats.
> + */
> +__rte_experimental
> +int
> +rte_dmadev_stats_get(uint16_t dev_id, uint16_t vchan,
> + struct rte_dmadev_stats *stats);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Reset basic statistics of a or all virtual DMA channel(s).
> + *
> + * @param dev_id
> + * The identifier of the device.
> + * @param vchan
> + * The identifier of virtual DMA channel.
> + * If equal RTE_DMADEV_ALL_VCHAN means all channels.
> + *
> + * @return
> + * - =0: Successfully reset stats.
> + * - <0: Failure to reset stats.
> + */
> +__rte_experimental
> +int
> +rte_dmadev_stats_reset(uint16_t dev_id, uint16_t vchan);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Dump DMA device info.
> + *
> + * @param dev_id
> + * The identifier of the device.
> + * @param f
> + * The file to write the output to.
> + *
> + * @return
> + * 0 on success. Non-zero otherwise.
> + */
> +__rte_experimental
> +int
> +rte_dmadev_dump(uint16_t dev_id, FILE *f);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Trigger the dmadev self test.
> + *
> + * @param dev_id
> + * The identifier of the device.
> + *
> + * @return
> + * - 0: Selftest successful.
> + * - -ENOTSUP if the device doesn't support selftest
> + * - other values < 0 on failure.
> + */
> +__rte_experimental
> +int
> +rte_dmadev_selftest(uint16_t dev_id);
> +
> +/**
> + * rte_dma_status_code - DMA transfer result status code defines.
> + */
> +enum rte_dma_status_code {
> + RTE_DMA_STATUS_SUCCESSFUL,
> + /**< The operation completed successfully. */
> + RTE_DMA_STATUS_USRER_ABORT,
> + /**< The operation failed to complete due abort by user.
> + * This is mainly used when processing dev_stop, user could modidy the
> + * descriptors (e.g. change one bit to tell hardware abort this job),
> + * it allows outstanding requests to be complete as much as possible,
> + * so reduce the time to stop the device.
> + */
> + RTE_DMA_STATUS_NOT_ATTEMPTED,
> + /**< The operation failed to complete due to following scenarios:
> + * The jobs in a particular batch are not attempted because they
> + * appeared after a fence where a previous job failed. In some HW
> + * implementation it's possible for jobs from later batches would be
> + * completed, though, so report the status from the not attempted jobs
> + * before reporting those newer completed jobs.
> + */
> + RTE_DMA_STATUS_INVALID_SRC_ADDR,
> + /**< The operation failed to complete due invalid source address. */
> + RTE_DMA_STATUS_INVALID_DST_ADDR,
> + /**< The operation failed to complete due invalid destination
> + * address.
> + */
> + RTE_DMA_STATUS_INVALID_LENGTH,
> + /**< The operation failed to complete due invalid length. */
> + RTE_DMA_STATUS_INVALID_OPCODE,
> + /**< The operation failed to complete due invalid opcode.
> + * The DMA descriptor could have multiple format, which are
> + * distinguished by the opcode field.
> + */
> + RTE_DMA_STATUS_BUS_ERROR,
> + /**< The operation failed to complete due bus err. */
> + RTE_DMA_STATUS_DATA_POISION,
> + /**< The operation failed to complete due data poison. */
> + RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR,
> + /**< The operation failed to complete due descriptor read error. */
> + RTE_DMA_STATUS_DEV_LINK_ERROR,
> + /**< The operation failed to complete due device link error.
> + * Used to indicates that the link error in the memory-to-device/
> + * device-to-memory/device-to-device transfer scenario.
> + */
> + RTE_DMA_STATUS_UNKNOWN = 0x100,
> + /**< The operation failed to complete due unknown reason.
> + * The initial value is 256, which reserves space for future errors.
> + */
> +};
> +
> +/**
> + * rte_dmadev_sge - can hold scatter DMA operation request entry.
> + */
> +struct rte_dmadev_sge {
> + rte_iova_t addr; /**< The DMA operation address. */
> + uint32_t length; /**< The DMA operation length. */
> +};
> +
> +#include "rte_dmadev_core.h"
> +
> +/* DMA flags to augment operation preparation. */
> +#define RTE_DMA_OP_FLAG_FENCE (1ull << 0)
> +/**< DMA fence flag.
> + * It means the operation with this flag must be processed only after all
> + * previous operations are completed.
> + * If the specify DMA HW works in-order (it means it has default fence between
> + * operations), this flag could be NOP.
> + *
> + * @see rte_dmadev_copy()
> + * @see rte_dmadev_copy_sg()
> + * @see rte_dmadev_fill()
> + */
> +
> +#define RTE_DMA_OP_FLAG_SUBMIT (1ull << 1)
> +/**< DMA submit flag.
> + * It means the operation with this flag must issue doorbell to hardware after
> + * enqueued jobs.
> + */
> +
> +#define RTE_DMA_OP_FLAG_LLC (1ull << 2)
> +/**< DMA write data to low level cache hint.
> + * Used for performance optimization, this is just a hint, and there is no
> + * capability bit for this, driver should not return error if this flag was set.
> + */
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Enqueue a copy operation onto the virtual DMA channel.
> + *
> + * This queues up a copy operation to be performed by hardware, if the 'flags'
> + * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin
> + * this operation, otherwise do not trigger doorbell.
> + *
> + * @param dev_id
> + * The identifier of the device.
> + * @param vchan
> + * The identifier of virtual DMA channel.
> + * @param src
> + * The address of the source buffer.
> + * @param dst
> + * The address of the destination buffer.
> + * @param length
> + * The length of the data to be copied.
> + * @param flags
> + * An flags for this operation.
> + * @see RTE_DMA_OP_FLAG_*
> + *
> + * @return
> + * - 0..UINT16_MAX: index of enqueued copy job.
> + * - <0: Error code returned by the driver copy function.
> + */
> +__rte_experimental
> +static inline int
> +rte_dmadev_copy(uint16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst,
> + uint32_t length, uint64_t flags)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> +
> +#ifdef RTE_DMADEV_DEBUG
> + if (!rte_dmadev_is_valid_dev(dev_id) ||
> + vchan >= dev->data->dev_conf.max_vchans || length == 0)
> + return -EINVAL;
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->copy, -ENOTSUP);
> +#endif
> +
> + return (*dev->copy)(dev, vchan, src, dst, length, flags);
> +}
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Enqueue a scatter list copy operation onto the virtual DMA channel.
> + *
> + * This queues up a scatter list copy operation to be performed by hardware, if
> + * the 'flags' parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell
> + * to begin this operation, otherwise do not trigger doorbell.
> + *
> + * @param dev_id
> + * The identifier of the device.
> + * @param vchan
> + * The identifier of virtual DMA channel.
> + * @param src
> + * The pointer of source scatter entry array.
> + * @param dst
> + * The pointer of destination scatter entry array.
> + * @param nb_src
> + * The number of source scatter entry.
> + * @param nb_dst
> + * The number of destination scatter entry.
> + * @param flags
> + * An flags for this operation.
> + * @see RTE_DMA_OP_FLAG_*
> + *
> + * @return
> + * - 0..UINT16_MAX: index of enqueued copy scatterlist job.
> + * - <0: Error code returned by the driver copy scatterlist function.
> + */
> +__rte_experimental
> +static inline int
> +rte_dmadev_copy_sg(uint16_t dev_id, uint16_t vchan, struct rte_dmadev_sge *src,
> + struct rte_dmadev_sge *dst, uint16_t nb_src, uint16_t nb_dst,
> + uint64_t flags)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> +
> +#ifdef RTE_DMADEV_DEBUG
> + if (!rte_dmadev_is_valid_dev(dev_id) ||
> + vchan >= dev->data->dev_conf.max_vchans ||
> + src == NULL || dst == NULL || nb_src == 0 || nb_dst == 0)
> + return -EINVAL;
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->copy_sg, -ENOTSUP);
> +#endif
> +
> + return (*dev->copy_sg)(dev, vchan, src, dst, nb_src, nb_dst, flags);
> +}
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Enqueue a fill operation onto the virtual DMA channel.
> + *
> + * This queues up a fill operation to be performed by hardware, if the 'flags'
> + * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin
> + * this operation, otherwise do not trigger doorbell.
> + *
> + * @param dev_id
> + * The identifier of the device.
> + * @param vchan
> + * The identifier of virtual DMA channel.
> + * @param pattern
> + * The pattern to populate the destination buffer with.
> + * @param dst
> + * The address of the destination buffer.
> + * @param length
> + * The length of the destination buffer.
> + * @param flags
> + * An flags for this operation.
> + * @see RTE_DMA_OP_FLAG_*
> + *
> + * @return
> + * - 0..UINT16_MAX: index of enqueued fill job.
> + * - <0: Error code returned by the driver fill function.
> + */
> +__rte_experimental
> +static inline int
> +rte_dmadev_fill(uint16_t dev_id, uint16_t vchan, uint64_t pattern,
> + rte_iova_t dst, uint32_t length, uint64_t flags)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> +
> +#ifdef RTE_DMADEV_DEBUG
> + if (!rte_dmadev_is_valid_dev(dev_id) ||
> + vchan >= dev->data->dev_conf.max_vchans || length == 0)
> + return -EINVAL;
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->fill, -ENOTSUP);
> +#endif
> +
> + return (*dev->fill)(dev, vchan, pattern, dst, length, flags);
> +}
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Trigger hardware to begin performing enqueued operations.
> + *
> + * This API is used to write the "doorbell" to the hardware to trigger it
> + * to begin the operations previously enqueued by rte_dmadev_copy/fill().
> + *
> + * @param dev_id
> + * The identifier of the device.
> + * @param vchan
> + * The identifier of virtual DMA channel.
> + *
> + * @return
> + * - =0: Successfully trigger hardware.
> + * - <0: Failure to trigger hardware.
> + */
> +__rte_experimental
> +static inline int
> +rte_dmadev_submit(uint16_t dev_id, uint16_t vchan)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> +
> +#ifdef RTE_DMADEV_DEBUG
> + if (!rte_dmadev_is_valid_dev(dev_id) ||
> + vchan >= dev->data->dev_conf.max_vchans)
> + return -EINVAL;
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->submit, -ENOTSUP);
> +#endif
> +
> + return (*dev->submit)(dev, vchan);
> +}
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Returns the number of operations that have been successfully completed.
> + *
> + * @param dev_id
> + * The identifier of the device.
> + * @param vchan
> + * The identifier of virtual DMA channel.
> + * @param nb_cpls
> + * The maximum number of completed operations that can be processed.
> + * @param[out] last_idx
> + * The last completed operation's index.
> + * If not required, NULL can be passed in.
> + * @param[out] has_error
> + * Indicates if there are transfer error.
> + * If not required, NULL can be passed in.
> + *
> + * @return
> + * The number of operations that successfully completed. This return value
> + * must be less than or equal to the value of nb_cpls.
> + */
> +__rte_experimental
> +static inline uint16_t
> +rte_dmadev_completed(uint16_t dev_id, uint16_t vchan, const uint16_t nb_cpls,
> + uint16_t *last_idx, bool *has_error)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> + uint16_t idx;
> + bool err;
> +
> +#ifdef RTE_DMADEV_DEBUG
> + if (!rte_dmadev_is_valid_dev(dev_id) ||
> + vchan >= dev->data->dev_conf.max_vchans || nb_cpls == 0)
> + return 0;
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->completed, 0);
> +#endif
> +
> + /* Ensure the pointer values are non-null to simplify drivers.
> + * In most cases these should be compile time evaluated, since this is
> + * an inline function.
> + * - If NULL is explicitly passed as parameter, then compiler knows the
> + * value is NULL
> + * - If address of local variable is passed as parameter, then compiler
> + * can know it's non-NULL.
> + */
> + if (last_idx == NULL)
> + last_idx = &idx;
> + if (has_error == NULL)
> + has_error = &err;
> +
> + *has_error = false;
> + return (*dev->completed)(dev, vchan, nb_cpls, last_idx, has_error);
> +}
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Returns the number of operations that have been completed, and the
> + * operations result may succeed or fail.
> + *
> + * @param dev_id
> + * The identifier of the device.
> + * @param vchan
> + * The identifier of virtual DMA channel.
> + * @param nb_cpls
> + * Indicates the size of status array.
> + * @param[out] last_idx
> + * The last completed operation's index.
> + * If not required, NULL can be passed in.
> + * @param[out] status
> + * The error code of operations that completed.
> + * @see enum rte_dma_status_code
> + *
> + * @return
> + * The number of operations that completed. This return value must be less
> + * than or equal to the value of nb_cpls.
> + */
> +__rte_experimental
> +static inline uint16_t
> +rte_dmadev_completed_status(uint16_t dev_id, uint16_t vchan,
> + const uint16_t nb_cpls, uint16_t *last_idx,
> + enum rte_dma_status_code *status)
> +{
> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> + uint16_t idx;
> +
> +#ifdef RTE_DMADEV_DEBUG
> + if (!rte_dmadev_is_valid_dev(dev_id) ||
> + vchan >= dev->data->dev_conf.max_vchans ||
> + nb_cpls == 0 || status == NULL)
> + return 0;
> + RTE_FUNC_PTR_OR_ERR_RET(*dev->completed_status, 0);
> +#endif
> +
> + if (last_idx == NULL)
> + last_idx = &idx;
> +
> + return (*dev->completed_status)(dev, vchan, nb_cpls, last_idx, status);
> +}
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif /* _RTE_DMADEV_H_ */
> diff --git a/lib/dmadev/rte_dmadev_core.h b/lib/dmadev/rte_dmadev_core.h
> new file mode 100644
> index 0000000..0122f67
> --- /dev/null
> +++ b/lib/dmadev/rte_dmadev_core.h
> @@ -0,0 +1,182 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2021 HiSilicon Limited.
> + * Copyright(c) 2021 Intel Corporation.
> + */
> +
> +#ifndef _RTE_DMADEV_CORE_H_
> +#define _RTE_DMADEV_CORE_H_
> +
> +/**
> + * @file
> + *
> + * RTE DMA Device internal header.
> + *
> + * This header contains internal data types, that are used by the DMA devices
> + * in order to expose their ops to the class.
> + *
> + * Applications should not use these API directly.
> + *
> + */
> +
> +struct rte_dmadev;
> +
> +typedef int (*rte_dmadev_info_get_t)(const struct rte_dmadev *dev,
> + struct rte_dmadev_info *dev_info,
> + uint32_t info_sz);
> +/**< @internal Used to get device information of a device. */
> +
> +typedef int (*rte_dmadev_configure_t)(struct rte_dmadev *dev,
> + const struct rte_dmadev_conf *dev_conf);
> +/**< @internal Used to configure a device. */
> +
> +typedef int (*rte_dmadev_start_t)(struct rte_dmadev *dev);
> +/**< @internal Used to start a configured device. */
> +
> +typedef int (*rte_dmadev_stop_t)(struct rte_dmadev *dev);
> +/**< @internal Used to stop a configured device. */
> +
> +typedef int (*rte_dmadev_close_t)(struct rte_dmadev *dev);
> +/**< @internal Used to close a configured device. */
> +
> +typedef int (*rte_dmadev_vchan_setup_t)(struct rte_dmadev *dev,
> + const struct rte_dmadev_vchan_conf *conf);
> +/**< @internal Used to allocate and set up a virtual DMA channel. */
> +
> +typedef int (*rte_dmadev_stats_get_t)(const struct rte_dmadev *dev,
> + uint16_t vchan, struct rte_dmadev_stats *stats,
> + uint32_t stats_sz);
> +/**< @internal Used to retrieve basic statistics. */
> +
> +typedef int (*rte_dmadev_stats_reset_t)(struct rte_dmadev *dev, uint16_t vchan);
> +/**< @internal Used to reset basic statistics. */
> +
> +typedef int (*rte_dmadev_dump_t)(const struct rte_dmadev *dev, FILE *f);
> +/**< @internal Used to dump internal information. */
> +
> +typedef int (*rte_dmadev_selftest_t)(uint16_t dev_id);
> +/**< @internal Used to start dmadev selftest. */
> +
> +typedef int (*rte_dmadev_copy_t)(struct rte_dmadev *dev, uint16_t vchan,
> + rte_iova_t src, rte_iova_t dst,
> + uint32_t length, uint64_t flags);
> +/**< @internal Used to enqueue a copy operation. */
> +
> +typedef int (*rte_dmadev_copy_sg_t)(struct rte_dmadev *dev, uint16_t vchan,
> + const struct rte_dmadev_sge *src,
> + const struct rte_dmadev_sge *dst,
> + uint16_t nb_src, uint16_t nb_dst,
> + uint64_t flags);
> +/**< @internal Used to enqueue a scatter list copy operation. */
> +
> +typedef int (*rte_dmadev_fill_t)(struct rte_dmadev *dev, uint16_t vchan,
> + uint64_t pattern, rte_iova_t dst,
> + uint32_t length, uint64_t flags);
> +/**< @internal Used to enqueue a fill operation. */
> +
> +typedef int (*rte_dmadev_submit_t)(struct rte_dmadev *dev, uint16_t vchan);
> +/**< @internal Used to trigger hardware to begin working. */
> +
> +typedef uint16_t (*rte_dmadev_completed_t)(struct rte_dmadev *dev,
> + uint16_t vchan, const uint16_t nb_cpls,
> + uint16_t *last_idx, bool *has_error);
> +/**< @internal Used to return number of successful completed operations. */
> +
> +typedef uint16_t (*rte_dmadev_completed_status_t)(struct rte_dmadev *dev,
> + uint16_t vchan, const uint16_t nb_cpls,
> + uint16_t *last_idx, enum rte_dma_status_code *status);
> +/**< @internal Used to return number of completed operations. */
> +
> +/**
> + * Possible states of a DMA device.
> + */
> +enum rte_dmadev_state {
> + RTE_DMADEV_UNUSED = 0,
> + /**< Device is unused before being probed. */
> + RTE_DMADEV_ATTACHED,
> + /**< Device is attached when allocated in probing. */
> +};
> +
> +/**
> + * DMA device operations function pointer table
> + */
> +struct rte_dmadev_ops {
> + rte_dmadev_info_get_t dev_info_get;
> + rte_dmadev_configure_t dev_configure;
> + rte_dmadev_start_t dev_start;
> + rte_dmadev_stop_t dev_stop;
> + rte_dmadev_close_t dev_close;
> + rte_dmadev_vchan_setup_t vchan_setup;
> + rte_dmadev_stats_get_t stats_get;
> + rte_dmadev_stats_reset_t stats_reset;
> + rte_dmadev_dump_t dev_dump;
> + rte_dmadev_selftest_t dev_selftest;
> +};
> +
> +/**
> + * @internal
> + * The data part, with no function pointers, associated with each DMA device.
> + *
> + * This structure is safe to place in shared memory to be common among different
> + * processes in a multi-process configuration.
> + */
> +struct rte_dmadev_data {
> + void *dev_private;
> + /**< PMD-specific private data.
> + * This is a copy of the 'dev_private' field in the 'struct rte_dmadev'
> + * from primary process, it is used by the secondary process to get
> + * dev_private information.
> + */
> + uint16_t dev_id; /**< Device [external] identifier. */
> + char dev_name[RTE_DMADEV_NAME_MAX_LEN]; /**< Unique identifier name */
> + struct rte_dmadev_conf dev_conf; /**< DMA device configuration. */
> + uint8_t dev_started : 1; /**< Device state: STARTED(1)/STOPPED(0). */
> + uint64_t reserved[2]; /**< Reserved for future fields */
> +} __rte_cache_aligned;
> +
> +/**
> + * @internal
> + * The generic data structure associated with each DMA device.
> + *
> + * The dataplane APIs are located at the beginning of the structure, along
> + * with the pointer to where all the data elements for the particular device
> + * are stored in shared memory. This split scheme allows the function pointer
> + * and driver data to be per-process, while the actual configuration data for
> + * the device is shared.
> + * And the 'dev_private' field was placed in the first cache line to optimize
> + * performance because the PMD driver mainly depends on this field.
> + */
> +struct rte_dmadev {
> + rte_dmadev_copy_t copy;
> + rte_dmadev_copy_sg_t copy_sg;
> + rte_dmadev_fill_t fill;
> + rte_dmadev_submit_t submit;
> + rte_dmadev_completed_t completed;
> + rte_dmadev_completed_status_t completed_status;
> + void *reserved_ptr; /**< Reserved for future IO function. */
> + void *dev_private;
> + /**< PMD-specific private data.
> + *
> + * - If is the primary process, after dmadev allocated by
> + * rte_dmadev_pmd_allocate(), the PCI/SoC device probing should
> + * initialize this field, and copy it's value to the 'dev_private'
> + * field of 'struct rte_dmadev_data' which pointer by 'data' filed.
> + *
> + * - If is the secondary process, dmadev framework will initialize this
> + * field by copy from 'dev_private' field of 'struct rte_dmadev_data'
> + * which initialized by primary process.
> + *
> + * @note It's the primary process responsibility to deinitialize this
> + * field after invoke rte_dmadev_pmd_release() in the PCI/SoC device
> + * removing stage.
> + */
> + struct rte_dmadev_data *data; /**< Pointer to device data. */
> + const struct rte_dmadev_ops *dev_ops; /**< Functions exported by PMD. */
> + struct rte_device *device;
> + /**< Device info which supplied during device initialization. */
> + enum rte_dmadev_state state; /**< Flag indicating the device state. */
> + uint64_t reserved[2]; /**< Reserved for future fields. */
> +} __rte_cache_aligned;
> +
> +extern struct rte_dmadev rte_dmadevices[];
> +
> +#endif /* _RTE_DMADEV_CORE_H_ */
> diff --git a/lib/dmadev/rte_dmadev_pmd.h b/lib/dmadev/rte_dmadev_pmd.h
> new file mode 100644
> index 0000000..45141f9
> --- /dev/null
> +++ b/lib/dmadev/rte_dmadev_pmd.h
> @@ -0,0 +1,72 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2021 HiSilicon Limited.
> + */
> +
> +#ifndef _RTE_DMADEV_PMD_H_
> +#define _RTE_DMADEV_PMD_H_
> +
> +/**
> + * @file
> + *
> + * RTE DMA Device PMD APIs
> + *
> + * Driver facing APIs for a DMA device. These are not to be called directly by
> + * any application.
> + */
> +
> +#include "rte_dmadev.h"
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +/**
> + * @internal
> + * Allocates a new dmadev slot for an DMA device and returns the pointer
> + * to that slot for the driver to use.
> + *
> + * @param name
> + * DMA device name.
> + *
> + * @return
> + * A pointer to the DMA device slot case of success,
> + * NULL otherwise.
> + */
> +__rte_internal
> +struct rte_dmadev *
> +rte_dmadev_pmd_allocate(const char *name);
> +
> +/**
> + * @internal
> + * Release the specified dmadev.
> + *
> + * @param dev
> + * Device to be released.
> + *
> + * @return
> + * - 0 on success, negative on error
> + */
> +__rte_internal
> +int
> +rte_dmadev_pmd_release(struct rte_dmadev *dev);
> +
> +/**
> + * @internal
> + * Return the DMA device based on the device name.
> + *
> + * @param name
> + * DMA device name.
> + *
> + * @return
> + * A pointer to the DMA device slot case of success,
> + * NULL otherwise.
> + */
> +__rte_internal
> +struct rte_dmadev *
> +rte_dmadev_get_device_by_name(const char *name);
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif /* _RTE_DMADEV_PMD_H_ */
> diff --git a/lib/dmadev/version.map b/lib/dmadev/version.map
> new file mode 100644
> index 0000000..0f2ed4b
> --- /dev/null
> +++ b/lib/dmadev/version.map
> @@ -0,0 +1,37 @@
> +EXPERIMENTAL {
> + global:
> +
> + rte_dmadev_close;
> + rte_dmadev_completed;
> + rte_dmadev_completed_status;
> + rte_dmadev_configure;
> + rte_dmadev_copy;
> + rte_dmadev_copy_sg;
> + rte_dmadev_count;
> + rte_dmadev_dump;
> + rte_dmadev_fill;
> + rte_dmadev_get_dev_id;
> + rte_dmadev_info_get;
> + rte_dmadev_is_valid_dev;
> + rte_dmadev_selftest;
> + rte_dmadev_start;
> + rte_dmadev_stats_get;
> + rte_dmadev_stats_reset;
> + rte_dmadev_stop;
> + rte_dmadev_submit;
> + rte_dmadev_vchan_setup;
> +
> + local: *;
> +};
> +
> +INTERNAL {
> + global:
> +
> + rte_dmadevices;
> + rte_dmadev_get_device_by_name;
> + rte_dmadev_pmd_allocate;
> + rte_dmadev_pmd_release;
> +
> + local: *;
> +};
> +
> diff --git a/lib/meson.build b/lib/meson.build
> index 1673ca4..68d239f 100644
> --- a/lib/meson.build
> +++ b/lib/meson.build
> @@ -60,6 +60,7 @@ libraries = [
> 'bpf',
> 'graph',
> 'node',
> + 'dmadev',
> ]
>
> if is_windows
> --
> 2.8.1
>
Thanks Jerin, comment inline
On 2021/7/20 13:03, Jerin Jacob wrote:
> On Tue, Jul 20, 2021 at 6:48 AM Chengwen Feng <fengchengwen@huawei.com> wrote:
>>
>> This patch introduce 'dmadevice' which is a generic type of DMA
>> device.
>>
>> The APIs of dmadev library exposes some generic operations which can
>> enable configuration and I/O with the DMA devices.
>>
>> Signed-off-by: Chengwen Feng <fengchengwen@huawei.com>
[snip]
>> +int
>> +rte_dmadev_info_get(uint16_t dev_id, struct rte_dmadev_info *dev_info)
>> +{
>> + const struct rte_dmadev *dev = &rte_dmadevices[dev_id];
>> + int ret;
>> +
>> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
>> + if (dev_info == NULL)
>> + return -EINVAL;
>> +
>> + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_info_get, -ENOTSUP);
>> + memset(dev_info, 0, sizeof(struct rte_dmadev_info));
>> + ret = (*dev->dev_ops->dev_info_get)(dev, dev_info,
>> + sizeof(struct rte_dmadev_info));
>> + if (ret != 0)
>> + return ret;
>> +
>> + dev_info->device = dev->device;
>> + dev_info->nb_vchans = dev->data->dev_conf.max_vchans;
>
> This will be updated after configure stage.
Yes, the dev_info->nb_vchans hold the number of virtual DMA channel configured.
Do you mean add one comment here ?
>
>
>> +
>> + return 0;
>> +}
>> +
>> +int
>> +rte_dmadev_configure(uint16_t dev_id, const struct rte_dmadev_conf *dev_conf)
>> +{
>> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
>> + struct rte_dmadev_info info;
>> + int ret;
>> +
>> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
>> + if (dev_conf == NULL)
>> + return -EINVAL;
>> +
>> + ret = rte_dmadev_info_get(dev_id, &info);
>> + if (ret != 0) {
>> + RTE_DMADEV_LOG(ERR, "Device %u get device info fail\n", dev_id);
>> + return -EINVAL;
>> + }
>> + if (dev_conf->max_vchans == 0) {
>> + RTE_DMADEV_LOG(ERR,
>> + "Device %u configure zero vchans\n", dev_id);
>> + return -EINVAL;
>> + }
>> + if (dev_conf->max_vchans > info.max_vchans) {
>> + RTE_DMADEV_LOG(ERR,
>> + "Device %u configure too many vchans\n", dev_id);
>> + return -EINVAL;
>> + }
>> + if (dev_conf->enable_silent &&
>> + !(info.dev_capa & RTE_DMADEV_CAPA_SILENT)) {
>> + RTE_DMADEV_LOG(ERR, "Device %u don't support silent\n", dev_id);
>> + return -EINVAL;
>> + }
>> +
>> + if (dev->data->dev_started != 0) {
>> + RTE_DMADEV_LOG(ERR,
>> + "Device %u must be stopped to allow configuration\n",
>> + dev_id);
>> + return -EBUSY;
>> + }
>
> ethdev and other device class common code handles the reconfigure case. i.e
> the application configures N vchan first and reconfigures to N - M
> then free the resources
> attached to M - N. Please do the same here.
DMA is a simple device, I think it's OK to reconfigure at driver-level.
PS: If we need support reconfigure at dmadev-level, dmadev should hold the vchan-configuration,
and invoke driver's vchan_release to release resources. This may introduce more complexity.
>> +int
>> +rte_dmadev_dump(uint16_t dev_id, FILE *f)
>> +{
>> + const struct rte_dmadev *dev = &rte_dmadevices[dev_id];
>> + struct rte_dmadev_info info;
>> + int ret;
>> +
>> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
>> + if (f == NULL)
>> + return -EINVAL;
>> +
>> + ret = rte_dmadev_info_get(dev_id, &info);
>> + if (ret != 0) {
>> + RTE_DMADEV_LOG(ERR, "Device %u get device info fail\n", dev_id);
>> + return -EINVAL;
>> + }
>> +
>> + fprintf(f, "DMA Dev %u, '%s' [%s]\n",
>> + dev->data->dev_id,
>> + dev->data->dev_name,
>> + dev->data->dev_started ? "started" : "stopped");
>> + fprintf(f, " dev_capa: 0x%" PRIx64 "\n", info.dev_capa);
>> + fprintf(f, " max_vchans_supported: %u\n", info.max_vchans);
>> + fprintf(f, " max_vchans_configured: %u\n", info.nb_vchans);
>> + fprintf(f, " silent_mode: %s\n",
>> + dev->data->dev_conf.enable_silent ? "on" : "off");
>
> Probably iterate over each vchan and dumping the at least direction
> will be usefull.
dmadev hasn't hold vchan-configuration, Need more discussion.
On Tue, Jul 20, 2021 at 12:23 PM fengchengwen <fengchengwen@huawei.com> wrote:
>
> Thanks Jerin, comment inline
>
> On 2021/7/20 13:03, Jerin Jacob wrote:
> > On Tue, Jul 20, 2021 at 6:48 AM Chengwen Feng <fengchengwen@huawei.com> wrote:
> >>
> >> This patch introduce 'dmadevice' which is a generic type of DMA
> >> device.
> >>
> >> The APIs of dmadev library exposes some generic operations which can
> >> enable configuration and I/O with the DMA devices.
> >>
> >> Signed-off-by: Chengwen Feng <fengchengwen@huawei.com>
>
> [snip]
>
> >> +int
> >> +rte_dmadev_info_get(uint16_t dev_id, struct rte_dmadev_info *dev_info)
> >> +{
> >> + const struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> >> + int ret;
> >> +
> >> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> >> + if (dev_info == NULL)
> >> + return -EINVAL;
> >> +
> >> + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_info_get, -ENOTSUP);
> >> + memset(dev_info, 0, sizeof(struct rte_dmadev_info));
> >> + ret = (*dev->dev_ops->dev_info_get)(dev, dev_info,
> >> + sizeof(struct rte_dmadev_info));
> >> + if (ret != 0)
> >> + return ret;
> >> +
> >> + dev_info->device = dev->device;
> >> + dev_info->nb_vchans = dev->data->dev_conf.max_vchans;
> >
> > This will be updated after configure stage.
>
> Yes, the dev_info->nb_vchans hold the number of virtual DMA channel configured.
> Do you mean add one comment here ?
If are taking care of the case where rte_dmadev_info_get() called
first and then configure() then fine.
>
> >
> >
> >> +
> >> + return 0;
> >> +}
> >> +
> >> +int
> >> +rte_dmadev_configure(uint16_t dev_id, const struct rte_dmadev_conf *dev_conf)
> >> +{
> >> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> >> + struct rte_dmadev_info info;
> >> + int ret;
> >> +
> >> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> >> + if (dev_conf == NULL)
> >> + return -EINVAL;
> >> +
> >> + ret = rte_dmadev_info_get(dev_id, &info);
> >> + if (ret != 0) {
> >> + RTE_DMADEV_LOG(ERR, "Device %u get device info fail\n", dev_id);
> >> + return -EINVAL;
> >> + }
> >> + if (dev_conf->max_vchans == 0) {
> >> + RTE_DMADEV_LOG(ERR,
> >> + "Device %u configure zero vchans\n", dev_id);
> >> + return -EINVAL;
> >> + }
> >> + if (dev_conf->max_vchans > info.max_vchans) {
> >> + RTE_DMADEV_LOG(ERR,
> >> + "Device %u configure too many vchans\n", dev_id);
> >> + return -EINVAL;
> >> + }
> >> + if (dev_conf->enable_silent &&
> >> + !(info.dev_capa & RTE_DMADEV_CAPA_SILENT)) {
> >> + RTE_DMADEV_LOG(ERR, "Device %u don't support silent\n", dev_id);
> >> + return -EINVAL;
> >> + }
> >> +
> >> + if (dev->data->dev_started != 0) {
> >> + RTE_DMADEV_LOG(ERR,
> >> + "Device %u must be stopped to allow configuration\n",
> >> + dev_id);
> >> + return -EBUSY;
> >> + }
> >
> > ethdev and other device class common code handles the reconfigure case. i.e
> > the application configures N vchan first and reconfigures to N - M
> > then free the resources
> > attached to M - N. Please do the same here.
>
> DMA is a simple device, I think it's OK to reconfigure at driver-level.
OK. If everyone thinks that way it is OK. No strong opinion.
>
> PS: If we need support reconfigure at dmadev-level, dmadev should hold the vchan-configuration,
> and invoke driver's vchan_release to release resources. This may introduce more complexity.
>
>
> >> +int
> >> +rte_dmadev_dump(uint16_t dev_id, FILE *f)
> >> +{
> >> + const struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> >> + struct rte_dmadev_info info;
> >> + int ret;
> >> +
> >> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> >> + if (f == NULL)
> >> + return -EINVAL;
> >> +
> >> + ret = rte_dmadev_info_get(dev_id, &info);
> >> + if (ret != 0) {
> >> + RTE_DMADEV_LOG(ERR, "Device %u get device info fail\n", dev_id);
> >> + return -EINVAL;
> >> + }
> >> +
> >> + fprintf(f, "DMA Dev %u, '%s' [%s]\n",
> >> + dev->data->dev_id,
> >> + dev->data->dev_name,
> >> + dev->data->dev_started ? "started" : "stopped");
> >> + fprintf(f, " dev_capa: 0x%" PRIx64 "\n", info.dev_capa);
> >> + fprintf(f, " max_vchans_supported: %u\n", info.max_vchans);
> >> + fprintf(f, " max_vchans_configured: %u\n", info.nb_vchans);
> >> + fprintf(f, " silent_mode: %s\n",
> >> + dev->data->dev_conf.enable_silent ? "on" : "off");
> >
> > Probably iterate over each vchan and dumping the at least direction
> > will be usefull.
>
> dmadev hasn't hold vchan-configuration, Need more discussion.
Prefer to have that. Leaving to others.
>
On Tue, Jul 20, 2021 at 02:53:08PM +0800, fengchengwen wrote:
> Thanks Jerin, comment inline
>
> On 2021/7/20 13:03, Jerin Jacob wrote:
> > On Tue, Jul 20, 2021 at 6:48 AM Chengwen Feng <fengchengwen@huawei.com> wrote:
> >>
> >> This patch introduce 'dmadevice' which is a generic type of DMA
> >> device.
> >>
> >> The APIs of dmadev library exposes some generic operations which can
> >> enable configuration and I/O with the DMA devices.
> >>
> >> Signed-off-by: Chengwen Feng <fengchengwen@huawei.com>
>
> [snip]
>
> >> +int
> >> +rte_dmadev_info_get(uint16_t dev_id, struct rte_dmadev_info *dev_info)
> >> +{
> >> + const struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> >> + int ret;
> >> +
> >> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> >> + if (dev_info == NULL)
> >> + return -EINVAL;
> >> +
> >> + RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_info_get, -ENOTSUP);
> >> + memset(dev_info, 0, sizeof(struct rte_dmadev_info));
> >> + ret = (*dev->dev_ops->dev_info_get)(dev, dev_info,
> >> + sizeof(struct rte_dmadev_info));
> >> + if (ret != 0)
> >> + return ret;
> >> +
> >> + dev_info->device = dev->device;
> >> + dev_info->nb_vchans = dev->data->dev_conf.max_vchans;
> >
> > This will be updated after configure stage.
>
> Yes, the dev_info->nb_vchans hold the number of virtual DMA channel configured.
> Do you mean add one comment here ?
>
> >
> >
> >> +
> >> + return 0;
> >> +}
> >> +
> >> +int
> >> +rte_dmadev_configure(uint16_t dev_id, const struct rte_dmadev_conf *dev_conf)
> >> +{
> >> + struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> >> + struct rte_dmadev_info info;
> >> + int ret;
> >> +
> >> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> >> + if (dev_conf == NULL)
> >> + return -EINVAL;
> >> +
> >> + ret = rte_dmadev_info_get(dev_id, &info);
> >> + if (ret != 0) {
> >> + RTE_DMADEV_LOG(ERR, "Device %u get device info fail\n", dev_id);
> >> + return -EINVAL;
> >> + }
> >> + if (dev_conf->max_vchans == 0) {
> >> + RTE_DMADEV_LOG(ERR,
> >> + "Device %u configure zero vchans\n", dev_id);
> >> + return -EINVAL;
> >> + }
> >> + if (dev_conf->max_vchans > info.max_vchans) {
> >> + RTE_DMADEV_LOG(ERR,
> >> + "Device %u configure too many vchans\n", dev_id);
> >> + return -EINVAL;
> >> + }
> >> + if (dev_conf->enable_silent &&
> >> + !(info.dev_capa & RTE_DMADEV_CAPA_SILENT)) {
> >> + RTE_DMADEV_LOG(ERR, "Device %u don't support silent\n", dev_id);
> >> + return -EINVAL;
> >> + }
> >> +
> >> + if (dev->data->dev_started != 0) {
> >> + RTE_DMADEV_LOG(ERR,
> >> + "Device %u must be stopped to allow configuration\n",
> >> + dev_id);
> >> + return -EBUSY;
> >> + }
> >
> > ethdev and other device class common code handles the reconfigure case. i.e
> > the application configures N vchan first and reconfigures to N - M
> > then free the resources
> > attached to M - N. Please do the same here.
>
> DMA is a simple device, I think it's OK to reconfigure at driver-level.
>
> PS: If we need support reconfigure at dmadev-level, dmadev should hold the vchan-configuration,
> and invoke driver's vchan_release to release resources. This may introduce more complexity.
>
I would tend to agree to keep this as it is.
>
> >> +int
> >> +rte_dmadev_dump(uint16_t dev_id, FILE *f)
> >> +{
> >> + const struct rte_dmadev *dev = &rte_dmadevices[dev_id];
> >> + struct rte_dmadev_info info;
> >> + int ret;
> >> +
> >> + RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
> >> + if (f == NULL)
> >> + return -EINVAL;
> >> +
> >> + ret = rte_dmadev_info_get(dev_id, &info);
> >> + if (ret != 0) {
> >> + RTE_DMADEV_LOG(ERR, "Device %u get device info fail\n", dev_id);
> >> + return -EINVAL;
> >> + }
> >> +
> >> + fprintf(f, "DMA Dev %u, '%s' [%s]\n",
> >> + dev->data->dev_id,
> >> + dev->data->dev_name,
> >> + dev->data->dev_started ? "started" : "stopped");
> >> + fprintf(f, " dev_capa: 0x%" PRIx64 "\n", info.dev_capa);
> >> + fprintf(f, " max_vchans_supported: %u\n", info.max_vchans);
> >> + fprintf(f, " max_vchans_configured: %u\n", info.nb_vchans);
> >> + fprintf(f, " silent_mode: %s\n",
> >> + dev->data->dev_conf.enable_silent ? "on" : "off");
> >
> > Probably iterate over each vchan and dumping the at least direction
> > will be usefull.
>
> dmadev hasn't hold vchan-configuration, Need more discussion.
>
I think this is fine as-is. The use of vchans doesn't apply to all devices
- unlike ethdev queues, for example - so I think having the driver manage
them is good.
/Bruce
@@ -495,6 +495,10 @@ F: drivers/raw/skeleton/
F: app/test/test_rawdev.c
F: doc/guides/prog_guide/rawdev.rst
+DMA device API - EXPERIMENTAL
+M: Chengwen Feng <fengchengwen@huawei.com>
+F: lib/dmadev/
+
Memory Pool Drivers
-------------------
@@ -81,6 +81,9 @@
/* rawdev defines */
#define RTE_RAWDEV_MAX_DEVS 64
+/* dmadev defines */
+#define RTE_DMADEV_MAX_DEVS 64
+
/* ip_fragmentation defines */
#define RTE_LIBRTE_IP_FRAG_MAX_FRAG 4
#undef RTE_LIBRTE_IP_FRAG_TBL_STAT
@@ -27,6 +27,7 @@ The public API headers are grouped by topics:
[event_timer_adapter] (@ref rte_event_timer_adapter.h),
[event_crypto_adapter] (@ref rte_event_crypto_adapter.h),
[rawdev] (@ref rte_rawdev.h),
+ [dmadev] (@ref rte_dmadev.h),
[metrics] (@ref rte_metrics.h),
[bitrate] (@ref rte_bitrate.h),
[latency] (@ref rte_latencystats.h),
@@ -35,6 +35,7 @@ INPUT = @TOPDIR@/doc/api/doxy-api-index.md \
@TOPDIR@/lib/compressdev \
@TOPDIR@/lib/cryptodev \
@TOPDIR@/lib/distributor \
+ @TOPDIR@/lib/dmadev \
@TOPDIR@/lib/efd \
@TOPDIR@/lib/ethdev \
@TOPDIR@/lib/eventdev \
new file mode 100644
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 HiSilicon Limited.
+
+sources = files('rte_dmadev.c')
+headers = files('rte_dmadev.h')
+indirect_headers += files('rte_dmadev_core.h')
+driver_sdk_headers += files('rte_dmadev_pmd.h')
new file mode 100644
@@ -0,0 +1,545 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 HiSilicon Limited.
+ * Copyright(c) 2021 Intel Corporation.
+ */
+
+#include <ctype.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_eal.h>
+#include <rte_errno.h>
+#include <rte_lcore.h>
+#include <rte_log.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_malloc.h>
+#include <rte_string_fns.h>
+
+#include "rte_dmadev.h"
+#include "rte_dmadev_pmd.h"
+
+struct rte_dmadev rte_dmadevices[RTE_DMADEV_MAX_DEVS];
+
+static const char *MZ_RTE_DMADEV_DATA = "rte_dmadev_data";
+/* Shared memory between primary and secondary processes. */
+static struct {
+ struct rte_dmadev_data data[RTE_DMADEV_MAX_DEVS];
+} *dmadev_shared_data;
+
+RTE_LOG_REGISTER_DEFAULT(rte_dmadev_logtype, INFO);
+#define RTE_DMADEV_LOG(level, ...) \
+ rte_log(RTE_LOG_ ## level, rte_dmadev_logtype, "" __VA_ARGS__)
+
+/* Macros to check for valid device id */
+#define RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, retval) do { \
+ if (!rte_dmadev_is_valid_dev(dev_id)) { \
+ RTE_DMADEV_LOG(ERR, "Invalid dev_id=%u\n", dev_id); \
+ return retval; \
+ } \
+} while (0)
+
+static int
+dmadev_check_name(const char *name)
+{
+ size_t name_len;
+
+ if (name == NULL) {
+ RTE_DMADEV_LOG(ERR, "Name can't be NULL\n");
+ return -EINVAL;
+ }
+
+ name_len = strnlen(name, RTE_DMADEV_NAME_MAX_LEN);
+ if (name_len == 0) {
+ RTE_DMADEV_LOG(ERR, "Zero length DMA device name\n");
+ return -EINVAL;
+ }
+ if (name_len >= RTE_DMADEV_NAME_MAX_LEN) {
+ RTE_DMADEV_LOG(ERR, "DMA device name is too long\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static uint16_t
+dmadev_find_free_dev(void)
+{
+ uint16_t i;
+
+ for (i = 0; i < RTE_DMADEV_MAX_DEVS; i++) {
+ if (dmadev_shared_data->data[i].dev_name[0] == '\0') {
+ RTE_ASSERT(rte_dmadevices[i].state ==
+ RTE_DMADEV_UNUSED);
+ return i;
+ }
+ }
+
+ return RTE_DMADEV_MAX_DEVS;
+}
+
+static struct rte_dmadev*
+dmadev_find(const char *name)
+{
+ uint16_t i;
+
+ for (i = 0; i < RTE_DMADEV_MAX_DEVS; i++) {
+ if ((rte_dmadevices[i].state == RTE_DMADEV_ATTACHED) &&
+ (!strcmp(name, rte_dmadevices[i].data->dev_name)))
+ return &rte_dmadevices[i];
+ }
+
+ return NULL;
+}
+
+static int
+dmadev_shared_data_prepare(void)
+{
+ const struct rte_memzone *mz;
+
+ if (dmadev_shared_data == NULL) {
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ /* Allocate port data and ownership shared memory. */
+ mz = rte_memzone_reserve(MZ_RTE_DMADEV_DATA,
+ sizeof(*dmadev_shared_data),
+ rte_socket_id(), 0);
+ } else
+ mz = rte_memzone_lookup(MZ_RTE_DMADEV_DATA);
+ if (mz == NULL)
+ return -ENOMEM;
+
+ dmadev_shared_data = mz->addr;
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ memset(dmadev_shared_data->data, 0,
+ sizeof(dmadev_shared_data->data));
+ }
+
+ return 0;
+}
+
+static struct rte_dmadev *
+dmadev_allocate(const char *name)
+{
+ struct rte_dmadev *dev;
+ uint16_t dev_id;
+
+ dev = dmadev_find(name);
+ if (dev != NULL) {
+ RTE_DMADEV_LOG(ERR, "DMA device already allocated\n");
+ return NULL;
+ }
+
+ if (dmadev_shared_data_prepare() != 0) {
+ RTE_DMADEV_LOG(ERR, "Cannot allocate DMA shared data\n");
+ return NULL;
+ }
+
+ dev_id = dmadev_find_free_dev();
+ if (dev_id == RTE_DMADEV_MAX_DEVS) {
+ RTE_DMADEV_LOG(ERR, "Reached maximum number of DMA devices\n");
+ return NULL;
+ }
+
+ dev = &rte_dmadevices[dev_id];
+ dev->data = &dmadev_shared_data->data[dev_id];
+ dev->data->dev_id = dev_id;
+ rte_strscpy(dev->data->dev_name, name, sizeof(dev->data->dev_name));
+
+ return dev;
+}
+
+static struct rte_dmadev *
+dmadev_attach_secondary(const char *name)
+{
+ struct rte_dmadev *dev;
+ uint16_t i;
+
+ if (dmadev_shared_data_prepare() != 0) {
+ RTE_DMADEV_LOG(ERR, "Cannot allocate DMA shared data\n");
+ return NULL;
+ }
+
+ for (i = 0; i < RTE_DMADEV_MAX_DEVS; i++) {
+ if (!strcmp(dmadev_shared_data->data[i].dev_name, name))
+ break;
+ }
+ if (i == RTE_DMADEV_MAX_DEVS) {
+ RTE_DMADEV_LOG(ERR,
+ "Device %s is not driven by the primary process\n",
+ name);
+ return NULL;
+ }
+
+ dev = &rte_dmadevices[i];
+ dev->data = &dmadev_shared_data->data[i];
+ RTE_ASSERT(dev->data->dev_id == i);
+ dev->dev_private = dev->data->dev_private;
+
+ return dev;
+}
+
+struct rte_dmadev *
+rte_dmadev_pmd_allocate(const char *name)
+{
+ struct rte_dmadev *dev;
+
+ if (dmadev_check_name(name) != 0)
+ return NULL;
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ dev = dmadev_allocate(name);
+ else
+ dev = dmadev_attach_secondary(name);
+
+ if (dev == NULL)
+ return NULL;
+ dev->state = RTE_DMADEV_ATTACHED;
+
+ return dev;
+}
+
+int
+rte_dmadev_pmd_release(struct rte_dmadev *dev)
+{
+ void *dev_private_bak;
+
+ if (dev == NULL)
+ return -EINVAL;
+
+ if (dev->state == RTE_DMADEV_UNUSED)
+ return 0;
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ memset(dev->data, 0, sizeof(struct rte_dmadev_data));
+
+ dev_private_bak = dev->dev_private;
+ memset(dev, 0, sizeof(struct rte_dmadev));
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ dev->dev_private = dev_private_bak;
+ dev->state = RTE_DMADEV_UNUSED;
+
+ return 0;
+}
+
+struct rte_dmadev *
+rte_dmadev_get_device_by_name(const char *name)
+{
+ if (dmadev_check_name(name) != 0)
+ return NULL;
+ return dmadev_find(name);
+}
+
+int
+rte_dmadev_get_dev_id(const char *name)
+{
+ struct rte_dmadev *dev = rte_dmadev_get_device_by_name(name);
+ if (dev != NULL)
+ return dev->data->dev_id;
+ return -EINVAL;
+}
+
+bool
+rte_dmadev_is_valid_dev(uint16_t dev_id)
+{
+ return (dev_id < RTE_DMADEV_MAX_DEVS) &&
+ rte_dmadevices[dev_id].state == RTE_DMADEV_ATTACHED;
+}
+
+uint16_t
+rte_dmadev_count(void)
+{
+ uint16_t count = 0;
+ uint16_t i;
+
+ for (i = 0; i < RTE_DMADEV_MAX_DEVS; i++) {
+ if (rte_dmadevices[i].state == RTE_DMADEV_ATTACHED)
+ count++;
+ }
+
+ return count;
+}
+
+int
+rte_dmadev_info_get(uint16_t dev_id, struct rte_dmadev_info *dev_info)
+{
+ const struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+ int ret;
+
+ RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
+ if (dev_info == NULL)
+ return -EINVAL;
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_info_get, -ENOTSUP);
+ memset(dev_info, 0, sizeof(struct rte_dmadev_info));
+ ret = (*dev->dev_ops->dev_info_get)(dev, dev_info,
+ sizeof(struct rte_dmadev_info));
+ if (ret != 0)
+ return ret;
+
+ dev_info->device = dev->device;
+ dev_info->nb_vchans = dev->data->dev_conf.max_vchans;
+
+ return 0;
+}
+
+int
+rte_dmadev_configure(uint16_t dev_id, const struct rte_dmadev_conf *dev_conf)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+ struct rte_dmadev_info info;
+ int ret;
+
+ RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
+ if (dev_conf == NULL)
+ return -EINVAL;
+
+ ret = rte_dmadev_info_get(dev_id, &info);
+ if (ret != 0) {
+ RTE_DMADEV_LOG(ERR, "Device %u get device info fail\n", dev_id);
+ return -EINVAL;
+ }
+ if (dev_conf->max_vchans == 0) {
+ RTE_DMADEV_LOG(ERR,
+ "Device %u configure zero vchans\n", dev_id);
+ return -EINVAL;
+ }
+ if (dev_conf->max_vchans > info.max_vchans) {
+ RTE_DMADEV_LOG(ERR,
+ "Device %u configure too many vchans\n", dev_id);
+ return -EINVAL;
+ }
+ if (dev_conf->enable_silent &&
+ !(info.dev_capa & RTE_DMADEV_CAPA_SILENT)) {
+ RTE_DMADEV_LOG(ERR, "Device %u don't support silent\n", dev_id);
+ return -EINVAL;
+ }
+
+ if (dev->data->dev_started != 0) {
+ RTE_DMADEV_LOG(ERR,
+ "Device %u must be stopped to allow configuration\n",
+ dev_id);
+ return -EBUSY;
+ }
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
+ ret = (*dev->dev_ops->dev_configure)(dev, dev_conf);
+ if (ret == 0)
+ memcpy(&dev->data->dev_conf, dev_conf, sizeof(*dev_conf));
+
+ return ret;
+}
+
+int
+rte_dmadev_start(uint16_t dev_id)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+ int ret;
+
+ RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
+
+ if (dev->data->dev_started != 0) {
+ RTE_DMADEV_LOG(WARNING, "Device %u already started\n", dev_id);
+ return 0;
+ }
+
+ if (dev->dev_ops->dev_start == NULL)
+ goto mark_started;
+
+ ret = (*dev->dev_ops->dev_start)(dev);
+ if (ret != 0)
+ return ret;
+
+mark_started:
+ dev->data->dev_started = 1;
+ return 0;
+}
+
+int
+rte_dmadev_stop(uint16_t dev_id)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+ int ret;
+
+ RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
+
+ if (dev->data->dev_started == 0) {
+ RTE_DMADEV_LOG(WARNING, "Device %u already stopped\n", dev_id);
+ return 0;
+ }
+
+ if (dev->dev_ops->dev_stop == NULL)
+ goto mark_stopped;
+
+ ret = (*dev->dev_ops->dev_stop)(dev);
+ if (ret != 0)
+ return ret;
+
+mark_stopped:
+ dev->data->dev_started = 0;
+ return 0;
+}
+
+int
+rte_dmadev_close(uint16_t dev_id)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+
+ RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
+
+ /* Device must be stopped before it can be closed */
+ if (dev->data->dev_started == 1) {
+ RTE_DMADEV_LOG(ERR,
+ "Device %u must be stopped before closing\n", dev_id);
+ return -EBUSY;
+ }
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
+ return (*dev->dev_ops->dev_close)(dev);
+}
+
+int
+rte_dmadev_vchan_setup(uint16_t dev_id,
+ const struct rte_dmadev_vchan_conf *conf)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+ struct rte_dmadev_info info;
+ int ret;
+
+ RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
+ if (conf == NULL)
+ return -EINVAL;
+
+ dev = &rte_dmadevices[dev_id];
+
+ ret = rte_dmadev_info_get(dev_id, &info);
+ if (ret != 0) {
+ RTE_DMADEV_LOG(ERR, "Device %u get device info fail\n", dev_id);
+ return -EINVAL;
+ }
+ if (conf->direction != RTE_DMA_DIR_MEM_TO_MEM &&
+ conf->direction != RTE_DMA_DIR_MEM_TO_DEV &&
+ conf->direction != RTE_DMA_DIR_DEV_TO_MEM &&
+ conf->direction != RTE_DMA_DIR_DEV_TO_DEV) {
+ RTE_DMADEV_LOG(ERR, "Device %u direction invalid!\n", dev_id);
+ return -EINVAL;
+ }
+ if (conf->direction == RTE_DMA_DIR_MEM_TO_MEM &&
+ !(info.dev_capa & RTE_DMADEV_CAPA_MEM_TO_MEM)) {
+ RTE_DMADEV_LOG(ERR,
+ "Device %u don't support mem2mem transfer\n", dev_id);
+ return -EINVAL;
+ }
+ if (conf->direction == RTE_DMA_DIR_MEM_TO_DEV &&
+ !(info.dev_capa & RTE_DMADEV_CAPA_MEM_TO_DEV)) {
+ RTE_DMADEV_LOG(ERR,
+ "Device %u don't support mem2dev transfer\n", dev_id);
+ return -EINVAL;
+ }
+ if (conf->direction == RTE_DMA_DIR_DEV_TO_MEM &&
+ !(info.dev_capa & RTE_DMADEV_CAPA_DEV_TO_MEM)) {
+ RTE_DMADEV_LOG(ERR,
+ "Device %u don't support dev2mem transfer\n", dev_id);
+ return -EINVAL;
+ }
+ if (conf->direction == RTE_DMA_DIR_DEV_TO_DEV &&
+ !(info.dev_capa & RTE_DMADEV_CAPA_DEV_TO_DEV)) {
+ RTE_DMADEV_LOG(ERR,
+ "Device %u don't support dev2dev transfer\n", dev_id);
+ return -EINVAL;
+ }
+ if (conf->nb_desc < info.min_desc || conf->nb_desc > info.max_desc) {
+ RTE_DMADEV_LOG(ERR,
+ "Device %u number of descriptors invalid\n", dev_id);
+ return -EINVAL;
+ }
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vchan_setup, -ENOTSUP);
+ return (*dev->dev_ops->vchan_setup)(dev, conf);
+}
+
+int
+rte_dmadev_stats_get(uint16_t dev_id, uint16_t vchan,
+ struct rte_dmadev_stats *stats)
+{
+ const struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+
+ RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
+ if (stats == NULL)
+ return -EINVAL;
+ if (vchan >= dev->data->dev_conf.max_vchans &&
+ vchan != RTE_DMADEV_ALL_VCHAN) {
+ RTE_DMADEV_LOG(ERR,
+ "Device %u vchan %u out of range\n", dev_id, vchan);
+ return -EINVAL;
+ }
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
+ memset(stats, 0, sizeof(struct rte_dmadev_stats));
+ return (*dev->dev_ops->stats_get)(dev, vchan, stats,
+ sizeof(struct rte_dmadev_stats));
+}
+
+int
+rte_dmadev_stats_reset(uint16_t dev_id, uint16_t vchan)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+
+ RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
+ if (vchan >= dev->data->dev_conf.max_vchans &&
+ vchan != RTE_DMADEV_ALL_VCHAN) {
+ RTE_DMADEV_LOG(ERR,
+ "Device %u vchan %u out of range\n", dev_id, vchan);
+ return -EINVAL;
+ }
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP);
+ return (*dev->dev_ops->stats_reset)(dev, vchan);
+}
+
+int
+rte_dmadev_dump(uint16_t dev_id, FILE *f)
+{
+ const struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+ struct rte_dmadev_info info;
+ int ret;
+
+ RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
+ if (f == NULL)
+ return -EINVAL;
+
+ ret = rte_dmadev_info_get(dev_id, &info);
+ if (ret != 0) {
+ RTE_DMADEV_LOG(ERR, "Device %u get device info fail\n", dev_id);
+ return -EINVAL;
+ }
+
+ fprintf(f, "DMA Dev %u, '%s' [%s]\n",
+ dev->data->dev_id,
+ dev->data->dev_name,
+ dev->data->dev_started ? "started" : "stopped");
+ fprintf(f, " dev_capa: 0x%" PRIx64 "\n", info.dev_capa);
+ fprintf(f, " max_vchans_supported: %u\n", info.max_vchans);
+ fprintf(f, " max_vchans_configured: %u\n", info.nb_vchans);
+ fprintf(f, " silent_mode: %s\n",
+ dev->data->dev_conf.enable_silent ? "on" : "off");
+
+ if (dev->dev_ops->dev_dump != NULL)
+ return (*dev->dev_ops->dev_dump)(dev, f);
+
+ return 0;
+}
+
+int
+rte_dmadev_selftest(uint16_t dev_id)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+
+ RTE_DMADEV_VALID_DEV_ID_OR_ERR_RET(dev_id, -EINVAL);
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_selftest, -ENOTSUP);
+ return (*dev->dev_ops->dev_selftest)(dev_id);
+}
new file mode 100644
@@ -0,0 +1,1041 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 HiSilicon Limited.
+ * Copyright(c) 2021 Intel Corporation.
+ * Copyright(c) 2021 Marvell International Ltd.
+ * Copyright(c) 2021 SmartShare Systems.
+ */
+
+#ifndef _RTE_DMADEV_H_
+#define _RTE_DMADEV_H_
+
+/**
+ * @file rte_dmadev.h
+ *
+ * RTE DMA (Direct Memory Access) device APIs.
+ *
+ * The DMA framework is built on the following model:
+ *
+ * --------------- --------------- ---------------
+ * | virtual DMA | | virtual DMA | | virtual DMA |
+ * | channel | | channel | | channel |
+ * --------------- --------------- ---------------
+ * | | |
+ * ------------------ |
+ * | |
+ * ------------ ------------
+ * | dmadev | | dmadev |
+ * ------------ ------------
+ * | |
+ * ------------------ ------------------
+ * | HW-DMA-channel | | HW-DMA-channel |
+ * ------------------ ------------------
+ * | |
+ * --------------------------------
+ * |
+ * ---------------------
+ * | HW-DMA-Controller |
+ * ---------------------
+ *
+ * The DMA controller could have multiple HW-DMA-channels (aka. HW-DMA-queues),
+ * each HW-DMA-channel should be represented by a dmadev.
+ *
+ * The dmadev could create multiple virtual DMA channels, each virtual DMA
+ * channel represents a different transfer context. The DMA operation request
+ * must be submitted to the virtual DMA channel. e.g. Application could create
+ * virtual DMA channel 0 for memory-to-memory transfer scenario, and create
+ * virtual DMA channel 1 for memory-to-device transfer scenario.
+ *
+ * The dmadev are dynamically allocated by rte_dmadev_pmd_allocate() during the
+ * PCI/SoC device probing phase performed at EAL initialization time. And could
+ * be released by rte_dmadev_pmd_release() during the PCI/SoC device removing
+ * phase.
+ *
+ * This framework uses 'uint16_t dev_id' as the device identifier of a dmadev,
+ * and 'uint16_t vchan' as the virtual DMA channel identifier in one dmadev.
+ *
+ * The functions exported by the dmadev API to setup a device designated by its
+ * device identifier must be invoked in the following order:
+ * - rte_dmadev_configure()
+ * - rte_dmadev_vchan_setup()
+ * - rte_dmadev_start()
+ *
+ * Then, the application can invoke dataplane APIs to process jobs.
+ *
+ * If the application wants to change the configuration (i.e. invoke
+ * rte_dmadev_configure() or rte_dmadev_vchan_setup()), it must invoke
+ * rte_dmadev_stop() first to stop the device and then do the reconfiguration
+ * before invoking rte_dmadev_start() again. The dataplane APIs should not be
+ * invoked when the device is stopped.
+ *
+ * Finally, an application can close a dmadev by invoking the
+ * rte_dmadev_close() function.
+ *
+ * The dataplane APIs include two parts:
+ * The first part is the submission of operation requests:
+ * - rte_dmadev_copy()
+ * - rte_dmadev_copy_sg()
+ * - rte_dmadev_fill()
+ * - rte_dmadev_submit()
+ *
+ * These APIs could work with different virtual DMA channels which have
+ * different contexts.
+ *
+ * The first three APIs are used to submit the operation request to the virtual
+ * DMA channel, if the submission is successful, a uint16_t ring_idx is
+ * returned, otherwise a negative number is returned.
+ *
+ * The last API was used to issue doorbell to hardware, and also there are flags
+ * (@see RTE_DMA_OP_FLAG_SUBMIT) parameter of the first three APIs could do the
+ * same work.
+ *
+ * The second part is to obtain the result of requests:
+ * - rte_dmadev_completed()
+ * - return the number of operation requests completed successfully.
+ * - rte_dmadev_completed_status()
+ * - return the number of operation requests completed.
+ *
+ * @note If the dmadev works in silent mode (@see RTE_DMADEV_CAPA_SILENT),
+ * application does not invoke the above two completed APIs.
+ *
+ * About the ring_idx which enqueue APIs (e.g. rte_dmadev_copy()
+ * rte_dmadev_fill()) returned, the rules are as follows:
+ * - ring_idx for each virtual DMA channel are independent.
+ * - For a virtual DMA channel, the ring_idx is monotonically incremented,
+ * when it reach UINT16_MAX, it wraps back to zero.
+ * - This ring_idx can be used by applications to track per-operation
+ * metadata in an application-defined circular ring.
+ * - The initial ring_idx of a virtual DMA channel is zero, after the
+ * device is stopped, the ring_idx needs to be reset to zero.
+ *
+ * One example:
+ * - step-1: start one dmadev
+ * - step-2: enqueue a copy operation, the ring_idx return is 0
+ * - step-3: enqueue a copy operation again, the ring_idx return is 1
+ * - ...
+ * - step-101: stop the dmadev
+ * - step-102: start the dmadev
+ * - step-103: enqueue a copy operation, the cookie return is 0
+ * - ...
+ * - step-x+0: enqueue a fill operation, the ring_idx return is 65535
+ * - step-x+1: enqueue a copy operation, the ring_idx return is 0
+ * - ...
+ *
+ * The DMA operation address used in enqueue APIs (i.e. rte_dmadev_copy(),
+ * rte_dmadev_copy_sg(), rte_dmadev_fill()) defined as rte_iova_t type. The
+ * dmadev supports two types of address: memory address and device address.
+ *
+ * - memory address: the source and destination address of the memory-to-memory
+ * transfer type, or the source address of the memory-to-device transfer type,
+ * or the destination address of the device-to-memory transfer type.
+ * @note If the device support SVA (@see RTE_DMADEV_CAPA_SVA), the memory
+ * address can be any VA address, otherwise it must be an IOVA address.
+ *
+ * - device address: the source and destination address of the device-to-device
+ * transfer type, or the source address of the device-to-memory transfer type,
+ * or the destination address of the memory-to-device transfer type.
+ *
+ * By default, all the functions of the dmadev API exported by a PMD are
+ * lock-free functions which assume to not be invoked in parallel on different
+ * logical cores to work on the same target dmadev object.
+ * @note Different virtual DMA channels on the same dmadev *DO NOT* support
+ * parallel invocation because these virtual DMA channels share the same
+ * HW-DMA-channel.
+ *
+ */
+
+#include <rte_common.h>
+#include <rte_compat.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_memory.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define RTE_DMADEV_NAME_MAX_LEN RTE_DEV_NAME_MAX_LEN
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Get the device identifier for the named DMA device.
+ *
+ * @param name
+ * DMA device name.
+ *
+ * @return
+ * Returns DMA device identifier on success.
+ * - <0: Failure to find named DMA device.
+ */
+__rte_experimental
+int
+rte_dmadev_get_dev_id(const char *name);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * @param dev_id
+ * DMA device index.
+ *
+ * @return
+ * - If the device index is valid (true) or not (false).
+ */
+__rte_experimental
+bool
+rte_dmadev_is_valid_dev(uint16_t dev_id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Get the total number of DMA devices that have been successfully
+ * initialised.
+ *
+ * @return
+ * The total number of usable DMA devices.
+ */
+__rte_experimental
+uint16_t
+rte_dmadev_count(void);
+
+/* Enumerates DMA device capabilities. */
+#define RTE_DMADEV_CAPA_MEM_TO_MEM (1ull << 0)
+/**< DMA device support memory-to-memory transfer.
+ *
+ * @see struct rte_dmadev_info::dev_capa
+ */
+
+#define RTE_DMADEV_CAPA_MEM_TO_DEV (1ull << 1)
+/**< DMA device support memory-to-device transfer.
+ *
+ * @see struct rte_dmadev_info::dev_capa
+ * @see struct rte_dmadev_port_param::port_type
+ */
+
+#define RTE_DMADEV_CAPA_DEV_TO_MEM (1ull << 2)
+/**< DMA device support device-to-memory transfer.
+ *
+ * @see struct rte_dmadev_info::dev_capa
+ * @see struct rte_dmadev_port_param::port_type
+ */
+
+#define RTE_DMADEV_CAPA_DEV_TO_DEV (1ull << 3)
+/**< DMA device support device-to-device transfer.
+ *
+ * @see struct rte_dmadev_info::dev_capa
+ * @see struct rte_dmadev_port_param::port_type
+ */
+
+#define RTE_DMADEV_CAPA_SVA (1ull << 4)
+/**< DMA device support SVA which could use VA as DMA address.
+ * If device support SVA then application could pass any VA address like memory
+ * from rte_malloc(), rte_memzone(), malloc, stack memory.
+ * If device don't support SVA, then application should pass IOVA address which
+ * from rte_malloc(), rte_memzone().
+ *
+ * @see struct rte_dmadev_info::dev_capa
+ */
+
+#define RTE_DMADEV_CAPA_SILENT (1ull << 5)
+/**< DMA device support work in silent mode.
+ * In this mode, application don't required to invoke rte_dmadev_completed*()
+ * API.
+ *
+ * @see struct rte_dmadev_conf::silent_mode
+ */
+
+#define RTE_DMADEV_CAPA_OPS_COPY (1ull << 32)
+/**< DMA device support copy ops.
+ * This capability start with index of 32, so that it could leave gap between
+ * normal capability and ops capability.
+ *
+ * @see struct rte_dmadev_info::dev_capa
+ */
+
+#define RTE_DMADEV_CAPA_OPS_COPY_SG (1ull << 33)
+/**< DMA device support scatter-list copy ops.
+ *
+ * @see struct rte_dmadev_info::dev_capa
+ */
+
+#define RTE_DMADEV_CAPA_OPS_FILL (1ull << 34)
+/**< DMA device support fill ops.
+ *
+ * @see struct rte_dmadev_info::dev_capa
+ */
+
+/**
+ * A structure used to retrieve the information of a DMA device.
+ */
+struct rte_dmadev_info {
+ struct rte_device *device; /**< Generic Device information. */
+ uint64_t dev_capa; /**< Device capabilities (RTE_DMADEV_CAPA_*). */
+ uint16_t max_vchans;
+ /**< Maximum number of virtual DMA channels supported. */
+ uint16_t max_desc;
+ /**< Maximum allowed number of virtual DMA channel descriptors. */
+ uint16_t min_desc;
+ /**< Minimum allowed number of virtual DMA channel descriptors. */
+ uint16_t nb_vchans; /**< Number of virtual DMA channel configured. */
+};
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Retrieve information of a DMA device.
+ *
+ * @param dev_id
+ * The identifier of the device.
+ * @param[out] dev_info
+ * A pointer to a structure of type *rte_dmadev_info* to be filled with the
+ * information of the device.
+ *
+ * @return
+ * - =0: Success, driver updates the information of the DMA device.
+ * - <0: Error code returned by the driver info get function.
+ *
+ */
+__rte_experimental
+int
+rte_dmadev_info_get(uint16_t dev_id, struct rte_dmadev_info *dev_info);
+
+/**
+ * A structure used to configure a DMA device.
+ */
+struct rte_dmadev_conf {
+ uint16_t max_vchans;
+ /**< Maximum number of virtual DMA channel to use.
+ * This value cannot be greater than the field 'max_vchans' of struct
+ * rte_dmadev_info which get from rte_dmadev_info_get().
+ */
+ bool enable_silent;
+ /**< Indicates whether to enable silent mode.
+ * false-default mode, true-silent mode.
+ * This value can be set to true only when the SILENT capability is
+ * supported.
+ *
+ * @see RTE_DMADEV_CAPA_SILENT
+ */
+};
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Configure a DMA device.
+ *
+ * This function must be invoked first before any other function in the
+ * API. This function can also be re-invoked when a device is in the
+ * stopped state.
+ *
+ * @param dev_id
+ * The identifier of the device to configure.
+ * @param dev_conf
+ * The DMA device configuration structure encapsulated into rte_dmadev_conf
+ * object.
+ *
+ * @return
+ * - =0: Success, device configured.
+ * - <0: Error code returned by the driver configuration function.
+ */
+__rte_experimental
+int
+rte_dmadev_configure(uint16_t dev_id, const struct rte_dmadev_conf *dev_conf);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Start a DMA device.
+ *
+ * The device start step is the last one and consists of setting the DMA
+ * to start accepting jobs.
+ *
+ * @param dev_id
+ * The identifier of the device.
+ *
+ * @return
+ * - =0: Success, device started.
+ * - <0: Error code returned by the driver start function.
+ */
+__rte_experimental
+int
+rte_dmadev_start(uint16_t dev_id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Stop a DMA device.
+ *
+ * The device can be restarted with a call to rte_dmadev_start().
+ *
+ * @param dev_id
+ * The identifier of the device.
+ *
+ * @return
+ * - =0: Success, device stopped.
+ * - <0: Error code returned by the driver stop function.
+ */
+__rte_experimental
+int
+rte_dmadev_stop(uint16_t dev_id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Close a DMA device.
+ *
+ * The device cannot be restarted after this call.
+ *
+ * @param dev_id
+ * The identifier of the device.
+ *
+ * @return
+ * - =0: Successfully close device
+ * - <0: Failure to close device
+ */
+__rte_experimental
+int
+rte_dmadev_close(uint16_t dev_id);
+
+/**
+ * rte_dma_direction - DMA transfer direction defines.
+ */
+enum rte_dma_direction {
+ RTE_DMA_DIR_MEM_TO_MEM,
+ /**< DMA transfer direction - from memory to memory.
+ *
+ * @see struct rte_dmadev_vchan_conf::direction
+ */
+ RTE_DMA_DIR_MEM_TO_DEV,
+ /**< DMA transfer direction - from memory to device.
+ * In a typical scenario, the SoCs are installed on host servers as
+ * iNICs through the PCIe interface. In this case, the SoCs works in
+ * EP(endpoint) mode, it could initiate a DMA move request from memory
+ * (which is SoCs memory) to device (which is host memory).
+ *
+ * @see struct rte_dmadev_vchan_conf::direction
+ */
+ RTE_DMA_DIR_DEV_TO_MEM,
+ /**< DMA transfer direction - from device to memory.
+ * In a typical scenario, the SoCs are installed on host servers as
+ * iNICs through the PCIe interface. In this case, the SoCs works in
+ * EP(endpoint) mode, it could initiate a DMA move request from device
+ * (which is host memory) to memory (which is SoCs memory).
+ *
+ * @see struct rte_dmadev_vchan_conf::direction
+ */
+ RTE_DMA_DIR_DEV_TO_DEV,
+ /**< DMA transfer direction - from device to device.
+ * In a typical scenario, the SoCs are installed on host servers as
+ * iNICs through the PCIe interface. In this case, the SoCs works in
+ * EP(endpoint) mode, it could initiate a DMA move request from device
+ * (which is host memory) to the device (which is another host memory).
+ *
+ * @see struct rte_dmadev_vchan_conf::direction
+ */
+};
+
+/**
+ * enum rte_dmadev_port_type - DMA access port type defines.
+ *
+ * @see struct rte_dmadev_port_param::port_type
+ */
+enum rte_dmadev_port_type {
+ RTE_DMADEV_PORT_NONE,
+ RTE_DMADEV_PORT_PCIE, /**< The DMA access port is PCIe. */
+};
+
+/**
+ * A structure used to descript DMA access port parameters.
+ *
+ * @see struct rte_dmadev_vchan_conf::src_port
+ * @see struct rte_dmadev_vchan_conf::dst_port
+ */
+struct rte_dmadev_port_param {
+ enum rte_dmadev_port_type port_type;
+ /**< The device access port type.
+ * @see enum rte_dmadev_port_type
+ */
+ union {
+ /** PCIe access port parameters.
+ *
+ * The following model shows SoC's PCIe module connects to
+ * multiple PCIe hosts and multiple endpoints. The PCIe module
+ * has an integrated DMA controller.
+ *
+ * If the DMA wants to access the memory of host A, it can be
+ * initiated by PF1 in core0, or by VF0 of PF0 in core0.
+ *
+ * \code{.unparsed}
+ * System Bus
+ * | ----------PCIe module----------
+ * | Bus
+ * | Interface
+ * | ----- ------------------
+ * | | | | PCIe Core0 |
+ * | | | | | -----------
+ * | | | | PF-0 -- VF-0 | | Host A |
+ * | | |--------| |- VF-1 |--------| Root |
+ * | | | | PF-1 | | Complex |
+ * | | | | PF-2 | -----------
+ * | | | ------------------
+ * | | |
+ * | | | ------------------
+ * | | | | PCIe Core1 |
+ * | | | | | -----------
+ * | | | | PF-0 -- VF-0 | | Host B |
+ * |-----| |--------| PF-1 -- VF-0 |--------| Root |
+ * | | | | |- VF-1 | | Complex |
+ * | | | | PF-2 | -----------
+ * | | | ------------------
+ * | | |
+ * | | | ------------------
+ * | |DMA| | | ------
+ * | | | | |--------| EP |
+ * | | |--------| PCIe Core2 | ------
+ * | | | | | ------
+ * | | | | |--------| EP |
+ * | | | | | ------
+ * | ----- ------------------
+ *
+ * \endcode
+ *
+ * @note If some fields can not be supported by the
+ * hardware/driver, then the driver ignores those fields.
+ * Please check driver-specific documentation for limitations
+ * and capablites.
+ */
+ struct {
+ uint64_t coreid : 4; /**< PCIe core id used. */
+ uint64_t pfid : 8; /**< PF id used. */
+ uint64_t vfen : 1; /**< VF enable bit. */
+ uint64_t vfid : 16; /**< VF id used. */
+ uint64_t pasid : 20;
+ /**< The pasid filed in TLP packet. */
+ uint64_t attr : 3;
+ /**< The attributes filed in TLP packet. */
+ uint64_t ph : 2;
+ /**< The processing hint filed in TLP packet. */
+ uint64_t st : 16;
+ /**< The steering tag filed in TLP packet. */
+ } pcie;
+ };
+ uint64_t reserved[2]; /**< Reserved for future fields. */
+};
+
+/**
+ * A structure used to configure a virtual DMA channel.
+ */
+struct rte_dmadev_vchan_conf {
+ enum rte_dma_direction direction;
+ /**< Transfer direction
+ * @see enum rte_dma_direction
+ */
+ uint16_t nb_desc;
+ /**< Number of descriptor for the virtual DMA channel */
+ struct rte_dmadev_port_param src_port;
+ /**< 1) Used to describes the device access port parameter in the
+ * device-to-memory transfer scenario.
+ * 2) Used to describes the source device access port parameter in the
+ * device-to-device transfer scenario.
+ * @see struct rte_dmadev_port_param
+ */
+ struct rte_dmadev_port_param dst_port;
+ /**< 1) Used to describes the device access port parameter in the
+ * memory-to-device transfer scenario.
+ * 2) Used to describes the destination device access port parameter in
+ * the device-to-device transfer scenario.
+ * @see struct rte_dmadev_port_param
+ */
+};
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Allocate and set up a virtual DMA channel.
+ *
+ * @param dev_id
+ * The identifier of the device.
+ * @param conf
+ * The virtual DMA channel configuration structure encapsulated into
+ * rte_dmadev_vchan_conf object.
+ *
+ * @return
+ * - >=0: Allocate success, it is the virtual DMA channel id. This value must
+ * be less than the field 'max_vchans' of struct rte_dmadev_conf
+ * which configured by rte_dmadev_configure().
+ * - <0: Error code returned by the driver virtual channel setup function.
+ */
+__rte_experimental
+int
+rte_dmadev_vchan_setup(uint16_t dev_id,
+ const struct rte_dmadev_vchan_conf *conf);
+
+/**
+ * rte_dmadev_stats - running statistics.
+ */
+struct rte_dmadev_stats {
+ uint64_t submitted_count;
+ /**< Count of operations which were submitted to hardware. */
+ uint64_t completed_fail_count;
+ /**< Count of operations which failed to complete. */
+ uint64_t completed_count;
+ /**< Count of operations which successfully complete. */
+};
+
+#define RTE_DMADEV_ALL_VCHAN 0xFFFFu
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Retrieve basic statistics of a or all virtual DMA channel(s).
+ *
+ * @param dev_id
+ * The identifier of the device.
+ * @param vchan
+ * The identifier of virtual DMA channel.
+ * If equal RTE_DMADEV_ALL_VCHAN means all channels.
+ * @param[out] stats
+ * The basic statistics structure encapsulated into rte_dmadev_stats
+ * object.
+ *
+ * @return
+ * - =0: Successfully retrieve stats.
+ * - <0: Failure to retrieve stats.
+ */
+__rte_experimental
+int
+rte_dmadev_stats_get(uint16_t dev_id, uint16_t vchan,
+ struct rte_dmadev_stats *stats);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Reset basic statistics of a or all virtual DMA channel(s).
+ *
+ * @param dev_id
+ * The identifier of the device.
+ * @param vchan
+ * The identifier of virtual DMA channel.
+ * If equal RTE_DMADEV_ALL_VCHAN means all channels.
+ *
+ * @return
+ * - =0: Successfully reset stats.
+ * - <0: Failure to reset stats.
+ */
+__rte_experimental
+int
+rte_dmadev_stats_reset(uint16_t dev_id, uint16_t vchan);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Dump DMA device info.
+ *
+ * @param dev_id
+ * The identifier of the device.
+ * @param f
+ * The file to write the output to.
+ *
+ * @return
+ * 0 on success. Non-zero otherwise.
+ */
+__rte_experimental
+int
+rte_dmadev_dump(uint16_t dev_id, FILE *f);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Trigger the dmadev self test.
+ *
+ * @param dev_id
+ * The identifier of the device.
+ *
+ * @return
+ * - 0: Selftest successful.
+ * - -ENOTSUP if the device doesn't support selftest
+ * - other values < 0 on failure.
+ */
+__rte_experimental
+int
+rte_dmadev_selftest(uint16_t dev_id);
+
+/**
+ * rte_dma_status_code - DMA transfer result status code defines.
+ */
+enum rte_dma_status_code {
+ RTE_DMA_STATUS_SUCCESSFUL,
+ /**< The operation completed successfully. */
+ RTE_DMA_STATUS_USRER_ABORT,
+ /**< The operation failed to complete due abort by user.
+ * This is mainly used when processing dev_stop, user could modidy the
+ * descriptors (e.g. change one bit to tell hardware abort this job),
+ * it allows outstanding requests to be complete as much as possible,
+ * so reduce the time to stop the device.
+ */
+ RTE_DMA_STATUS_NOT_ATTEMPTED,
+ /**< The operation failed to complete due to following scenarios:
+ * The jobs in a particular batch are not attempted because they
+ * appeared after a fence where a previous job failed. In some HW
+ * implementation it's possible for jobs from later batches would be
+ * completed, though, so report the status from the not attempted jobs
+ * before reporting those newer completed jobs.
+ */
+ RTE_DMA_STATUS_INVALID_SRC_ADDR,
+ /**< The operation failed to complete due invalid source address. */
+ RTE_DMA_STATUS_INVALID_DST_ADDR,
+ /**< The operation failed to complete due invalid destination
+ * address.
+ */
+ RTE_DMA_STATUS_INVALID_LENGTH,
+ /**< The operation failed to complete due invalid length. */
+ RTE_DMA_STATUS_INVALID_OPCODE,
+ /**< The operation failed to complete due invalid opcode.
+ * The DMA descriptor could have multiple format, which are
+ * distinguished by the opcode field.
+ */
+ RTE_DMA_STATUS_BUS_ERROR,
+ /**< The operation failed to complete due bus err. */
+ RTE_DMA_STATUS_DATA_POISION,
+ /**< The operation failed to complete due data poison. */
+ RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR,
+ /**< The operation failed to complete due descriptor read error. */
+ RTE_DMA_STATUS_DEV_LINK_ERROR,
+ /**< The operation failed to complete due device link error.
+ * Used to indicates that the link error in the memory-to-device/
+ * device-to-memory/device-to-device transfer scenario.
+ */
+ RTE_DMA_STATUS_UNKNOWN = 0x100,
+ /**< The operation failed to complete due unknown reason.
+ * The initial value is 256, which reserves space for future errors.
+ */
+};
+
+/**
+ * rte_dmadev_sge - can hold scatter DMA operation request entry.
+ */
+struct rte_dmadev_sge {
+ rte_iova_t addr; /**< The DMA operation address. */
+ uint32_t length; /**< The DMA operation length. */
+};
+
+#include "rte_dmadev_core.h"
+
+/* DMA flags to augment operation preparation. */
+#define RTE_DMA_OP_FLAG_FENCE (1ull << 0)
+/**< DMA fence flag.
+ * It means the operation with this flag must be processed only after all
+ * previous operations are completed.
+ * If the specify DMA HW works in-order (it means it has default fence between
+ * operations), this flag could be NOP.
+ *
+ * @see rte_dmadev_copy()
+ * @see rte_dmadev_copy_sg()
+ * @see rte_dmadev_fill()
+ */
+
+#define RTE_DMA_OP_FLAG_SUBMIT (1ull << 1)
+/**< DMA submit flag.
+ * It means the operation with this flag must issue doorbell to hardware after
+ * enqueued jobs.
+ */
+
+#define RTE_DMA_OP_FLAG_LLC (1ull << 2)
+/**< DMA write data to low level cache hint.
+ * Used for performance optimization, this is just a hint, and there is no
+ * capability bit for this, driver should not return error if this flag was set.
+ */
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Enqueue a copy operation onto the virtual DMA channel.
+ *
+ * This queues up a copy operation to be performed by hardware, if the 'flags'
+ * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin
+ * this operation, otherwise do not trigger doorbell.
+ *
+ * @param dev_id
+ * The identifier of the device.
+ * @param vchan
+ * The identifier of virtual DMA channel.
+ * @param src
+ * The address of the source buffer.
+ * @param dst
+ * The address of the destination buffer.
+ * @param length
+ * The length of the data to be copied.
+ * @param flags
+ * An flags for this operation.
+ * @see RTE_DMA_OP_FLAG_*
+ *
+ * @return
+ * - 0..UINT16_MAX: index of enqueued copy job.
+ * - <0: Error code returned by the driver copy function.
+ */
+__rte_experimental
+static inline int
+rte_dmadev_copy(uint16_t dev_id, uint16_t vchan, rte_iova_t src, rte_iova_t dst,
+ uint32_t length, uint64_t flags)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+
+#ifdef RTE_DMADEV_DEBUG
+ if (!rte_dmadev_is_valid_dev(dev_id) ||
+ vchan >= dev->data->dev_conf.max_vchans || length == 0)
+ return -EINVAL;
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->copy, -ENOTSUP);
+#endif
+
+ return (*dev->copy)(dev, vchan, src, dst, length, flags);
+}
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Enqueue a scatter list copy operation onto the virtual DMA channel.
+ *
+ * This queues up a scatter list copy operation to be performed by hardware, if
+ * the 'flags' parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell
+ * to begin this operation, otherwise do not trigger doorbell.
+ *
+ * @param dev_id
+ * The identifier of the device.
+ * @param vchan
+ * The identifier of virtual DMA channel.
+ * @param src
+ * The pointer of source scatter entry array.
+ * @param dst
+ * The pointer of destination scatter entry array.
+ * @param nb_src
+ * The number of source scatter entry.
+ * @param nb_dst
+ * The number of destination scatter entry.
+ * @param flags
+ * An flags for this operation.
+ * @see RTE_DMA_OP_FLAG_*
+ *
+ * @return
+ * - 0..UINT16_MAX: index of enqueued copy scatterlist job.
+ * - <0: Error code returned by the driver copy scatterlist function.
+ */
+__rte_experimental
+static inline int
+rte_dmadev_copy_sg(uint16_t dev_id, uint16_t vchan, struct rte_dmadev_sge *src,
+ struct rte_dmadev_sge *dst, uint16_t nb_src, uint16_t nb_dst,
+ uint64_t flags)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+
+#ifdef RTE_DMADEV_DEBUG
+ if (!rte_dmadev_is_valid_dev(dev_id) ||
+ vchan >= dev->data->dev_conf.max_vchans ||
+ src == NULL || dst == NULL || nb_src == 0 || nb_dst == 0)
+ return -EINVAL;
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->copy_sg, -ENOTSUP);
+#endif
+
+ return (*dev->copy_sg)(dev, vchan, src, dst, nb_src, nb_dst, flags);
+}
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Enqueue a fill operation onto the virtual DMA channel.
+ *
+ * This queues up a fill operation to be performed by hardware, if the 'flags'
+ * parameter contains RTE_DMA_OP_FLAG_SUBMIT then trigger doorbell to begin
+ * this operation, otherwise do not trigger doorbell.
+ *
+ * @param dev_id
+ * The identifier of the device.
+ * @param vchan
+ * The identifier of virtual DMA channel.
+ * @param pattern
+ * The pattern to populate the destination buffer with.
+ * @param dst
+ * The address of the destination buffer.
+ * @param length
+ * The length of the destination buffer.
+ * @param flags
+ * An flags for this operation.
+ * @see RTE_DMA_OP_FLAG_*
+ *
+ * @return
+ * - 0..UINT16_MAX: index of enqueued fill job.
+ * - <0: Error code returned by the driver fill function.
+ */
+__rte_experimental
+static inline int
+rte_dmadev_fill(uint16_t dev_id, uint16_t vchan, uint64_t pattern,
+ rte_iova_t dst, uint32_t length, uint64_t flags)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+
+#ifdef RTE_DMADEV_DEBUG
+ if (!rte_dmadev_is_valid_dev(dev_id) ||
+ vchan >= dev->data->dev_conf.max_vchans || length == 0)
+ return -EINVAL;
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->fill, -ENOTSUP);
+#endif
+
+ return (*dev->fill)(dev, vchan, pattern, dst, length, flags);
+}
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Trigger hardware to begin performing enqueued operations.
+ *
+ * This API is used to write the "doorbell" to the hardware to trigger it
+ * to begin the operations previously enqueued by rte_dmadev_copy/fill().
+ *
+ * @param dev_id
+ * The identifier of the device.
+ * @param vchan
+ * The identifier of virtual DMA channel.
+ *
+ * @return
+ * - =0: Successfully trigger hardware.
+ * - <0: Failure to trigger hardware.
+ */
+__rte_experimental
+static inline int
+rte_dmadev_submit(uint16_t dev_id, uint16_t vchan)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+
+#ifdef RTE_DMADEV_DEBUG
+ if (!rte_dmadev_is_valid_dev(dev_id) ||
+ vchan >= dev->data->dev_conf.max_vchans)
+ return -EINVAL;
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->submit, -ENOTSUP);
+#endif
+
+ return (*dev->submit)(dev, vchan);
+}
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Returns the number of operations that have been successfully completed.
+ *
+ * @param dev_id
+ * The identifier of the device.
+ * @param vchan
+ * The identifier of virtual DMA channel.
+ * @param nb_cpls
+ * The maximum number of completed operations that can be processed.
+ * @param[out] last_idx
+ * The last completed operation's index.
+ * If not required, NULL can be passed in.
+ * @param[out] has_error
+ * Indicates if there are transfer error.
+ * If not required, NULL can be passed in.
+ *
+ * @return
+ * The number of operations that successfully completed. This return value
+ * must be less than or equal to the value of nb_cpls.
+ */
+__rte_experimental
+static inline uint16_t
+rte_dmadev_completed(uint16_t dev_id, uint16_t vchan, const uint16_t nb_cpls,
+ uint16_t *last_idx, bool *has_error)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+ uint16_t idx;
+ bool err;
+
+#ifdef RTE_DMADEV_DEBUG
+ if (!rte_dmadev_is_valid_dev(dev_id) ||
+ vchan >= dev->data->dev_conf.max_vchans || nb_cpls == 0)
+ return 0;
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->completed, 0);
+#endif
+
+ /* Ensure the pointer values are non-null to simplify drivers.
+ * In most cases these should be compile time evaluated, since this is
+ * an inline function.
+ * - If NULL is explicitly passed as parameter, then compiler knows the
+ * value is NULL
+ * - If address of local variable is passed as parameter, then compiler
+ * can know it's non-NULL.
+ */
+ if (last_idx == NULL)
+ last_idx = &idx;
+ if (has_error == NULL)
+ has_error = &err;
+
+ *has_error = false;
+ return (*dev->completed)(dev, vchan, nb_cpls, last_idx, has_error);
+}
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Returns the number of operations that have been completed, and the
+ * operations result may succeed or fail.
+ *
+ * @param dev_id
+ * The identifier of the device.
+ * @param vchan
+ * The identifier of virtual DMA channel.
+ * @param nb_cpls
+ * Indicates the size of status array.
+ * @param[out] last_idx
+ * The last completed operation's index.
+ * If not required, NULL can be passed in.
+ * @param[out] status
+ * The error code of operations that completed.
+ * @see enum rte_dma_status_code
+ *
+ * @return
+ * The number of operations that completed. This return value must be less
+ * than or equal to the value of nb_cpls.
+ */
+__rte_experimental
+static inline uint16_t
+rte_dmadev_completed_status(uint16_t dev_id, uint16_t vchan,
+ const uint16_t nb_cpls, uint16_t *last_idx,
+ enum rte_dma_status_code *status)
+{
+ struct rte_dmadev *dev = &rte_dmadevices[dev_id];
+ uint16_t idx;
+
+#ifdef RTE_DMADEV_DEBUG
+ if (!rte_dmadev_is_valid_dev(dev_id) ||
+ vchan >= dev->data->dev_conf.max_vchans ||
+ nb_cpls == 0 || status == NULL)
+ return 0;
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->completed_status, 0);
+#endif
+
+ if (last_idx == NULL)
+ last_idx = &idx;
+
+ return (*dev->completed_status)(dev, vchan, nb_cpls, last_idx, status);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_DMADEV_H_ */
new file mode 100644
@@ -0,0 +1,182 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 HiSilicon Limited.
+ * Copyright(c) 2021 Intel Corporation.
+ */
+
+#ifndef _RTE_DMADEV_CORE_H_
+#define _RTE_DMADEV_CORE_H_
+
+/**
+ * @file
+ *
+ * RTE DMA Device internal header.
+ *
+ * This header contains internal data types, that are used by the DMA devices
+ * in order to expose their ops to the class.
+ *
+ * Applications should not use these API directly.
+ *
+ */
+
+struct rte_dmadev;
+
+typedef int (*rte_dmadev_info_get_t)(const struct rte_dmadev *dev,
+ struct rte_dmadev_info *dev_info,
+ uint32_t info_sz);
+/**< @internal Used to get device information of a device. */
+
+typedef int (*rte_dmadev_configure_t)(struct rte_dmadev *dev,
+ const struct rte_dmadev_conf *dev_conf);
+/**< @internal Used to configure a device. */
+
+typedef int (*rte_dmadev_start_t)(struct rte_dmadev *dev);
+/**< @internal Used to start a configured device. */
+
+typedef int (*rte_dmadev_stop_t)(struct rte_dmadev *dev);
+/**< @internal Used to stop a configured device. */
+
+typedef int (*rte_dmadev_close_t)(struct rte_dmadev *dev);
+/**< @internal Used to close a configured device. */
+
+typedef int (*rte_dmadev_vchan_setup_t)(struct rte_dmadev *dev,
+ const struct rte_dmadev_vchan_conf *conf);
+/**< @internal Used to allocate and set up a virtual DMA channel. */
+
+typedef int (*rte_dmadev_stats_get_t)(const struct rte_dmadev *dev,
+ uint16_t vchan, struct rte_dmadev_stats *stats,
+ uint32_t stats_sz);
+/**< @internal Used to retrieve basic statistics. */
+
+typedef int (*rte_dmadev_stats_reset_t)(struct rte_dmadev *dev, uint16_t vchan);
+/**< @internal Used to reset basic statistics. */
+
+typedef int (*rte_dmadev_dump_t)(const struct rte_dmadev *dev, FILE *f);
+/**< @internal Used to dump internal information. */
+
+typedef int (*rte_dmadev_selftest_t)(uint16_t dev_id);
+/**< @internal Used to start dmadev selftest. */
+
+typedef int (*rte_dmadev_copy_t)(struct rte_dmadev *dev, uint16_t vchan,
+ rte_iova_t src, rte_iova_t dst,
+ uint32_t length, uint64_t flags);
+/**< @internal Used to enqueue a copy operation. */
+
+typedef int (*rte_dmadev_copy_sg_t)(struct rte_dmadev *dev, uint16_t vchan,
+ const struct rte_dmadev_sge *src,
+ const struct rte_dmadev_sge *dst,
+ uint16_t nb_src, uint16_t nb_dst,
+ uint64_t flags);
+/**< @internal Used to enqueue a scatter list copy operation. */
+
+typedef int (*rte_dmadev_fill_t)(struct rte_dmadev *dev, uint16_t vchan,
+ uint64_t pattern, rte_iova_t dst,
+ uint32_t length, uint64_t flags);
+/**< @internal Used to enqueue a fill operation. */
+
+typedef int (*rte_dmadev_submit_t)(struct rte_dmadev *dev, uint16_t vchan);
+/**< @internal Used to trigger hardware to begin working. */
+
+typedef uint16_t (*rte_dmadev_completed_t)(struct rte_dmadev *dev,
+ uint16_t vchan, const uint16_t nb_cpls,
+ uint16_t *last_idx, bool *has_error);
+/**< @internal Used to return number of successful completed operations. */
+
+typedef uint16_t (*rte_dmadev_completed_status_t)(struct rte_dmadev *dev,
+ uint16_t vchan, const uint16_t nb_cpls,
+ uint16_t *last_idx, enum rte_dma_status_code *status);
+/**< @internal Used to return number of completed operations. */
+
+/**
+ * Possible states of a DMA device.
+ */
+enum rte_dmadev_state {
+ RTE_DMADEV_UNUSED = 0,
+ /**< Device is unused before being probed. */
+ RTE_DMADEV_ATTACHED,
+ /**< Device is attached when allocated in probing. */
+};
+
+/**
+ * DMA device operations function pointer table
+ */
+struct rte_dmadev_ops {
+ rte_dmadev_info_get_t dev_info_get;
+ rte_dmadev_configure_t dev_configure;
+ rte_dmadev_start_t dev_start;
+ rte_dmadev_stop_t dev_stop;
+ rte_dmadev_close_t dev_close;
+ rte_dmadev_vchan_setup_t vchan_setup;
+ rte_dmadev_stats_get_t stats_get;
+ rte_dmadev_stats_reset_t stats_reset;
+ rte_dmadev_dump_t dev_dump;
+ rte_dmadev_selftest_t dev_selftest;
+};
+
+/**
+ * @internal
+ * The data part, with no function pointers, associated with each DMA device.
+ *
+ * This structure is safe to place in shared memory to be common among different
+ * processes in a multi-process configuration.
+ */
+struct rte_dmadev_data {
+ void *dev_private;
+ /**< PMD-specific private data.
+ * This is a copy of the 'dev_private' field in the 'struct rte_dmadev'
+ * from primary process, it is used by the secondary process to get
+ * dev_private information.
+ */
+ uint16_t dev_id; /**< Device [external] identifier. */
+ char dev_name[RTE_DMADEV_NAME_MAX_LEN]; /**< Unique identifier name */
+ struct rte_dmadev_conf dev_conf; /**< DMA device configuration. */
+ uint8_t dev_started : 1; /**< Device state: STARTED(1)/STOPPED(0). */
+ uint64_t reserved[2]; /**< Reserved for future fields */
+} __rte_cache_aligned;
+
+/**
+ * @internal
+ * The generic data structure associated with each DMA device.
+ *
+ * The dataplane APIs are located at the beginning of the structure, along
+ * with the pointer to where all the data elements for the particular device
+ * are stored in shared memory. This split scheme allows the function pointer
+ * and driver data to be per-process, while the actual configuration data for
+ * the device is shared.
+ * And the 'dev_private' field was placed in the first cache line to optimize
+ * performance because the PMD driver mainly depends on this field.
+ */
+struct rte_dmadev {
+ rte_dmadev_copy_t copy;
+ rte_dmadev_copy_sg_t copy_sg;
+ rte_dmadev_fill_t fill;
+ rte_dmadev_submit_t submit;
+ rte_dmadev_completed_t completed;
+ rte_dmadev_completed_status_t completed_status;
+ void *reserved_ptr; /**< Reserved for future IO function. */
+ void *dev_private;
+ /**< PMD-specific private data.
+ *
+ * - If is the primary process, after dmadev allocated by
+ * rte_dmadev_pmd_allocate(), the PCI/SoC device probing should
+ * initialize this field, and copy it's value to the 'dev_private'
+ * field of 'struct rte_dmadev_data' which pointer by 'data' filed.
+ *
+ * - If is the secondary process, dmadev framework will initialize this
+ * field by copy from 'dev_private' field of 'struct rte_dmadev_data'
+ * which initialized by primary process.
+ *
+ * @note It's the primary process responsibility to deinitialize this
+ * field after invoke rte_dmadev_pmd_release() in the PCI/SoC device
+ * removing stage.
+ */
+ struct rte_dmadev_data *data; /**< Pointer to device data. */
+ const struct rte_dmadev_ops *dev_ops; /**< Functions exported by PMD. */
+ struct rte_device *device;
+ /**< Device info which supplied during device initialization. */
+ enum rte_dmadev_state state; /**< Flag indicating the device state. */
+ uint64_t reserved[2]; /**< Reserved for future fields. */
+} __rte_cache_aligned;
+
+extern struct rte_dmadev rte_dmadevices[];
+
+#endif /* _RTE_DMADEV_CORE_H_ */
new file mode 100644
@@ -0,0 +1,72 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 HiSilicon Limited.
+ */
+
+#ifndef _RTE_DMADEV_PMD_H_
+#define _RTE_DMADEV_PMD_H_
+
+/**
+ * @file
+ *
+ * RTE DMA Device PMD APIs
+ *
+ * Driver facing APIs for a DMA device. These are not to be called directly by
+ * any application.
+ */
+
+#include "rte_dmadev.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @internal
+ * Allocates a new dmadev slot for an DMA device and returns the pointer
+ * to that slot for the driver to use.
+ *
+ * @param name
+ * DMA device name.
+ *
+ * @return
+ * A pointer to the DMA device slot case of success,
+ * NULL otherwise.
+ */
+__rte_internal
+struct rte_dmadev *
+rte_dmadev_pmd_allocate(const char *name);
+
+/**
+ * @internal
+ * Release the specified dmadev.
+ *
+ * @param dev
+ * Device to be released.
+ *
+ * @return
+ * - 0 on success, negative on error
+ */
+__rte_internal
+int
+rte_dmadev_pmd_release(struct rte_dmadev *dev);
+
+/**
+ * @internal
+ * Return the DMA device based on the device name.
+ *
+ * @param name
+ * DMA device name.
+ *
+ * @return
+ * A pointer to the DMA device slot case of success,
+ * NULL otherwise.
+ */
+__rte_internal
+struct rte_dmadev *
+rte_dmadev_get_device_by_name(const char *name);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_DMADEV_PMD_H_ */
new file mode 100644
@@ -0,0 +1,37 @@
+EXPERIMENTAL {
+ global:
+
+ rte_dmadev_close;
+ rte_dmadev_completed;
+ rte_dmadev_completed_status;
+ rte_dmadev_configure;
+ rte_dmadev_copy;
+ rte_dmadev_copy_sg;
+ rte_dmadev_count;
+ rte_dmadev_dump;
+ rte_dmadev_fill;
+ rte_dmadev_get_dev_id;
+ rte_dmadev_info_get;
+ rte_dmadev_is_valid_dev;
+ rte_dmadev_selftest;
+ rte_dmadev_start;
+ rte_dmadev_stats_get;
+ rte_dmadev_stats_reset;
+ rte_dmadev_stop;
+ rte_dmadev_submit;
+ rte_dmadev_vchan_setup;
+
+ local: *;
+};
+
+INTERNAL {
+ global:
+
+ rte_dmadevices;
+ rte_dmadev_get_device_by_name;
+ rte_dmadev_pmd_allocate;
+ rte_dmadev_pmd_release;
+
+ local: *;
+};
+
@@ -60,6 +60,7 @@ libraries = [
'bpf',
'graph',
'node',
+ 'dmadev',
]
if is_windows