[dpdk-dev] compressdev: implement API
Checks
Commit Message
With the vast amounts of data being transported around networks
and stored in storage systems, reducing data size is becoming ever
more important.
There are both software libraries and hardware devices available
that provide compression, but no common API.
This API is proposed in this commit, which supports the following features:
- Deflate Algorithm (https://tools.ietf.org/html/rfc1951)
- LZS algorithm (https://tools.ietf.org/html/rfc2395)
- Static and Dynamic Huffman encoding.
- Compression levels
- Checksum generation
- Asynchronous burst API
- Session-based (a session contains immutable data only and is useable across devices)
- stream-based to maintain state and history data for stateful flows.
Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
Changes since RFCv3:
- Added missing dependencies for shared lib
- Used SPDX license header
- Used dynamic logging
- Changed window size type to uint32_t
- Removed some unnecessary API
- Replaced phys_addr_t with rte_iova_t
- Resolved checkpatch issues
- Resolved Doxygen issues
- Added default mx nb qps for virtual pmds
- Removed unnecessary extern keywords
- Resolved enqueue/dequeued prototype issue
- Resolved capabilities issues
- Completed API documentation
- Added missing API in version.map file
- Added experimental tag
- Added compressdev Doxygen doc in index
- Deleted TODOs
- Deleted unused event callback mechanism
- Clarified flush flag and window size behaviour
- Added capability RTE_COMP_FF_NONCOMPRESSED_BLOCKS
- Simplified xform struct - no need for separate
common and stateful structs
- Renamed device features to COMPDEV to distinguish from
service features
Items to be addressed in v2
- Add hash feature
- Implement stream functions
- Add int rte_comp_stream_create_in_op_priv()
- Add meson build
- Set/clear cache-aligned keyword
- Description of stateless/stateful behaviour in
rte_comp_enqueue_burst function header
- Add capability helper functions
config/common_base | 6 +
doc/api/doxy-api-index.md | 1 +
doc/api/doxy-api.conf | 1 +
lib/Makefile | 3 +
lib/librte_compressdev/Makefile | 29 +
lib/librte_compressdev/rte_comp.h | 503 ++++++++++++
lib/librte_compressdev/rte_compressdev.c | 902 +++++++++++++++++++++
lib/librte_compressdev/rte_compressdev.h | 757 +++++++++++++++++
lib/librte_compressdev/rte_compressdev_pmd.c | 163 ++++
lib/librte_compressdev/rte_compressdev_pmd.h | 439 ++++++++++
lib/librte_compressdev/rte_compressdev_version.map | 47 ++
lib/librte_eal/common/include/rte_log.h | 1 +
mk/rte.app.mk | 1 +
13 files changed, 2853 insertions(+)
create mode 100644 lib/librte_compressdev/Makefile
create mode 100644 lib/librte_compressdev/rte_comp.h
create mode 100644 lib/librte_compressdev/rte_compressdev.c
create mode 100644 lib/librte_compressdev/rte_compressdev.h
create mode 100644 lib/librte_compressdev/rte_compressdev_pmd.c
create mode 100644 lib/librte_compressdev/rte_compressdev_pmd.h
create mode 100644 lib/librte_compressdev/rte_compressdev_version.map
Comments
02/02/2018 19:25, Fiona Trahe:
> - Used dynamic logging
[...]
> --- a/lib/librte_eal/common/include/rte_log.h
> +++ b/lib/librte_eal/common/include/rte_log.h
> @@ -60,6 +60,7 @@ extern struct rte_logs rte_logs;
> #define RTE_LOGTYPE_EFD 18 /**< Log related to EFD. */
> #define RTE_LOGTYPE_EVENTDEV 19 /**< Log related to eventdev. */
> #define RTE_LOGTYPE_GSO 20 /**< Log related to GSO. */
> +#define RTE_LOGTYPE_COMPRESSDEV 21 /**< Log related to compressdev. */
You don't need this define with dynamic logging.
02/02/2018 19:25, Fiona Trahe:
> config/common_base | 6 +
> doc/api/doxy-api-index.md | 1 +
> doc/api/doxy-api.conf | 1 +
> lib/Makefile | 3 +
> lib/librte_compressdev/Makefile | 29 +
> lib/librte_compressdev/rte_comp.h | 503 ++++++++++++
Why rte_comp.h instead of the more consistent rte_compress.h?
> lib/librte_compressdev/rte_compressdev.c | 902 +++++++++++++++++++++
> lib/librte_compressdev/rte_compressdev.h | 757 +++++++++++++++++
> lib/librte_compressdev/rte_compressdev_pmd.c | 163 ++++
> lib/librte_compressdev/rte_compressdev_pmd.h | 439 ++++++++++
> lib/librte_compressdev/rte_compressdev_version.map | 47 ++
> lib/librte_eal/common/include/rte_log.h | 1 +
> mk/rte.app.mk | 1 +
> 13 files changed, 2853 insertions(+)
Please update MAINTAINERS file and release notes.
Maybe it is worth splitting this patch in few shorter parts?
> --- a/config/common_base
> +++ b/config/common_base
> @@ -535,6 +535,12 @@ CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO=n
> CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO_DEBUG=n
>
> #
> +# Compile generic compression device library
> +#
> +CONFIG_RTE_LIBRTE_COMPRESSDEV=y
> +CONFIG_RTE_COMPRESS_MAX_DEVS=64
> +
> +#
> # Compile generic security library
> #
> CONFIG_RTE_LIBRTE_SECURITY=y
> diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
> index d77f205..07b8e75 100644
> --- a/doc/api/doxy-api-index.md
> +++ b/doc/api/doxy-api-index.md
> @@ -43,6 +43,7 @@ The public API headers are grouped by topics:
> [rte_tm] (@ref rte_tm.h),
> [rte_mtr] (@ref rte_mtr.h),
> [bbdev] (@ref rte_bbdev.h),
> + [compressdev] (@ref rte_compressdev.h),
> [cryptodev] (@ref rte_cryptodev.h),
> [security] (@ref rte_security.h),
> [eventdev] (@ref rte_eventdev.h),
Please move it between security and eventdev in these lists.
We've been struggling with the idea of session in compressdev.
Is it really a session?
- It's not in the same sense as cryptodev where it's used to hold a key, and maps to a Security Association.
- It's a set of immutable data that is needed with the op and stream to perform the operation.
- It inherited from cryptodev the ability to be set up for multiple driver types and used across any
devices of those types. For stateful ops this facility can't be used.
For stateless we don't think it's important, and think it's unlikely to be used.
- Drivers use it to prepare private data, set up resources, do pre-work, so there's
less work to be done on the data path. Initially we didn't have a stream, we do now,
this may be a better alternative place for that work.
So we've been toying with the idea of getting rid of the session.
We also struggle with the idea of setting up a stream for stateless ops.
- Well, really I just think the name is misleading, i.e. there's no problem with setting
up some private PMD data to use with stateless operations, just calling it a
stream doesn't seem right.
So putting above thoughts together I want to propose:
- Removal of the session and all associated APIs.
- Passing in one of three data types in the rte_comp_op
union {
struct rte_comp_xform *xform;
/**< Immutable compress/decompress params */
void *pmd_stateless_data;
/**< Stateless private PMD data derived from an rte_comp_xform
* rte_comp_stateless_data_init() must be called on a device
* before sending any STATELESS operations. If the PMD returns a non-NULL
* value the handle must be attached to subsequent STATELESS operations.
* If a PMD returns NULL, then the xform should be passed directly to each op
*/
void *stream;
/* Private PMD data derived initially from an rte_comp_xform, which holds state
* and history data and evolves as operations are processed.
* rte_comp_stream_create() must be called on a device for all STATEFUL
* data streams and the resulting stream attached
* to the one or more operations associated with the data stream.
* All operations in a stream must be sent to the same device.
*/
}
Notes:
1. Internally if a PMD wants to use the exact same data structure for both it can do,
just on the API I think it's better if they're named differently with
different comments.
2. I'm not clear of the constraints if any, which attach to the pmd_stateless_data
For our PMD it would only hold immutable data as the session did, and so
could be attached to many ops in parallel.
Is this true for all PMDs or are there constraints which should be called out?
Is it limited to a specific device, qp, or to be used on one op at a time?
3. Am open to other naming suggestions, just trying to capture the essence
of these data structs better than our current API does.
We would put some more helper fns and structure around the above code if people
are in agreement, just want to see if the concept flies before going further?
Fiona
Hi Fiona,
Thanks for starting this discussion. In the current API the user must
make 12 API calls just to get information to compress. Maybe there is a
way to simplify. At least for some use cases (stateless). I think a call
sometime next week would be good to help clarify coalesce some of the
complexity.
I added specific comments inline.
Thanks,
Ahmed
On 2/21/2018 2:12 PM, Trahe, Fiona wrote:
> We've been struggling with the idea of session in compressdev.
>
> Is it really a session?
> - It's not in the same sense as cryptodev where it's used to hold a key, and maps to a Security Association.
> - It's a set of immutable data that is needed with the op and stream to perform the operation.
> - It inherited from cryptodev the ability to be set up for multiple driver types and used across any
> devices of those types. For stateful ops this facility can't be used.
> For stateless we don't think it's important, and think it's unlikely to be used.
> - Drivers use it to prepare private data, set up resources, do pre-work, so there's
> less work to be done on the data path. Initially we didn't have a stream, we do now,
> this may be a better alternative place for that work.
> So we've been toying with the idea of getting rid of the session.
[Ahmed] In our proprietary API the stream and session are one. A session
holds many properties like the op-type, instead of having this
information in the op itself. This way we lower the per op setup cost.
This also allows rapid reuse of stateful infrastructure, once a stream
is closed on a stateful session, the next op (stream) on this session
reuses the stateful storage. Obviously if a stream is in "pause mode" on
a session, all following ops that may be unrelated to this
stream/session must also wait until this current stream is closed or
aborted before the infrastructure can be reused.
>
> We also struggle with the idea of setting up a stream for stateless ops.
> - Well, really I just think the name is misleading, i.e. there's no problem with setting
> up some private PMD data to use with stateless operations, just calling it a
> stream doesn't seem right.
[Ahmed] I agree. The op has all the necessary information to process it
in the current API? Both the stream and the op are one time use. We
can't attach multiple similar ops to a single stream/session and rely on
their properties to simplify op setup, so why the hassle.
>
> So putting above thoughts together I want to propose:
> - Removal of the session and all associated APIs.
> - Passing in one of three data types in the rte_comp_op
>
> union {
> struct rte_comp_xform *xform;
> /**< Immutable compress/decompress params */
> void *pmd_stateless_data;
> /**< Stateless private PMD data derived from an rte_comp_xform
> * rte_comp_stateless_data_init() must be called on a device
> * before sending any STATELESS operations. If the PMD returns a non-NULL
> * value the handle must be attached to subsequent STATELESS operations.
> * If a PMD returns NULL, then the xform should be passed directly to each op
> */
> void *stream;
> /* Private PMD data derived initially from an rte_comp_xform, which holds state
> * and history data and evolves as operations are processed.
> * rte_comp_stream_create() must be called on a device for all STATEFUL
> * data streams and the resulting stream attached
> * to the one or more operations associated with the data stream.
> * All operations in a stream must be sent to the same device.
> */
> }
[Ahmed] I like this setup, but I am not sure in what cases the xform
immutable would be used. I understand the other two.
> Notes:
> 1. Internally if a PMD wants to use the exact same data structure for both it can do,
> just on the API I think it's better if they're named differently with
> different comments.
> 2. I'm not clear of the constraints if any, which attach to the pmd_stateless_data
> For our PMD it would only hold immutable data as the session did, and so
> could be attached to many ops in parallel.
> Is this true for all PMDs or are there constraints which should be called out?
> Is it limited to a specific device, qp, or to be used on one op at a time?
> 3. Am open to other naming suggestions, just trying to capture the essence
> of these data structs better than our current API does.
>
> We would put some more helper fns and structure around the above code if people
> are in agreement, just want to see if the concept flies before going further?
>
> Fiona
>
>
>
> -----Original Message-----
> From: Ahmed Mansour [mailto:ahmed.mansour@nxp.com]
> Sent: Saturday, February 24, 2018 1:17 AM
> To: Trahe, Fiona <fiona.trahe@intel.com>; dev@dpdk.org; Shally.Verma@cavium.com
> Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad
> <NarayanaPrasad.Athreya@cavium.com>; Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila
> <Sunila.Sahu@cavium.com>; Challa, Mahipal <Mahipal.Challa@cavium.com>; Jain, Deepak K
> <deepak.k.jain@intel.com>; Hemant Agrawal <hemant.agrawal@nxp.com>; Roy Pledge
> <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
> Subject: Re: [dpdk-dev] [PATCH] compressdev: implement API
>
> Hi Fiona,
>
> Thanks for starting this discussion. In the current API the user must
> make 12 API calls just to get information to compress. Maybe there is a
> way to simplify. At least for some use cases (stateless). I think a call
> sometime next week would be good to help clarify coalesce some of the
> complexity.
[Fiona] Would 10:30 GMT on Wednesday 28th Feb suit?
>
> I added specific comments inline.
>
> Thanks,
>
> Ahmed
>
> On 2/21/2018 2:12 PM, Trahe, Fiona wrote:
> > We've been struggling with the idea of session in compressdev.
> >
> > Is it really a session?
> > - It's not in the same sense as cryptodev where it's used to hold a key, and maps to a Security
> Association.
> > - It's a set of immutable data that is needed with the op and stream to perform the operation.
> > - It inherited from cryptodev the ability to be set up for multiple driver types and used across any
> > devices of those types. For stateful ops this facility can't be used.
> > For stateless we don't think it's important, and think it's unlikely to be used.
> > - Drivers use it to prepare private data, set up resources, do pre-work, so there's
> > less work to be done on the data path. Initially we didn't have a stream, we do now,
> > this may be a better alternative place for that work.
> > So we've been toying with the idea of getting rid of the session.
> [Ahmed] In our proprietary API the stream and session are one. A session
> holds many properties like the op-type, instead of having this
> information in the op itself. This way we lower the per op setup cost.
> This also allows rapid reuse of stateful infrastructure, once a stream
> is closed on a stateful session, the next op (stream) on this session
> reuses the stateful storage. Obviously if a stream is in "pause mode" on
> a session, all following ops that may be unrelated to this
> stream/session must also wait until this current stream is closed or
> aborted before the infrastructure can be reused.
> >
> > We also struggle with the idea of setting up a stream for stateless ops.
> > - Well, really I just think the name is misleading, i.e. there's no problem with setting
> > up some private PMD data to use with stateless operations, just calling it a
> > stream doesn't seem right.
> [Ahmed] I agree. The op has all the necessary information to process it
> in the current API? Both the stream and the op are one time use. We
> can't attach multiple similar ops to a single stream/session and rely on
> their properties to simplify op setup, so why the hassle.
> >
> > So putting above thoughts together I want to propose:
> > - Removal of the session and all associated APIs.
> > - Passing in one of three data types in the rte_comp_op
> >
> > union {
> > struct rte_comp_xform *xform;
> > /**< Immutable compress/decompress params */
> > void *pmd_stateless_data;
> > /**< Stateless private PMD data derived from an rte_comp_xform
> > * rte_comp_stateless_data_init() must be called on a device
> > * before sending any STATELESS operations. If the PMD returns a non-NULL
> > * value the handle must be attached to subsequent STATELESS operations.
> > * If a PMD returns NULL, then the xform should be passed directly to each op
> > */
> > void *stream;
> > /* Private PMD data derived initially from an rte_comp_xform, which holds state
> > * and history data and evolves as operations are processed.
> > * rte_comp_stream_create() must be called on a device for all STATEFUL
> > * data streams and the resulting stream attached
> > * to the one or more operations associated with the data stream.
> > * All operations in a stream must be sent to the same device.
> > */
> > }
> [Ahmed] I like this setup, but I am not sure in what cases the xform
> immutable would be used. I understand the other two.
[Fiona] The xform is there because I don't know yet what limitations may apply to the
pmd_stateless_data. If it has no limitation and once set up once on a device
can be attached simultaneously to any op sent to any qp on that device
then we don't need the xform. But I understood from Shally's earlier request for
setting up a stream on a stateless request that some resources are
allocated, so we may need to document some limitations.
In this case the xform may be a better path for PMDs which don't have the same
limitations.
> > Notes:
> > 1. Internally if a PMD wants to use the exact same data structure for both it can do,
> > just on the API I think it's better if they're named differently with
> > different comments.
> > 2. I'm not clear of the constraints if any, which attach to the pmd_stateless_data
> > For our PMD it would only hold immutable data as the session did, and so
> > could be attached to many ops in parallel.
> > Is this true for all PMDs or are there constraints which should be called out?
> > Is it limited to a specific device, qp, or to be used on one op at a time?
> > 3. Am open to other naming suggestions, just trying to capture the essence
> > of these data structs better than our current API does.
> >
> > We would put some more helper fns and structure around the above code if people
> > are in agreement, just want to see if the concept flies before going further?
> >
> > Fiona
> >
> >
> >
Hi Fiona, Ahmed
>-----Original Message-----
>From: Ahmed Mansour [mailto:ahmed.mansour@nxp.com]
>Sent: 24 February 2018 06:47
>To: Trahe, Fiona <fiona.trahe@intel.com>; dev@dpdk.org; Verma, Shally <Shally.Verma@cavium.com>
>Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad <NarayanaPrasad.Athreya@cavium.com>;
>Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila <Sunila.Sahu@cavium.com>; Challa, Mahipal
><Mahipal.Challa@cavium.com>; Jain, Deepak K <deepak.k.jain@intel.com>; Hemant Agrawal <hemant.agrawal@nxp.com>; Roy
>Pledge <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
>Subject: Re: [dpdk-dev] [PATCH] compressdev: implement API
>
>Hi Fiona,
>
>Thanks for starting this discussion. In the current API the user must
>make 12 API calls just to get information to compress. Maybe there is a
>way to simplify. At least for some use cases (stateless). I think a call
>sometime next week would be good to help clarify coalesce some of the
>complexity.
>
>I added specific comments inline.
>
>Thanks,
>
>Ahmed
>
>On 2/21/2018 2:12 PM, Trahe, Fiona wrote:
>> We've been struggling with the idea of session in compressdev.
>>
>> Is it really a session?
>> - It's not in the same sense as cryptodev where it's used to hold a key, and maps to a Security Association.
>> - It's a set of immutable data that is needed with the op and stream to perform the operation.
>> - It inherited from cryptodev the ability to be set up for multiple driver types and used across any
>> devices of those types. For stateful ops this facility can't be used.
>> For stateless we don't think it's important, and think it's unlikely to be used.
>> - Drivers use it to prepare private data, set up resources, do pre-work, so there's
>> less work to be done on the data path. Initially we didn't have a stream, we do now,
>> this may be a better alternative place for that work.
>> So we've been toying with the idea of getting rid of the session.
>[Ahmed] In our proprietary API the stream and session are one. A session
>holds many properties like the op-type, instead of having this
>information in the op itself. This way we lower the per op setup cost.
>This also allows rapid reuse of stateful infrastructure, once a stream
>is closed on a stateful session, the next op (stream) on this session
>reuses the stateful storage. Obviously if a stream is in "pause mode" on
>a session, all following ops that may be unrelated to this
>stream/session must also wait until this current stream is closed or
>aborted before the infrastructure can be reused.
>>
>> We also struggle with the idea of setting up a stream for stateless ops.
>> - Well, really I just think the name is misleading, i.e. there's no problem with setting
>> up some private PMD data to use with stateless operations, just calling it a
>> stream doesn't seem right.
>[Ahmed] I agree. The op has all the necessary information to process it
>in the current API? Both the stream and the op are one time use. We
>can't attach multiple similar ops to a single stream/session and rely on
>their properties to simplify op setup, so why the hassle.
[Shally] As per my knowledge, session came with idea in DPDK, if system has multiple devices setup to do similar jobs then application can fan out ops to any of them for load-balancing. Though it is not possible for stateful ops but it still can work for stateless. If there's an application which only have stateless ops to process then I see this is still useful feature to support.
In current proposal, stream logically represent data and hold its specific information and session is generic information that can be applied on multiple data. If we want to combine stream and session. Then one way to look at this is:
"let application only allocate and initialize session with rte_comp_xform (and possibly op type) information so that PMD can do one-time setup and allocate enough resources. Once attached to op, cannot be reused until that op is fully processed. So, if app has 16 data elements to process in a burst, it will setup 16 sessions."
This is same as what Ahmed suggested. For a particular load-balancing case suggested above, If application want, can initialize different sessions on multiple devices with same xform so that each is prepared to process ops. Application can then fanout stateless ops to multiple devices for load-balancing but then it would need to keep map of device & a session map.
If this sound feasible, then I too believe we can rather get rid of either and keep one (possibly session but am open with stream as well).
However, regardless of case whether we live with name stream or session, I don't see much deviation from current API spec except description and few modifications/additions as identified.
So, then I see it as:
- A stream(or session whichever name is chosen) can be used with only one-op at-a-time
- It can be re-used when previously attached op is processed
- if it is stream then currently it is allocated from PMD managed pool whereas Sessions are allocated from application created mempool.
In either of case, I would expect to review pool management API
With this in mind, below are few of my comments
>>
>> So putting above thoughts together I want to propose:
>> - Removal of the session and all associated APIs.
>> - Passing in one of three data types in the rte_comp_op
>>
>> union {
>> struct rte_comp_xform *xform;
>> /**< Immutable compress/decompress params */
>> void *pmd_stateless_data;
>> /**< Stateless private PMD data derived from an rte_comp_xform
>> * rte_comp_stateless_data_init() must be called on a device
>> * before sending any STATELESS operations. If the PMD returns a non-NULL
>> * value the handle must be attached to subsequent STATELESS operations.
>> * If a PMD returns NULL, then the xform should be passed directly to each op
>> */
[Shally] It sounds like stateless_data_init() nothing more than a replacement of session_init().
So, this is needed neither if we retain session concept nor if we retain stream concept ( rte_comp_stream_create() with op_type: stateless can serve same purpose).
It should be sufficient to provide either stream (or session) pointer.
>> void *stream;
>> /* Private PMD data derived initially from an rte_comp_xform, which holds state
>> * and history data and evolves as operations are processed.
>> * rte_comp_stream_create() must be called on a device for all STATEFUL
>> * data streams and the resulting stream attached
>> * to the one or more operations associated with the data stream.
>> * All operations in a stream must be sent to the same device.
>> */
>> }
>[Ahmed] I like this setup, but I am not sure in what cases the xform
>immutable would be used. I understand the other two.
[Shally] my understanding is xform will be mapped by PMD to its internally managed stream(or session data structure). And then we can remove STATEFUL reference here and just say stream(or session) it belongs to. However, This condition still apply:
*All operations that belong to same stream must be sent to the same device.*
>> Notes:
>> 1. Internally if a PMD wants to use the exact same data structure for both it can do,
>> just on the API I think it's better if they're named differently with
>> different comments.
>> 2. I'm not clear of the constraints if any, which attach to the pmd_stateless_data
>> For our PMD it would only hold immutable data as the session did, and so
>> could be attached to many ops in parallel.
>> Is this true for all PMDs or are there constraints which should be called out?
>> Is it limited to a specific device, qp, or to be used on one op at a time?
>> 3. Am open to other naming suggestions, just trying to capture the essence
>> of these data structs better than our current API does.
>>
>> We would put some more helper fns and structure around the above code if people
>> are in agreement, just want to see if the concept flies before going further?
>>
>> Fiona
>>
>>
>>
>> Hi Fiona,
>>
>> Thanks for starting this discussion. In the current API the user must
>> make 12 API calls just to get information to compress. Maybe there is a
>> way to simplify. At least for some use cases (stateless). I think a call
>> sometime next week would be good to help clarify coalesce some of the
>> complexity.
> [Fiona] Would 10:30 GMT on Wednesday 28th Feb suit?
[Ahmed] I am in Ottawa Canada so Wednesday 12:00 or 1:00 GMT would be
better. Does that work?
>> I added specific comments inline.
>>
>> Thanks,
>>
>> Ahmed
>>
>> On 2/21/2018 2:12 PM, Trahe, Fiona wrote:
>>> We've been struggling with the idea of session in compressdev.
>>>
>>> Is it really a session?
>>> - It's not in the same sense as cryptodev where it's used to hold a key, and maps to a Security
>> Association.
>>> - It's a set of immutable data that is needed with the op and stream to perform the operation.
>>> - It inherited from cryptodev the ability to be set up for multiple driver types and used across any
>>> devices of those types. For stateful ops this facility can't be used.
>>> For stateless we don't think it's important, and think it's unlikely to be used.
>>> - Drivers use it to prepare private data, set up resources, do pre-work, so there's
>>> less work to be done on the data path. Initially we didn't have a stream, we do now,
>>> this may be a better alternative place for that work.
>>> So we've been toying with the idea of getting rid of the session.
>> [Ahmed] In our proprietary API the stream and session are one. A session
>> holds many properties like the op-type, instead of having this
>> information in the op itself. This way we lower the per op setup cost.
>> This also allows rapid reuse of stateful infrastructure, once a stream
>> is closed on a stateful session, the next op (stream) on this session
>> reuses the stateful storage. Obviously if a stream is in "pause mode" on
>> a session, all following ops that may be unrelated to this
>> stream/session must also wait until this current stream is closed or
>> aborted before the infrastructure can be reused.
>>> We also struggle with the idea of setting up a stream for stateless ops.
>>> - Well, really I just think the name is misleading, i.e. there's no problem with setting
>>> up some private PMD data to use with stateless operations, just calling it a
>>> stream doesn't seem right.
>> [Ahmed] I agree. The op has all the necessary information to process it
>> in the current API? Both the stream and the op are one time use. We
>> can't attach multiple similar ops to a single stream/session and rely on
>> their properties to simplify op setup, so why the hassle.
>>> So putting above thoughts together I want to propose:
>>> - Removal of the session and all associated APIs.
>>> - Passing in one of three data types in the rte_comp_op
>>>
>>> union {
>>> struct rte_comp_xform *xform;
>>> /**< Immutable compress/decompress params */
>>> void *pmd_stateless_data;
>>> /**< Stateless private PMD data derived from an rte_comp_xform
>>> * rte_comp_stateless_data_init() must be called on a device
>>> * before sending any STATELESS operations. If the PMD returns a non-NULL
>>> * value the handle must be attached to subsequent STATELESS operations.
>>> * If a PMD returns NULL, then the xform should be passed directly to each op
>>> */
>>> void *stream;
>>> /* Private PMD data derived initially from an rte_comp_xform, which holds state
>>> * and history data and evolves as operations are processed.
>>> * rte_comp_stream_create() must be called on a device for all STATEFUL
>>> * data streams and the resulting stream attached
>>> * to the one or more operations associated with the data stream.
>>> * All operations in a stream must be sent to the same device.
>>> */
>>> }
>> [Ahmed] I like this setup, but I am not sure in what cases the xform
>> immutable would be used. I understand the other two.
> [Fiona] The xform is there because I don't know yet what limitations may apply to the
> pmd_stateless_data. If it has no limitation and once set up once on a device
> can be attached simultaneously to any op sent to any qp on that device
> then we don't need the xform. But I understood from Shally's earlier request for
> setting up a stream on a stateless request that some resources are
> allocated, so we may need to document some limitations.
> In this case the xform may be a better path for PMDs which don't have the same
> limitations.
>
>
>>> Notes:
>>> 1. Internally if a PMD wants to use the exact same data structure for both it can do,
>>> just on the API I think it's better if they're named differently with
>>> different comments.
>>> 2. I'm not clear of the constraints if any, which attach to the pmd_stateless_data
>>> For our PMD it would only hold immutable data as the session did, and so
>>> could be attached to many ops in parallel.
>>> Is this true for all PMDs or are there constraints which should be called out?
>>> Is it limited to a specific device, qp, or to be used on one op at a time?
>>> 3. Am open to other naming suggestions, just trying to capture the essence
>>> of these data structs better than our current API does.
>>>
>>> We would put some more helper fns and structure around the above code if people
>>> are in agreement, just want to see if the concept flies before going further?
>>>
>>> Fiona
>>>
>>>
>>>
>
> Hi Fiona, Ahmed
>> Hi Fiona,
>>
>> Thanks for starting this discussion. In the current API the user must
>> make 12 API calls just to get information to compress. Maybe there is a
>> way to simplify. At least for some use cases (stateless). I think a call
>> sometime next week would be good to help clarify coalesce some of the
>> complexity.
>>
>> I added specific comments inline.
>>
>> Thanks,
>>
>> Ahmed
>>
>> On 2/21/2018 2:12 PM, Trahe, Fiona wrote:
>>> We've been struggling with the idea of session in compressdev.
>>>
>>> Is it really a session?
>>> - It's not in the same sense as cryptodev where it's used to hold a key, and maps to a Security Association.
>>> - It's a set of immutable data that is needed with the op and stream to perform the operation.
>>> - It inherited from cryptodev the ability to be set up for multiple driver types and used across any
>>> devices of those types. For stateful ops this facility can't be used.
>>> For stateless we don't think it's important, and think it's unlikely to be used.
>>> - Drivers use it to prepare private data, set up resources, do pre-work, so there's
>>> less work to be done on the data path. Initially we didn't have a stream, we do now,
>>> this may be a better alternative place for that work.
>>> So we've been toying with the idea of getting rid of the session.
>> [Ahmed] In our proprietary API the stream and session are one. A session
>> holds many properties like the op-type, instead of having this
>> information in the op itself. This way we lower the per op setup cost.
>> This also allows rapid reuse of stateful infrastructure, once a stream
>> is closed on a stateful session, the next op (stream) on this session
>> reuses the stateful storage. Obviously if a stream is in "pause mode" on
>> a session, all following ops that may be unrelated to this
>> stream/session must also wait until this current stream is closed or
>> aborted before the infrastructure can be reused.
>>> We also struggle with the idea of setting up a stream for stateless ops.
>>> - Well, really I just think the name is misleading, i.e. there's no problem with setting
>>> up some private PMD data to use with stateless operations, just calling it a
>>> stream doesn't seem right.
>> [Ahmed] I agree. The op has all the necessary information to process it
>> in the current API? Both the stream and the op are one time use. We
>> can't attach multiple similar ops to a single stream/session and rely on
>> their properties to simplify op setup, so why the hassle.
> [Shally] As per my knowledge, session came with idea in DPDK, if system has multiple devices setup to do similar jobs then application can fan out ops to any of them for load-balancing. Though it is not possible for stateful ops but it still can work for stateless. If there's an application which only have stateless ops to process then I see this is still useful feature to support.
[Ahmed] Is there an advantage to exposing load balancing to the user? I
do not see load balancing as a feature within itself. Can the PMD take
care of this? I guess a system that has
> In current proposal, stream logically represent data and hold its specific information and session is generic information that can be applied on multiple data. If we want to combine stream and session. Then one way to look at this is:
>
> "let application only allocate and initialize session with rte_comp_xform (and possibly op type) information so that PMD can do one-time setup and allocate enough resources. Once attached to op, cannot be reused until that op is fully processed. So, if app has 16 data elements to process in a burst, it will setup 16 sessions."
[Ahmed] Why not allow multiple inflight stateless ops with the same
session? Stateless by definition guarantees that the resources used to
work on one up will be free after the op is processed. That means that
even if an op fails to process correctly on a session, it will have no
effect on the next op since there is not interdependence. This assumes
that the resources are shareable between hardware instances for
stateless. That is not a bad assumption since hardware should not need
more than the data of the op itself to work on a statelss op.
> This is same as what Ahmed suggested. For a particular load-balancing case suggested above, If application want, can initialize different sessions on multiple devices with same xform so that each is prepared to process ops. Application can then fanout stateless ops to multiple devices for load-balancing but then it would need to keep map of device & a session map.
>
> If this sound feasible, then I too believe we can rather get rid of either and keep one (possibly session but am open with stream as well).
> However, regardless of case whether we live with name stream or session, I don't see much deviation from current API spec except description and few modifications/additions as identified.
> So, then I see it as:
>
> - A stream(or session whichever name is chosen) can be used with only one-op at-a-time
> - It can be re-used when previously attached op is processed
> - if it is stream then currently it is allocated from PMD managed pool whereas Sessions are allocated from application created mempool.
> In either of case, I would expect to review pool management API
>
> With this in mind, below are few of my comments
>
>>> So putting above thoughts together I want to propose:
>>> - Removal of the session and all associated APIs.
>>> - Passing in one of three data types in the rte_comp_op
>>>
>>> union {
>>> struct rte_comp_xform *xform;
>>> /**< Immutable compress/decompress params */
>>> void *pmd_stateless_data;
>>> /**< Stateless private PMD data derived from an rte_comp_xform
>>> * rte_comp_stateless_data_init() must be called on a device
>>> * before sending any STATELESS operations. If the PMD returns a non-NULL
>>> * value the handle must be attached to subsequent STATELESS operations.
>>> * If a PMD returns NULL, then the xform should be passed directly to each op
>>> */
> [Shally] It sounds like stateless_data_init() nothing more than a replacement of session_init().
> So, this is needed neither if we retain session concept nor if we retain stream concept ( rte_comp_stream_create() with op_type: stateless can serve same purpose).
> It should be sufficient to provide either stream (or session) pointer.
>
>>> void *stream;
>>> /* Private PMD data derived initially from an rte_comp_xform, which holds state
>>> * and history data and evolves as operations are processed.
>>> * rte_comp_stream_create() must be called on a device for all STATEFUL
>>> * data streams and the resulting stream attached
>>> * to the one or more operations associated with the data stream.
>>> * All operations in a stream must be sent to the same device.
>>> */
>>> }
>> [Ahmed] I like this setup, but I am not sure in what cases the xform
>> immutable would be used. I understand the other two.
> [Shally] my understanding is xform will be mapped by PMD to its internally managed stream(or session data structure). And then we can remove STATEFUL reference here and just say stream(or session) it belongs to. However, This condition still apply:
> *All operations that belong to same stream must be sent to the same device.*
>
>>> Notes:
>>> 1. Internally if a PMD wants to use the exact same data structure for both it can do,
>>> just on the API I think it's better if they're named differently with
>>> different comments.
>>> 2. I'm not clear of the constraints if any, which attach to the pmd_stateless_data
>>> For our PMD it would only hold immutable data as the session did, and so
>>> could be attached to many ops in parallel.
>>> Is this true for all PMDs or are there constraints which should be called out?
>>> Is it limited to a specific device, qp, or to be used on one op at a time?
>>> 3. Am open to other naming suggestions, just trying to capture the essence
>>> of these data structs better than our current API does.
>>>
>>> We would put some more helper fns and structure around the above code if people
>>> are in agreement, just want to see if the concept flies before going further?
>>>
>>> Fiona
>>>
>>>
>>>
>
>-----Original Message-----
>From: Ahmed Mansour [mailto:ahmed.mansour@nxp.com]
>Sent: 27 February 2018 00:46
>To: Trahe, Fiona <fiona.trahe@intel.com>; dev@dpdk.org; Verma, Shally <Shally.Verma@cavium.com>
>Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad <NarayanaPrasad.Athreya@cavium.com>;
>Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila <Sunila.Sahu@cavium.com>; Challa, Mahipal
><Mahipal.Challa@cavium.com>; Jain, Deepak K <deepak.k.jain@intel.com>; Hemant Agrawal <hemant.agrawal@nxp.com>; Roy
>Pledge <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
>Subject: Re: [dpdk-dev] [PATCH] compressdev: implement API
>
>>> Hi Fiona,
>>>
>>> Thanks for starting this discussion. In the current API the user must
>>> make 12 API calls just to get information to compress. Maybe there is a
>>> way to simplify. At least for some use cases (stateless). I think a call
>>> sometime next week would be good to help clarify coalesce some of the
>>> complexity.
>> [Fiona] Would 10:30 GMT on Wednesday 28th Feb suit?
>[Ahmed] I am in Ottawa Canada so Wednesday 12:00 or 1:00 GMT would be
>better. Does that work?
[Shally] Am in India time zone. Either works for me.
//snip
>-----Original Message-----
>From: Ahmed Mansour [mailto:ahmed.mansour@nxp.com]
>Sent: 27 February 2018 03:05
>To: Verma, Shally <Shally.Verma@cavium.com>; Trahe, Fiona <fiona.trahe@intel.com>; dev@dpdk.org
>Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad <NarayanaPrasad.Athreya@cavium.com>;
>Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila <Sunila.Sahu@cavium.com>; Challa, Mahipal
><Mahipal.Challa@cavium.com>; Jain, Deepak K <deepak.k.jain@intel.com>; Hemant Agrawal <hemant.agrawal@nxp.com>; Roy
>Pledge <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
>Subject: Re: [dpdk-dev] [PATCH] compressdev: implement API
>
>> Hi Fiona, Ahmed
>>> Hi Fiona,
>>>
>>> Thanks for starting this discussion. In the current API the user must
>>> make 12 API calls just to get information to compress. Maybe there is a
>>> way to simplify. At least for some use cases (stateless). I think a call
>>> sometime next week would be good to help clarify coalesce some of the
>>> complexity.
>>>
>>> I added specific comments inline.
>>>
>>> Thanks,
>>>
>>> Ahmed
>>>
>>> On 2/21/2018 2:12 PM, Trahe, Fiona wrote:
>>>> We've been struggling with the idea of session in compressdev.
>>>>
>>>> Is it really a session?
>>>> - It's not in the same sense as cryptodev where it's used to hold a key, and maps to a Security Association.
>>>> - It's a set of immutable data that is needed with the op and stream to perform the operation.
>>>> - It inherited from cryptodev the ability to be set up for multiple driver types and used across any
>>>> devices of those types. For stateful ops this facility can't be used.
>>>> For stateless we don't think it's important, and think it's unlikely to be used.
>>>> - Drivers use it to prepare private data, set up resources, do pre-work, so there's
>>>> less work to be done on the data path. Initially we didn't have a stream, we do now,
>>>> this may be a better alternative place for that work.
>>>> So we've been toying with the idea of getting rid of the session.
>>> [Ahmed] In our proprietary API the stream and session are one. A session
>>> holds many properties like the op-type, instead of having this
>>> information in the op itself. This way we lower the per op setup cost.
>>> This also allows rapid reuse of stateful infrastructure, once a stream
>>> is closed on a stateful session, the next op (stream) on this session
>>> reuses the stateful storage. Obviously if a stream is in "pause mode" on
>>> a session, all following ops that may be unrelated to this
>>> stream/session must also wait until this current stream is closed or
>>> aborted before the infrastructure can be reused.
>>>> We also struggle with the idea of setting up a stream for stateless ops.
>>>> - Well, really I just think the name is misleading, i.e. there's no problem with setting
>>>> up some private PMD data to use with stateless operations, just calling it a
>>>> stream doesn't seem right.
>>> [Ahmed] I agree. The op has all the necessary information to process it
>>> in the current API? Both the stream and the op are one time use. We
>>> can't attach multiple similar ops to a single stream/session and rely on
>>> their properties to simplify op setup, so why the hassle.
>> [Shally] As per my knowledge, session came with idea in DPDK, if system has multiple devices setup to do similar jobs then
>application can fan out ops to any of them for load-balancing. Though it is not possible for stateful ops but it still can work for stateless.
>If there's an application which only have stateless ops to process then I see this is still useful feature to support.
>[Ahmed] Is there an advantage to exposing load balancing to the user? I
>do not see load balancing as a feature within itself. Can the PMD take
>care of this? I guess a system that has
[Shally] I assume idea was to leverage multiple PMDs that are available in system (say QAT+SW ZLIB) and I believe matter of load-balancing came out of one of the earlier discussion with Fiona on RFC v1. http://dev.dpdk.narkive.com/CHS5l01B/dpdk-dev-rfc-v1-doc-compression-api-for-dpdk#post3
So, I wait for her comments on this. But in any case, with changed notion too it looks achievable to me, if so is desired.
>> In current proposal, stream logically represent data and hold its specific information and session is generic information that can be
>applied on multiple data. If we want to combine stream and session. Then one way to look at this is:
>>
>> "let application only allocate and initialize session with rte_comp_xform (and possibly op type) information so that PMD can do one-
>time setup and allocate enough resources. Once attached to op, cannot be reused until that op is fully processed. So, if app has 16
>data elements to process in a burst, it will setup 16 sessions."
>[Ahmed] Why not allow multiple inflight stateless ops with the same
>session? Stateless by definition guarantees that the resources used to
>work on one up will be free after the op is processed. That means that
>even if an op fails to process correctly on a session, it will have no
>effect on the next op since there is not interdependence. This assumes
>that the resources are shareable between hardware instances for
>stateless. That is not a bad assumption since hardware should not need
>more than the data of the op itself to work on a statelss op.
[Shally] multiple ops in-flight can connect to same session but I assume you agree then they cannot execute in parallel i.e. only one op at-a-time can use session here? And as far as I understand your PMD works this way. Your HW execute one op at-a-time from queue?!
>> This is same as what Ahmed suggested. For a particular load-balancing case suggested above, If application want, can initialize
>different sessions on multiple devices with same xform so that each is prepared to process ops. Application can then fanout stateless
>ops to multiple devices for load-balancing but then it would need to keep map of device & a session map.
>>
>> If this sound feasible, then I too believe we can rather get rid of either and keep one (possibly session but am open with stream as
>well).
>> However, regardless of case whether we live with name stream or session, I don't see much deviation from current API spec except
>description and few modifications/additions as identified.
>> So, then I see it as:
>>
>> - A stream(or session whichever name is chosen) can be used with only one-op at-a-time
>> - It can be re-used when previously attached op is processed
>> - if it is stream then currently it is allocated from PMD managed pool whereas Sessions are allocated from application created
>mempool.
>> In either of case, I would expect to review pool management API
>>
>> With this in mind, below are few of my comments
>>
>>>> So putting above thoughts together I want to propose:
>>>> - Removal of the session and all associated APIs.
>>>> - Passing in one of three data types in the rte_comp_op
>>>>
>>>> union {
>>>> struct rte_comp_xform *xform;
>>>> /**< Immutable compress/decompress params */
>>>> void *pmd_stateless_data;
>>>> /**< Stateless private PMD data derived from an rte_comp_xform
>>>> * rte_comp_stateless_data_init() must be called on a device
>>>> * before sending any STATELESS operations. If the PMD returns a non-NULL
>>>> * value the handle must be attached to subsequent STATELESS operations.
>>>> * If a PMD returns NULL, then the xform should be passed directly to each op
>>>> */
>> [Shally] It sounds like stateless_data_init() nothing more than a replacement of session_init().
>> So, this is needed neither if we retain session concept nor if we retain stream concept ( rte_comp_stream_create() with
>op_type: stateless can serve same purpose).
>> It should be sufficient to provide either stream (or session) pointer.
>>
>>>> void *stream;
>>>> /* Private PMD data derived initially from an rte_comp_xform, which holds state
>>>> * and history data and evolves as operations are processed.
>>>> * rte_comp_stream_create() must be called on a device for all STATEFUL
>>>> * data streams and the resulting stream attached
>>>> * to the one or more operations associated with the data stream.
>>>> * All operations in a stream must be sent to the same device.
>>>> */
>>>> }
>>> [Ahmed] I like this setup, but I am not sure in what cases the xform
>>> immutable would be used. I understand the other two.
>> [Shally] my understanding is xform will be mapped by PMD to its internally managed stream(or session data structure). And then we
>can remove STATEFUL reference here and just say stream(or session) it belongs to. However, This condition still apply:
>> *All operations that belong to same stream must be sent to the same device.*
>>
>>>> Notes:
>>>> 1. Internally if a PMD wants to use the exact same data structure for both it can do,
>>>> just on the API I think it's better if they're named differently with
>>>> different comments.
>>>> 2. I'm not clear of the constraints if any, which attach to the pmd_stateless_data
>>>> For our PMD it would only hold immutable data as the session did, and so
>>>> could be attached to many ops in parallel.
>>>> Is this true for all PMDs or are there constraints which should be called out?
>>>> Is it limited to a specific device, qp, or to be used on one op at a time?
>>>> 3. Am open to other naming suggestions, just trying to capture the essence
>>>> of these data structs better than our current API does.
>>>>
>>>> We would put some more helper fns and structure around the above code if people
>>>> are in agreement, just want to see if the concept flies before going further?
>>>>
>>>> Fiona
>>>>
>>>>
>>>>
>>
Hi Ahmed, Shally,
So just to capture what we concluded in the call today:
- There's no requirement for a device-agnostic session to facilitate load-balancing.
- For stateful data a stream is compulsory. Xform is passed to stream on creation.
So no need for a session in stateful op.
Re session data for stateless ops:
- All PMDs could cope with just passing in a xform with a stateless op. But it might
not be performant.
- Some PMDs need to allocate some resources which can only be used by one op
at a time. For stateful ops these resources can be attached to a stream. For stateless
they could allocate the resources on each op enqueue, but it would be better if
the resources were setup once based on the xform and could be re-used on ops,
though only by one op at a time.
- Some PMDs don't need to allocate such resources, but could benefit by
setting up some pmd data based on the xform. This data would not be
constrained, could be used in parallel by any op or qp of the device.
- The name pmd_stateless_data was not popular, maybe something like
xform_private_data can be used. On creation of this data, the PMD can return
an indication of whether it should be used by one op at a time or shared.
So I'll
- remove the session completely from the API.
- add an initialiser API for the data to be attached to stateless ops
- add a union to the op:
union {
void *pmd_private_xform;
/**< Stateless private PMD data derived from an rte_comp_xform
* rte_comp_xform_init() must be called on a device
* before sending any STATELESS operations. The PMD returns a handle
* which must be attached to subsequent STATELESS operations.
* The PMD also returns a flag, if this is COMP_PRIVATE_XFORM_SHAREABLE
* then the xform can be attached to multiple ops at the same time,
* if it's COMP_PRIVATE_XFORM_SINGLE_OP then it can only be
* be used on one op at a time, other private xforms must be initialised
* to send other ops in parallel.
*/
void *stream;
/* Private PMD data derived initially from an rte_comp_xform, which holds state
* and history data and evolves as operations are processed.
* rte_comp_stream_create() must be called on a device for all STATEFUL
* data streams and the resulting stream attached
* to the one or more operations associated with the data stream.
* All operations in a stream must be sent to the same device.
*/
}
Previous startup flow before sending a stateful op:
rte_comp_get_private_size(devid)
rte_comp_mempool_create() - returns sess_pool
rte_comp_session_create(sess_pool)
rte_comp_session_init(devid, sess, sess_pool, xform)
rte_comp_stream_create(devid, sess, **stream, op_type)
simplified to:
rte_comp_xform_init(devid, xform, **priv_xform, *flag) - returns handle and flag
(pool is within the PMD)
Note, I don't think we bottomed out on removing the xform from the union, but I don't
think we need it with above solution.
Other discussion:
- we should document on API that qp is not thread-safe, so enqueue
and dequeue should be performed by same thread.
device and qp flow:
- dev_info_get() - application reads device capabilities, including the max qps the device can support.
- dev_config() - application specifies how many qps it intends to use - typically one per thread, must be < device max
- qp_setup() - called per qp. Creates the qp based on the size indicated by max_inflights
- dev_start() - once started device can't be reconfigured, must call dev_stop to reconfigure.
Regards,
Fiona
> -----Original Message-----
> From: Verma, Shally [mailto:Shally.Verma@cavium.com]
> Sent: Tuesday, February 27, 2018 5:54 AM
> To: Ahmed Mansour <ahmed.mansour@nxp.com>; Trahe, Fiona <fiona.trahe@intel.com>;
> dev@dpdk.org
> Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad
> <NarayanaPrasad.Athreya@cavium.com>; Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila
> <Sunila.Sahu@cavium.com>; Challa, Mahipal <Mahipal.Challa@cavium.com>; Jain, Deepak K
> <deepak.k.jain@intel.com>; Hemant Agrawal <hemant.agrawal@nxp.com>; Roy Pledge
> <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
> Subject: RE: [dpdk-dev] [PATCH] compressdev: implement API
>
>
>
> >-----Original Message-----
> >From: Ahmed Mansour [mailto:ahmed.mansour@nxp.com]
> >Sent: 27 February 2018 03:05
> >To: Verma, Shally <Shally.Verma@cavium.com>; Trahe, Fiona <fiona.trahe@intel.com>; dev@dpdk.org
> >Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad
> <NarayanaPrasad.Athreya@cavium.com>;
> >Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila <Sunila.Sahu@cavium.com>; Challa,
> Mahipal
> ><Mahipal.Challa@cavium.com>; Jain, Deepak K <deepak.k.jain@intel.com>; Hemant Agrawal
> <hemant.agrawal@nxp.com>; Roy
> >Pledge <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
> >Subject: Re: [dpdk-dev] [PATCH] compressdev: implement API
> >
> >> Hi Fiona, Ahmed
> >>> Hi Fiona,
> >>>
> >>> Thanks for starting this discussion. In the current API the user must
> >>> make 12 API calls just to get information to compress. Maybe there is a
> >>> way to simplify. At least for some use cases (stateless). I think a call
> >>> sometime next week would be good to help clarify coalesce some of the
> >>> complexity.
> >>>
> >>> I added specific comments inline.
> >>>
> >>> Thanks,
> >>>
> >>> Ahmed
> >>>
> >>> On 2/21/2018 2:12 PM, Trahe, Fiona wrote:
> >>>> We've been struggling with the idea of session in compressdev.
> >>>>
> >>>> Is it really a session?
> >>>> - It's not in the same sense as cryptodev where it's used to hold a key, and maps to a Security
> Association.
> >>>> - It's a set of immutable data that is needed with the op and stream to perform the operation.
> >>>> - It inherited from cryptodev the ability to be set up for multiple driver types and used across any
> >>>> devices of those types. For stateful ops this facility can't be used.
> >>>> For stateless we don't think it's important, and think it's unlikely to be used.
> >>>> - Drivers use it to prepare private data, set up resources, do pre-work, so there's
> >>>> less work to be done on the data path. Initially we didn't have a stream, we do now,
> >>>> this may be a better alternative place for that work.
> >>>> So we've been toying with the idea of getting rid of the session.
> >>> [Ahmed] In our proprietary API the stream and session are one. A session
> >>> holds many properties like the op-type, instead of having this
> >>> information in the op itself. This way we lower the per op setup cost.
> >>> This also allows rapid reuse of stateful infrastructure, once a stream
> >>> is closed on a stateful session, the next op (stream) on this session
> >>> reuses the stateful storage. Obviously if a stream is in "pause mode" on
> >>> a session, all following ops that may be unrelated to this
> >>> stream/session must also wait until this current stream is closed or
> >>> aborted before the infrastructure can be reused.
> >>>> We also struggle with the idea of setting up a stream for stateless ops.
> >>>> - Well, really I just think the name is misleading, i.e. there's no problem with setting
> >>>> up some private PMD data to use with stateless operations, just calling it a
> >>>> stream doesn't seem right.
> >>> [Ahmed] I agree. The op has all the necessary information to process it
> >>> in the current API? Both the stream and the op are one time use. We
> >>> can't attach multiple similar ops to a single stream/session and rely on
> >>> their properties to simplify op setup, so why the hassle.
> >> [Shally] As per my knowledge, session came with idea in DPDK, if system has multiple devices setup
> to do similar jobs then
> >application can fan out ops to any of them for load-balancing. Though it is not possible for stateful ops
> but it still can work for stateless.
> >If there's an application which only have stateless ops to process then I see this is still useful feature to
> support.
> >[Ahmed] Is there an advantage to exposing load balancing to the user? I
> >do not see load balancing as a feature within itself. Can the PMD take
> >care of this? I guess a system that has
>
> [Shally] I assume idea was to leverage multiple PMDs that are available in system (say QAT+SW ZLIB)
> and I believe matter of load-balancing came out of one of the earlier discussion with Fiona on RFC v1.
> http://dev.dpdk.narkive.com/CHS5l01B/dpdk-dev-rfc-v1-doc-compression-api-for-dpdk#post3
> So, I wait for her comments on this. But in any case, with changed notion too it looks achievable to me,
> if so is desired.
>
> >> In current proposal, stream logically represent data and hold its specific information and session is
> generic information that can be
> >applied on multiple data. If we want to combine stream and session. Then one way to look at this is:
> >>
> >> "let application only allocate and initialize session with rte_comp_xform (and possibly op type)
> information so that PMD can do one-
> >time setup and allocate enough resources. Once attached to op, cannot be reused until that op is fully
> processed. So, if app has 16
> >data elements to process in a burst, it will setup 16 sessions."
> >[Ahmed] Why not allow multiple inflight stateless ops with the same
> >session? Stateless by definition guarantees that the resources used to
> >work on one up will be free after the op is processed. That means that
> >even if an op fails to process correctly on a session, it will have no
> >effect on the next op since there is not interdependence. This assumes
> >that the resources are shareable between hardware instances for
> >stateless. That is not a bad assumption since hardware should not need
> >more than the data of the op itself to work on a statelss op.
>
> [Shally] multiple ops in-flight can connect to same session but I assume you agree then they cannot
> execute in parallel i.e. only one op at-a-time can use session here? And as far as I understand your PMD
> works this way. Your HW execute one op at-a-time from queue?!
>
> >> This is same as what Ahmed suggested. For a particular load-balancing case suggested above, If
> application want, can initialize
> >different sessions on multiple devices with same xform so that each is prepared to process ops.
> Application can then fanout stateless
> >ops to multiple devices for load-balancing but then it would need to keep map of device & a session
> map.
> >>
> >> If this sound feasible, then I too believe we can rather get rid of either and keep one (possibly session
> but am open with stream as
> >well).
> >> However, regardless of case whether we live with name stream or session, I don't see much deviation
> from current API spec except
> >description and few modifications/additions as identified.
> >> So, then I see it as:
> >>
> >> - A stream(or session whichever name is chosen) can be used with only one-op at-a-time
> >> - It can be re-used when previously attached op is processed
> >> - if it is stream then currently it is allocated from PMD managed pool whereas Sessions are allocated
> from application created
> >mempool.
> >> In either of case, I would expect to review pool management API
> >>
> >> With this in mind, below are few of my comments
> >>
> >>>> So putting above thoughts together I want to propose:
> >>>> - Removal of the session and all associated APIs.
> >>>> - Passing in one of three data types in the rte_comp_op
> >>>>
> >>>> union {
> >>>> struct rte_comp_xform *xform;
> >>>> /**< Immutable compress/decompress params */
> >>>> void *pmd_stateless_data;
> >>>> /**< Stateless private PMD data derived from an rte_comp_xform
> >>>> * rte_comp_stateless_data_init() must be called on a device
> >>>> * before sending any STATELESS operations. If the PMD returns a non-NULL
> >>>> * value the handle must be attached to subsequent STATELESS operations.
> >>>> * If a PMD returns NULL, then the xform should be passed directly to each op
> >>>> */
> >> [Shally] It sounds like stateless_data_init() nothing more than a replacement of session_init().
> >> So, this is needed neither if we retain session concept nor if we retain stream concept (
> rte_comp_stream_create() with
> >op_type: stateless can serve same purpose).
> >> It should be sufficient to provide either stream (or session) pointer.
> >>
> >>>> void *stream;
> >>>> /* Private PMD data derived initially from an rte_comp_xform, which holds state
> >>>> * and history data and evolves as operations are processed.
> >>>> * rte_comp_stream_create() must be called on a device for all STATEFUL
> >>>> * data streams and the resulting stream attached
> >>>> * to the one or more operations associated with the data stream.
> >>>> * All operations in a stream must be sent to the same device.
> >>>> */
> >>>> }
> >>> [Ahmed] I like this setup, but I am not sure in what cases the xform
> >>> immutable would be used. I understand the other two.
> >> [Shally] my understanding is xform will be mapped by PMD to its internally managed stream(or
> session data structure). And then we
> >can remove STATEFUL reference here and just say stream(or session) it belongs to. However, This
> condition still apply:
> >> *All operations that belong to same stream must be sent to the same device.*
> >>
> >>>> Notes:
> >>>> 1. Internally if a PMD wants to use the exact same data structure for both it can do,
> >>>> just on the API I think it's better if they're named differently with
> >>>> different comments.
> >>>> 2. I'm not clear of the constraints if any, which attach to the pmd_stateless_data
> >>>> For our PMD it would only hold immutable data as the session did, and so
> >>>> could be attached to many ops in parallel.
> >>>> Is this true for all PMDs or are there constraints which should be called out?
> >>>> Is it limited to a specific device, qp, or to be used on one op at a time?
> >>>> 3. Am open to other naming suggestions, just trying to capture the essence
> >>>> of these data structs better than our current API does.
> >>>>
> >>>> We would put some more helper fns and structure around the above code if people
> >>>> are in agreement, just want to see if the concept flies before going further?
> >>>>
> >>>> Fiona
> >>>>
> >>>>
> >>>>
> >>
Hi Fiona
>-----Original Message-----
>From: Trahe, Fiona [mailto:fiona.trahe@intel.com]
>Sent: 01 March 2018 00:09
>To: Verma, Shally <Shally.Verma@cavium.com>; Ahmed Mansour <ahmed.mansour@nxp.com>; dev@dpdk.org
>Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad <NarayanaPrasad.Athreya@cavium.com>;
>Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila <Sunila.Sahu@cavium.com>; Challa, Mahipal
><Mahipal.Challa@cavium.com>; Jain, Deepak K <deepak.k.jain@intel.com>; Hemant Agrawal <hemant.agrawal@nxp.com>; Roy
>Pledge <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>; Trahe, Fiona <fiona.trahe@intel.com>
>Subject: RE: [dpdk-dev] [PATCH] compressdev: implement API
>
>Hi Ahmed, Shally,
>
>So just to capture what we concluded in the call today:
>
> - There's no requirement for a device-agnostic session to facilitate load-balancing.
> - For stateful data a stream is compulsory. Xform is passed to stream on creation.
> So no need for a session in stateful op.
>
>Re session data for stateless ops:
> - All PMDs could cope with just passing in a xform with a stateless op. But it might
> not be performant.
> - Some PMDs need to allocate some resources which can only be used by one op
> at a time. For stateful ops these resources can be attached to a stream. For stateless
> they could allocate the resources on each op enqueue, but it would be better if
> the resources were setup once based on the xform and could be re-used on ops,
> though only by one op at a time.
> - Some PMDs don't need to allocate such resources, but could benefit by
> setting up some pmd data based on the xform. This data would not be
> constrained, could be used in parallel by any op or qp of the device.
> - The name pmd_stateless_data was not popular, maybe something like
> xform_private_data can be used. On creation of this data, the PMD can return
> an indication of whether it should be used by one op at a time or shared.
>
>So I'll
> - remove the session completely from the API.
> - add an initialiser API for the data to be attached to stateless ops
> - add a union to the op:
>
> union {
> void *pmd_private_xform;
> /**< Stateless private PMD data derived from an rte_comp_xform
> * rte_comp_xform_init() must be called on a device
> * before sending any STATELESS operations. The PMD returns a handle
> * which must be attached to subsequent STATELESS operations.
> * The PMD also returns a flag, if this is COMP_PRIVATE_XFORM_SHAREABLE
> * then the xform can be attached to multiple ops at the same time,
> * if it's COMP_PRIVATE_XFORM_SINGLE_OP then it can only be
> * be used on one op at a time, other private xforms must be initialised
> * to send other ops in parallel.
> */
> void *stream;
> /* Private PMD data derived initially from an rte_comp_xform, which holds state
> * and history data and evolves as operations are processed.
> * rte_comp_stream_create() must be called on a device for all STATEFUL
> * data streams and the resulting stream attached
> * to the one or more operations associated with the data stream.
> * All operations in a stream must be sent to the same device.
> */
> }
>
>Previous startup flow before sending a stateful op:
>rte_comp_get_private_size(devid)
>rte_comp_mempool_create() - returns sess_pool
>rte_comp_session_create(sess_pool)
>rte_comp_session_init(devid, sess, sess_pool, xform)
>rte_comp_stream_create(devid, sess, **stream, op_type)
>
>simplified to:
>rte_comp_xform_init(devid, xform, **priv_xform, *flag) - returns handle and flag
>(pool is within the PMD)
>
>Note, I don't think we bottomed out on removing the xform from the union, but I don't
>think we need it with above solution.
[Shally] This looks better to me. So it mean app would always call xform_init() for stateless and attach an updated priv_xform to ops (depending upon if there's shareable or not). So it does not need to have NULL pointer on priv_xform. right?
>
>Other discussion:
> - we should document on API that qp is not thread-safe, so enqueue
> and dequeue should be performed by same thread.
>
>device and qp flow:
> - dev_info_get() - application reads device capabilities, including the max qps the device can support.
> - dev_config() - application specifies how many qps it intends to use - typically one per thread, must be < device max
> - qp_setup() - called per qp. Creates the qp based on the size indicated by max_inflights
> - dev_start() - once started device can't be reconfigured, must call dev_stop to reconfigure.
>
>
>Regards,
>Fiona
>
>> -----Original Message-----
>> From: Verma, Shally [mailto:Shally.Verma@cavium.com]
>> Sent: Tuesday, February 27, 2018 5:54 AM
>> To: Ahmed Mansour <ahmed.mansour@nxp.com>; Trahe, Fiona <fiona.trahe@intel.com>;
>> dev@dpdk.org
>> Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad
>> <NarayanaPrasad.Athreya@cavium.com>; Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila
>> <Sunila.Sahu@cavium.com>; Challa, Mahipal <Mahipal.Challa@cavium.com>; Jain, Deepak K
>> <deepak.k.jain@intel.com>; Hemant Agrawal <hemant.agrawal@nxp.com>; Roy Pledge
>> <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
>> Subject: RE: [dpdk-dev] [PATCH] compressdev: implement API
>>
>>
>>
>> >-----Original Message-----
>> >From: Ahmed Mansour [mailto:ahmed.mansour@nxp.com]
>> >Sent: 27 February 2018 03:05
>> >To: Verma, Shally <Shally.Verma@cavium.com>; Trahe, Fiona <fiona.trahe@intel.com>; dev@dpdk.org
>> >Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad
>> <NarayanaPrasad.Athreya@cavium.com>;
>> >Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila <Sunila.Sahu@cavium.com>; Challa,
>> Mahipal
>> ><Mahipal.Challa@cavium.com>; Jain, Deepak K <deepak.k.jain@intel.com>; Hemant Agrawal
>> <hemant.agrawal@nxp.com>; Roy
>> >Pledge <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
>> >Subject: Re: [dpdk-dev] [PATCH] compressdev: implement API
>> >
>> >> Hi Fiona, Ahmed
>> >>> Hi Fiona,
>> >>>
>> >>> Thanks for starting this discussion. In the current API the user must
>> >>> make 12 API calls just to get information to compress. Maybe there is a
>> >>> way to simplify. At least for some use cases (stateless). I think a call
>> >>> sometime next week would be good to help clarify coalesce some of the
>> >>> complexity.
>> >>>
>> >>> I added specific comments inline.
>> >>>
>> >>> Thanks,
>> >>>
>> >>> Ahmed
>> >>>
>> >>> On 2/21/2018 2:12 PM, Trahe, Fiona wrote:
>> >>>> We've been struggling with the idea of session in compressdev.
>> >>>>
>> >>>> Is it really a session?
>> >>>> - It's not in the same sense as cryptodev where it's used to hold a key, and maps to a Security
>> Association.
>> >>>> - It's a set of immutable data that is needed with the op and stream to perform the operation.
>> >>>> - It inherited from cryptodev the ability to be set up for multiple driver types and used across any
>> >>>> devices of those types. For stateful ops this facility can't be used.
>> >>>> For stateless we don't think it's important, and think it's unlikely to be used.
>> >>>> - Drivers use it to prepare private data, set up resources, do pre-work, so there's
>> >>>> less work to be done on the data path. Initially we didn't have a stream, we do now,
>> >>>> this may be a better alternative place for that work.
>> >>>> So we've been toying with the idea of getting rid of the session.
>> >>> [Ahmed] In our proprietary API the stream and session are one. A session
>> >>> holds many properties like the op-type, instead of having this
>> >>> information in the op itself. This way we lower the per op setup cost.
>> >>> This also allows rapid reuse of stateful infrastructure, once a stream
>> >>> is closed on a stateful session, the next op (stream) on this session
>> >>> reuses the stateful storage. Obviously if a stream is in "pause mode" on
>> >>> a session, all following ops that may be unrelated to this
>> >>> stream/session must also wait until this current stream is closed or
>> >>> aborted before the infrastructure can be reused.
>> >>>> We also struggle with the idea of setting up a stream for stateless ops.
>> >>>> - Well, really I just think the name is misleading, i.e. there's no problem with setting
>> >>>> up some private PMD data to use with stateless operations, just calling it a
>> >>>> stream doesn't seem right.
>> >>> [Ahmed] I agree. The op has all the necessary information to process it
>> >>> in the current API? Both the stream and the op are one time use. We
>> >>> can't attach multiple similar ops to a single stream/session and rely on
>> >>> their properties to simplify op setup, so why the hassle.
>> >> [Shally] As per my knowledge, session came with idea in DPDK, if system has multiple devices setup
>> to do similar jobs then
>> >application can fan out ops to any of them for load-balancing. Though it is not possible for stateful ops
>> but it still can work for stateless.
>> >If there's an application which only have stateless ops to process then I see this is still useful feature to
>> support.
>> >[Ahmed] Is there an advantage to exposing load balancing to the user? I
>> >do not see load balancing as a feature within itself. Can the PMD take
>> >care of this? I guess a system that has
>>
>> [Shally] I assume idea was to leverage multiple PMDs that are available in system (say QAT+SW ZLIB)
>> and I believe matter of load-balancing came out of one of the earlier discussion with Fiona on RFC v1.
>> http://dev.dpdk.narkive.com/CHS5l01B/dpdk-dev-rfc-v1-doc-compression-api-for-dpdk#post3
>> So, I wait for her comments on this. But in any case, with changed notion too it looks achievable to me,
>> if so is desired.
>>
>> >> In current proposal, stream logically represent data and hold its specific information and session is
>> generic information that can be
>> >applied on multiple data. If we want to combine stream and session. Then one way to look at this is:
>> >>
>> >> "let application only allocate and initialize session with rte_comp_xform (and possibly op type)
>> information so that PMD can do one-
>> >time setup and allocate enough resources. Once attached to op, cannot be reused until that op is fully
>> processed. So, if app has 16
>> >data elements to process in a burst, it will setup 16 sessions."
>> >[Ahmed] Why not allow multiple inflight stateless ops with the same
>> >session? Stateless by definition guarantees that the resources used to
>> >work on one up will be free after the op is processed. That means that
>> >even if an op fails to process correctly on a session, it will have no
>> >effect on the next op since there is not interdependence. This assumes
>> >that the resources are shareable between hardware instances for
>> >stateless. That is not a bad assumption since hardware should not need
>> >more than the data of the op itself to work on a statelss op.
>>
>> [Shally] multiple ops in-flight can connect to same session but I assume you agree then they cannot
>> execute in parallel i.e. only one op at-a-time can use session here? And as far as I understand your PMD
>> works this way. Your HW execute one op at-a-time from queue?!
>>
>> >> This is same as what Ahmed suggested. For a particular load-balancing case suggested above, If
>> application want, can initialize
>> >different sessions on multiple devices with same xform so that each is prepared to process ops.
>> Application can then fanout stateless
>> >ops to multiple devices for load-balancing but then it would need to keep map of device & a session
>> map.
>> >>
>> >> If this sound feasible, then I too believe we can rather get rid of either and keep one (possibly session
>> but am open with stream as
>> >well).
>> >> However, regardless of case whether we live with name stream or session, I don't see much deviation
>> from current API spec except
>> >description and few modifications/additions as identified.
>> >> So, then I see it as:
>> >>
>> >> - A stream(or session whichever name is chosen) can be used with only one-op at-a-time
>> >> - It can be re-used when previously attached op is processed
>> >> - if it is stream then currently it is allocated from PMD managed pool whereas Sessions are allocated
>> from application created
>> >mempool.
>> >> In either of case, I would expect to review pool management API
>> >>
>> >> With this in mind, below are few of my comments
>> >>
>> >>>> So putting above thoughts together I want to propose:
>> >>>> - Removal of the session and all associated APIs.
>> >>>> - Passing in one of three data types in the rte_comp_op
>> >>>>
>> >>>> union {
>> >>>> struct rte_comp_xform *xform;
>> >>>> /**< Immutable compress/decompress params */
>> >>>> void *pmd_stateless_data;
>> >>>> /**< Stateless private PMD data derived from an rte_comp_xform
>> >>>> * rte_comp_stateless_data_init() must be called on a device
>> >>>> * before sending any STATELESS operations. If the PMD returns a non-NULL
>> >>>> * value the handle must be attached to subsequent STATELESS operations.
>> >>>> * If a PMD returns NULL, then the xform should be passed directly to each op
>> >>>> */
>> >> [Shally] It sounds like stateless_data_init() nothing more than a replacement of session_init().
>> >> So, this is needed neither if we retain session concept nor if we retain stream concept (
>> rte_comp_stream_create() with
>> >op_type: stateless can serve same purpose).
>> >> It should be sufficient to provide either stream (or session) pointer.
>> >>
>> >>>> void *stream;
>> >>>> /* Private PMD data derived initially from an rte_comp_xform, which holds state
>> >>>> * and history data and evolves as operations are processed.
>> >>>> * rte_comp_stream_create() must be called on a device for all STATEFUL
>> >>>> * data streams and the resulting stream attached
>> >>>> * to the one or more operations associated with the data stream.
>> >>>> * All operations in a stream must be sent to the same device.
>> >>>> */
>> >>>> }
>> >>> [Ahmed] I like this setup, but I am not sure in what cases the xform
>> >>> immutable would be used. I understand the other two.
>> >> [Shally] my understanding is xform will be mapped by PMD to its internally managed stream(or
>> session data structure). And then we
>> >can remove STATEFUL reference here and just say stream(or session) it belongs to. However, This
>> condition still apply:
>> >> *All operations that belong to same stream must be sent to the same device.*
>> >>
>> >>>> Notes:
>> >>>> 1. Internally if a PMD wants to use the exact same data structure for both it can do,
>> >>>> just on the API I think it's better if they're named differently with
>> >>>> different comments.
>> >>>> 2. I'm not clear of the constraints if any, which attach to the pmd_stateless_data
>> >>>> For our PMD it would only hold immutable data as the session did, and so
>> >>>> could be attached to many ops in parallel.
>> >>>> Is this true for all PMDs or are there constraints which should be called out?
>> >>>> Is it limited to a specific device, qp, or to be used on one op at a time?
>> >>>> 3. Am open to other naming suggestions, just trying to capture the essence
>> >>>> of these data structs better than our current API does.
>> >>>>
>> >>>> We would put some more helper fns and structure around the above code if people
>> >>>> are in agreement, just want to see if the concept flies before going further?
>> >>>>
>> >>>> Fiona
>> >>>>
>> >>>>
>> >>>>
>> >>
Hi Shally
//snip//
> [Shally] This looks better to me. So it mean app would always call xform_init() for stateless and attach an
> updated priv_xform to ops (depending upon if there's shareable or not). So it does not need to have
> NULL pointer on priv_xform. right?
>
[Fiona] yes. The PMD must return a valid priv_xform pointer.
On 3/1/2018 9:41 AM, Trahe, Fiona wrote:
> Hi Shally
>
> //snip//
>> [Shally] This looks better to me. So it mean app would always call xform_init() for stateless and attach an
>> updated priv_xform to ops (depending upon if there's shareable or not). So it does not need to have
>> NULL pointer on priv_xform. right?
>>
> [Fiona] yes. The PMD must return a valid priv_xform pointer.
[Ahmed] What I understood is that the xform_init will be called once
initially. if the @flag returned is NONE_SHAREABLE then the application
must not attach two inflight ops to the same @priv_xform? Otherwise the
application can attach many ops in flight to the @priv_xform?
> -----Original Message-----
> From: Ahmed Mansour [mailto:ahmed.mansour@nxp.com]
> Sent: Friday, March 2, 2018 12:56 AM
> To: Trahe, Fiona <fiona.trahe@intel.com>; Verma, Shally <Shally.Verma@cavium.com>; dev@dpdk.org
> Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad
> <NarayanaPrasad.Athreya@cavium.com>; Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila
> <Sunila.Sahu@cavium.com>; Challa, Mahipal <Mahipal.Challa@cavium.com>; Jain, Deepak K
> <deepak.k.jain@intel.com>; Hemant Agrawal <hemant.agrawal@nxp.com>; Roy Pledge
> <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
> Subject: Re: [dpdk-dev] [PATCH] compressdev: implement API
>
> On 3/1/2018 9:41 AM, Trahe, Fiona wrote:
> > Hi Shally
> >
> > //snip//
> >> [Shally] This looks better to me. So it mean app would always call xform_init() for stateless and attach
> an
> >> updated priv_xform to ops (depending upon if there's shareable or not). So it does not need to have
> >> NULL pointer on priv_xform. right?
> >>
> > [Fiona] yes. The PMD must return a valid priv_xform pointer.
>
> [Ahmed] What I understood is that the xform_init will be called once
> initially. if the @flag returned is NONE_SHAREABLE then the application
> must not attach two inflight ops to the same @priv_xform? Otherwise the
> application can attach many ops in flight to the @priv_xform?
[Fiona Yes. App calls the xform_init() once on a device where it plans to send stateless ops.
If PMD returns shareable, then it doesn't need to call again and can attach this to every stateless op going to that device.
If PMD returns SINGLE_OP then it must call xform_init() before every other
stateless op it wants to have inflight simultaneously. This does not mean it must be called before every op,
but probably will set up a batch of priv_xforms - it can reuse each priv_xform once the op finishes with it.
On 3/2/2018 4:53 AM, Trahe, Fiona wrote:
>
>> On 3/1/2018 9:41 AM, Trahe, Fiona wrote:
>>> Hi Shally
>>>
>>> //snip//
>>>> [Shally] This looks better to me. So it mean app would always call xform_init() for stateless and attach
>> an
>>>> updated priv_xform to ops (depending upon if there's shareable or not). So it does not need to have
>>>> NULL pointer on priv_xform. right?
>>>>
>>> [Fiona] yes. The PMD must return a valid priv_xform pointer.
>> [Ahmed] What I understood is that the xform_init will be called once
>> initially. if the @flag returned is NONE_SHAREABLE then the application
>> must not attach two inflight ops to the same @priv_xform? Otherwise the
>> application can attach many ops in flight to the @priv_xform?
> [Fiona Yes. App calls the xform_init() once on a device where it plans to send stateless ops.
> If PMD returns shareable, then it doesn't need to call again and can attach this to every stateless op going to that device.
> If PMD returns SINGLE_OP then it must call xform_init() before every other
> stateless op it wants to have inflight simultaneously. This does not mean it must be called before every op,
> but probably will set up a batch of priv_xforms - it can reuse each priv_xform once the op finishes with it.
[Ahmed] @Shally Can this complexity of managing the NONE_SHAREABLE mode
be pushed into the PMD? A flexible stockpile can be kept and maintained
by the PMD and it can be increased or decreased based on
low-water/high-water thresholds
On 2/28/2018 1:39 PM, Trahe, Fiona wrote:
> Hi Ahmed, Shally,
>
> So just to capture what we concluded in the call today:
>
> - There's no requirement for a device-agnostic session to facilitate load-balancing.
> - For stateful data a stream is compulsory. Xform is passed to stream on creation.
> So no need for a session in stateful op.
>
> Re session data for stateless ops:
> - All PMDs could cope with just passing in a xform with a stateless op. But it might
> not be performant.
> - Some PMDs need to allocate some resources which can only be used by one op
> at a time. For stateful ops these resources can be attached to a stream. For stateless
> they could allocate the resources on each op enqueue, but it would be better if
> the resources were setup once based on the xform and could be re-used on ops,
> though only by one op at a time.
> - Some PMDs don't need to allocate such resources, but could benefit by
> setting up some pmd data based on the xform. This data would not be
> constrained, could be used in parallel by any op or qp of the device.
> - The name pmd_stateless_data was not popular, maybe something like
> xform_private_data can be used. On creation of this data, the PMD can return
> an indication of whether it should be used by one op at a time or shared.
>
> So I'll
> - remove the session completely from the API.
> - add an initialiser API for the data to be attached to stateless ops
> - add a union to the op:
>
> union {
> void *pmd_private_xform;
> /**< Stateless private PMD data derived from an rte_comp_xform
> * rte_comp_xform_init() must be called on a device
> * before sending any STATELESS operations. The PMD returns a handle
> * which must be attached to subsequent STATELESS operations.
> * The PMD also returns a flag, if this is COMP_PRIVATE_XFORM_SHAREABLE
> * then the xform can be attached to multiple ops at the same time,
> * if it's COMP_PRIVATE_XFORM_SINGLE_OP then it can only be
> * be used on one op at a time, other private xforms must be initialised
> * to send other ops in parallel.
> */
> void *stream;
> /* Private PMD data derived initially from an rte_comp_xform, which holds state
> * and history data and evolves as operations are processed.
> * rte_comp_stream_create() must be called on a device for all STATEFUL
> * data streams and the resulting stream attached
> * to the one or more operations associated with the data stream.
> * All operations in a stream must be sent to the same device.
> */
> }
>
> Previous startup flow before sending a stateful op:
> rte_comp_get_private_size(devid)
> rte_comp_mempool_create() - returns sess_pool
> rte_comp_session_create(sess_pool)
> rte_comp_session_init(devid, sess, sess_pool, xform)
> rte_comp_stream_create(devid, sess, **stream, op_type)
>
> simplified to:
> rte_comp_xform_init(devid, xform, **priv_xform, *flag) - returns handle and flag
> (pool is within the PMD)
>
> Note, I don't think we bottomed out on removing the xform from the union, but I don't
> think we need it with above solution.
>
> Other discussion:
> - we should document on API that qp is not thread-safe, so enqueue
> and dequeue should be performed by same thread.
[Ahmed] - I understand a qp should represent a single software user.
This is good because we will not have to add locking as you mentioned,
but are you sure that dequeues cannot be performed by another thread
without adding significant overhead? it would enable producer consumer
applications, and the name queue pair implies some independence.
- Another question. I want to be sure. A qp can be used to send both
compress and decompress ops.
- Nitpick: The name queue pair implies order preservation to the user.
Maybe we should change it to something that does not imply that.
>
> device and qp flow:
> - dev_info_get() - application reads device capabilities, including the max qps the device can support.
> - dev_config() - application specifies how many qps it intends to use - typically one per thread, must be < device max
> - qp_setup() - called per qp. Creates the qp based on the size indicated by max_inflights
> - dev_start() - once started device can't be reconfigured, must call dev_stop to reconfigure.
>
>
> Regards,
> Fiona
>
>> -----Original Message-----
>> From: Verma, Shally [mailto:Shally.Verma@cavium.com]
>> Sent: Tuesday, February 27, 2018 5:54 AM
>> To: Ahmed Mansour <ahmed.mansour@nxp.com>; Trahe, Fiona <fiona.trahe@intel.com>;
>> dev@dpdk.org
>> Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad
>> <NarayanaPrasad.Athreya@cavium.com>; Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila
>> <Sunila.Sahu@cavium.com>; Challa, Mahipal <Mahipal.Challa@cavium.com>; Jain, Deepak K
>> <deepak.k.jain@intel.com>; Hemant Agrawal <hemant.agrawal@nxp.com>; Roy Pledge
>> <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
>> Subject: RE: [dpdk-dev] [PATCH] compressdev: implement API
>>
>>
>>
>>> -----Original Message-----
>>> From: Ahmed Mansour [mailto:ahmed.mansour@nxp.com]
>>> Sent: 27 February 2018 03:05
>>> To: Verma, Shally <Shally.Verma@cavium.com>; Trahe, Fiona <fiona.trahe@intel.com>; dev@dpdk.org
>>> Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad
>> <NarayanaPrasad.Athreya@cavium.com>;
>>> Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila <Sunila.Sahu@cavium.com>; Challa,
>> Mahipal
>>> <Mahipal.Challa@cavium.com>; Jain, Deepak K <deepak.k.jain@intel.com>; Hemant Agrawal
>> <hemant.agrawal@nxp.com>; Roy
>>> Pledge <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
>>> Subject: Re: [dpdk-dev] [PATCH] compressdev: implement API
>>>
>>>> Hi Fiona, Ahmed
>>>>> Hi Fiona,
>>>>>
>>>>> Thanks for starting this discussion. In the current API the user must
>>>>> make 12 API calls just to get information to compress. Maybe there is a
>>>>> way to simplify. At least for some use cases (stateless). I think a call
>>>>> sometime next week would be good to help clarify coalesce some of the
>>>>> complexity.
>>>>>
>>>>> I added specific comments inline.
>>>>>
>>>>> Thanks,
>>>>>
>>>>> Ahmed
>>>>>
>>>>> On 2/21/2018 2:12 PM, Trahe, Fiona wrote:
>>>>>> We've been struggling with the idea of session in compressdev.
>>>>>>
>>>>>> Is it really a session?
>>>>>> - It's not in the same sense as cryptodev where it's used to hold a key, and maps to a Security
>> Association.
>>>>>> - It's a set of immutable data that is needed with the op and stream to perform the operation.
>>>>>> - It inherited from cryptodev the ability to be set up for multiple driver types and used across any
>>>>>> devices of those types. For stateful ops this facility can't be used.
>>>>>> For stateless we don't think it's important, and think it's unlikely to be used.
>>>>>> - Drivers use it to prepare private data, set up resources, do pre-work, so there's
>>>>>> less work to be done on the data path. Initially we didn't have a stream, we do now,
>>>>>> this may be a better alternative place for that work.
>>>>>> So we've been toying with the idea of getting rid of the session.
>>>>> [Ahmed] In our proprietary API the stream and session are one. A session
>>>>> holds many properties like the op-type, instead of having this
>>>>> information in the op itself. This way we lower the per op setup cost.
>>>>> This also allows rapid reuse of stateful infrastructure, once a stream
>>>>> is closed on a stateful session, the next op (stream) on this session
>>>>> reuses the stateful storage. Obviously if a stream is in "pause mode" on
>>>>> a session, all following ops that may be unrelated to this
>>>>> stream/session must also wait until this current stream is closed or
>>>>> aborted before the infrastructure can be reused.
>>>>>> We also struggle with the idea of setting up a stream for stateless ops.
>>>>>> - Well, really I just think the name is misleading, i.e. there's no problem with setting
>>>>>> up some private PMD data to use with stateless operations, just calling it a
>>>>>> stream doesn't seem right.
>>>>> [Ahmed] I agree. The op has all the necessary information to process it
>>>>> in the current API? Both the stream and the op are one time use. We
>>>>> can't attach multiple similar ops to a single stream/session and rely on
>>>>> their properties to simplify op setup, so why the hassle.
>>>> [Shally] As per my knowledge, session came with idea in DPDK, if system has multiple devices setup
>> to do similar jobs then
>>> application can fan out ops to any of them for load-balancing. Though it is not possible for stateful ops
>> but it still can work for stateless.
>>> If there's an application which only have stateless ops to process then I see this is still useful feature to
>> support.
>>> [Ahmed] Is there an advantage to exposing load balancing to the user? I
>>> do not see load balancing as a feature within itself. Can the PMD take
>>> care of this? I guess a system that has
>> [Shally] I assume idea was to leverage multiple PMDs that are available in system (say QAT+SW ZLIB)
>> and I believe matter of load-balancing came out of one of the earlier discussion with Fiona on RFC v1.
>> https://emea01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fdev.dpdk.narkive.com%2FCHS5l01B%2Fdpdk-dev-rfc-v1-doc-compression-api-for-dpdk%23post3&data=02%7C01%7Cahmed.mansour%40nxp.com%7C4299d16d58144e417f5208d57eda9ba9%7C686ea1d3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C636554399768115871&sdata=MHWrD0qD%2FMKL%2FjX4j1kOdDeElh0cG1gQj9d1862K4V0%3D&reserved=0
>> So, I wait for her comments on this. But in any case, with changed notion too it looks achievable to me,
>> if so is desired.
>>
>>>> In current proposal, stream logically represent data and hold its specific information and session is
>> generic information that can be
>>> applied on multiple data. If we want to combine stream and session. Then one way to look at this is:
>>>> "let application only allocate and initialize session with rte_comp_xform (and possibly op type)
>> information so that PMD can do one-
>>> time setup and allocate enough resources. Once attached to op, cannot be reused until that op is fully
>> processed. So, if app has 16
>>> data elements to process in a burst, it will setup 16 sessions."
>>> [Ahmed] Why not allow multiple inflight stateless ops with the same
>>> session? Stateless by definition guarantees that the resources used to
>>> work on one up will be free after the op is processed. That means that
>>> even if an op fails to process correctly on a session, it will have no
>>> effect on the next op since there is not interdependence. This assumes
>>> that the resources are shareable between hardware instances for
>>> stateless. That is not a bad assumption since hardware should not need
>>> more than the data of the op itself to work on a statelss op.
>> [Shally] multiple ops in-flight can connect to same session but I assume you agree then they cannot
>> execute in parallel i.e. only one op at-a-time can use session here? And as far as I understand your PMD
>> works this way. Your HW execute one op at-a-time from queue?!
>>
>>>> This is same as what Ahmed suggested. For a particular load-balancing case suggested above, If
>> application want, can initialize
>>> different sessions on multiple devices with same xform so that each is prepared to process ops.
>> Application can then fanout stateless
>>> ops to multiple devices for load-balancing but then it would need to keep map of device & a session
>> map.
>>>> If this sound feasible, then I too believe we can rather get rid of either and keep one (possibly session
>> but am open with stream as
>>> well).
>>>> However, regardless of case whether we live with name stream or session, I don't see much deviation
>> from current API spec except
>>> description and few modifications/additions as identified.
>>>> So, then I see it as:
>>>>
>>>> - A stream(or session whichever name is chosen) can be used with only one-op at-a-time
>>>> - It can be re-used when previously attached op is processed
>>>> - if it is stream then currently it is allocated from PMD managed pool whereas Sessions are allocated
>> from application created
>>> mempool.
>>>> In either of case, I would expect to review pool management API
>>>>
>>>> With this in mind, below are few of my comments
>>>>
>>>>>> So putting above thoughts together I want to propose:
>>>>>> - Removal of the session and all associated APIs.
>>>>>> - Passing in one of three data types in the rte_comp_op
>>>>>>
>>>>>> union {
>>>>>> struct rte_comp_xform *xform;
>>>>>> /**< Immutable compress/decompress params */
>>>>>> void *pmd_stateless_data;
>>>>>> /**< Stateless private PMD data derived from an rte_comp_xform
>>>>>> * rte_comp_stateless_data_init() must be called on a device
>>>>>> * before sending any STATELESS operations. If the PMD returns a non-NULL
>>>>>> * value the handle must be attached to subsequent STATELESS operations.
>>>>>> * If a PMD returns NULL, then the xform should be passed directly to each op
>>>>>> */
>>>> [Shally] It sounds like stateless_data_init() nothing more than a replacement of session_init().
>>>> So, this is needed neither if we retain session concept nor if we retain stream concept (
>> rte_comp_stream_create() with
>>> op_type: stateless can serve same purpose).
>>>> It should be sufficient to provide either stream (or session) pointer.
>>>>
>>>>>> void *stream;
>>>>>> /* Private PMD data derived initially from an rte_comp_xform, which holds state
>>>>>> * and history data and evolves as operations are processed.
>>>>>> * rte_comp_stream_create() must be called on a device for all STATEFUL
>>>>>> * data streams and the resulting stream attached
>>>>>> * to the one or more operations associated with the data stream.
>>>>>> * All operations in a stream must be sent to the same device.
>>>>>> */
>>>>>> }
>>>>> [Ahmed] I like this setup, but I am not sure in what cases the xform
>>>>> immutable would be used. I understand the other two.
>>>> [Shally] my understanding is xform will be mapped by PMD to its internally managed stream(or
>> session data structure). And then we
>>> can remove STATEFUL reference here and just say stream(or session) it belongs to. However, This
>> condition still apply:
>>>> *All operations that belong to same stream must be sent to the same device.*
>>>>
>>>>>> Notes:
>>>>>> 1. Internally if a PMD wants to use the exact same data structure for both it can do,
>>>>>> just on the API I think it's better if they're named differently with
>>>>>> different comments.
>>>>>> 2. I'm not clear of the constraints if any, which attach to the pmd_stateless_data
>>>>>> For our PMD it would only hold immutable data as the session did, and so
>>>>>> could be attached to many ops in parallel.
>>>>>> Is this true for all PMDs or are there constraints which should be called out?
>>>>>> Is it limited to a specific device, qp, or to be used on one op at a time?
>>>>>> 3. Am open to other naming suggestions, just trying to capture the essence
>>>>>> of these data structs better than our current API does.
>>>>>>
>>>>>> We would put some more helper fns and structure around the above code if people
>>>>>> are in agreement, just want to see if the concept flies before going further?
>>>>>>
>>>>>> Fiona
>>>>>>
>>>>>>
>>>>>>
>
>-----Original Message-----
>From: Ahmed Mansour [mailto:ahmed.mansour@nxp.com]
>Sent: 03 March 2018 01:19
>To: Trahe, Fiona <fiona.trahe@intel.com>; Verma, Shally <Shally.Verma@cavium.com>; dev@dpdk.org
>Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad <NarayanaPrasad.Athreya@cavium.com>;
>Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila <Sunila.Sahu@cavium.com>; Challa, Mahipal
><Mahipal.Challa@cavium.com>; Jain, Deepak K <deepak.k.jain@intel.com>; Hemant Agrawal <hemant.agrawal@nxp.com>; Roy
>Pledge <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
>Subject: Re: [dpdk-dev] [PATCH] compressdev: implement API
>
>On 3/2/2018 4:53 AM, Trahe, Fiona wrote:
>>
>>> On 3/1/2018 9:41 AM, Trahe, Fiona wrote:
>>>> Hi Shally
>>>>
>>>> //snip//
>>>>> [Shally] This looks better to me. So it mean app would always call xform_init() for stateless and attach
>>> an
>>>>> updated priv_xform to ops (depending upon if there's shareable or not). So it does not need to have
>>>>> NULL pointer on priv_xform. right?
>>>>>
>>>> [Fiona] yes. The PMD must return a valid priv_xform pointer.
>>> [Ahmed] What I understood is that the xform_init will be called once
>>> initially. if the @flag returned is NONE_SHAREABLE then the application
>>> must not attach two inflight ops to the same @priv_xform? Otherwise the
>>> application can attach many ops in flight to the @priv_xform?
>> [Fiona Yes. App calls the xform_init() once on a device where it plans to send stateless ops.
>> If PMD returns shareable, then it doesn't need to call again and can attach this to every stateless op going to that device.
>> If PMD returns SINGLE_OP then it must call xform_init() before every other
>> stateless op it wants to have inflight simultaneously. This does not mean it must be called before every op,
>> but probably will set up a batch of priv_xforms - it can reuse each priv_xform once the op finishes with it.
>[Ahmed] @Shally Can this complexity of managing the NONE_SHAREABLE mode
>be pushed into the PMD? A flexible stockpile can be kept and maintained
>by the PMD and it can be increased or decreased based on
>low-water/high-water thresholds
[Shally] It is doable to manage within PMD but need to do hands on to evaluate effectiveness. So far, we have never exercised this way and left it to application to attach different session (or stream) to op for maximum performance gain. So, I would say, may it be ok to have flag feature in first place and deprecate later, if it not required?! Or just have API without any flag option and add a feature flag to indicate PMD support for SHAREABLE/NON-SHAREABLE xform_priv handle?!
On 3/5/2018 9:32 AM, Verma, Shally wrote:
>
>> -----Original Message-----
>> From: Ahmed Mansour [mailto:ahmed.mansour@nxp.com]
>> Sent: 03 March 2018 01:19
>> To: Trahe, Fiona <fiona.trahe@intel.com>; Verma, Shally <Shally.Verma@cavium.com>; dev@dpdk.org
>> Cc: De Lara Guarch, Pablo <pablo.de.lara.guarch@intel.com>; Athreya, Narayana Prasad <NarayanaPrasad.Athreya@cavium.com>;
>> Gupta, Ashish <Ashish.Gupta@cavium.com>; Sahu, Sunila <Sunila.Sahu@cavium.com>; Challa, Mahipal
>> <Mahipal.Challa@cavium.com>; Jain, Deepak K <deepak.k.jain@intel.com>; Hemant Agrawal <hemant.agrawal@nxp.com>; Roy
>> Pledge <roy.pledge@nxp.com>; Youri Querry <youri.querry_1@nxp.com>
>> Subject: Re: [dpdk-dev] [PATCH] compressdev: implement API
>>
>> On 3/2/2018 4:53 AM, Trahe, Fiona wrote:
>>>> On 3/1/2018 9:41 AM, Trahe, Fiona wrote:
>>>>> Hi Shally
>>>>>
>>>>> //snip//
>>>>>> [Shally] This looks better to me. So it mean app would always call xform_init() for stateless and attach
>>>> an
>>>>>> updated priv_xform to ops (depending upon if there's shareable or not). So it does not need to have
>>>>>> NULL pointer on priv_xform. right?
>>>>>>
>>>>> [Fiona] yes. The PMD must return a valid priv_xform pointer.
>>>> [Ahmed] What I understood is that the xform_init will be called once
>>>> initially. if the @flag returned is NONE_SHAREABLE then the application
>>>> must not attach two inflight ops to the same @priv_xform? Otherwise the
>>>> application can attach many ops in flight to the @priv_xform?
>>> [Fiona Yes. App calls the xform_init() once on a device where it plans to send stateless ops.
>>> If PMD returns shareable, then it doesn't need to call again and can attach this to every stateless op going to that device.
>>> If PMD returns SINGLE_OP then it must call xform_init() before every other
>>> stateless op it wants to have inflight simultaneously. This does not mean it must be called before every op,
>>> but probably will set up a batch of priv_xforms - it can reuse each priv_xform once the op finishes with it.
>> [Ahmed] @Shally Can this complexity of managing the NONE_SHAREABLE mode
>> be pushed into the PMD? A flexible stockpile can be kept and maintained
>> by the PMD and it can be increased or decreased based on
>> low-water/high-water thresholds
> [Shally] It is doable to manage within PMD but need to do hands on to evaluate effectiveness. So far, we have never exercised this way and left it to application to attach different session (or stream) to op for maximum performance gain. So, I would say, may it be ok to have flag feature in first place and deprecate later, if it not required?! Or just have API without any flag option and add a feature flag to indicate PMD support for SHAREABLE/NON-SHAREABLE xform_priv handle?!
[Ahmed] Either way looks ok to me. I see your point about performance.
If this is in the PMD it will have to constantly guess how much memory
the user needs and accommodate dynamically. The user can implement a
similar scheme or if the application is simple they can pre-allocate and
reduce CPU allocation de-allocation overhead.
Hi Tomas,
Sorry for the delay in replying to this
> -----Original Message-----
> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> Sent: Sunday, February 4, 2018 2:25 PM
> To: Trahe, Fiona <fiona.trahe@intel.com>
> Cc: dev@dpdk.org; ahmed.mansour@nxp.com; Shally.Verma@cavium.com; De Lara Guarch, Pablo
> <pablo.de.lara.guarch@intel.com>
> Subject: Re: [dpdk-dev] [PATCH] compressdev: implement API
>
> 02/02/2018 19:25, Fiona Trahe:
> > config/common_base | 6 +
> > doc/api/doxy-api-index.md | 1 +
> > doc/api/doxy-api.conf | 1 +
> > lib/Makefile | 3 +
> > lib/librte_compressdev/Makefile | 29 +
> > lib/librte_compressdev/rte_comp.h | 503 ++++++++++++
>
> Why rte_comp.h instead of the more consistent rte_compress.h?
[Fiona] I did originally... but ran into difficulty with horribly names like
RTE_COMPRESS_COMPRESS
RTE_COMPRESS_DECOMPRESS
rte_compress_compress_xform
rte_compress_decompress_xform
So compress is both the module prefix and the name of one of the actions.
I could have used compressdev - but names were very long.
So decided to opt for using
_compressdev_ in names to do with the device and
_comp_ in names to do with the compression service
Also I could have used compdev instead of compressdev,
but I felt compress should be in the lib name
>
> > lib/librte_compressdev/rte_compressdev.c | 902 +++++++++++++++++++++
> > lib/librte_compressdev/rte_compressdev.h | 757 +++++++++++++++++
> > lib/librte_compressdev/rte_compressdev_pmd.c | 163 ++++
> > lib/librte_compressdev/rte_compressdev_pmd.h | 439 ++++++++++
> > lib/librte_compressdev/rte_compressdev_version.map | 47 ++
> > lib/librte_eal/common/include/rte_log.h | 1 +
> > mk/rte.app.mk | 1 +
> > 13 files changed, 2853 insertions(+)
>
> Please update MAINTAINERS file and release notes.
>
> Maybe it is worth splitting this patch in few shorter parts?
[Fiona] will do
>
>
> > --- a/config/common_base
> > +++ b/config/common_base
> > @@ -535,6 +535,12 @@ CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO=n
> > CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO_DEBUG=n
> >
> > #
> > +# Compile generic compression device library
> > +#
> > +CONFIG_RTE_LIBRTE_COMPRESSDEV=y
> > +CONFIG_RTE_COMPRESS_MAX_DEVS=64
> > +
> > +#
> > # Compile generic security library
> > #
> > CONFIG_RTE_LIBRTE_SECURITY=y
> > diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
> > index d77f205..07b8e75 100644
> > --- a/doc/api/doxy-api-index.md
> > +++ b/doc/api/doxy-api-index.md
> > @@ -43,6 +43,7 @@ The public API headers are grouped by topics:
> > [rte_tm] (@ref rte_tm.h),
> > [rte_mtr] (@ref rte_mtr.h),
> > [bbdev] (@ref rte_bbdev.h),
> > + [compressdev] (@ref rte_compressdev.h),
> > [cryptodev] (@ref rte_cryptodev.h),
> > [security] (@ref rte_security.h),
> > [eventdev] (@ref rte_eventdev.h),
>
> Please move it between security and eventdev in these lists.
ok
23/03/2018 19:08, Trahe, Fiona:
> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> > 02/02/2018 19:25, Fiona Trahe:
> > > lib/librte_compressdev/rte_comp.h | 503 ++++++++++++
> >
> > Why rte_comp.h instead of the more consistent rte_compress.h?
> [Fiona] I did originally... but ran into difficulty with horribly names like
> RTE_COMPRESS_COMPRESS
> RTE_COMPRESS_DECOMPRESS
> rte_compress_compress_xform
> rte_compress_decompress_xform
> So compress is both the module prefix and the name of one of the actions.
> I could have used compressdev - but names were very long.
> So decided to opt for using
> _compressdev_ in names to do with the device and
> _comp_ in names to do with the compression service
>
> Also I could have used compdev instead of compressdev,
> but I felt compress should be in the lib name
I understand your concerns.
I don't like "comp" very much because it sounds like "comparison".
However, I don't have a better idea.
Sometimes naming is more difficult than coding :)
> -----Original Message-----
> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> Sent: Saturday, March 24, 2018 1:02 AM
> To: Trahe, Fiona <fiona.trahe@intel.com>
> Cc: dev@dpdk.org; ahmed.mansour@nxp.com; Shally.Verma@cavium.com; De Lara Guarch, Pablo
> <pablo.de.lara.guarch@intel.com>; fiona.trahe@gmail.com
> Subject: Re: [dpdk-dev] [PATCH] compressdev: implement API
>
> 23/03/2018 19:08, Trahe, Fiona:
> > From: Thomas Monjalon [mailto:thomas@monjalon.net]
> > > 02/02/2018 19:25, Fiona Trahe:
> > > > lib/librte_compressdev/rte_comp.h | 503 ++++++++++++
> > >
> > > Why rte_comp.h instead of the more consistent rte_compress.h?
> > [Fiona] I did originally... but ran into difficulty with horribly names like
> > RTE_COMPRESS_COMPRESS
> > RTE_COMPRESS_DECOMPRESS
> > rte_compress_compress_xform
> > rte_compress_decompress_xform
> > So compress is both the module prefix and the name of one of the actions.
> > I could have used compressdev - but names were very long.
> > So decided to opt for using
> > _compressdev_ in names to do with the device and
> > _comp_ in names to do with the compression service
> >
> > Also I could have used compdev instead of compressdev,
> > but I felt compress should be in the lib name
>
> I understand your concerns.
> I don't like "comp" very much because it sounds like "comparison".
> However, I don't have a better idea.
> Sometimes naming is more difficult than coding :)
[Fiona] True :)
With the vast amounts of data being transported around networks
and stored in storage systems, reducing data size is becoming ever
more important.
There are both software libraries and hardware devices available
that provide compression, but no common API. Such an API is proposed
in this commit, which supports the following features:
- Deflate Algorithm (https://tools.ietf.org/html/rfc1951)
- LZS algorithm (https://tools.ietf.org/html/rfc2395)
- Static and Dynamic Huffman encoding.
- Compression levels
- Checksum generation
- Asynchronous burst API
- private_xform - a place for PMDs to hold private data derived from
a xform and used by stateless operations.
- stream - a place for PMDs to hold private data derived from
a xform and also maintain state and history data. For
stateful flows.
Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Pablo de Lara pablo.de.lara.guarch@intel.com
Signed-off-by: Shally Verma <shally.verma@caviumnetworks.com>
Signed-off-by: Ashish Gupta <ashish.gupta@caviumnetworks.com>
Opens:
- creation of private_xform and stream mempools. In v2, config
added to give PMD the size params needed to create mempools.
Still open as to whether the pools should be created by
application, in API layer or by PMDs. Expect to resolve in a v3.
- alternative to mbufs for passing data between application
and accelerator. A lot of varied opinions on this, expect to
address in a future release.
- addition of hash feature - to be added by Cavium
- addition of feature to add capability to PMD to allow more than one
inflight operation from a stateful stream - to be added by NXP
Changes in v2:
- Add stream APIs
- Remove session
- Add SHAREABLE / NON_SHAREABLE private_xform types
- Add algo enum 'UNSPECIFIED' to fix warning in capabilities
- Change one remaining log to use dynamic logging.
- Add rte_cache_aligned keyword to op
- Rename enums with better names _ALGO, __CHECKSUM, _HUFFMAN_
- Use const keyword when passing xform
- Remove qp_count fn from dev_ops as never used
- Remove max_nb_queue-pairs from compressdev_init_param as never used
- Clarify device configure and start sequence
- Replace OUT_OF_SPACE with OUT_OF_SPACE_RECOVERABLE and TERMINATED
and clarified usage.
- Add stream and private_xform sizes to device config for use in
mempool creation
- Add capability helper fn
- Use Base2 log value for window size on xforms
- Add Meson build
- Update MAINTAINERS
- Update Map file
- Change order in doxy file
- Update Release note
Fiona Trahe (3):
compressdev: add structs and enum for compression service
compressdev: implement API
doc: update doxy and release note for compressdev
MAINTAINERS | 7 +
config/common_base | 6 +
config/rte_config.h | 3 +
doc/api/doxy-api-index.md | 1 +
doc/api/doxy-api.conf | 1 +
doc/guides/rel_notes/release_18_05.rst | 6 +
lib/Makefile | 3 +
lib/librte_compressdev/Makefile | 29 +
lib/librte_compressdev/meson.build | 9 +
lib/librte_compressdev/rte_comp.h | 528 ++++++++++++
lib/librte_compressdev/rte_compressdev.c | 904 +++++++++++++++++++++
lib/librte_compressdev/rte_compressdev.h | 727 +++++++++++++++++
lib/librte_compressdev/rte_compressdev_pmd.c | 156 ++++
lib/librte_compressdev/rte_compressdev_pmd.h | 450 ++++++++++
lib/librte_compressdev/rte_compressdev_version.map | 43 +
lib/meson.build | 2 +-
mk/rte.app.mk | 1 +
17 files changed, 2875 insertions(+), 1 deletion(-)
create mode 100644 lib/librte_compressdev/Makefile
create mode 100644 lib/librte_compressdev/meson.build
create mode 100644 lib/librte_compressdev/rte_comp.h
create mode 100644 lib/librte_compressdev/rte_compressdev.c
create mode 100644 lib/librte_compressdev/rte_compressdev.h
create mode 100644 lib/librte_compressdev/rte_compressdev_pmd.c
create mode 100644 lib/librte_compressdev/rte_compressdev_pmd.h
create mode 100644 lib/librte_compressdev/rte_compressdev_version.map
With the vast amounts of data being transported around networks
and stored in storage systems, reducing data size is becoming ever more important.
There are both software libraries and hardware devices available that
provide compression, but no common API.
Such an API is proposed in this patchset, which supports the following features:
- Deflate Algorithm (https://tools.ietf.org/html/rfc1951)
- LZS algorithm (https://tools.ietf.org/html/rfc2395)
- Static and Dynamic Huffman encoding.
- Compression levels
- Checksum generation
- Asynchronous burst API
- private_xform - a place for PMDs to hold private data derived from
a xform and used by stateless operations.
- stream - a place for PMDs to hold private data derived from
a xform and also maintain state and history data. For
stateful flows.
Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Pablo de Lara pablo.de.lara.guarch@intel.com
Signed-off-by: Shally Verma <shally.verma@caviumnetworks.com>
Signed-off-by: Ashish Gupta <ashish.gupta@caviumnetworks.com>
Opens:
- creation of private_xform and stream mempools. In v3, config
added to give PMD the size params needed to create mempools.
Still open as to whether the pools should be created by
application, in API layer or by PMDs. Expect to resolve in a v4.
- alternative to mbufs for passing data between application
and accelerator. A lot of varied opinions on this, expect to
address in a future release.
- addition of feature to add capability to PMD to allow more than one
inflight operation from a stateful stream - to be added by NXP
Changes in v3:
- Remove rte_comp_op_ctod helper functions
- Remove param_range_check macro
- Rename from phys_addr to iova_addr
- Remove rte_comp_algo_strings
- Rename rte_compressdev_pmd_is_valid_dev to rte_compressdev_is_valid_dev
- Remove feature flags from compressdev
- Support hash operations
- Add shareable priv xform in feature flags, instead of returnin it
on xform creation
- Allow max number of queue pairs to be 0, meaning that there is no
limit.
- Add invalid configuration checks
- Add capability helper functions
Changes in v2:
- Add stream APIs
- Remove session
- Add SHAREABLE / NON_SHAREABLE private_xform types
- Add algo enum 'UNSPECIFIED' to fix warning in capabilities
- Change one remaining log to use dynamic logging.
- Add rte_cache_aligned keyword to op
- Rename enums with better names _ALGO, __CHECKSUM, _HUFFMAN_
- Use const keyword when passing xform
- Remove qp_count fn from dev_ops as never used
- Remove max_nb_queue-pairs from compressdev_init_param as never used
- Clarify device configure and start sequence
- Replace OUT_OF_SPACE with OUT_OF_SPACE_RECOVERABLE and TERMINATED
and clarified usage.
- Add stream and private_xform sizes to device config for use in
mempool creation
- Add capability helper fn
- Use Base2 log value for window size on xforms
- Add Meson build
- Update MAINTAINERS
- Update Map file
- Change order in doxy file
- Update Release note
Fiona Trahe (12):
compressdev: add basic device management
compressdev: add queue pair management
compressdev: add compression specific data
compressdev: add enqueue/dequeue functions
compressdev: add operation management
compressdev: support stateless operations
compressdev: support stateful operations
compressdev: add device feature flags
compressdev: add compression service feature flags
compressdev: add device stats
compressdev: add device capabilities
compressdev: get device id from name
Shally Verma (1):
compressdev: support hash operations
MAINTAINERS | 7 +
config/common_base | 6 +
config/rte_config.h | 3 +
doc/api/doxy-api-index.md | 1 +
doc/api/doxy-api.conf | 1 +
doc/guides/rel_notes/release_18_05.rst | 6 +
lib/Makefile | 3 +
lib/librte_compressdev/Makefile | 29 +
lib/librte_compressdev/meson.build | 9 +
lib/librte_compressdev/rte_comp.h | 539 ++++++++++++
lib/librte_compressdev/rte_compressdev.c | 902 +++++++++++++++++++++
lib/librte_compressdev/rte_compressdev.h | 701 ++++++++++++++++
lib/librte_compressdev/rte_compressdev_pmd.c | 157 ++++
lib/librte_compressdev/rte_compressdev_pmd.h | 437 ++++++++++
lib/librte_compressdev/rte_compressdev_version.map | 44 +
lib/meson.build | 2 +-
mk/rte.app.mk | 1 +
17 files changed, 2847 insertions(+), 1 deletion(-)
create mode 100644 lib/librte_compressdev/Makefile
create mode 100644 lib/librte_compressdev/meson.build
create mode 100644 lib/librte_compressdev/rte_comp.h
create mode 100644 lib/librte_compressdev/rte_compressdev.c
create mode 100644 lib/librte_compressdev/rte_compressdev.h
create mode 100644 lib/librte_compressdev/rte_compressdev_pmd.c
create mode 100644 lib/librte_compressdev/rte_compressdev_pmd.h
create mode 100644 lib/librte_compressdev/rte_compressdev_version.map
With the vast amounts of data being transported around networks
and stored in storage systems, reducing data size is becoming ever more important.
There are both software libraries and hardware devices available that
provide compression, but no common API.
Such an API is proposed in this patchset, which supports the following features:
- Deflate Algorithm (https://tools.ietf.org/html/rfc1951)
- LZS algorithm (https://tools.ietf.org/html/rfc2395)
- Static and Dynamic Huffman encoding.
- Compression levels
- Checksum generation
- Asynchronous burst API
- private_xform - a place for PMDs to hold private data derived from
a xform and used by stateless operations.
- stream - a place for PMDs to hold private data derived from
a xform and also maintain state and history data. For
stateful flows.
Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Pablo de Lara pablo.de.lara.guarch@intel.com
Signed-off-by: Shally Verma <shally.verma@caviumnetworks.com>
Signed-off-by: Ashish Gupta <ashish.gupta@caviumnetworks.com>
Opens:
- creation of private_xform and stream mempools. In v3, config
added to give PMD the size params needed to create mempools.
Still open as to whether the pools should be created by
application, in API layer or by PMDs. Expect to resolve in a v4.
- alternative to mbufs for passing data between application
and accelerator. A lot of varied opinions on this, expect to
address in a future release.
- addition of feature to add capability to PMD to allow more than one
inflight operation from a stateful stream - to be added by NXP
Changes in v4:
- Fix build (missing ";")
- Change order in common_base, so compression is placed after security
Changes in v3:
- Remove rte_comp_op_ctod helper functions
- Remove param_range_check macro
- Rename from phys_addr to iova_addr
- Remove rte_comp_algo_strings
- Rename rte_compressdev_pmd_is_valid_dev to rte_compressdev_is_valid_dev
- Remove feature flags from compressdev
- Support hash operations
- Add shareable priv xform in feature flags, instead of returnin it
on xform creation
- Allow max number of queue pairs to be 0, meaning that there is no
limit.
- Add invalid configuration checks
- Add capability helper functions
Changes in v2:
- Add stream APIs
- Remove session
- Add SHAREABLE / NON_SHAREABLE private_xform types
- Add algo enum 'UNSPECIFIED' to fix warning in capabilities
- Change one remaining log to use dynamic logging.
- Add rte_cache_aligned keyword to op
- Rename enums with better names _ALGO, __CHECKSUM, _HUFFMAN_
- Use const keyword when passing xform
- Remove qp_count fn from dev_ops as never used
- Remove max_nb_queue-pairs from compressdev_init_param as never used
- Clarify device configure and start sequence
- Replace OUT_OF_SPACE with OUT_OF_SPACE_RECOVERABLE and TERMINATED
and clarified usage.
- Add stream and private_xform sizes to device config for use in
mempool creation
- Add capability helper fn
- Use Base2 log value for window size on xforms
- Add Meson build
- Update MAINTAINERS
- Update Map file
- Change order in doxy file
- Update Release note
Fiona Trahe (12):
compressdev: add basic device management
compressdev: add queue pair management
compressdev: add compression specific data
compressdev: add enqueue/dequeue functions
compressdev: add operation management
compressdev: support stateless operations
compressdev: support stateful operations
compressdev: add device feature flags
compressdev: add compression service feature flags
compressdev: add device stats
compressdev: add device capabilities
compressdev: get device id from name
Shally Verma (1):
compressdev: support hash operations
MAINTAINERS | 7 +
config/common_base | 6 +
config/rte_config.h | 3 +
doc/api/doxy-api-index.md | 1 +
doc/api/doxy-api.conf | 1 +
doc/guides/rel_notes/release_18_05.rst | 6 +
lib/Makefile | 3 +
lib/librte_compressdev/Makefile | 29 +
lib/librte_compressdev/meson.build | 9 +
lib/librte_compressdev/rte_comp.h | 539 ++++++++++++
lib/librte_compressdev/rte_compressdev.c | 902 +++++++++++++++++++++
lib/librte_compressdev/rte_compressdev.h | 701 ++++++++++++++++
lib/librte_compressdev/rte_compressdev_pmd.c | 157 ++++
lib/librte_compressdev/rte_compressdev_pmd.h | 437 ++++++++++
lib/librte_compressdev/rte_compressdev_version.map | 44 +
lib/meson.build | 2 +-
mk/rte.app.mk | 1 +
17 files changed, 2847 insertions(+), 1 deletion(-)
create mode 100644 lib/librte_compressdev/Makefile
create mode 100644 lib/librte_compressdev/meson.build
create mode 100644 lib/librte_compressdev/rte_comp.h
create mode 100644 lib/librte_compressdev/rte_compressdev.c
create mode 100644 lib/librte_compressdev/rte_compressdev.h
create mode 100644 lib/librte_compressdev/rte_compressdev_pmd.c
create mode 100644 lib/librte_compressdev/rte_compressdev_pmd.h
create mode 100644 lib/librte_compressdev/rte_compressdev_version.map
With the vast amounts of data being transported around networks and stored in storage systems, reducing data size is becoming ever more important.
There are both software libraries and hardware devices available that provide compression, but no common API.
Such an API is proposed in this patchset, which supports the following features:
- Deflate Algorithm (https://tools.ietf.org/html/rfc1951)
- LZS algorithm (https://tools.ietf.org/html/rfc2395)
- Static and Dynamic Huffman encoding.
- Compression levels
- Checksum generation
- Asynchronous burst API
- private_xform - a place for PMDs to hold private data derived from
a xform and used by stateless operations.
- stream - a place for PMDs to hold private data derived from
a xform and also maintain state and history data. For
stateful flows.
Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Pablo de Lara pablo.de.lara.guarch@intel.com
Signed-off-by: Shally Verma <shally.verma@caviumnetworks.com>
Signed-off-by: Ashish Gupta <ashish.gupta@caviumnetworks.com>
Changes in v5:
- Make rte_compressdev_is_valid_dev, rte_compressdev_pmd_get_dev
functions internal
- Add rte_comp.h to API doc
- Remove unused rte_comp_op_get_user_data,
rte_comp_op_get_user_data_size functions
- Fix comment
- Move private data size arg out of PMD init params structure
- Split out of space error status into two
- Make rte_comp_op_alloc, rte_comp_op_bulk_alloc, rte_comp_free,
rte_compressdev_enqueue_burst and rte_compressdev_dequeue_burst
functions not inline.
- Move rte_compressdev and rte_compressdev_data structures out of
rte_compressdev.h, making them internal to the API
- Remove unneeded driver registration (used in cryptodev, for multi PMD
sessions)
- Remove unneeded includes
- rte_comp_op_bulk_alloc returns now 0 on success, instead of the
number of allocated operations
- Move internals macros to rte_compressdev_internals.h, to avoid
exposing them unnecessarily
Changes in v4:
- Fix build (missing ";")
- Change order in common_base, so compression is placed after security
Changes in v3:
- Remove rte_comp_op_ctod helper functions
- Remove param_range_check macro
- Rename from phys_addr to iova_addr
- Remove rte_comp_algo_strings
- Rename rte_compressdev_pmd_is_valid_dev to rte_compressdev_is_valid_dev
- Remove feature flags from compressdev
- Support hash operations
- Add shareable priv xform in feature flags, instead of returnin it
on xform creation
- Allow max number of queue pairs to be 0, meaning that there is no
limit.
- Add invalid configuration checks
- Add capability helper functions
Changes in v2:
- Add stream APIs
- Remove session
- Add SHAREABLE / NON_SHAREABLE private_xform types
- Add algo enum 'UNSPECIFIED' to fix warning in capabilities
- Change one remaining log to use dynamic logging.
- Add rte_cache_aligned keyword to op
- Rename enums with better names _ALGO, __CHECKSUM, _HUFFMAN_
- Use const keyword when passing xform
- Remove qp_count fn from dev_ops as never used
- Remove max_nb_queue-pairs from compressdev_init_param as never used
- Clarify device configure and start sequence
- Replace OUT_OF_SPACE with OUT_OF_SPACE_RECOVERABLE and TERMINATED
and clarified usage.
- Add stream and private_xform sizes to device config for use in
mempool creation
- Add capability helper fn
- Use Base2 log value for window size on xforms
- Add Meson build
- Update MAINTAINERS
- Update Map file
- Change order in doxy file
- Update Release note
Fiona Trahe (12):
compressdev: add basic device management
compressdev: add queue pair management
compressdev: add compression specific data
compressdev: add enqueue/dequeue functions
compressdev: add operation management
compressdev: support stateless operations
compressdev: support stateful operations
compressdev: add device feature flags
compressdev: add compression service feature flags
compressdev: add device stats
compressdev: add device capabilities
compressdev: get device id from name
Shally Verma (1):
compressdev: support hash operations
MAINTAINERS | 7 +
config/common_base | 6 +
config/rte_config.h | 3 +
doc/api/doxy-api-index.md | 2 +
doc/api/doxy-api.conf | 1 +
doc/guides/rel_notes/release_18_05.rst | 6 +
lib/Makefile | 3 +
lib/librte_compressdev/Makefile | 31 +
lib/librte_compressdev/meson.build | 12 +
lib/librte_compressdev/rte_comp.c | 207 ++++++
lib/librte_compressdev/rte_comp.h | 464 +++++++++++++
lib/librte_compressdev/rte_compressdev.c | 734 +++++++++++++++++++++
lib/librte_compressdev/rte_compressdev.h | 536 +++++++++++++++
lib/librte_compressdev/rte_compressdev_internal.h | 123 ++++
lib/librte_compressdev/rte_compressdev_pmd.c | 160 +++++
lib/librte_compressdev/rte_compressdev_pmd.h | 390 +++++++++++
lib/librte_compressdev/rte_compressdev_version.map | 42 ++
lib/meson.build | 2 +-
mk/rte.app.mk | 1 +
19 files changed, 2729 insertions(+), 1 deletion(-)
create mode 100644 lib/librte_compressdev/Makefile
create mode 100644 lib/librte_compressdev/meson.build
create mode 100644 lib/librte_compressdev/rte_comp.c
create mode 100644 lib/librte_compressdev/rte_comp.h
create mode 100644 lib/librte_compressdev/rte_compressdev.c
create mode 100644 lib/librte_compressdev/rte_compressdev.h
create mode 100644 lib/librte_compressdev/rte_compressdev_internal.h
create mode 100644 lib/librte_compressdev/rte_compressdev_pmd.c
create mode 100644 lib/librte_compressdev/rte_compressdev_pmd.h
create mode 100644 lib/librte_compressdev/rte_compressdev_version.map
With the vast amounts of data being transported around networks
and stored in storage systems, reducing data size is becoming
ever more important.
There are both software libraries and hardware devices available
that provide compression, but no common API.
Such an API is proposed in this patchset, which supports
the following features:
- Deflate Algorithm (https://tools.ietf.org/html/rfc1951)
- LZS algorithm (https://tools.ietf.org/html/rfc2395)
- Static and Dynamic Huffman encoding.
- Compression levels
- Checksum generation
- Asynchronous burst API
- private_xform - a place for PMDs to hold private data derived from
a xform and used by stateless operations.
- stream - a place for PMDs to hold private data derived from
a xform and also maintain state and history data. For
stateful flows.
Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
Signed-off-by: Shally Verma <shally.verma@caviumnetworks.com>
Signed-off-by: Ashish Gupta <ashish.gupta@caviumnetworks.com>
Changes in v6:
- Add programmer's guide section for library
- Change macro for no hash
- Fix compilation in patch 4
- Remove next pointer in xform
- Add queue pair release in device closure
Changes in v5:
- Make rte_compressdev_is_valid_dev, rte_compressdev_pmd_get_dev
functions internal
- Add rte_comp.h to API doc
- Remove unused rte_comp_op_get_user_data,
rte_comp_op_get_user_data_size functions
- Fix comment
- Move private data size arg out of PMD init params structure
- Split out of space error status into two
- Make rte_comp_op_alloc, rte_comp_op_bulk_alloc, rte_comp_free,
rte_compressdev_enqueue_burst and rte_compressdev_dequeue_burst
functions not inline.
- Move rte_compressdev and rte_compressdev_data structures out of
rte_compressdev.h, making them internal to the API
- Remove unneeded driver registration (used in cryptodev, for multi PMD
sessions)
- Remove unneeded includes
- rte_comp_op_bulk_alloc returns now 0 on success, instead of the
number of allocated operations
- Move internals macros to rte_compressdev_internals.h, to avoid
exposing them unnecessarily
Changes in v4:
- Fix build (missing ";")
- Change order in common_base, so compression is placed after security
Changes in v3:
- Remove rte_comp_op_ctod helper functions
- Remove param_range_check macro
- Rename from phys_addr to iova_addr
- Remove rte_comp_algo_strings
- Rename rte_compressdev_pmd_is_valid_dev to rte_compressdev_is_valid_dev
- Remove feature flags from compressdev
- Support hash operations
- Add shareable priv xform in feature flags, instead of returnin it
on xform creation
- Allow max number of queue pairs to be 0, meaning that there is no
limit.
- Add invalid configuration checks
- Add capability helper functions
Changes in v2:
- Add stream APIs
- Remove session
- Add SHAREABLE / NON_SHAREABLE private_xform types
- Add algo enum 'UNSPECIFIED' to fix warning in capabilities
- Change one remaining log to use dynamic logging.
- Add rte_cache_aligned keyword to op
- Rename enums with better names _ALGO, __CHECKSUM, _HUFFMAN_
- Use const keyword when passing xform
- Remove qp_count fn from dev_ops as never used
- Remove max_nb_queue-pairs from compressdev_init_param as never used
- Clarify device configure and start sequence
- Replace OUT_OF_SPACE with OUT_OF_SPACE_RECOVERABLE and TERMINATED
and clarified usage.
- Add stream and private_xform sizes to device config for use in
mempool creation
- Add capability helper fn
- Use Base2 log value for window size on xforms
- Add Meson build
- Update MAINTAINERS
- Update Map file
- Change order in doxy file
- Update Release note
Ashish Gupta (1):
doc: add compressdev library guide
Fiona Trahe (12):
compressdev: add basic device management
compressdev: add queue pair management
compressdev: add compression specific data
compressdev: add enqueue/dequeue functions
compressdev: add operation management
compressdev: support stateless operations
compressdev: support stateful operations
compressdev: add device feature flags
compressdev: add compression service feature flags
compressdev: add device stats
compressdev: add device capabilities
compressdev: get device id from name
Shally Verma (1):
compressdev: support hash operations
MAINTAINERS | 8 +
config/common_base | 6 +
config/rte_config.h | 3 +
doc/api/doxy-api-index.md | 2 +
doc/api/doxy-api.conf | 1 +
doc/guides/prog_guide/compressdev_lib.rst | 623 +++++++++++++++++
doc/guides/prog_guide/img/stateful-op.svg | 116 +++
doc/guides/prog_guide/img/stateless-op-shared.svg | 124 ++++
doc/guides/prog_guide/img/stateless-op.svg | 140 ++++
doc/guides/prog_guide/index.rst | 1 +
doc/guides/rel_notes/release_18_05.rst | 6 +
lib/Makefile | 3 +
lib/librte_compressdev/Makefile | 31 +
lib/librte_compressdev/meson.build | 12 +
lib/librte_compressdev/rte_comp.c | 207 ++++++
lib/librte_compressdev/rte_comp.h | 458 ++++++++++++
lib/librte_compressdev/rte_compressdev.c | 775 +++++++++++++++++++++
lib/librte_compressdev/rte_compressdev.h | 539 ++++++++++++++
lib/librte_compressdev/rte_compressdev_internal.h | 117 ++++
lib/librte_compressdev/rte_compressdev_pmd.c | 160 +++++
lib/librte_compressdev/rte_compressdev_pmd.h | 390 +++++++++++
lib/librte_compressdev/rte_compressdev_version.map | 42 ++
lib/meson.build | 2 +-
mk/rte.app.mk | 1 +
24 files changed, 3766 insertions(+), 1 deletion(-)
create mode 100644 doc/guides/prog_guide/compressdev_lib.rst
create mode 100644 doc/guides/prog_guide/img/stateful-op.svg
create mode 100644 doc/guides/prog_guide/img/stateless-op-shared.svg
create mode 100644 doc/guides/prog_guide/img/stateless-op.svg
create mode 100644 lib/librte_compressdev/Makefile
create mode 100644 lib/librte_compressdev/meson.build
create mode 100644 lib/librte_compressdev/rte_comp.c
create mode 100644 lib/librte_compressdev/rte_comp.h
create mode 100644 lib/librte_compressdev/rte_compressdev.c
create mode 100644 lib/librte_compressdev/rte_compressdev.h
create mode 100644 lib/librte_compressdev/rte_compressdev_internal.h
create mode 100644 lib/librte_compressdev/rte_compressdev_pmd.c
create mode 100644 lib/librte_compressdev/rte_compressdev_pmd.h
create mode 100644 lib/librte_compressdev/rte_compressdev_version.map
> -----Original Message-----
> From: De Lara Guarch, Pablo
> Sent: Friday, April 27, 2018 2:24 PM
> To: dev@dpdk.org
> Cc: Trahe, Fiona <fiona.trahe@intel.com>; shally.verma@cavium.com;
> ahmed.mansour@nxp.com; Ashish.Gupta@cavium.com; De Lara Guarch, Pablo
> <pablo.de.lara.guarch@intel.com>; Shally Verma
> <shally.verma@caviumnetworks.com>; Ashish Gupta
> <ashish.gupta@caviumnetworks.com>
> Subject: [PATCH v6 00/14] Implement compression API
>
> With the vast amounts of data being transported around networks and stored in
> storage systems, reducing data size is becoming ever more important.
>
> There are both software libraries and hardware devices available that provide
> compression, but no common API.
> Such an API is proposed in this patchset, which supports the following features:
>
> - Deflate Algorithm (https://tools.ietf.org/html/rfc1951)
> - LZS algorithm (https://tools.ietf.org/html/rfc2395)
> - Static and Dynamic Huffman encoding.
> - Compression levels
> - Checksum generation
> - Asynchronous burst API
> - private_xform - a place for PMDs to hold private data derived from
> a xform and used by stateless operations.
> - stream - a place for PMDs to hold private data derived from
> a xform and also maintain state and history data. For
> stateful flows.
>
> Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
> Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
> Signed-off-by: Shally Verma <shally.verma@caviumnetworks.com>
> Signed-off-by: Ashish Gupta <ashish.gupta@caviumnetworks.com>
Applied to dpdk-next-crypto.
Thanks,
Pablo
@@ -535,6 +535,12 @@ CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO=n
CONFIG_RTE_LIBRTE_PMD_MRVL_CRYPTO_DEBUG=n
#
+# Compile generic compression device library
+#
+CONFIG_RTE_LIBRTE_COMPRESSDEV=y
+CONFIG_RTE_COMPRESS_MAX_DEVS=64
+
+#
# Compile generic security library
#
CONFIG_RTE_LIBRTE_SECURITY=y
@@ -43,6 +43,7 @@ The public API headers are grouped by topics:
[rte_tm] (@ref rte_tm.h),
[rte_mtr] (@ref rte_mtr.h),
[bbdev] (@ref rte_bbdev.h),
+ [compressdev] (@ref rte_compressdev.h),
[cryptodev] (@ref rte_cryptodev.h),
[security] (@ref rte_security.h),
[eventdev] (@ref rte_eventdev.h),
@@ -45,6 +45,7 @@ INPUT = doc/api/doxy-api-index.md \
lib/librte_cfgfile \
lib/librte_cmdline \
lib/librte_compat \
+ lib/librte_compressdev \
lib/librte_cryptodev \
lib/librte_distributor \
lib/librte_efd \
@@ -26,6 +26,9 @@ DEPDIRS-librte_bbdev := librte_eal librte_mempool librte_mbuf
DIRS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += librte_cryptodev
DEPDIRS-librte_cryptodev := librte_eal librte_mempool librte_ring librte_mbuf
DEPDIRS-librte_cryptodev += librte_kvargs
+DIRS-$(CONFIG_RTE_LIBRTE_COMPRESSDEV) += librte_compressdev
+DEPDIRS-librte_compressdev := librte_eal librte_mempool librte_ring librte_mbuf
+DEPDIRS-librte_compressdev += librte_kvargs
DIRS-$(CONFIG_RTE_LIBRTE_SECURITY) += librte_security
DEPDIRS-librte_security := librte_eal librte_mempool librte_ring librte_mbuf
DEPDIRS-librte_security += librte_ether
new file mode 100644
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2017-2018 Intel Corporation
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_compressdev.a
+
+# library version
+LIBABIVER := 1
+
+# build flags
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+LDLIBS += -lrte_eal -lrte_mempool -lrte_kvargs
+
+# library source files
+SRCS-y += rte_compressdev.c rte_compressdev_pmd.c
+
+# export include files
+SYMLINK-y-include += rte_comp.h
+SYMLINK-y-include += rte_compressdev.h
+SYMLINK-y-include += rte_compressdev_pmd.h
+
+# versioning export map
+EXPORT_MAP := rte_compressdev_version.map
+
+include $(RTE_SDK)/mk/rte.lib.mk
new file mode 100644
@@ -0,0 +1,503 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017-2018 Intel Corporation
+ */
+
+#ifndef _RTE_COMP_H_
+#define _RTE_COMP_H_
+
+/**
+ * @file rte_comp.h
+ *
+ * RTE definitions for Data Compression Service
+ *
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <string.h>
+#include <rte_mempool.h>
+
+
+/** Status of comp operation */
+enum rte_comp_op_status {
+ RTE_COMP_OP_STATUS_SUCCESS = 0,
+ /**< Operation completed successfully */
+ RTE_COMP_OP_STATUS_NOT_PROCESSED,
+ /**< Operation has not yet been processed by the device */
+ RTE_COMP_OP_STATUS_INVALID_SESSION,
+ /**< Operation failed due to invalid session arguments */
+ RTE_COMP_OP_STATUS_INVALID_ARGS,
+ /**< Operation failed due to invalid arguments in request */
+ RTE_COMP_OP_STATUS_ERROR,
+ /**< Error handling operation */
+ RTE_COMP_OP_STATUS_INVALID_STATE,
+ /**< Operation is invoked in invalid state */
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE,
+ /**< Output buffer ran out of space before operation completed */
+};
+
+
+/** Compression Algorithms */
+enum rte_comp_algorithm {
+ RTE_COMP_NULL = 0,
+ /**< No compression.
+ * Pass-through, data is copied unchanged from source buffer to
+ * destination buffer.
+ */
+ RTE_COMP_DEFLATE,
+ /**< DEFLATE compression algorithm
+ * https://tools.ietf.org/html/rfc1951
+ */
+ RTE_COMP_LZS,
+ /**< LZS compression algorithm
+ * https://tools.ietf.org/html/rfc2395
+ */
+ RTE_COMP_ALGO_LIST_END
+};
+
+/**< Compression Level.
+ * The number is interpreted by each PMD differently. However, lower numbers
+ * give fastest compression, at the expense of compression ratio while
+ * higher numbers may give better compression ratios but are likely slower.
+ */
+#define RTE_COMP_LEVEL_PMD_DEFAULT (-1)
+/** Use PMD Default */
+#define RTE_COMP_LEVEL_NONE (0)
+/** Output uncompressed blocks if supported by the specified algorithm */
+#define RTE_COMP_LEVEL_MIN (1)
+/** Use minimum compression level supported by the PMD */
+#define RTE_COMP_LEVEL_MAX (9)
+/** Use maximum compression level supported by the PMD */
+
+/** Compression checksum types */
+enum rte_comp_checksum_type {
+ RTE_COMP_NONE,
+ /**< No checksum generated */
+ RTE_COMP_CRC32,
+ /**< Generates a CRC32 checksum, as used by gzip */
+ RTE_COMP_ADLER32,
+ /**< Generates an Adler-32 checksum, as used by zlib */
+ RTE_COMP_CRC32_ADLER32,
+ /**< Generates both Adler-32 and CRC32 checksums, concatenated.
+ * CRC32 is in the lower 32bits, Adler-32 in the upper 32 bits.
+ */
+};
+
+
+/** Compression Huffman Type - used by DEFLATE algorithm */
+enum rte_comp_huffman {
+ RTE_COMP_DEFAULT,
+ /**< PMD may choose which Huffman codes to use */
+ RTE_COMP_FIXED,
+ /**< Use Fixed Huffman codes */
+ RTE_COMP_DYNAMIC,
+ /**< Use Dynamic Huffman codes */
+};
+
+enum rte_comp_flush_flag {
+ RTE_COMP_FLUSH_NONE,
+ /**< Data is not flushed. Output may remain in the compressor and be
+ * processed during a following op. It may not be possible to decompress
+ * output until a later op with some other flush flag has been sent.
+ */
+ RTE_COMP_FLUSH_SYNC,
+ /**< All data should be flushed to output buffer. Output data can be
+ * decompressed. However state and history is not cleared, so future
+ * operations may use history from this operation.
+ */
+ RTE_COMP_FLUSH_FULL,
+ /**< All data should be flushed to output buffer. Output data can be
+ * decompressed. State and history data is cleared, so future
+ * ops will be independent of ops processed before this.
+ */
+ RTE_COMP_FLUSH_FINAL
+ /**< Same as RTE_COMP_FLUSH_FULL but if op.algo is RTE_COMP_DEFLATE
+ * then bfinal bit is set in the last block.
+ */
+};
+
+/** Compression transform types */
+enum rte_comp_xform_type {
+ RTE_COMP_COMPRESS,
+ /**< Compression service - compress */
+ RTE_COMP_DECOMPRESS,
+ /**< Compression service - decompress */
+};
+
+enum rte_comp_op_type {
+ RTE_COMP_OP_STATELESS,
+ /**< All data to be processed is submitted in the op, no state or history
+ * from previous ops is used and none will be stored for future ops.
+ * Flush flag must be set to either FLUSH_FULL or FLUSH_FINAL.
+ */
+ RTE_COMP_OP_STATEFUL
+ /**< There may be more data to be processed after this op, it's part of a
+ * stream of data. State and history from previous ops can be used
+ * and resulting state and history can be stored for future ops,
+ * depending on flush flag.
+ */
+};
+
+
+/** Parameters specific to the deflate algorithm */
+struct rte_comp_deflate_params {
+ enum rte_comp_huffman huffman;
+ /**< Compression huffman encoding type */
+};
+
+/** Session Setup Data for compression */
+struct rte_comp_compress_xform {
+ enum rte_comp_algorithm algo;
+ /**< Algorithm to use for compress operation */
+ union {
+ struct rte_comp_deflate_params deflate;
+ /**< Parameters specific to the deflate algorithm */
+ }; /**< Algorithm specific parameters */
+ int level;
+ /**< Compression level */
+ uint32_t window_size;
+ /**< Depth of sliding window to be used. If window size can't be
+ * supported by the PMD then it may fall back to a smaller size. This
+ * is likely to result in a worse compression ratio.
+ */
+ enum rte_comp_checksum_type chksum;
+ /**< Type of checksum to generate on the uncompressed data */
+};
+
+/**
+ * Session Setup Data for decompression.
+ */
+struct rte_comp_decompress_xform {
+ enum rte_comp_algorithm algo;
+ /**< Algorithm to use for decompression */
+ enum rte_comp_checksum_type chksum;
+ /**< Type of checksum to generate on the decompressed data */
+ uint32_t window_size;
+ /**< Max depth of sliding window which was used to generate compressed
+ * data. If window size can't be supported by the PMD then session
+ * setup should fail.
+ */
+};
+
+/**
+ * Compression transform structure.
+ *
+ * This is used to specify the compression transforms required.
+ * Each transform structure can hold a single transform, the type field is
+ * used to specify which transform is contained within the union.
+ * There are no chain cases currently supported, just single xforms of
+ * - compress-only
+ * - decompress-only
+ *
+ */
+struct rte_comp_xform {
+ struct rte_comp_xform *next;
+ /**< next xform in chain */
+ enum rte_comp_xform_type type;
+ /**< xform type */
+ union {
+ struct rte_comp_compress_xform compress;
+ /**< xform for compress operation */
+ struct rte_comp_decompress_xform decompress;
+ /**< decompress xform */
+ };
+};
+
+
+struct rte_comp_session;
+/**
+ * Compression Operation.
+ *
+ * This structure contains data relating to performing a compression
+ * operation on the referenced mbuf data buffers.
+ *
+ * Comp operations are enqueued and dequeued in comp PMDs using the
+ * rte_compressdev_enqueue_burst() / rte_compressdev_dequeue_burst() APIs
+ */
+struct rte_comp_op {
+
+ enum rte_comp_op_type op_type;
+ void *stream_private;
+ /* Handle for an initialised stream, which holds state and history data.
+ * rte_comp_stream_create() must be called for all data streams whether
+ * op_type is STATEFUL or STATELESS and the resulting stream attached
+ * to the one or more operations associated with the data stream.
+ */
+ struct rte_comp_session *session;
+ /**< Handle for the initialised session context */
+ struct rte_mempool *mempool;
+ /**< Pool from which operation is allocated */
+ rte_iova_t phys_addr;
+ /**< Physical address of this operation */
+ struct rte_mbuf *m_src;
+ /**< source mbuf
+ * The total size of the input buffer(s) can be retrieved using
+ * rte_pktmbuf_data_len(m_src)
+ */
+ struct rte_mbuf *m_dst;
+ /**< destination mbuf
+ * The total size of the output buffer(s) can be retrieved using
+ * rte_pktmbuf_data_len(m_dst)
+ */
+
+ struct {
+ uint32_t offset;
+ /**< Starting point for compression or decompression,
+ * specified as number of bytes from start of packet in
+ * source buffer.
+ * Starting point for checksum generation in compress direction.
+ */
+ uint32_t length;
+ /**< The length, in bytes, of the data in source buffer
+ * to be compressed or decompressed.
+ * Also the length of the data over which the checksum
+ * should be generated in compress direction
+ */
+ } src;
+ struct {
+ uint32_t offset;
+ /**< Starting point for writing output data, specified as
+ * number of bytes from start of packet in dest
+ * buffer. Starting point for checksum generation in
+ * decompress direction.
+ */
+ } dst;
+ enum rte_comp_flush_flag flush_flag;
+ /**< Defines flush characteristics for the output data.
+ * Only applicable in compress direction
+ */
+ uint64_t input_chksum;
+ /**< An input checksum can be provided to generate a
+ * cumulative checksum across sequential blocks in a STATELESS stream.
+ * Checksum type is as specified in xform chksum_type
+ */
+ uint64_t output_chksum;
+ /**< If a checksum is generated it will be written in here.
+ * Checksum type is as specified in xform chksum_type.
+ */
+ uint32_t consumed;
+ /**< The number of bytes from the source buffer
+ * which were compressed/decompressed.
+ */
+ uint32_t produced;
+ /**< The number of bytes written to the destination buffer
+ * which were compressed/decompressed.
+ */
+ uint64_t debug_status;
+ /**<
+ * Status of the operation is returned in the status param.
+ * This field allows the PMD to pass back extra
+ * pmd-specific debug information. Value is not defined on the API.
+ */
+ uint8_t status;
+ /**<
+ * Operation status - use values from enum rte_comp_status.
+ * This is reset to
+ * RTE_COMP_OP_STATUS_NOT_PROCESSED on allocation from mempool and
+ * will be set to RTE_COMP_OP_STATUS_SUCCESS after operation
+ * is successfully processed by a PMD
+ */
+};
+
+
+/**
+ * Reset the fields of an operation to their default values.
+ *
+ * @note The private data associated with the operation is not zeroed.
+ *
+ * @param op
+ * The operation to be reset
+ */
+static inline void
+__rte_comp_op_reset(struct rte_comp_op *op)
+{
+ struct rte_mempool *tmp_mp = op->mempool;
+ phys_addr_t tmp_phys_addr = op->phys_addr;
+
+ memset(op, 0, sizeof(struct rte_comp_op));
+ op->status = RTE_COMP_OP_STATUS_NOT_PROCESSED;
+ op->phys_addr = tmp_phys_addr;
+ op->mempool = tmp_mp;
+}
+
+/**
+ * Private data structure belonging to an operation pool.
+ */
+struct rte_comp_op_pool_private {
+ uint16_t user_size;
+ /**< Size of private user data with each operation. */
+};
+
+
+/**
+ * Returns the size of private user data allocated with each object in
+ * the mempool
+ *
+ * @param mempool
+ * Mempool for operations
+ * @return
+ * user data size
+ */
+static inline uint16_t
+__rte_comp_op_get_user_data_size(struct rte_mempool *mempool)
+{
+ struct rte_comp_op_pool_private *priv =
+ (struct rte_comp_op_pool_private *)rte_mempool_get_priv(mempool);
+
+ return priv->user_size;
+}
+
+
+/**
+ * Creates an operation pool
+ *
+ * @param name
+ * Compress pool name
+ * @param nb_elts
+ * Number of elements in pool
+ * @param cache_size
+ * Number of elements to cache on lcore, see
+ * *rte_mempool_create* for further details about cache size
+ * @param user_size
+ * Size of private data to allocate for user with each operation
+ * @param socket_id
+ * Socket to identifier allocate memory on
+ * @return
+ * - On success pointer to mempool
+ * - On failure NULL
+ */
+struct rte_mempool *
+rte_comp_op_pool_create(const char *name,
+ unsigned int nb_elts, unsigned int cache_size,
+ uint16_t user_size, int socket_id);
+
+/**
+ * Bulk allocate raw element from mempool and return as comp operations
+ *
+ * @param mempool
+ * Compress operation mempool
+ * @param ops
+ * Array to place allocated operations
+ * @param nb_ops
+ * Number of operations to allocate
+ * @return
+ * - On success returns number of ops allocated
+ */
+static inline int
+__rte_comp_op_raw_bulk_alloc(struct rte_mempool *mempool,
+ struct rte_comp_op **ops, uint16_t nb_ops)
+{
+ if (rte_mempool_get_bulk(mempool, (void **)ops, nb_ops) == 0)
+ return nb_ops;
+
+ return 0;
+}
+
+/**
+ * Allocate an operation from a mempool with default parameters set
+ *
+ * @param mempool
+ * Compress operation mempool
+ *
+ * @return
+ * - On success returns a valid rte_comp_op structure
+ * - On failure returns NULL
+ */
+static inline struct rte_comp_op *
+rte_comp_op_alloc(struct rte_mempool *mempool)
+{
+ struct rte_comp_op *op = NULL;
+ int retval;
+
+ retval = __rte_comp_op_raw_bulk_alloc(mempool, &op, 1);
+ if (unlikely(retval != 1))
+ return NULL;
+
+ __rte_comp_op_reset(op);
+
+ return op;
+}
+
+
+/**
+ * Bulk allocate operations from a mempool with default parameters set
+ *
+ * @param mempool
+ * Compress operation mempool
+ * @param ops
+ * Array to place allocated operations
+ * @param nb_ops
+ * Number of operations to allocate
+ * @return
+ * - nb_ops if the number of operations requested were allocated.
+ * - 0 if the requested number of ops are not available.
+ * None are allocated in this case.
+ */
+static inline unsigned
+rte_comp_op_bulk_alloc(struct rte_mempool *mempool,
+ struct rte_comp_op **ops, uint16_t nb_ops)
+{
+ int i;
+
+ if (unlikely(__rte_comp_op_raw_bulk_alloc(mempool, ops, nb_ops)
+ != nb_ops))
+ return 0;
+
+ for (i = 0; i < nb_ops; i++)
+ __rte_comp_op_reset(ops[i]);
+
+ return nb_ops;
+}
+
+
+
+/**
+ * Returns a pointer to the private user data of an operation if
+ * that operation has enough capacity for requested size.
+ *
+ * @param op
+ * Compress operation
+ * @param size
+ * Size of space requested in private data
+ * @return
+ * - if sufficient space available returns pointer to start of user data
+ * - if insufficient space returns NULL
+ */
+static inline void *
+__rte_comp_op_get_user_data(struct rte_comp_op *op, uint32_t size)
+{
+ uint32_t user_size;
+
+ if (likely(op->mempool != NULL)) {
+ user_size = __rte_comp_op_get_user_data_size(op->mempool);
+
+ if (likely(user_size >= size))
+ return (void *)(op + 1);
+
+ }
+
+ return NULL;
+}
+
+/**
+ * free operation structure
+ * If operation has been allocate from a rte_mempool, then the operation will
+ * be returned to the mempool.
+ *
+ * @param op
+ * Compress operation
+ */
+static inline void
+rte_comp_op_free(struct rte_comp_op *op)
+{
+ if (op != NULL && op->mempool != NULL)
+ rte_mempool_put(op->mempool, op);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_COMP_H_ */
new file mode 100644
@@ -0,0 +1,902 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017-2018 Intel Corporation
+ */
+
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <netinet/in.h>
+
+#include <rte_byteorder.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_interrupts.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_tailq.h>
+#include <rte_eal.h>
+#include <rte_per_lcore.h>
+#include <rte_lcore.h>
+#include <rte_atomic.h>
+#include <rte_branch_prediction.h>
+#include <rte_common.h>
+#include <rte_mempool.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_errno.h>
+#include <rte_spinlock.h>
+#include <rte_string_fns.h>
+
+#include "rte_comp.h"
+#include "rte_compressdev.h"
+#include "rte_compressdev_pmd.h"
+
+static uint8_t nb_drivers;
+
+struct rte_compressdev rte_comp_devices[RTE_COMPRESS_MAX_DEVS];
+
+struct rte_compressdev *rte_compressdevs = &rte_comp_devices[0];
+
+static struct rte_compressdev_global compressdev_globals = {
+ .devs = &rte_comp_devices[0],
+ .data = { NULL },
+ .nb_devs = 0,
+ .max_devs = RTE_COMPRESS_MAX_DEVS
+};
+
+struct rte_compressdev_global *rte_compressdev_globals = &compressdev_globals;
+
+/**
+ * The compression algorithm strings identifiers.
+ * It could be used in application command line.
+ */
+const char *
+rte_comp_algorithm_strings[] = {
+ [RTE_COMP_DEFLATE] = "deflate",
+ [RTE_COMP_LZS] = "lzs"
+};
+
+
+#define param_range_check(x, y) \
+ (((x < y.min) || (x > y.max)) || \
+ (y.increment != 0 && (x % y.increment) != 0))
+
+
+const char * __rte_experimental
+rte_compressdev_get_feature_name(uint64_t flag)
+{
+ switch (flag) {
+ case RTE_COMPDEV_FF_HW_ACCELERATED:
+ return "HW_ACCELERATED";
+ case RTE_COMPDEV_FF_CPU_SSE:
+ return "CPU_SSE";
+ case RTE_COMPDEV_FF_CPU_AVX:
+ return "CPU_AVX";
+ case RTE_COMPDEV_FF_CPU_AVX2:
+ return "CPU_AVX2";
+ case RTE_COMPDEV_FF_CPU_AVX512:
+ return "CPU_AVX512";
+ case RTE_COMPDEV_FF_CPU_NEON:
+ return "CPU_NEON";
+ default:
+ return NULL;
+ }
+}
+
+const char * __rte_experimental
+rte_comp_get_feature_name(uint64_t flag)
+{
+ switch (flag) {
+ case RTE_COMP_FF_STATEFUL_COMPRESSION:
+ return "STATEFUL_COMPRESSION";
+ case RTE_COMP_FF_STATEFUL_DECOMPRESSION:
+ return "STATEFUL_DECOMPRESSION";
+ case RTE_COMP_FF_MBUF_SCATTER_GATHER:
+ return "MBUF_SCATTER_GATHER";
+ case RTE_COMP_FF_MULTI_PKT_CHECKSUM:
+ return "MULTI_PKT_CHECKSUM";
+ case RTE_COMP_FF_ADLER32_CHECKSUM:
+ return "ADLER32_CHECKSUM";
+ case RTE_COMP_FF_CRC32_CHECKSUM:
+ return "CRC32_CHECKSUM";
+ case RTE_COMP_FF_CRC32_ADLER32_CHECKSUM:
+ return "CRC32_ADLER32_CHECKSUM";
+ case RTE_COMP_FF_NONCOMPRESSED_BLOCKS:
+ return "NONCOMPRESSED_BLOCKS";
+ default:
+ return NULL;
+ }
+}
+
+struct rte_compressdev * __rte_experimental
+rte_compressdev_pmd_get_dev(uint8_t dev_id)
+{
+ return &rte_compressdev_globals->devs[dev_id];
+}
+
+struct rte_compressdev * __rte_experimental
+rte_compressdev_pmd_get_named_dev(const char *name)
+{
+ struct rte_compressdev *dev;
+ unsigned int i;
+
+ if (name == NULL)
+ return NULL;
+
+ for (i = 0; i < rte_compressdev_globals->max_devs; i++) {
+ dev = &rte_compressdev_globals->devs[i];
+
+ if ((dev->attached == RTE_COMPRESSDEV_ATTACHED) &&
+ (strcmp(dev->data->name, name) == 0))
+ return dev;
+ }
+
+ return NULL;
+}
+
+unsigned int __rte_experimental
+rte_compressdev_pmd_is_valid_dev(uint8_t dev_id)
+{
+ struct rte_compressdev *dev = NULL;
+
+ if (dev_id >= rte_compressdev_globals->nb_devs)
+ return 0;
+
+ dev = rte_compressdev_pmd_get_dev(dev_id);
+ if (dev->attached != RTE_COMPRESSDEV_ATTACHED)
+ return 0;
+ else
+ return 1;
+}
+
+
+int __rte_experimental
+rte_compressdev_get_dev_id(const char *name)
+{
+ unsigned int i;
+
+ if (name == NULL)
+ return -1;
+
+ for (i = 0; i < rte_compressdev_globals->nb_devs; i++)
+ if ((strcmp(rte_compressdev_globals->devs[i].data->name, name)
+ == 0) &&
+ (rte_compressdev_globals->devs[i].attached ==
+ RTE_COMPRESSDEV_ATTACHED))
+ return i;
+
+ return -1;
+}
+
+uint8_t __rte_experimental
+rte_compressdev_count(void)
+{
+ return rte_compressdev_globals->nb_devs;
+}
+
+uint8_t __rte_experimental
+rte_compressdev_device_count_by_driver(uint8_t driver_id)
+{
+ uint8_t i, dev_count = 0;
+
+ for (i = 0; i < rte_compressdev_globals->max_devs; i++)
+ if (rte_compressdev_globals->devs[i].driver_id == driver_id &&
+ rte_compressdev_globals->devs[i].attached ==
+ RTE_COMPRESSDEV_ATTACHED)
+ dev_count++;
+
+ return dev_count;
+}
+
+uint8_t __rte_experimental
+rte_compressdev_devices_get(const char *driver_name, uint8_t *devices,
+ uint8_t nb_devices)
+{
+ uint8_t i, count = 0;
+ struct rte_compressdev *devs = rte_compressdev_globals->devs;
+ uint8_t max_devs = rte_compressdev_globals->max_devs;
+
+ for (i = 0; i < max_devs && count < nb_devices; i++) {
+
+ if (devs[i].attached == RTE_COMPRESSDEV_ATTACHED) {
+ int cmp;
+
+ cmp = strncmp(devs[i].device->driver->name,
+ driver_name,
+ strlen(driver_name));
+
+ if (cmp == 0)
+ devices[count++] = devs[i].data->dev_id;
+ }
+ }
+
+ return count;
+}
+
+
+int __rte_experimental
+rte_compressdev_socket_id(uint8_t dev_id)
+{
+ struct rte_compressdev *dev;
+
+ if (!rte_compressdev_pmd_is_valid_dev(dev_id))
+ return -1;
+
+ dev = rte_compressdev_pmd_get_dev(dev_id);
+
+ return dev->data->socket_id;
+}
+
+static inline int
+rte_compressdev_data_alloc(uint8_t dev_id, struct rte_compressdev_data **data,
+ int socket_id)
+{
+ char mz_name[RTE_COMPRESSDEV_NAME_MAX_LEN];
+ const struct rte_memzone *mz;
+ int n;
+
+ /* generate memzone name */
+ n = snprintf(mz_name, sizeof(mz_name),
+ "rte_compressdev_data_%u", dev_id);
+ if (n >= (int)sizeof(mz_name))
+ return -EINVAL;
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ mz = rte_memzone_reserve(mz_name,
+ sizeof(struct rte_compressdev_data),
+ socket_id, 0);
+ } else
+ mz = rte_memzone_lookup(mz_name);
+
+ if (mz == NULL)
+ return -ENOMEM;
+
+ *data = mz->addr;
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ memset(*data, 0, sizeof(struct rte_compressdev_data));
+
+ return 0;
+}
+
+static uint8_t
+rte_compressdev_find_free_device_index(void)
+{
+ uint8_t dev_id;
+
+ for (dev_id = 0; dev_id < RTE_COMPRESS_MAX_DEVS; dev_id++) {
+ if (rte_comp_devices[dev_id].attached ==
+ RTE_COMPRESSDEV_DETACHED)
+ return dev_id;
+ }
+ return RTE_COMPRESS_MAX_DEVS;
+}
+
+struct rte_compressdev * __rte_experimental
+rte_compressdev_pmd_allocate(const char *name, int socket_id)
+{
+ struct rte_compressdev *compressdev;
+ uint8_t dev_id;
+
+ if (rte_compressdev_pmd_get_named_dev(name) != NULL) {
+ COMPRESSDEV_LOG(ERR,
+ "comp device with name %s already allocated!", name);
+ return NULL;
+ }
+
+ dev_id = rte_compressdev_find_free_device_index();
+ if (dev_id == RTE_COMPRESS_MAX_DEVS) {
+ COMPRESSDEV_LOG(ERR, "Reached maximum number of comp devices");
+ return NULL;
+ }
+
+ compressdev = rte_compressdev_pmd_get_dev(dev_id);
+
+ if (compressdev->data == NULL) {
+ struct rte_compressdev_data *compressdev_data =
+ compressdev_globals.data[dev_id];
+
+ int retval = rte_compressdev_data_alloc(dev_id,
+ &compressdev_data, socket_id);
+
+ if (retval < 0 || compressdev_data == NULL)
+ return NULL;
+
+ compressdev->data = compressdev_data;
+
+ snprintf(compressdev->data->name, RTE_COMPRESSDEV_NAME_MAX_LEN,
+ "%s", name);
+
+ compressdev->data->dev_id = dev_id;
+ compressdev->data->socket_id = socket_id;
+ compressdev->data->dev_started = 0;
+
+ compressdev->attached = RTE_COMPRESSDEV_ATTACHED;
+
+ compressdev_globals.nb_devs++;
+ }
+
+ return compressdev;
+}
+
+int __rte_experimental
+rte_compressdev_pmd_release_device(struct rte_compressdev *compressdev)
+{
+ int ret;
+
+ if (compressdev == NULL)
+ return -EINVAL;
+
+ /* Close device only if device operations have been set */
+ if (compressdev->dev_ops) {
+ ret = rte_compressdev_close(compressdev->data->dev_id);
+ if (ret < 0)
+ return ret;
+ }
+
+ compressdev->attached = RTE_COMPRESSDEV_DETACHED;
+ compressdev_globals.nb_devs--;
+ return 0;
+}
+
+uint16_t __rte_experimental
+rte_compressdev_queue_pair_count(uint8_t dev_id)
+{
+ struct rte_compressdev *dev;
+
+ dev = &rte_comp_devices[dev_id];
+ return dev->data->nb_queue_pairs;
+}
+
+static int
+rte_compressdev_queue_pairs_config(struct rte_compressdev *dev,
+ uint16_t nb_qpairs, int socket_id)
+{
+ struct rte_compressdev_info dev_info;
+ void **qp;
+ unsigned int i;
+
+ if ((dev == NULL) || (nb_qpairs < 1)) {
+ COMPRESSDEV_LOG(ERR, "invalid param: dev %p, nb_queues %u",
+ dev, nb_qpairs);
+ return -EINVAL;
+ }
+
+ COMPRESSDEV_LOG(DEBUG, "Setup %d queues pairs on device %u",
+ nb_qpairs, dev->data->dev_id);
+
+ memset(&dev_info, 0, sizeof(struct rte_compressdev_info));
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
+ (*dev->dev_ops->dev_infos_get)(dev, &dev_info);
+
+ if (nb_qpairs > (dev_info.max_nb_queue_pairs)) {
+ COMPRESSDEV_LOG(ERR, "Invalid num queue_pairs (%u) for dev %u",
+ nb_qpairs, dev->data->dev_id);
+ return -EINVAL;
+ }
+
+ if (dev->data->queue_pairs == NULL) { /* first time configuration */
+ dev->data->queue_pairs = rte_zmalloc_socket(
+ "compressdev->queue_pairs",
+ sizeof(dev->data->queue_pairs[0]) * nb_qpairs,
+ RTE_CACHE_LINE_SIZE, socket_id);
+
+ if (dev->data->queue_pairs == NULL) {
+ dev->data->nb_queue_pairs = 0;
+ COMPRESSDEV_LOG(ERR,
+ "failed to get memory for qp meta data, nb_queues %u",
+ nb_qpairs);
+ return -(ENOMEM);
+ }
+ } else { /* re-configure */
+ int ret;
+ uint16_t old_nb_queues = dev->data->nb_queue_pairs;
+
+ qp = dev->data->queue_pairs;
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_release,
+ -ENOTSUP);
+
+ for (i = nb_qpairs; i < old_nb_queues; i++) {
+ ret = (*dev->dev_ops->queue_pair_release)(dev, i);
+ if (ret < 0)
+ return ret;
+ }
+
+ qp = rte_realloc(qp, sizeof(qp[0]) * nb_qpairs,
+ RTE_CACHE_LINE_SIZE);
+ if (qp == NULL) {
+ COMPRESSDEV_LOG(ERR,
+ "failed to realloc qp meta data, nb_queues %u",
+ nb_qpairs);
+ return -(ENOMEM);
+ }
+
+ if (nb_qpairs > old_nb_queues) {
+ uint16_t new_qs = nb_qpairs - old_nb_queues;
+
+ memset(qp + old_nb_queues, 0,
+ sizeof(qp[0]) * new_qs);
+ }
+
+ dev->data->queue_pairs = qp;
+
+ }
+ dev->data->nb_queue_pairs = nb_qpairs;
+ return 0;
+}
+
+int __rte_experimental
+rte_compressdev_configure(uint8_t dev_id, struct rte_compressdev_config *config)
+{
+ struct rte_compressdev *dev;
+ int diag;
+
+ if (!rte_compressdev_pmd_is_valid_dev(dev_id)) {
+ COMPRESSDEV_LOG(ERR, "Invalid dev_id=%" PRIu8, dev_id);
+ return -EINVAL;
+ }
+
+ dev = &rte_comp_devices[dev_id];
+
+ if (dev->data->dev_started) {
+ COMPRESSDEV_LOG(ERR,
+ "device %d must be stopped to allow configuration", dev_id);
+ return -EBUSY;
+ }
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
+
+ /* Setup new number of queue pairs and reconfigure device. */
+ diag = rte_compressdev_queue_pairs_config(dev, config->nb_queue_pairs,
+ config->socket_id);
+ if (diag != 0) {
+ COMPRESSDEV_LOG(ERR,
+ "dev%d rte_comp_dev_queue_pairs_config = %d",
+ dev_id, diag);
+ return diag;
+ }
+
+ return (*dev->dev_ops->dev_configure)(dev, config);
+}
+
+
+int __rte_experimental
+rte_compressdev_start(uint8_t dev_id)
+{
+ struct rte_compressdev *dev;
+ int diag;
+
+ COMPRESSDEV_LOG(DEBUG, "Start dev_id=%" PRIu8, dev_id);
+
+ if (!rte_compressdev_pmd_is_valid_dev(dev_id)) {
+ COMPRESSDEV_LOG(ERR, "Invalid dev_id=%" PRIu8, dev_id);
+ return -EINVAL;
+ }
+
+ dev = &rte_comp_devices[dev_id];
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
+
+ if (dev->data->dev_started != 0) {
+ COMPRESSDEV_LOG(ERR,
+ "Device with dev_id=%" PRIu8 " already started", dev_id);
+ return 0;
+ }
+
+ diag = (*dev->dev_ops->dev_start)(dev);
+ if (diag == 0)
+ dev->data->dev_started = 1;
+ else
+ return diag;
+
+ return 0;
+}
+
+void __rte_experimental
+rte_compressdev_stop(uint8_t dev_id)
+{
+ struct rte_compressdev *dev;
+
+ if (!rte_compressdev_pmd_is_valid_dev(dev_id)) {
+ COMPRESSDEV_LOG(ERR, "Invalid dev_id=%" PRIu8, dev_id);
+ return;
+ }
+
+ dev = &rte_comp_devices[dev_id];
+
+ RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
+
+ if (dev->data->dev_started == 0) {
+ COMPRESSDEV_LOG(ERR,
+ "Device with dev_id=%" PRIu8 " already stopped", dev_id);
+ return;
+ }
+
+ (*dev->dev_ops->dev_stop)(dev);
+ dev->data->dev_started = 0;
+}
+
+int __rte_experimental
+rte_compressdev_close(uint8_t dev_id)
+{
+ struct rte_compressdev *dev;
+ int retval;
+
+ if (!rte_compressdev_pmd_is_valid_dev(dev_id)) {
+ COMPRESSDEV_LOG(ERR, "Invalid dev_id=%" PRIu8, dev_id);
+ return -1;
+ }
+
+ dev = &rte_comp_devices[dev_id];
+
+ /* Device must be stopped before it can be closed */
+ if (dev->data->dev_started == 1) {
+ COMPRESSDEV_LOG(ERR, "Device %u must be stopped before closing",
+ dev_id);
+ return -EBUSY;
+ }
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
+ retval = (*dev->dev_ops->dev_close)(dev);
+
+ if (retval < 0)
+ return retval;
+
+ return 0;
+}
+
+int __rte_experimental
+rte_compressdev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
+ uint32_t max_inflight_ops, int socket_id)
+{
+ struct rte_compressdev *dev;
+
+ if (!rte_compressdev_pmd_is_valid_dev(dev_id)) {
+ COMPRESSDEV_LOG(ERR, "Invalid dev_id=%" PRIu8, dev_id);
+ return -EINVAL;
+ }
+
+ dev = &rte_comp_devices[dev_id];
+ if (queue_pair_id >= dev->data->nb_queue_pairs) {
+ COMPRESSDEV_LOG(ERR, "Invalid queue_pair_id=%d", queue_pair_id);
+ return -EINVAL;
+ }
+
+ if (dev->data->dev_started) {
+ COMPRESSDEV_LOG(ERR,
+ "device %d must be stopped to allow configuration", dev_id);
+ return -EBUSY;
+ }
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_setup, -ENOTSUP);
+
+ return (*dev->dev_ops->queue_pair_setup)(dev, queue_pair_id,
+ max_inflight_ops, socket_id);
+}
+
+
+int __rte_experimental
+rte_compressdev_stats_get(uint8_t dev_id, struct rte_compressdev_stats *stats)
+{
+ struct rte_compressdev *dev;
+
+ if (!rte_compressdev_pmd_is_valid_dev(dev_id)) {
+ COMPRESSDEV_LOG(ERR, "Invalid dev_id=%d", dev_id);
+ return -ENODEV;
+ }
+
+ if (stats == NULL) {
+ COMPRESSDEV_LOG(ERR, "Invalid stats ptr");
+ return -EINVAL;
+ }
+
+ dev = &rte_comp_devices[dev_id];
+ memset(stats, 0, sizeof(*stats));
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
+ (*dev->dev_ops->stats_get)(dev, stats);
+ return 0;
+}
+
+void __rte_experimental
+rte_compressdev_stats_reset(uint8_t dev_id)
+{
+ struct rte_compressdev *dev;
+
+ if (!rte_compressdev_pmd_is_valid_dev(dev_id)) {
+ COMPRESSDEV_LOG(ERR, "Invalid dev_id=%" PRIu8, dev_id);
+ return;
+ }
+
+ dev = &rte_comp_devices[dev_id];
+
+ RTE_FUNC_PTR_OR_RET(*dev->dev_ops->stats_reset);
+ (*dev->dev_ops->stats_reset)(dev);
+}
+
+
+void __rte_experimental
+rte_compressdev_info_get(uint8_t dev_id, struct rte_compressdev_info *dev_info)
+{
+ struct rte_compressdev *dev;
+
+ if (dev_id >= compressdev_globals.nb_devs) {
+ COMPRESSDEV_LOG(ERR, "Invalid dev_id=%d", dev_id);
+ return;
+ }
+
+ dev = &rte_comp_devices[dev_id];
+
+ memset(dev_info, 0, sizeof(struct rte_compressdev_info));
+
+ RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
+ (*dev->dev_ops->dev_infos_get)(dev, dev_info);
+
+ dev_info->driver_name = dev->device->driver->name;
+}
+
+
+int __rte_experimental
+rte_compressdev_session_init(uint8_t dev_id,
+ struct rte_comp_session *sess,
+ struct rte_comp_xform *xforms,
+ struct rte_mempool *mp)
+{
+ struct rte_compressdev *dev;
+ uint8_t index;
+ int ret;
+
+ dev = rte_compressdev_pmd_get_dev(dev_id);
+
+ if (sess == NULL || xforms == NULL || dev == NULL)
+ return -EINVAL;
+
+ index = dev->driver_id;
+
+ if (sess->sess_private_data[index] == NULL) {
+ ret = dev->dev_ops->session_configure(dev, xforms, sess, mp);
+ if (ret < 0) {
+ COMPRESSDEV_LOG(ERR,
+ "dev_id %d failed to configure session details",
+ dev_id);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+struct rte_comp_session * __rte_experimental
+rte_compressdev_session_create(struct rte_mempool *mp)
+{
+ struct rte_comp_session *sess;
+
+ /* Allocate a session structure from the session pool */
+ if (rte_mempool_get(mp, (void *)&sess)) {
+ COMPRESSDEV_LOG(ERR, "couldn't get object from session mempool");
+ return NULL;
+ }
+
+ /* Clear device session pointer */
+ memset(sess, 0, (sizeof(void *) * nb_drivers));
+
+ return sess;
+}
+
+int __rte_experimental
+rte_compressdev_session_clear(uint8_t dev_id,
+ struct rte_comp_session *sess)
+{
+ struct rte_compressdev *dev;
+
+ dev = rte_compressdev_pmd_get_dev(dev_id);
+
+ if (dev == NULL || sess == NULL)
+ return -EINVAL;
+
+ dev->dev_ops->session_clear(dev, sess);
+
+ return 0;
+}
+
+int __rte_experimental
+rte_compressdev_session_terminate(struct rte_comp_session *sess)
+{
+ uint8_t i;
+ void *sess_priv;
+ struct rte_mempool *sess_mp;
+
+ if (sess == NULL)
+ return -EINVAL;
+
+ /* Check that all device private data has been freed */
+ for (i = 0; i < nb_drivers; i++) {
+ sess_priv = get_session_private_data(sess, i);
+ if (sess_priv != NULL)
+ return -EBUSY;
+ }
+
+ /* Return session to mempool */
+ sess_mp = rte_mempool_from_obj(sess);
+ rte_mempool_put(sess_mp, sess);
+
+ return 0;
+}
+
+unsigned int __rte_experimental
+rte_compressdev_get_header_session_size(void)
+{
+ /*
+ * Header contains pointers to the private data
+ * of all registered drivers
+ */
+ return (sizeof(void *) * nb_drivers);
+}
+
+unsigned int __rte_experimental
+rte_compressdev_get_private_session_size(uint8_t dev_id)
+{
+ struct rte_compressdev *dev;
+
+ if (!rte_compressdev_pmd_is_valid_dev(dev_id))
+ return 0;
+
+ dev = rte_compressdev_pmd_get_dev(dev_id);
+
+ if (*dev->dev_ops->session_get_size == NULL)
+ return 0;
+
+ return (*dev->dev_ops->session_get_size)(dev);
+
+}
+
+/** Initialise rte_comp_op mempool element */
+static void
+rte_comp_op_init(struct rte_mempool *mempool,
+ __rte_unused void *opaque_arg,
+ void *_op_data,
+ __rte_unused unsigned int i)
+{
+ struct rte_comp_op *op = _op_data;
+
+ memset(_op_data, 0, mempool->elt_size);
+
+ op->status = RTE_COMP_OP_STATUS_NOT_PROCESSED;
+ op->phys_addr = rte_mem_virt2iova(_op_data);
+ op->mempool = mempool;
+}
+
+
+struct rte_mempool * __rte_experimental
+rte_comp_op_pool_create(const char *name,
+ unsigned int nb_elts, unsigned int cache_size,
+ uint16_t user_size, int socket_id)
+{
+ struct rte_comp_op_pool_private *priv;
+
+ unsigned int elt_size = sizeof(struct rte_comp_op) + user_size;
+
+ /* lookup mempool in case already allocated */
+ struct rte_mempool *mp = rte_mempool_lookup(name);
+
+ if (mp != NULL) {
+ priv = (struct rte_comp_op_pool_private *)
+ rte_mempool_get_priv(mp);
+
+ if (mp->elt_size != elt_size ||
+ mp->cache_size < cache_size ||
+ mp->size < nb_elts ||
+ priv->user_size < user_size) {
+ mp = NULL;
+ COMPRESSDEV_LOG(ERR,
+ "Mempool %s already exists but with incompatible parameters",
+ name);
+ return NULL;
+ }
+ return mp;
+ }
+
+ mp = rte_mempool_create(
+ name,
+ nb_elts,
+ elt_size,
+ cache_size,
+ sizeof(struct rte_comp_op_pool_private),
+ NULL,
+ NULL,
+ rte_comp_op_init,
+ NULL,
+ socket_id,
+ 0);
+
+ if (mp == NULL) {
+ COMPRESSDEV_LOG(ERR, "Failed to create mempool %s", name);
+ return NULL;
+ }
+
+ priv = (struct rte_comp_op_pool_private *)
+ rte_mempool_get_priv(mp);
+
+ priv->user_size = user_size;
+
+ return mp;
+}
+
+TAILQ_HEAD(compressdev_driver_list, compressdev_driver);
+
+static struct compressdev_driver_list compressdev_driver_list =
+ TAILQ_HEAD_INITIALIZER(compressdev_driver_list);
+
+int __rte_experimental
+rte_compressdev_driver_id_get(const char *name)
+{
+ struct compressdev_driver *driver;
+ const char *driver_name;
+
+ if (name == NULL) {
+ RTE_LOG(DEBUG, COMPRESSDEV, "name pointer NULL");
+ return -1;
+ }
+
+ TAILQ_FOREACH(driver, &compressdev_driver_list, next) {
+ driver_name = driver->driver->name;
+ if (strncmp(driver_name, name, strlen(driver_name)) == 0)
+ return driver->id;
+ }
+ return -1;
+}
+
+const char * __rte_experimental
+rte_compressdev_name_get(uint8_t dev_id)
+{
+ struct rte_compressdev *dev = rte_compressdev_pmd_get_dev(dev_id);
+
+ if (dev == NULL)
+ return NULL;
+
+ return dev->data->name;
+}
+
+const char * __rte_experimental
+rte_compressdev_driver_name_get(uint8_t driver_id)
+{
+ struct compressdev_driver *driver;
+
+ TAILQ_FOREACH(driver, &compressdev_driver_list, next)
+ if (driver->id == driver_id)
+ return driver->driver->name;
+ return NULL;
+}
+
+uint8_t
+rte_compressdev_allocate_driver(struct compressdev_driver *comp_drv,
+ const struct rte_driver *drv)
+{
+ comp_drv->driver = drv;
+ comp_drv->id = nb_drivers;
+
+ TAILQ_INSERT_TAIL(&compressdev_driver_list, comp_drv, next);
+
+ return nb_drivers++;
+}
+
+RTE_INIT(rte_compressdev_log);
+
+static void
+rte_compressdev_log(void)
+{
+ compressdev_logtype = rte_log_register("librte.compressdev");
+ if (compressdev_logtype >= 0)
+ rte_log_set_level(compressdev_logtype, RTE_LOG_NOTICE);
+}
new file mode 100644
@@ -0,0 +1,757 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017-2018 Intel Corporation
+ */
+
+#ifndef _RTE_COMPRESSDEV_H_
+#define _RTE_COMPRESSDEV_H_
+
+/**
+ * @file rte_compressdev.h
+ *
+ * RTE Compression Device APIs
+ *
+ * Defines comp device APIs for the provisioning of compression operations.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "rte_kvargs.h"
+#include "rte_comp.h"
+#include "rte_dev.h"
+#include <rte_common.h>
+
+extern const char **rte_cyptodev_names;
+
+/* Logging Macros */
+extern int compressdev_logtype;
+#define COMPRESSDEV_LOG(level, fmt, args...) \
+ rte_log(RTE_LOG_ ## level, compressdev_logtype, "%s(): "fmt "\n", \
+ __func__, ##args)
+
+#define RTE_COMPRESSDEV_DETACHED (0)
+#define RTE_COMPRESSDEV_ATTACHED (1)
+
+#define RTE_COMPRESSDEV_NAME_MAX_LEN (64)
+
+/**< Max length of name of comp PMD */
+/**
+ * A macro that points to an offset from the start
+ * of the comp operation structure (rte_comp_op)
+ *
+ * The returned pointer is cast to type t.
+ *
+ * @param c
+ * The comp operation
+ * @param o
+ * The offset from the start of the comp operation
+ * @param t
+ * The type to cast the result into
+ */
+#define rte_comp_op_ctod_offset(c, t, o) \
+ ((t)((char *)(c) + (o)))
+
+/**
+ * A macro that returns the physical address that points
+ * to an offset from the start of the comp operation
+ * (rte_comp_op).
+ *
+ * @param c
+ * The comp operation
+ * @param o
+ * The offset from the start of the comp operation
+ * to calculate address from
+ */
+#define rte_comp_op_ctophys_offset(c, o) \
+ (rte_iova_t)((c)->phys_addr + (o))
+
+/**
+ * Parameter log base 2 range description.
+ * Final value will be 2^value.
+ */
+struct rte_param_log2_range {
+ uint8_t min; /**< Minimum log2 value */
+ uint8_t max; /**< Maximum log2 value */
+ uint8_t increment;
+ /**< If a range of sizes are supported,
+ * this parameter is used to indicate
+ * increments in log base 2 byte value
+ * that are supported between the minimum and maximum
+ */
+};
+
+/** Structure used to capture a capability of a comp device */
+struct rte_compressdev_capabilities {
+ enum rte_comp_algorithm algo;
+ /* Compression algorithm */
+ uint64_t comp_feature_flags;
+ /**< Bitmask of flags for compression service features */
+ struct rte_param_log2_range window_size;
+ /**< Window size range in bytes */
+};
+
+
+/** Macro used at end of comp PMD list */
+#define RTE_COMP_END_OF_CAPABILITIES_LIST() \
+ { RTE_COMP_ALGO_LIST_END }
+
+/**
+ * compression device supported feature flags
+ *
+ * @note New features flags should be added to the end of the list
+ *
+ * Keep these flags synchronised with rte_compressdev_get_feature_name()
+ */
+
+#define RTE_COMPDEV_FF_HW_ACCELERATED (1ULL << 0)
+/**< Operations are off-loaded to an external hardware accelerator */
+#define RTE_COMPDEV_FF_CPU_SSE (1ULL << 1)
+/**< Utilises CPU SIMD SSE instructions */
+#define RTE_COMPDEV_FF_CPU_AVX (1ULL << 2)
+/**< Utilises CPU SIMD AVX instructions */
+#define RTE_COMPDEV_FF_CPU_AVX2 (1ULL << 3)
+/**< Utilises CPU SIMD AVX2 instructions */
+#define RTE_COMPDEV_FF_CPU_AVX512 (1ULL << 4)
+/**< Utilises CPU SIMD AVX512 instructions */
+#define RTE_COMPDEV_FF_CPU_NEON (1ULL << 5)
+/**< Utilises CPU NEON instructions */
+
+/**
+ * compression service feature flags
+ *
+ * @note New features flags should be added to the end of the list
+ *
+ * Keep these flags synchronised with rte_comp_get_feature_name()
+ */
+#define RTE_COMP_FF_STATEFUL_COMPRESSION (1ULL << 0)
+/**< Stateful compression is supported */
+#define RTE_COMP_FF_STATEFUL_DECOMPRESSION (1ULL << 1)
+/**< Stateful decompression is supported */
+#define RTE_COMP_FF_MBUF_SCATTER_GATHER (1ULL << 2)
+/**< Scatter-gather mbufs are supported */
+#define RTE_COMP_FF_ADLER32_CHECKSUM (1ULL << 3)
+/**< Adler-32 Checksum is supported */
+#define RTE_COMP_FF_CRC32_CHECKSUM (1ULL << 4)
+/**< CRC32 Checksum is supported */
+#define RTE_COMP_FF_CRC32_ADLER32_CHECKSUM (1ULL << 5)
+/**< Adler-32/CRC32 Checksum is supported */
+#define RTE_COMP_FF_MULTI_PKT_CHECKSUM (1ULL << 6)
+/**< Generation of checksum across multiple stateless packets is supported */
+#define RTE_COMP_FF_NONCOMPRESSED_BLOCKS (1ULL << 7)
+/**< Creation of non-compressed blocks using RTE_COMP_LEVEL_NONE is supported */
+
+/**
+ * Get the name of a compress device feature flag.
+ *
+ * @param flag
+ * The mask describing the flag
+ *
+ * @return
+ * The name of this flag, or NULL if it's not a valid feature flag.
+ */
+const char * __rte_experimental
+rte_compressdev_get_feature_name(uint64_t flag);
+
+/**
+ * Get the name of a compress service feature flag
+ *
+ * @param flag
+ * The mask describing the flag
+ *
+ * @return
+ * The name of this flag, or NULL if it's not a valid feature flag.
+ */
+const char * __rte_experimental
+rte_comp_get_feature_name(uint64_t flag);
+
+/** comp device information */
+struct rte_compressdev_info {
+ const char *driver_name; /**< Driver name. */
+ uint8_t driver_id; /**< Driver identifier */
+ uint64_t feature_flags; /**< Feature flags */
+ const struct rte_compressdev_capabilities *capabilities;
+ /**< Array of devices supported capabilities */
+ unsigned int max_nb_queue_pairs;
+ /**< Maximum number of queues pairs supported by device. */
+};
+
+
+/** comp device statistics */
+struct rte_compressdev_stats {
+ uint64_t enqueued_count;
+ /**< Count of all operations enqueued */
+ uint64_t dequeued_count;
+ /**< Count of all operations dequeued */
+
+ uint64_t enqueue_err_count;
+ /**< Total error count on operations enqueued */
+ uint64_t dequeue_err_count;
+ /**< Total error count on operations dequeued */
+};
+
+
+/**
+ * Get the device identifier for the named compress device.
+ *
+ * @param name
+ * Device name to select the device structure
+ * @return
+ * - Returns compress device identifier on success.
+ * - Return -1 on failure to find named compress device.
+ */
+int __rte_experimental
+rte_compressdev_get_dev_id(const char *name);
+
+/**
+ * Get the compress device name given a device identifier.
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @return
+ * - Returns compress device name.
+ * - Returns NULL if compress device is not present.
+ */
+const char * __rte_experimental
+rte_compressdev_name_get(uint8_t dev_id);
+
+/**
+ * Get the total number of compress devices that have been successfully
+ * initialised.
+ *
+ * @return
+ * - The total number of usable compress devices.
+ */
+uint8_t __rte_experimental
+rte_compressdev_count(void);
+
+/**
+ * Get number of comp device defined type.
+ *
+ * @param driver_id
+ * Driver identifier
+ * @return
+ * Returns number of comp device.
+ */
+uint8_t __rte_experimental
+rte_compressdev_device_count_by_driver(uint8_t driver_id);
+
+/**
+ * Get number and identifiers of attached comp devices that
+ * use the same compress driver.
+ *
+ * @param driver_name
+ * Driver name
+ * @param devices
+ * Output devices identifiers
+ * @param nb_devices
+ * Maximal number of devices
+ *
+ * @return
+ * Returns number of attached compress devices.
+ */
+uint8_t __rte_experimental
+rte_compressdev_devices_get(const char *driver_name, uint8_t *devices,
+ uint8_t nb_devices);
+/*
+ * Return the NUMA socket to which a device is connected.
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @return
+ * The NUMA socket id to which the device is connected or
+ * a default of zero if the socket could not be determined.
+ * -1 if returned is the dev_id value is out of range.
+ */
+int __rte_experimental
+rte_compressdev_socket_id(uint8_t dev_id);
+
+/** Compress device configuration structure */
+struct rte_compressdev_config {
+ int socket_id;
+ /**< Socket on which to allocate resources */
+ uint16_t nb_queue_pairs;
+ /**< Total number of queue pairs to configure on a device */
+};
+
+/**
+ * Configure a device.
+ *
+ * This function must be invoked first before any other function in the
+ * API. This function can also be re-invoked when a device is in the
+ * stopped state.
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @param config
+ * The compress device configuration
+ * @return
+ * - 0: Success, device configured.
+ * - <0: Error code returned by the driver configuration function.
+ */
+int __rte_experimental
+rte_compressdev_configure(uint8_t dev_id,
+ struct rte_compressdev_config *config);
+
+/**
+ * Start an device.
+ *
+ * The device start step is the last one and consists of setting the configured
+ * offload features and in starting the transmit and the receive units of the
+ * device.
+ * On success, all basic functions exported by the API (link status,
+ * receive/transmit, and so on) can be invoked.
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @return
+ * - 0: Success, device started.
+ * - <0: Error code of the driver device start function.
+ */
+int __rte_experimental
+rte_compressdev_start(uint8_t dev_id);
+
+/**
+ * Stop an device. The device can be restarted with a call to
+ * rte_compressdev_start()
+ *
+ * @param dev_id
+ * Compress device identifier
+ */
+void __rte_experimental
+rte_compressdev_stop(uint8_t dev_id);
+
+/**
+ * Close an device. The device cannot be restarted!
+ *
+ * @param dev_id
+ * Compress device identifier
+ *
+ * @return
+ * - 0 on successfully closing device
+ * - <0 on failure to close device
+ */
+int __rte_experimental
+rte_compressdev_close(uint8_t dev_id);
+
+/**
+ * Allocate and set up a receive queue pair for a device.
+ *
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @param queue_pair_id
+ * The index of the queue pairs to set up. The
+ * value must be in the range [0, nb_queue_pair - 1]
+ * previously supplied to rte_compressdev_configure()
+ * @param max_inflight_ops
+ * Max number of ops which the qp will have to
+ * accommodate simultaneously
+ * @param socket_id
+ * The *socket_id* argument is the socket identifier
+ * in case of NUMA. The value can be *SOCKET_ID_ANY*
+ * if there is no NUMA constraint for the DMA memory
+ * allocated for the receive queue pair
+ * @return
+ * - 0: Success, queue pair correctly set up.
+ * - <0: Queue pair configuration failed
+ */
+int __rte_experimental
+rte_compressdev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
+ uint32_t max_inflight_ops, int socket_id);
+
+/**
+ * Get the number of queue pairs on a specific comp device
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @return
+ * - The number of configured queue pairs.
+ */
+uint16_t __rte_experimental
+rte_compressdev_queue_pair_count(uint8_t dev_id);
+
+
+/**
+ * Retrieve the general I/O statistics of a device.
+ *
+ * @param dev_id
+ * The identifier of the device
+ * @param stats
+ * A pointer to a structure of type
+ * *rte_compressdev_stats* to be filled with the
+ * values of device counters
+ * @return
+ * - Zero if successful.
+ * - Non-zero otherwise.
+ */
+int __rte_experimental
+rte_compressdev_stats_get(uint8_t dev_id, struct rte_compressdev_stats *stats);
+
+/**
+ * Reset the general I/O statistics of a device.
+ *
+ * @param dev_id
+ * The identifier of the device.
+ */
+void __rte_experimental
+rte_compressdev_stats_reset(uint8_t dev_id);
+
+/**
+ * Retrieve the contextual information of a device.
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @param dev_info
+ * A pointer to a structure of type *rte_compressdev_info*
+ * to be filled with the contextual information of the device
+ *
+ * @note The capabilities field of dev_info is set to point to the first
+ * element of an array of struct rte_compressdev_capabilities.
+ * The element after the last valid element has it's op field set to
+ * RTE_COMP_ALGO_LIST_END.
+ */
+void __rte_experimental
+rte_compressdev_info_get(uint8_t dev_id, struct rte_compressdev_info *dev_info);
+
+
+typedef uint16_t (*compress_dequeue_pkt_burst_t)(void *qp,
+ struct rte_comp_op **ops, uint16_t nb_ops);
+/**< Dequeue processed packets from queue pair of a device. */
+
+typedef uint16_t (*compress_enqueue_pkt_burst_t)(void *qp,
+ struct rte_comp_op **ops, uint16_t nb_ops);
+/**< Enqueue packets for processing on queue pair of a device. */
+
+
+/** The data structure associated with each comp device. */
+struct rte_compressdev {
+ compress_dequeue_pkt_burst_t dequeue_burst;
+ /**< Pointer to PMD receive function */
+ compress_enqueue_pkt_burst_t enqueue_burst;
+ /**< Pointer to PMD transmit function */
+
+ struct rte_compressdev_data *data;
+ /**< Pointer to device data */
+ struct rte_compressdev_ops *dev_ops;
+ /**< Functions exported by PMD */
+ uint64_t feature_flags;
+ /**< Supported features */
+ struct rte_device *device;
+ /**< Backing device */
+
+ uint8_t driver_id;
+ /**< comp driver identifier*/
+
+ __extension__
+ uint8_t attached : 1;
+ /**< Flag indicating the device is attached */
+} __rte_cache_aligned;
+
+
+/**
+ *
+ * The data part, with no function pointers, associated with each device.
+ *
+ * This structure is safe to place in shared memory to be common among
+ * different processes in a multi-process configuration.
+ */
+struct rte_compressdev_data {
+ uint8_t dev_id;
+ /**< Compress device identifier */
+ uint8_t socket_id;
+ /**< Socket identifier where memory is allocated */
+ char name[RTE_COMPRESSDEV_NAME_MAX_LEN];
+ /**< Unique identifier name */
+
+ __extension__
+ uint8_t dev_started : 1;
+ /**< Device state: STARTED(1)/STOPPED(0) */
+
+ void **queue_pairs;
+ /**< Array of pointers to queue pairs. */
+ uint16_t nb_queue_pairs;
+ /**< Number of device queue pairs */
+
+ void *dev_private;
+ /**< PMD-specific private data */
+} __rte_cache_aligned;
+
+struct rte_compressdev *rte_compressdevs;
+/**
+ *
+ * Dequeue a burst of processed compression operations from a queue on the comp
+ * device. The dequeued operation are stored in *rte_comp_op* structures
+ * whose pointers are supplied in the *ops* array.
+ *
+ * The rte_compressdev_dequeue_burst() function returns the number of ops
+ * actually dequeued, which is the number of *rte_comp_op* data structures
+ * effectively supplied into the *ops* array.
+ *
+ * A return value equal to *nb_ops* indicates that the queue contained
+ * at least *nb_ops* operations, and this is likely to signify that other
+ * processed operations remain in the devices output queue. Applications
+ * implementing a "retrieve as many processed operations as possible" policy
+ * can check this specific case and keep invoking the
+ * rte_compressdev_dequeue_burst() function until a value less than
+ * *nb_ops* is returned.
+ *
+ * The rte_compressdev_dequeue_burst() function does not provide any error
+ * notification to avoid the corresponding overhead.
+ *
+ * Note: operation ordering is not maintained within the queue pair.
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @param qp_id
+ * The index of the queue pair from which to retrieve
+ * processed operations. The value must be in the range
+ * [0, nb_queue_pair - 1] previously supplied to
+ * rte_compressdev_configure()
+ * @param ops
+ * The address of an array of pointers to
+ * *rte_comp_op* structures that must be
+ * large enough to store *nb_ops* pointers in it
+ * @param nb_ops
+ * The maximum number of operations to dequeue
+ * @return
+ * - The number of operations actually dequeued, which is the number
+ * of pointers to *rte_comp_op* structures effectively supplied to the
+ * *ops* array.
+ */
+static inline uint16_t
+rte_compressdev_dequeue_burst(uint8_t dev_id, uint16_t qp_id,
+ struct rte_comp_op **ops, uint16_t nb_ops)
+{
+ struct rte_compressdev *dev = &rte_compressdevs[dev_id];
+
+ nb_ops = (*dev->dequeue_burst)
+ (dev->data->queue_pairs[qp_id], ops, nb_ops);
+
+ return nb_ops;
+}
+
+/**
+ * Enqueue a burst of operations for processing on a compression device.
+ *
+ * The rte_compressdev_enqueue_burst() function is invoked to place
+ * comp operations on the queue *qp_id* of the device designated by
+ * its *dev_id*.
+ *
+ * The *nb_ops* parameter is the number of operations to process which are
+ * supplied in the *ops* array of *rte_comp_op* structures.
+ *
+ * The rte_compressdev_enqueue_burst() function returns the number of
+ * operations it actually enqueued for processing. A return value equal to
+ * *nb_ops* means that all packets have been enqueued.
+ *
+ * @note All compression operations are Out-of-place (OOP) operations,
+ * as the size of the output data is different to the size of the input data.
+ *
+ * @note The flush flag only applies to operations which return SUCCESS.
+ * In OUT_OF_SPACE case whether STATEFUL or STATELESS, data in dest buffer
+ * is as if flush flag was FLUSH_NONE.
+ * @note flush flag only applies in compression direction. It has no meaning
+ * for decompression.
+ * @note: operation ordering is not maintained within the queue pair.
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @param qp_id
+ * The index of the queue pair on which operations
+ * are to be enqueued for processing. The value
+ * must be in the range [0, nb_queue_pairs - 1]
+ * previously supplied to *rte_compressdev_configure*
+ * @param ops
+ * The address of an array of *nb_ops* pointers
+ * to *rte_comp_op* structures which contain
+ * the operations to be processed
+ * @param nb_ops
+ * The number of operations to process
+ * @return
+ * The number of operations actually enqueued on the device. The return
+ * value can be less than the value of the *nb_ops* parameter when the
+ * comp devices queue is full or if invalid parameters are specified in
+ * a *rte_comp_op*.
+ */
+static inline uint16_t
+rte_compressdev_enqueue_burst(uint8_t dev_id, uint16_t qp_id,
+ struct rte_comp_op **ops, uint16_t nb_ops)
+{
+ struct rte_compressdev *dev = &rte_compressdevs[dev_id];
+
+ return (*dev->enqueue_burst)(
+ dev->data->queue_pairs[qp_id], ops, nb_ops);
+}
+
+
+/** Compressdev session */
+struct rte_comp_session {
+ __extension__ void *sess_private_data[0];
+ /**< Private session material */
+};
+
+
+/**
+ * Create compression session header (generic with no private data)
+ *
+ * @param mempool
+ * Symmetric session mempool to allocate session objects from
+ * @return
+ * - On success return pointer to compression session
+ * - On failure returns NULL
+ */
+struct rte_comp_session * __rte_experimental
+rte_compressdev_session_create(struct rte_mempool *mempool);
+
+/**
+ * Frees comp session header, after checking that all
+ * the device private data has been freed, returning it
+ * to its original mempool.
+ *
+ * @param sess
+ * Session header to be freed
+ *
+ * @return
+ * - 0 if successful.
+ * - -EINVAL if session is NULL.
+ * - -EBUSY if not all device private data has been freed.
+ */
+int __rte_experimental
+rte_compressdev_session_terminate(struct rte_comp_session *sess);
+
+/**
+ * Fill out private session data for the device, based on its device id.
+ * The same private session data is shared by all devices exposed by a driver
+ * so this only needs to be called for one device per driver-type.
+ * All private data stored must be shareable across devices, so read-only.
+ * A session initialised for more than one device (of different driver types)
+ * must used the same xform for each init.
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @param sess
+ * Session where the private data will be attached to
+ * @param xforms
+ * Compress transform operations to apply on flow
+ * processed with this session
+ * @param mempool
+ * Mempool from where the private data should be allocated
+ *
+ * @return
+ * - On success, zero.
+ * - -EINVAL if input parameters are invalid.
+ * - -ENOTSUP if comp device does not support the comp transform.
+ * - -ENOMEM if the private session could not be allocated.
+ */
+int __rte_experimental
+rte_compressdev_session_init(uint8_t dev_id,
+ struct rte_comp_session *sess,
+ struct rte_comp_xform *xforms,
+ struct rte_mempool *mempool);
+
+/**
+ * Frees private data for the device id, based on its device type,
+ * returning it to its mempool.
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @param sess
+ * Session containing the reference to the private data
+ *
+ * @return
+ * - 0 if successful.
+ * - -EINVAL if device is invalid or session is NULL.
+ */
+int __rte_experimental
+rte_compressdev_session_clear(uint8_t dev_id,
+ struct rte_comp_session *sess);
+
+/**
+ * Get the size of the header session, for all registered drivers.
+ *
+ * @return
+ * Size of the header session.
+ */
+unsigned int __rte_experimental
+rte_compressdev_get_header_session_size(void);
+
+/**
+ * Get the size of the private session data for a device.
+ *
+ * @param dev_id
+ * Compress device identifier
+ *
+ * @return
+ * - Size of the private data, if successful
+ * - 0 if device is invalid or does not have private session
+ */
+unsigned int __rte_experimental
+rte_compressdev_get_private_session_size(uint8_t dev_id);
+
+/**
+ * This should alloc a stream from the device's mempool and initialise it.
+ * This handle will be passed to the PMD with every op in the stream.
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @param session
+ * Session previously allocated by
+ * *rte_compressdev_session_create*
+ * @param stream
+ * Pointer to PMD's private stream data
+ * @param op_type
+ * Op type for which the stream will be used
+ *
+ * @return
+ *
+ * TODO: Should qp_id also be added, with constraint that all ops in the same
+ * stream should be sent to the same qp?
+ *
+ */
+int __rte_experimental
+rte_comp_stream_create(uint8_t dev_id, struct rte_comp_session *sess,
+ void **stream, enum rte_comp_op_type op_type);
+
+/**
+ * This should clear the stream and return it to the device's mempool.
+ *
+ * @param dev_id
+ * Compress device identifier
+ *
+ * @param stream
+ * PMD's private stream data
+ *
+ *
+ * @return
+ */
+int __rte_experimental
+rte_comp_stream_free(uint8_t dev_id, void *stream);
+
+/**
+ * Provide driver identifier.
+ *
+ * @param name
+ * Compress driver name
+ * @return
+ * The driver type identifier or -1 if no driver found
+ */
+int __rte_experimental
+rte_compressdev_driver_id_get(const char *name);
+
+/**
+ * Provide driver name.
+ *
+ * @param driver_id
+ * The driver identifier
+ * @return
+ * The driver name or null if no driver found
+ */
+const char * __rte_experimental
+rte_compressdev_driver_name_get(uint8_t driver_id);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_COMPRESSDEV_H_ */
new file mode 100644
@@ -0,0 +1,163 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017-2018 Intel Corporation
+ */
+
+#include <rte_malloc.h>
+
+#include "rte_compressdev_pmd.h"
+
+int compressdev_logtype;
+
+/**
+ * Parse name from argument
+ */
+static int
+rte_compressdev_pmd_parse_name_arg(const char *key __rte_unused,
+ const char *value, void *extra_args)
+{
+ struct rte_compressdev_pmd_init_params *params = extra_args;
+ int n;
+
+ n = snprintf(params->name, RTE_COMPRESSDEV_NAME_MAX_LEN, "%s", value);
+ if (n >= RTE_COMPRESSDEV_NAME_MAX_LEN)
+ return -EINVAL;
+
+ return 0;
+}
+
+/**
+ * Parse unsigned integer from argument
+ */
+static int
+rte_compressdev_pmd_parse_uint_arg(const char *key __rte_unused,
+ const char *value, void *extra_args)
+{
+ int i;
+ char *end;
+
+ errno = 0;
+ i = strtol(value, &end, 10);
+ if (*end != 0 || errno != 0 || i < 0)
+ return -EINVAL;
+
+ *((uint32_t *)extra_args) = i;
+ return 0;
+}
+
+int
+rte_compressdev_pmd_parse_input_args(
+ struct rte_compressdev_pmd_init_params *params,
+ const char *args)
+{
+ struct rte_kvargs *kvlist = NULL;
+ int ret = 0;
+
+ if (params == NULL)
+ return -EINVAL;
+
+ if (args) {
+ kvlist = rte_kvargs_parse(args, compressdev_pmd_valid_params);
+ if (kvlist == NULL)
+ return -EINVAL;
+
+ ret = rte_kvargs_process(kvlist,
+ RTE_COMPRESSDEV_PMD_MAX_NB_QP_ARG,
+ &rte_compressdev_pmd_parse_uint_arg,
+ ¶ms->max_nb_queue_pairs);
+ if (ret < 0)
+ goto free_kvlist;
+
+ ret = rte_kvargs_process(kvlist,
+ RTE_COMPRESSDEV_PMD_SOCKET_ID_ARG,
+ &rte_compressdev_pmd_parse_uint_arg,
+ ¶ms->socket_id);
+ if (ret < 0)
+ goto free_kvlist;
+
+ ret = rte_kvargs_process(kvlist,
+ RTE_COMPRESSDEV_PMD_NAME_ARG,
+ &rte_compressdev_pmd_parse_name_arg,
+ params);
+ if (ret < 0)
+ goto free_kvlist;
+ }
+
+free_kvlist:
+ rte_kvargs_free(kvlist);
+ return ret;
+}
+
+struct rte_compressdev * __rte_experimental
+rte_compressdev_pmd_create(const char *name,
+ struct rte_device *device,
+ struct rte_compressdev_pmd_init_params *params)
+{
+ struct rte_compressdev *compressdev;
+
+ if (params->name[0] != '\0') {
+ COMPRESSDEV_LOG(INFO, "[%s] User specified device name = %s\n",
+ device->driver->name, params->name);
+ name = params->name;
+ }
+
+ COMPRESSDEV_LOG(INFO, "[%s] - Creating compressdev %s\n",
+ device->driver->name, name);
+
+ COMPRESSDEV_LOG(INFO,
+ "[%s] - Init parameters - name: %s, socket id: %d, max queue pairs: %u",
+ device->driver->name, name,
+ params->socket_id, params->max_nb_queue_pairs);
+
+ /* allocate device structure */
+ compressdev = rte_compressdev_pmd_allocate(name, params->socket_id);
+ if (compressdev == NULL) {
+ COMPRESSDEV_LOG(ERR, "[%s] Failed to allocate comp device for %s",
+ device->driver->name, name);
+ return NULL;
+ }
+
+ /* allocate private device structure */
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ compressdev->data->dev_private =
+ rte_zmalloc_socket("compressdev device private",
+ params->private_data_size,
+ RTE_CACHE_LINE_SIZE,
+ params->socket_id);
+
+ if (compressdev->data->dev_private == NULL) {
+ COMPRESSDEV_LOG(ERR,
+ "[%s] Cannot allocate memory for compressdev %s private data",
+ device->driver->name, name);
+
+ rte_compressdev_pmd_release_device(compressdev);
+ return NULL;
+ }
+ }
+
+ compressdev->device = device;
+
+ return compressdev;
+}
+
+int __rte_experimental
+rte_compressdev_pmd_destroy(struct rte_compressdev *compressdev)
+{
+ int retval;
+
+ COMPRESSDEV_LOG(INFO, "[%s] Closing comp device %s",
+ compressdev->device->driver->name,
+ compressdev->device->name);
+
+ /* free comp device */
+ retval = rte_compressdev_pmd_release_device(compressdev);
+ if (retval)
+ return retval;
+
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ rte_free(compressdev->data->dev_private);
+
+ compressdev->device = NULL;
+ compressdev->data = NULL;
+
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,439 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017-2018 Intel Corporation
+ */
+
+#ifndef _RTE_COMPRESSDEV_PMD_H_
+#define _RTE_COMPRESSDEV_PMD_H_
+
+/** @file
+ * RTE comp PMD APIs
+ *
+ * @note
+ * These APIs are for comp PMDs only and user applications should not call
+ * them directly.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <string.h>
+
+#include <rte_dev.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_log.h>
+#include <rte_common.h>
+
+#include "rte_comp.h"
+#include "rte_compressdev.h"
+
+#define RTE_COMPRESSDEV_PMD_DEFAULT_MAX_NB_QPS 8
+
+#define RTE_COMPRESSDEV_PMD_NAME_ARG ("name")
+#define RTE_COMPRESSDEV_PMD_MAX_NB_QP_ARG ("max_nb_queue_pairs")
+#define RTE_COMPRESSDEV_PMD_SOCKET_ID_ARG ("socket_id")
+
+static const char * const compressdev_pmd_valid_params[] = {
+ RTE_COMPRESSDEV_PMD_NAME_ARG,
+ RTE_COMPRESSDEV_PMD_MAX_NB_QP_ARG,
+ RTE_COMPRESSDEV_PMD_SOCKET_ID_ARG
+};
+
+/**
+ * @internal
+ * Initialisation parameters for comp devices
+ */
+struct rte_compressdev_pmd_init_params {
+ char name[RTE_COMPRESSDEV_NAME_MAX_LEN];
+ size_t private_data_size;
+ int socket_id;
+ unsigned int max_nb_queue_pairs;
+};
+
+/** Global structure used for maintaining state of allocated comp devices */
+struct rte_compressdev_global {
+ struct rte_compressdev *devs; /**< Device information array */
+ struct rte_compressdev_data *data[RTE_COMPRESS_MAX_DEVS];
+ /**< Device private data */
+ uint8_t nb_devs; /**< Number of devices found */
+ uint8_t max_devs; /**< Max number of devices */
+};
+
+/* compressdev driver, containing the driver identifier */
+struct compressdev_driver {
+ TAILQ_ENTRY(compressdev_driver) next; /**< Next in list. */
+ const struct rte_driver *driver;
+ uint8_t id;
+};
+
+/** Pointer to global array of comp devices */
+extern struct rte_compressdev *rte_compressdevs;
+/** Pointer to global comp devices data structure */
+extern struct rte_compressdev_global *rte_compressdev_globals;
+
+/**
+ * Get the rte_compressdev structure device pointer for the device. Assumes a
+ * valid device index.
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @return
+ * - The rte_compressdev structure pointer for the given device identifier.
+ */
+struct rte_compressdev * __rte_experimental
+rte_compressdev_pmd_get_dev(uint8_t dev_id);
+
+/**
+ * Get the rte_compressdev structure device pointer for the named device.
+ *
+ * @param name
+ * Compress device name
+ * @return
+ * - The rte_compressdev structure pointer for the given device identifier.
+ */
+struct rte_compressdev * __rte_experimental
+rte_compressdev_pmd_get_named_dev(const char *name);
+
+/**
+ * Validate if the comp device index is valid attached comp device.
+ *
+ * @param dev_id
+ * Compress device identifier
+ * @return
+ * - If the device index is valid (1) or not (0).
+ */
+unsigned int __rte_experimental
+rte_compressdev_pmd_is_valid_dev(uint8_t dev_id);
+
+/**
+ * Definitions of all functions exported by a driver through the
+ * the generic structure of type *comp_dev_ops* supplied in the
+ * *rte_compressdev* structure associated with a device.
+ */
+
+/**
+ * Function used to configure device.
+ *
+ * @param dev
+ * Compress device
+ * @param config
+ * Compress device configurations
+ * @return
+ * Returns 0 on success
+ */
+typedef int (*compressdev_configure_t)(struct rte_compressdev *dev,
+ struct rte_compressdev_config *config);
+
+/**
+ * Function used to start a configured device.
+ *
+ * @param dev
+ * Compress device
+ * @return
+ * Returns 0 on success
+ */
+typedef int (*compressdev_start_t)(struct rte_compressdev *dev);
+
+/**
+ * Function used to stop a configured device.
+ *
+ * @param dev
+ * Compress device
+ */
+typedef void (*compressdev_stop_t)(struct rte_compressdev *dev);
+
+/**
+ * Function used to close a configured device.
+ *
+ * @param dev
+ * Compress device
+ * @return
+ * - 0 on success.
+ * - EAGAIN if can't close as device is busy
+ */
+typedef int (*compressdev_close_t)(struct rte_compressdev *dev);
+
+
+/**
+ * Function used to get statistics of a device.
+ *
+ * @param dev
+ * Compress device
+ * @param stats
+ * Compress device stats to populate
+ */
+typedef void (*compressdev_stats_get_t)(struct rte_compressdev *dev,
+ struct rte_compressdev_stats *stats);
+
+
+/**
+ * Function used to reset statistics of a device.
+ *
+ * @param dev
+ * Compress device
+ */
+typedef void (*compressdev_stats_reset_t)(struct rte_compressdev *dev);
+
+
+/**
+ * Function used to get specific information of a device.
+ *
+ * @param dev
+ * Compress device
+ */
+typedef void (*compressdev_info_get_t)(struct rte_compressdev *dev,
+ struct rte_compressdev_info *dev_info);
+
+/**
+ * Setup a queue pair for a device.
+ *
+ * @param dev
+ * Compress device
+ * @param qp_id
+ * Queue pair identifier
+ * @param max_inflight_ops
+ * Max inflight ops which qp must accommodate
+ * @param socket_id
+ * Socket identifier
+ * @return
+ * Returns 0 on success.
+ */
+typedef int (*compressdev_queue_pair_setup_t)(struct rte_compressdev *dev,
+ uint16_t qp_id, uint32_t max_inflight_ops, int socket_id);
+
+/**
+ * Release memory resources allocated by given queue pair.
+ *
+ * @param dev
+ * Compress device
+ * @param qp_id
+ * Queue pair identifier
+ * @return
+ * - 0 on success.
+ * - EAGAIN if can't close as device is busy
+ */
+typedef int (*compressdev_queue_pair_release_t)(struct rte_compressdev *dev,
+ uint16_t qp_id);
+
+/**
+ * Get number of available queue pairs of a device.
+ *
+ * @param dev
+ * Compress device
+ * @return
+ * Returns number of queue pairs on success.
+ */
+typedef uint32_t (*compressdev_queue_pair_count_t)(struct rte_compressdev *dev);
+
+/**
+ * Get the size of a compressdev session
+ *
+ * @param dev
+ * Compress device
+ * @return
+ * - On success returns the size of the session structure for device
+ * - On failure returns 0
+ */
+typedef unsigned int (*compressdev_get_session_private_size_t)(
+ struct rte_compressdev *dev);
+
+/**
+ * Configure a comp session on a device.
+ *
+ * @param dev
+ * Compress device
+ * @param xform
+ * Single or chain of comp xforms
+ * @param session
+ * Compressdev session
+ * @param mp
+ * Mempool where the private session is allocated
+ * @return
+ * - Returns 0 if private session structure have been created successfully.
+ * - Returns -EINVAL if input parameters are invalid.
+ * - Returns -ENOTSUP if comp device does not support the comp transform.
+ * - Returns -ENOMEM if the private session could not be allocated.
+ */
+typedef int (*compressdev_configure_session_t)(struct rte_compressdev *dev,
+ struct rte_comp_xform *xform,
+ struct rte_comp_session *session,
+ struct rte_mempool *mp);
+
+/**
+ * Free driver private session data.
+ *
+ * @param dev
+ * Compress device
+ * @param sess
+ * Compressdev session
+ */
+typedef void (*compressdev_free_session_t)(struct rte_compressdev *dev,
+ struct rte_comp_session *sess);
+
+/** comp device operations function pointer table */
+struct rte_compressdev_ops {
+ compressdev_configure_t dev_configure; /**< Configure device. */
+ compressdev_start_t dev_start; /**< Start device. */
+ compressdev_stop_t dev_stop; /**< Stop device. */
+ compressdev_close_t dev_close; /**< Close device. */
+
+ compressdev_info_get_t dev_infos_get; /**< Get device info. */
+
+ compressdev_stats_get_t stats_get;
+ /**< Get device statistics. */
+ compressdev_stats_reset_t stats_reset;
+ /**< Reset device statistics. */
+
+ compressdev_queue_pair_setup_t queue_pair_setup;
+ /**< Set up a device queue pair. */
+ compressdev_queue_pair_release_t queue_pair_release;
+ /**< Release a queue pair. */
+ compressdev_queue_pair_count_t queue_pair_count;
+ /**< Get count of the queue pairs. */
+
+ compressdev_get_session_private_size_t session_get_size;
+ /**< Return private session. */
+ compressdev_configure_session_t session_configure;
+ /**< Configure a comp session. */
+ compressdev_free_session_t session_clear;
+ /**< Clear a comp sessions private data. */
+};
+
+
+/**
+ * @internal
+ *
+ * Function for internal use by dummy drivers primarily, e.g. ring-based
+ * driver.
+ * Allocates a new compressdev slot for an comp device and returns the pointer
+ * to that slot for the driver to use.
+ *
+ * @param name
+ * Unique identifier name for each device
+ * @param socket_id
+ * Socket to allocate resources on
+ * @return
+ * - Slot in the rte_dev_devices array for a new device;
+ */
+struct rte_compressdev * __rte_experimental
+rte_compressdev_pmd_allocate(const char *name, int socket_id);
+
+/**
+ * @internal
+ *
+ * Function for internal use by dummy drivers primarily, e.g. ring-based
+ * driver.
+ * Release the specified compressdev device.
+ *
+ * @param dev
+ * Compress device
+ * @return
+ * - 0 on success, negative on error
+ */
+int __rte_experimental
+rte_compressdev_pmd_release_device(struct rte_compressdev *dev);
+
+
+/**
+ * @internal
+ *
+ * PMD assist function to parse initialisation arguments for comp driver
+ * when creating a new comp PMD device instance.
+ *
+ * PMD driver should set default values for that PMD before calling function,
+ * these default values will be over-written with successfully parsed values
+ * from args string.
+ *
+ * @param params
+ * Parsed PMD initialisation parameters
+ * @param args
+ * Input argument string to parse
+ * @return
+ * - 0 on success
+ * - errno on failure
+ */
+int __rte_experimental
+rte_compressdev_pmd_parse_input_args(
+ struct rte_compressdev_pmd_init_params *params,
+ const char *args);
+
+/**
+ * @internal
+ *
+ * PMD assist function to provide boiler plate code for comp driver to create
+ * and allocate resources for a new comp PMD device instance.
+ *
+ * @param name
+ * Compress device name
+ * @param device
+ * Base device instance
+ * @param params
+ * PMD initialisation parameters
+ * @return
+ * - comp device instance on success
+ * - NULL on creation failure
+ */
+struct rte_compressdev * __rte_experimental
+rte_compressdev_pmd_create(const char *name,
+ struct rte_device *device,
+ struct rte_compressdev_pmd_init_params *params);
+
+/**
+ * @internal
+ *
+ * PMD assist function to provide boiler plate code for comp driver to
+ * destroy and free resources associated with a comp PMD device instance.
+ *
+ * @param dev
+ * Compress device
+ * @return
+ * - 0 on success
+ * - errno on failure
+ */
+int __rte_experimental
+rte_compressdev_pmd_destroy(struct rte_compressdev *dev);
+
+
+/**
+ * @internal
+ * Allocate compressdev driver.
+ *
+ * @param comp_drv
+ * Compressdev driver
+ * @param drv
+ * Rte_driver
+ * @return
+ * The driver type identifier
+ */
+uint8_t __rte_experimental
+rte_compressdev_allocate_driver(struct compressdev_driver *comp_drv,
+ const struct rte_driver *drv);
+
+
+#define RTE_PMD_REGISTER_COMPRESSDEV_DRIVER(comp_drv, drv, driver_id)\
+RTE_INIT(init_ ##driver_id);\
+static void init_ ##driver_id(void)\
+{\
+ driver_id = rte_compressdev_allocate_driver(&comp_drv, &(drv).driver);\
+}
+
+static inline void *
+get_session_private_data(const struct rte_comp_session *sess,
+ uint8_t driver_id) {
+ return sess->sess_private_data[driver_id];
+}
+
+static inline void
+set_session_private_data(struct rte_comp_session *sess,
+ uint8_t driver_id, void *private_data)
+{
+ sess->sess_private_data[driver_id] = private_data;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_COMPRESSDEV_PMD_H_ */
new file mode 100644
@@ -0,0 +1,47 @@
+EXPERIMENTAL {
+ global:
+
+ rte_compressdevs;
+ rte_compressdev_allocate_driver;
+ rte_compressdev_callback_register;
+ rte_compressdev_callback_unregister;
+ rte_compressdev_close;
+ rte_compressdev_configure;
+ rte_compressdev_count;
+ rte_compressdev_device_count_by_driver;
+ rte_compressdev_devices_get;
+ rte_compressdev_driver_id_get;
+ rte_compressdev_driver_name_get;
+ rte_compressdev_get_dev_id;
+ rte_compressdev_get_feature_name;
+ rte_compressdev_get_header_session_size;
+ rte_compressdev_get_private_session_size;
+ rte_compressdev_info_get;
+ rte_compressdev_name_get;
+ rte_compressdev_pmd_allocate;
+ rte_compressdev_pmd_callback_process;
+ rte_compressdev_pmd_create;
+ rte_compressdev_pmd_destroy;
+ rte_compressdev_pmd_get_dev;
+ rte_compressdev_pmd_get_named_dev;
+ rte_compressdev_pmd_is_valid_dev;
+ rte_compressdev_pmd_parse_input_args;
+ rte_compressdev_pmd_release_device;
+ rte_compressdev_queue_pair_count;
+ rte_compressdev_queue_pair_setup;
+ rte_compressdev_session_create;
+ rte_compressdev_session_init;
+ rte_compressdev_session_clear;
+ rte_compressdev_session_terminate;
+ rte_compressdev_socket_id;
+ rte_compressdev_start;
+ rte_compressdev_stats_get;
+ rte_compressdev_stats_reset;
+ rte_compressdev_stop;
+ rte_comp_get_feature_name;
+ rte_comp_op_pool_create;
+ rte_comp_stream_create;
+ rte_comp_stream_free;
+
+ local: *;
+};
@@ -60,6 +60,7 @@ extern struct rte_logs rte_logs;
#define RTE_LOGTYPE_EFD 18 /**< Log related to EFD. */
#define RTE_LOGTYPE_EVENTDEV 19 /**< Log related to eventdev. */
#define RTE_LOGTYPE_GSO 20 /**< Log related to GSO. */
+#define RTE_LOGTYPE_COMPRESSDEV 21 /**< Log related to compressdev. */
/* these log types can be used in an application */
#define RTE_LOGTYPE_USER1 24 /**< User-defined log type 1. */
@@ -94,6 +94,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_MBUF) += -lrte_mbuf
_LDLIBS-$(CONFIG_RTE_LIBRTE_NET) += -lrte_net
_LDLIBS-$(CONFIG_RTE_LIBRTE_ETHER) += -lrte_ethdev
_LDLIBS-$(CONFIG_RTE_LIBRTE_BBDEV) += -lrte_bbdev
+_LDLIBS-$(CONFIG_RTE_LIBRTE_COMPRESSDEV) += -lrte_compressdev
_LDLIBS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += -lrte_cryptodev
_LDLIBS-$(CONFIG_RTE_LIBRTE_SECURITY) += -lrte_security
_LDLIBS-$(CONFIG_RTE_LIBRTE_EVENTDEV) += -lrte_eventdev