[v2] examples: add multi process crypto application
diff mbox series

Message ID 20200624142344.3152-1-arkadiuszx.kusztal@intel.com
State Changes Requested
Delegated to: akhil goyal
Headers show
Series
  • [v2] examples: add multi process crypto application
Related show

Checks

Context Check Description
ci/Intel-compilation success Compilation OK
ci/iol-testing success Testing PASS
ci/iol-mellanox-Performance success Performance Testing PASS
ci/travis-robot success Travis build: passed
ci/iol-nxp-Performance success Performance Testing PASS
ci/iol-intel-Performance success Performance Testing PASS
ci/iol-broadcom-Performance success Performance Testing PASS
ci/checkpatch warning coding style issues

Commit Message

Arek Kusztal June 24, 2020, 2:23 p.m. UTC
This patch adds example application that can test
usage of cryptodev in multi process environment.
More can be found in mp_crypto.rst in sample app guides.

Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
---
 doc/guides/sample_app_ug/index.rst     |    1 +
 doc/guides/sample_app_ug/mp_crypto.rst |  153 +++++
 examples/mp_crypto/Makefile            |   58 ++
 examples/mp_crypto/main.c              | 1109 ++++++++++++++++++++++++++++++++
 examples/mp_crypto/meson.build         |   18 +
 examples/mp_crypto/mp_crypto.c         |  139 ++++
 examples/mp_crypto/mp_crypto.h         |  224 +++++++
 examples/mp_crypto/mp_crypto_ipc.c     |   32 +
 examples/mp_crypto/mp_crypto_parser.c  |  511 +++++++++++++++
 examples/mp_crypto/mp_crypto_parser.h  |  149 +++++
 examples/mp_crypto/mp_crypto_vectors.c |  175 +++++
 examples/mp_crypto/mp_crypto_vectors.h |   66 ++
 12 files changed, 2635 insertions(+)
 create mode 100644 doc/guides/sample_app_ug/mp_crypto.rst
 create mode 100644 examples/mp_crypto/Makefile
 create mode 100644 examples/mp_crypto/main.c
 create mode 100644 examples/mp_crypto/meson.build
 create mode 100644 examples/mp_crypto/mp_crypto.c
 create mode 100644 examples/mp_crypto/mp_crypto.h
 create mode 100644 examples/mp_crypto/mp_crypto_ipc.c
 create mode 100644 examples/mp_crypto/mp_crypto_parser.c
 create mode 100644 examples/mp_crypto/mp_crypto_parser.h
 create mode 100644 examples/mp_crypto/mp_crypto_vectors.c
 create mode 100644 examples/mp_crypto/mp_crypto_vectors.h

Comments

Arek Kusztal June 24, 2020, 2:23 p.m. UTC | #1
Due to increasing interest in multi process support for crypto PMDs
new sample app can be added so in overview we can say that:

The Multi-process Crypto application is a simple application that
allows to run crypto related operations in a multiple process environment. It
builds on the EAL primary/secondary process infrastructure.

v2:
- fixed error when first process dequeueing
- fixed multiple checkpach issues


Arek Kusztal (1):
  examples: add multi process crypto application

 doc/guides/sample_app_ug/index.rst     |    1 +
 doc/guides/sample_app_ug/mp_crypto.rst |  153 +++++
 examples/mp_crypto/Makefile            |   58 ++
 examples/mp_crypto/main.c              | 1109 ++++++++++++++++++++++++++++++++
 examples/mp_crypto/meson.build         |   18 +
 examples/mp_crypto/mp_crypto.c         |  139 ++++
 examples/mp_crypto/mp_crypto.h         |  224 +++++++
 examples/mp_crypto/mp_crypto_ipc.c     |   32 +
 examples/mp_crypto/mp_crypto_parser.c  |  511 +++++++++++++++
 examples/mp_crypto/mp_crypto_parser.h  |  149 +++++
 examples/mp_crypto/mp_crypto_vectors.c |  175 +++++
 examples/mp_crypto/mp_crypto_vectors.h |   66 ++
 12 files changed, 2635 insertions(+)
 create mode 100644 doc/guides/sample_app_ug/mp_crypto.rst
 create mode 100644 examples/mp_crypto/Makefile
 create mode 100644 examples/mp_crypto/main.c
 create mode 100644 examples/mp_crypto/meson.build
 create mode 100644 examples/mp_crypto/mp_crypto.c
 create mode 100644 examples/mp_crypto/mp_crypto.h
 create mode 100644 examples/mp_crypto/mp_crypto_ipc.c
 create mode 100644 examples/mp_crypto/mp_crypto_parser.c
 create mode 100644 examples/mp_crypto/mp_crypto_parser.h
 create mode 100644 examples/mp_crypto/mp_crypto_vectors.c
 create mode 100644 examples/mp_crypto/mp_crypto_vectors.h
Akhil Goyal July 2, 2020, 7:29 p.m. UTC | #2
Hi Arek/Fiona,

> This patch adds example application that can test
> usage of cryptodev in multi process environment.
> More can be found in mp_crypto.rst in sample app guides.
> 
> Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
> ---

Shouldn't this be part of examples/multi_process?

>  doc/guides/sample_app_ug/index.rst     |    1 +
>  doc/guides/sample_app_ug/mp_crypto.rst |  153 +++++
>  examples/mp_crypto/Makefile            |   58 ++
>  examples/mp_crypto/main.c              | 1109
> ++++++++++++++++++++++++++++++++
>  examples/mp_crypto/meson.build         |   18 +
>  examples/mp_crypto/mp_crypto.c         |  139 ++++
>  examples/mp_crypto/mp_crypto.h         |  224 +++++++
>  examples/mp_crypto/mp_crypto_ipc.c     |   32 +
>  examples/mp_crypto/mp_crypto_parser.c  |  511 +++++++++++++++
>  examples/mp_crypto/mp_crypto_parser.h  |  149 +++++
>  examples/mp_crypto/mp_crypto_vectors.c |  175 +++++
>  examples/mp_crypto/mp_crypto_vectors.h |   66 ++
>  12 files changed, 2635 insertions(+)

Can we split this patch in some logical way so that it can be reviewed easily?

>  create mode 100644 doc/guides/sample_app_ug/mp_crypto.rst
>  create mode 100644 examples/mp_crypto/Makefile
>  create mode 100644 examples/mp_crypto/main.c
>  create mode 100644 examples/mp_crypto/meson.build
>  create mode 100644 examples/mp_crypto/mp_crypto.c
>  create mode 100644 examples/mp_crypto/mp_crypto.h
>  create mode 100644 examples/mp_crypto/mp_crypto_ipc.c
>  create mode 100644 examples/mp_crypto/mp_crypto_parser.c
>  create mode 100644 examples/mp_crypto/mp_crypto_parser.h
>  create mode 100644 examples/mp_crypto/mp_crypto_vectors.c
>  create mode 100644 examples/mp_crypto/mp_crypto_vectors.h
> 
> diff --git a/doc/guides/sample_app_ug/index.rst
> b/doc/guides/sample_app_ug/index.rst
> index affa9c5..ff033e4 100644
> --- a/doc/guides/sample_app_ug/index.rst
> +++ b/doc/guides/sample_app_ug/index.rst
> @@ -35,6 +35,7 @@ Sample Applications User Guides
>      link_status_intr
>      server_node_efd
>      service_cores
> +    mp_crypto
>      multi_process
>      qos_metering
>      qos_scheduler
> diff --git a/doc/guides/sample_app_ug/mp_crypto.rst
> b/doc/guides/sample_app_ug/mp_crypto.rst
> new file mode 100644
> index 0000000..d3cb1d9
> --- /dev/null
> +++ b/doc/guides/sample_app_ug/mp_crypto.rst
> @@ -0,0 +1,153 @@
> +..  SPDX-License-Identifier: BSD-3-Clause
> +    Copyright(c) 2020 Intel Corporation.
> +
> +.. _mp_crypto:
> +
> +Multi-process Crypto Sample Application
> +=======================================
> +
> +The Multi-process Crypto application is a simple application that
> +allows to run crypto related operations in a multiple process environment. It
> +builds on the EAL primary/secondary process infrastructure.
> +
> +The application allows a user to configure devices, setup queue-pairs, create
> +and init sessions and specify data-path flow (enqueue/dequeue) in different
> +processes. The app can help to check if the PMD behaves correctly
> +in scenarios like the following:
> +
> +* device is configured in primary process, queue-pairs are setup in secondary
> process
> +
> +* queue pair is shared across processes, i.e. enqueue in one process and
> dequeue in another
> +
> +
Extra line

> +Compiling the Application
> +-------------------------
> +
> +To compile the sample application see :doc:`compiling`.
> +
> +The application is located in the ``mp_crypto`` sub-directory.
> +
> +Running the Application
> +-----------------------
> +
> +App binary: mp_crypto (in mp_crypto/build/app)
> +
> +For running PRIMARY or SECONDARY process standard EAL options apply:
> +
> +.. code-block:: console
> +
> +    ./mp_crypto --proc-type primary
> +
> +    ./mp_crypto --proc-type secondary
> +
> +.. Note::
> +
> +	The same set of BDFs must be passed to all processes.
> +
> +.. Note::
> +	The same crypto devices must be created in all processes, e.g. in qat
> +	case if asym and sym devices are enabled in the primary process, they
> +	must be enabled in all secondary processes.
> +
> +General help can by checked by running:
> +
> +.. code-block:: console
> +
> +    ./mp_crypto -- -h
> +
> +The application has a number of command line options:
> +
> +.. code-block:: console
> +
> +    ./mp_crypto -- --devtype [dev-name]
> +
> +This option specifies which driver to use by its name (for example "crypto_qat").
> +The same name must be passed to all processes.
> +
> +.. code-block:: console
> +
> +    ./mp_crypto -- --config_dev [devA, devB,]
> +
> +This option specifies the list of devices that should be configured by this
> process,
> +this results in a call to the ``rte_cryptodev_configure`` API. devX is a positive
> +integer (including zero), the value is according to probe order (from the
> smallest
> +BDF number), not necessarily the cmdline order.
> +
> +Example command:
> +
> +.. code-block:: console
> +
> +    ./mp_crypto -w 03:01.2 -w 03:01.1 -w 03:01.3 --config-dev 0,2
> +
> +will configure devices 03:01.1 and 03:01.3.
> +
> +.. code-block:: console
> +
> +    ./mp_crypto -- --qp-config=[devA]:[qp_A, qp_B,];[devB]:[qp_A, qp_C];
> +
> +devX - positive integer (including zero), as in config_dev command
> +
> +qp_X - positive integer (including zero), specifies which queue pair should be
> setup
> +
> +This command specifies which queue pairs should be setup, resulting in a call to
> +``rte_cryptodev_queue_pair_setup`` API.
> +
> +.. code-block:: console
> +
> +    ./mp_crypto -w 03:01.2 -w 03:01.1 -w 03:01.3 --qp-config="0:0,1;1:1;2:0,1;"
> +
> +This command will configure queue pairs 0 and 1 on device 0 (03:01.1), queue
> pair 1
> +on device 1 (03:01.2), queue pairs 0 and 1 on device 2 (03:01.3). The device in
> question
> +should be configured before that, though not necessarily by the same process.
> +
> +.. code-block:: console
> +
> +    ./mp_crypto -- --enq=[devX]:[qpX]:[ops]:[vector_id]
> +    ./mp_crypto -- --deq=[devX]:[qpX]:[ops]:[vector_id]
> +
> +devX - positive integer (including zero), as in config_dev command
> +
> +qp_X - positive integer (including zero), as in qp-config command
qpX

> +
> +ops - when positive integer - number of operations to enqueue/dequeue, when
> 0 infinite loop
> +
> +vector_id - positive integer (including zero), vector_id used by this process
What is vector_id? How is it meant to be used?

> +
> +This commands will enqueue/dequeue "ops" number of packets to qp_X on
> devX.
> +Example usage:
> +
> +.. code-block:: console
> +
> +    ./mp_crypto -- --enq=2:0:0:0, --deq=2:0:0:0,
> +
> +Note. ',' comma character is necessary at the end due to some parser
> shortcomings.
> +
> +To close the application when running in an infinite loop a signal handler is
> +registered to catch interrupt signals i.e. ``ctrl-c`` should be used. When
> +used in primary process other processes will be notified about exiting
> +intention and will close after collecting remaining packets (if dequeuing).
> +
> +Example commands
> +----------------
> +
> +Use two different devices on 3 separate queues:
> +
> +.. code-block:: console
> +
> +    ./mp_crypto --proc-type primary -c 1 -w 03:01.1 -w 03:01.2 -- --devtype
> "crypto_qat" --config-dev 0,1   --qp-config="0:0,1;1:0,1;" --session-mask=0x3  --
> enq=0:0:0:0, --deq=0:0:0:0,  --print-stats
> +    ./mp_crypto --proc-type secondary -c 2 -w 03:01.1 -w 03:01.2 -- --devtype
> "crypto_qat"  --enq=0:1:0:0, --deq=0:1:0:0,  --print-stats
> +    ./mp_crypto --proc-type secondary -c 4 -w 03:01.1 -w 03:01.2 -- --devtype
> "crypto_qat"  --enq=1:0:0:0, --deq=1:0:0:0,  --print-stats
> +
> +Use different processes to enqueue and dequeue to one queue pair:
> +
> +.. code-block:: console
> +
> +    ./mp_crypto --proc-type primary -c 1 -w 03:01.1 -- --devtype "crypto_qat" --
> config-dev 0    --session-mask=0x3 --qp-config="0:1;"   --enq=0:1:0:0,   --print-
> stats
> +    ./mp_crypto --proc-type secondary -c 2 -w 03:01.1 -- --devtype "crypto_qat"
> --deq=0:1:0:0,   --print-stats

What are the expected results out of this app. Can we have some logs in this doc?

Can we kill the primary app first? Will the secondary get killed automatically or will throw error?
Can we open and kill secondary apps multiple times without killing the primary?

This multi process app is only taking care of  crypto queues while others are for NICs.
Is it not worth to have crypto+NIC multi process app instead of this app?
I believe most common usecases of crypto are with network traffic.
Can we modify l2fwd-crypto for multi process?

> +
> +Limitations
> +-----------
> +
> +Software devices are not supported currently, but small changes in code suffice
> to enable it.
> +
> +Only one crypto vector and session type is possible to chose right now and it is
> AES-GCM test case.

Regards,
Akhil
Arek Kusztal July 3, 2020, 7:47 a.m. UTC | #3
Hi Akhil,

> -----Original Message-----
> From: Akhil Goyal <akhil.goyal@nxp.com>
> Sent: Thursday, July 2, 2020 9:30 PM
> To: Kusztal, ArkadiuszX <arkadiuszx.kusztal@intel.com>; dev@dpdk.org;
> Trahe, Fiona <fiona.trahe@intel.com>; Burakov, Anatoly
> <anatoly.burakov@intel.com>
> Subject: RE: [PATCH v2] examples: add multi process crypto application
> 
> Hi Arek/Fiona,
> 
> > This patch adds example application that can test usage of cryptodev
> > in multi process environment.
> > More can be found in mp_crypto.rst in sample app guides.
> >
> > Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
> > ---
> 
> Shouldn't this be part of examples/multi_process?
> 
> >  doc/guides/sample_app_ug/index.rst     |    1 +
> >  doc/guides/sample_app_ug/mp_crypto.rst |  153 +++++
> >  examples/mp_crypto/Makefile            |   58 ++
> >  examples/mp_crypto/main.c              | 1109
> > ++++++++++++++++++++++++++++++++
> >  examples/mp_crypto/meson.build         |   18 +
> >  examples/mp_crypto/mp_crypto.c         |  139 ++++
> >  examples/mp_crypto/mp_crypto.h         |  224 +++++++
> >  examples/mp_crypto/mp_crypto_ipc.c     |   32 +
> >  examples/mp_crypto/mp_crypto_parser.c  |  511 +++++++++++++++
> > examples/mp_crypto/mp_crypto_parser.h  |  149 +++++
> > examples/mp_crypto/mp_crypto_vectors.c |  175 +++++
> >  examples/mp_crypto/mp_crypto_vectors.h |   66 ++
> >  12 files changed, 2635 insertions(+)
> 
> Can we split this patch in some logical way so that it can be reviewed easily?
> 
> >  create mode 100644 doc/guides/sample_app_ug/mp_crypto.rst
> >  create mode 100644 examples/mp_crypto/Makefile  create mode 100644
> > examples/mp_crypto/main.c  create mode 100644
> > examples/mp_crypto/meson.build  create mode 100644
> > examples/mp_crypto/mp_crypto.c  create mode 100644
> > examples/mp_crypto/mp_crypto.h  create mode 100644
> > examples/mp_crypto/mp_crypto_ipc.c
> >  create mode 100644 examples/mp_crypto/mp_crypto_parser.c
> >  create mode 100644 examples/mp_crypto/mp_crypto_parser.h
> >  create mode 100644 examples/mp_crypto/mp_crypto_vectors.c
> >  create mode 100644 examples/mp_crypto/mp_crypto_vectors.h
> >
> > diff --git a/doc/guides/sample_app_ug/index.rst
> > b/doc/guides/sample_app_ug/index.rst
> > index affa9c5..ff033e4 100644
> > --- a/doc/guides/sample_app_ug/index.rst
> > +++ b/doc/guides/sample_app_ug/index.rst
> > @@ -35,6 +35,7 @@ Sample Applications User Guides
> >      link_status_intr
> >      server_node_efd
> >      service_cores
> > +    mp_crypto
> >      multi_process
> >      qos_metering
> >      qos_scheduler
> > diff --git a/doc/guides/sample_app_ug/mp_crypto.rst
> > b/doc/guides/sample_app_ug/mp_crypto.rst
> > new file mode 100644
> > index 0000000..d3cb1d9
> > --- /dev/null
> > +++ b/doc/guides/sample_app_ug/mp_crypto.rst
> > @@ -0,0 +1,153 @@
> > +..  SPDX-License-Identifier: BSD-3-Clause
> > +    Copyright(c) 2020 Intel Corporation.
> > +
> > +.. _mp_crypto:
> > +
> > +Multi-process Crypto Sample Application
> > +=======================================
> > +
> > +The Multi-process Crypto application is a simple application that
> > +allows to run crypto related operations in a multiple process
> > +environment. It builds on the EAL primary/secondary process
> infrastructure.
> > +
> > +The application allows a user to configure devices, setup
> > +queue-pairs, create and init sessions and specify data-path flow
> > +(enqueue/dequeue) in different processes. The app can help to check
> > +if the PMD behaves correctly in scenarios like the following:
> > +
> > +* device is configured in primary process, queue-pairs are setup in
> > +secondary
> > process
> > +
> > +* queue pair is shared across processes, i.e. enqueue in one process
> > +and
> > dequeue in another
> > +
> > +
> Extra line
> 
> > +Compiling the Application
> > +-------------------------
> > +
> > +To compile the sample application see :doc:`compiling`.
> > +
> > +The application is located in the ``mp_crypto`` sub-directory.
> > +
> > +Running the Application
> > +-----------------------
> > +
> > +App binary: mp_crypto (in mp_crypto/build/app)
> > +
> > +For running PRIMARY or SECONDARY process standard EAL options apply:
> > +
> > +.. code-block:: console
> > +
> > +    ./mp_crypto --proc-type primary
> > +
> > +    ./mp_crypto --proc-type secondary
> > +
> > +.. Note::
> > +
> > +	The same set of BDFs must be passed to all processes.
> > +
> > +.. Note::
> > +	The same crypto devices must be created in all processes, e.g. in qat
> > +	case if asym and sym devices are enabled in the primary process,
> they
> > +	must be enabled in all secondary processes.
> > +
> > +General help can by checked by running:
> > +
> > +.. code-block:: console
> > +
> > +    ./mp_crypto -- -h
> > +
> > +The application has a number of command line options:
> > +
> > +.. code-block:: console
> > +
> > +    ./mp_crypto -- --devtype [dev-name]
> > +
> > +This option specifies which driver to use by its name (for example
> "crypto_qat").
> > +The same name must be passed to all processes.
> > +
> > +.. code-block:: console
> > +
> > +    ./mp_crypto -- --config_dev [devA, devB,]
> > +
> > +This option specifies the list of devices that should be configured
> > +by this
> > process,
> > +this results in a call to the ``rte_cryptodev_configure`` API. devX
> > +is a positive integer (including zero), the value is according to
> > +probe order (from the
> > smallest
> > +BDF number), not necessarily the cmdline order.
> > +
> > +Example command:
> > +
> > +.. code-block:: console
> > +
> > +    ./mp_crypto -w 03:01.2 -w 03:01.1 -w 03:01.3 --config-dev 0,2
> > +
> > +will configure devices 03:01.1 and 03:01.3.
> > +
> > +.. code-block:: console
> > +
> > +    ./mp_crypto -- --qp-config=[devA]:[qp_A, qp_B,];[devB]:[qp_A,
> > + qp_C];
> > +
> > +devX - positive integer (including zero), as in config_dev command
> > +
> > +qp_X - positive integer (including zero), specifies which queue pair
> > +should be
> > setup
> > +
> > +This command specifies which queue pairs should be setup, resulting
> > +in a call to ``rte_cryptodev_queue_pair_setup`` API.
> > +
> > +.. code-block:: console
> > +
> > +    ./mp_crypto -w 03:01.2 -w 03:01.1 -w 03:01.3 --qp-
> config="0:0,1;1:1;2:0,1;"
> > +
> > +This command will configure queue pairs 0 and 1 on device 0
> > +(03:01.1), queue
> > pair 1
> > +on device 1 (03:01.2), queue pairs 0 and 1 on device 2 (03:01.3). The
> > +device in
> > question
> > +should be configured before that, though not necessarily by the same
> process.
> > +
> > +.. code-block:: console
> > +
> > +    ./mp_crypto -- --enq=[devX]:[qpX]:[ops]:[vector_id]
> > +    ./mp_crypto -- --deq=[devX]:[qpX]:[ops]:[vector_id]
> > +
> > +devX - positive integer (including zero), as in config_dev command
> > +
> > +qp_X - positive integer (including zero), as in qp-config command
> qpX
> 
> > +
> > +ops - when positive integer - number of operations to
> > +enqueue/dequeue, when
> > 0 infinite loop
> > +
> > +vector_id - positive integer (including zero), vector_id used by this
> > +process
> What is vector_id? How is it meant to be used?
> 
> > +
> > +This commands will enqueue/dequeue "ops" number of packets to qp_X
> on
> > devX.
> > +Example usage:
> > +
> > +.. code-block:: console
> > +
> > +    ./mp_crypto -- --enq=2:0:0:0, --deq=2:0:0:0,
> > +
> > +Note. ',' comma character is necessary at the end due to some parser
> > shortcomings.
> > +
> > +To close the application when running in an infinite loop a signal
> > +handler is registered to catch interrupt signals i.e. ``ctrl-c``
> > +should be used. When used in primary process other processes will be
> > +notified about exiting intention and will close after collecting remaining
> packets (if dequeuing).
> > +
> > +Example commands
> > +----------------
> > +
> > +Use two different devices on 3 separate queues:
> > +
> > +.. code-block:: console
> > +
> > +    ./mp_crypto --proc-type primary -c 1 -w 03:01.1 -w 03:01.2 --
> > + --devtype
> > "crypto_qat" --config-dev 0,1   --qp-config="0:0,1;1:0,1;" --session-
> mask=0x3  --
> > enq=0:0:0:0, --deq=0:0:0:0,  --print-stats
> > +    ./mp_crypto --proc-type secondary -c 2 -w 03:01.1 -w 03:01.2 --
> > + --devtype
> > "crypto_qat"  --enq=0:1:0:0, --deq=0:1:0:0,  --print-stats
> > +    ./mp_crypto --proc-type secondary -c 4 -w 03:01.1 -w 03:01.2 --
> > + --devtype
> > "crypto_qat"  --enq=1:0:0:0, --deq=1:0:0:0,  --print-stats
> > +
> > +Use different processes to enqueue and dequeue to one queue pair:
> > +
> > +.. code-block:: console
> > +
> > +    ./mp_crypto --proc-type primary -c 1 -w 03:01.1 -- --devtype
> > + "crypto_qat" --
> > config-dev 0    --session-mask=0x3 --qp-config="0:1;"   --enq=0:1:0:0,   --
> print-
> > stats
> > +    ./mp_crypto --proc-type secondary -c 2 -w 03:01.1 -- --devtype
> "crypto_qat"
> > --deq=0:1:0:0,   --print-stats
> 
> What are the expected results out of this app. Can we have some logs in this
> doc?
[Arek] - yes we will add (soon we will send v3), example output would look something like that:

For following command (enqueue only in primary process, dequeue in one of secondary's)
./mp_crypto --proc-type primary -c 1 -w 03:01.1 -w 03:01.2 -w 03:02.7 --log-level 7 -- --devtype "crypto_qat" --config-dev 2    --session-mask=0x3 --qp-config="2:1;"   --enq=2:1:0:0,   --print-stats
---------------------------------------------------------------------
USER1: Run enqueue on device 2
USER1: Run enqueue on qp 1
USER1: - Setting driver 2 for this process
USER1: - Starting PRIMARY process
USER1: Number of processes = 1
USER1: Configure devices according to mask: 0x4
USER1: - Configure Device id 2
CRYPTODEV: elt_size 16 is expanded to 192

CRYPTODEV: elt_size 16 is expanded to 192

USER1: - Configuring queues:
CRYPTODEV: qp 1 on dev 2 is not initialised
USER1: Created qp 1 on dev 2
USER1: - Configured 1 queues.
USER1: Initialized session = 0
USER1: Initialized session = 1
USER1: Configuring vector 0, using session 0
USER1: Start enqueuing packets on dev 2 qp 1
CRYPTODEV: qp 1 on dev 2 is initialised
USER1: Enqueuing /^C
Interrupted, finalizing...USER1: STATS: Enqueued on dev 2 = 16337024
USER1: STATS: Enqueue err count on dev 2 = 0

Press 'q' to quit: q
USER1: Waiting for 1 out of 10 seconds
USER1: All secondary processes exited normally
---------------------------------------------------------------------

> 
> Can we kill the primary app first? Will the secondary get killed automatically
> or will throw error?
[Arek] - primary will inform all secondary processes to collect remaining packets (if possible) and finish. Primary will wait at most 10 seconds. SIGINT is registered so actually it will work correctly with ctrl-c.

> Can we open and kill secondary apps multiple times without killing the
> primary?
[Arek] - yes.
> 
> This multi process app is only taking care of  crypto queues while others are
> for NICs.
> Is it not worth to have crypto+NIC multi process app instead of this app?
[Arek] - initially main purpose was to check PMD behavior when:
1) configure cryptodev, sessions, queues and do enqueue/dequeue from another different processes
2) run enqueue/dequeue from different processes on the same queue pair.
If it can be done with one app I think it is ok.
> I believe most common usecases of crypto are with network traffic.
> Can we modify l2fwd-crypto for multi process?
> 
> > +
> > +Limitations
> > +-----------
> > +
> > +Software devices are not supported currently, but small changes in
> > +code suffice
> > to enable it.
> > +
> > +Only one crypto vector and session type is possible to chose right
> > +now and it is
> > AES-GCM test case.
> 
> Regards,
> Akhil
Trahe, Fiona July 3, 2020, 3:16 p.m. UTC | #4
Hi Akhil, Arek,

> -----Original Message-----
> From: Kusztal, ArkadiuszX <arkadiuszx.kusztal@intel.com>
> Sent: Friday, July 3, 2020 8:48 AM
> To: Akhil Goyal <akhil.goyal@nxp.com>; dev@dpdk.org; Trahe, Fiona <fiona.trahe@intel.com>; Burakov,
> Anatoly <anatoly.burakov@intel.com>
> Subject: RE: [PATCH v2] examples: add multi process crypto application


////
> > Shouldn't this be part of examples/multi_process?
[Fiona] Yes, good idea - we can move in there.
(pending agreement on another comment below)


> > Can we split this patch in some logical way so that it can be reviewed easily?
[Fiona] yes, patch will be split in v3

> > What are the expected results out of this app. Can we have some logs in this
> > doc?
> [Arek] - yes we will add (soon we will send v3), example output would look something like that:
[Fiona] I agree it would be good to have expected output, but think there's too much noise in 
pasting in the whole output. I'd suggest to trim to something like:

Scenario: enqueue only in primary process, dequeue in a secondary
Commands:
 - Xxx primary
 - Xxx secondary
Expected output will include text like:
On Primary:
    > USER1: - Starting PRIMARY process
    > USER1: - Configure Device id 2
    > USER1: Created qp 1 on dev 2
    > USER1: Initialized session = 0
    > USER1: Initialized session = 1
    > USER1: Configuring vector 0, using session 0
    > USER1: Start enqueuing packets on dev 2 qp 1
    > USER1: Enqueuing / (liveness symbol keeps changing until Ctrl-C is input)
    > USER1: STATS: Enqueued on dev 2 = 16337024
On Secondary:
    > USER1: - Starting SECONDARY process
    > USER1: Found Device id 2
    > USER1: Found qp 1 on dev 2
    > USER1: Start dequeueing packets on dev 2 qp 1
    > USER1: Dequeuing / (liveness symbol keeps changing until Ctrl-C is input)
    > USER1: STATS: Dequeued on dev 2 = 16337024
Note, the STATS will be in sync if the process enqueuing is stopped before the process dequeueing, in other cases they may differ.


> >
> > This multi process app is only taking care of  crypto queues while others are
> > for NICs.
> > Is it not worth to have crypto+NIC multi process app instead of this app?
> [Arek] - initially main purpose was to check PMD behavior when:
> 1) configure cryptodev, sessions, queues and do enqueue/dequeue from another different processes
> 2) run enqueue/dequeue from different processes on the same queue pair.
> If it can be done with one app I think it is ok.
> > I believe most common usecases of crypto are with network traffic.
> > Can we modify l2fwd-crypto for multi process?
[Fiona] Yes, it would be a good idea to do that sample app too.
However this app allows standalone validation of cryptodev lib and PMDs,
running in multiple processes, without introducing dependencies on
ethdev APIs, traffic generator, NIC, etc. I think this is useful as is.
One consideration is whether it would be better to treat this as a test tool
and move to the test directory - as you're right, it's not showing
a typical complete application, just allowing to play around with the crypto part.
My preference is to move to under the examples/multi-process, but up to you.
Akhil Goyal July 4, 2020, 6:29 p.m. UTC | #5
> 
> > >
> > > This multi process app is only taking care of  crypto queues while others are
> > > for NICs.
> > > Is it not worth to have crypto+NIC multi process app instead of this app?
> > [Arek] - initially main purpose was to check PMD behavior when:
> > 1) configure cryptodev, sessions, queues and do enqueue/dequeue from
> another different processes
> > 2) run enqueue/dequeue from different processes on the same queue pair.
> > If it can be done with one app I think it is ok.
> > > I believe most common usecases of crypto are with network traffic.
> > > Can we modify l2fwd-crypto for multi process?
> [Fiona] Yes, it would be a good idea to do that sample app too.
> However this app allows standalone validation of cryptodev lib and PMDs,
> running in multiple processes, without introducing dependencies on
> ethdev APIs, traffic generator, NIC, etc. I think this is useful as is.
> One consideration is whether it would be better to treat this as a test tool
> and move to the test directory - as you're right, it's not showing
> a typical complete application, just allowing to play around with the crypto part.
> My preference is to move to under the examples/multi-process, but up to you.
> 
Example applications are also not close to real work application. But they should
Have end to end functionality available with some missing processing in between.
I would say:
- modifying l2fwd-crypto can be preference 1
- moving multiprocess crypto to test app can be preference 2 as it is just a unit test application for multi process.
- moving to examples/multi-process will be preference 3.

Patch
diff mbox series

diff --git a/doc/guides/sample_app_ug/index.rst b/doc/guides/sample_app_ug/index.rst
index affa9c5..ff033e4 100644
--- a/doc/guides/sample_app_ug/index.rst
+++ b/doc/guides/sample_app_ug/index.rst
@@ -35,6 +35,7 @@  Sample Applications User Guides
     link_status_intr
     server_node_efd
     service_cores
+    mp_crypto
     multi_process
     qos_metering
     qos_scheduler
diff --git a/doc/guides/sample_app_ug/mp_crypto.rst b/doc/guides/sample_app_ug/mp_crypto.rst
new file mode 100644
index 0000000..d3cb1d9
--- /dev/null
+++ b/doc/guides/sample_app_ug/mp_crypto.rst
@@ -0,0 +1,153 @@ 
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+.. _mp_crypto:
+
+Multi-process Crypto Sample Application
+=======================================
+
+The Multi-process Crypto application is a simple application that
+allows to run crypto related operations in a multiple process environment. It
+builds on the EAL primary/secondary process infrastructure.
+
+The application allows a user to configure devices, setup queue-pairs, create
+and init sessions and specify data-path flow (enqueue/dequeue) in different
+processes. The app can help to check if the PMD behaves correctly
+in scenarios like the following:
+
+* device is configured in primary process, queue-pairs are setup in secondary process
+
+* queue pair is shared across processes, i.e. enqueue in one process and dequeue in another
+
+
+Compiling the Application
+-------------------------
+
+To compile the sample application see :doc:`compiling`.
+
+The application is located in the ``mp_crypto`` sub-directory.
+
+Running the Application
+-----------------------
+
+App binary: mp_crypto (in mp_crypto/build/app)
+
+For running PRIMARY or SECONDARY process standard EAL options apply:
+
+.. code-block:: console
+
+    ./mp_crypto --proc-type primary
+
+    ./mp_crypto --proc-type secondary
+
+.. Note::
+
+	The same set of BDFs must be passed to all processes.
+
+.. Note::
+	The same crypto devices must be created in all processes, e.g. in qat
+	case if asym and sym devices are enabled in the primary process, they
+	must be enabled in all secondary processes.
+
+General help can by checked by running:
+
+.. code-block:: console
+
+    ./mp_crypto -- -h
+
+The application has a number of command line options:
+
+.. code-block:: console
+
+    ./mp_crypto -- --devtype [dev-name]
+
+This option specifies which driver to use by its name (for example "crypto_qat").
+The same name must be passed to all processes.
+
+.. code-block:: console
+
+    ./mp_crypto -- --config_dev [devA, devB,]
+
+This option specifies the list of devices that should be configured by this process,
+this results in a call to the ``rte_cryptodev_configure`` API. devX is a positive
+integer (including zero), the value is according to probe order (from the smallest
+BDF number), not necessarily the cmdline order.
+
+Example command:
+
+.. code-block:: console
+
+    ./mp_crypto -w 03:01.2 -w 03:01.1 -w 03:01.3 --config-dev 0,2
+
+will configure devices 03:01.1 and 03:01.3.
+
+.. code-block:: console
+
+    ./mp_crypto -- --qp-config=[devA]:[qp_A, qp_B,];[devB]:[qp_A, qp_C];
+
+devX - positive integer (including zero), as in config_dev command
+
+qp_X - positive integer (including zero), specifies which queue pair should be setup
+
+This command specifies which queue pairs should be setup, resulting in a call to
+``rte_cryptodev_queue_pair_setup`` API.
+
+.. code-block:: console
+
+    ./mp_crypto -w 03:01.2 -w 03:01.1 -w 03:01.3 --qp-config="0:0,1;1:1;2:0,1;"
+
+This command will configure queue pairs 0 and 1 on device 0 (03:01.1), queue pair 1
+on device 1 (03:01.2), queue pairs 0 and 1 on device 2 (03:01.3). The device in question
+should be configured before that, though not necessarily by the same process.
+
+.. code-block:: console
+
+    ./mp_crypto -- --enq=[devX]:[qpX]:[ops]:[vector_id]
+    ./mp_crypto -- --deq=[devX]:[qpX]:[ops]:[vector_id]
+
+devX - positive integer (including zero), as in config_dev command
+
+qp_X - positive integer (including zero), as in qp-config command
+
+ops - when positive integer - number of operations to enqueue/dequeue, when 0 infinite loop
+
+vector_id - positive integer (including zero), vector_id used by this process
+
+This commands will enqueue/dequeue "ops" number of packets to qp_X on devX.
+Example usage:
+
+.. code-block:: console
+
+    ./mp_crypto -- --enq=2:0:0:0, --deq=2:0:0:0,
+
+Note. ',' comma character is necessary at the end due to some parser shortcomings.
+
+To close the application when running in an infinite loop a signal handler is
+registered to catch interrupt signals i.e. ``ctrl-c`` should be used. When
+used in primary process other processes will be notified about exiting
+intention and will close after collecting remaining packets (if dequeuing).
+
+Example commands
+----------------
+
+Use two different devices on 3 separate queues:
+
+.. code-block:: console
+
+    ./mp_crypto --proc-type primary -c 1 -w 03:01.1 -w 03:01.2 -- --devtype "crypto_qat" --config-dev 0,1   --qp-config="0:0,1;1:0,1;" --session-mask=0x3  --enq=0:0:0:0, --deq=0:0:0:0,  --print-stats
+    ./mp_crypto --proc-type secondary -c 2 -w 03:01.1 -w 03:01.2 -- --devtype "crypto_qat"  --enq=0:1:0:0, --deq=0:1:0:0,  --print-stats
+    ./mp_crypto --proc-type secondary -c 4 -w 03:01.1 -w 03:01.2 -- --devtype "crypto_qat"  --enq=1:0:0:0, --deq=1:0:0:0,  --print-stats
+
+Use different processes to enqueue and dequeue to one queue pair:
+
+.. code-block:: console
+
+    ./mp_crypto --proc-type primary -c 1 -w 03:01.1 -- --devtype "crypto_qat" --config-dev 0    --session-mask=0x3 --qp-config="0:1;"   --enq=0:1:0:0,   --print-stats
+    ./mp_crypto --proc-type secondary -c 2 -w 03:01.1 -- --devtype "crypto_qat"  --deq=0:1:0:0,   --print-stats
+
+Limitations
+-----------
+
+Software devices are not supported currently, but small changes in code suffice to enable it.
+
+Only one crypto vector and session type is possible to chose right now and it is AES-GCM test case.
diff --git a/examples/mp_crypto/Makefile b/examples/mp_crypto/Makefile
new file mode 100644
index 0000000..4c75446
--- /dev/null
+++ b/examples/mp_crypto/Makefile
@@ -0,0 +1,58 @@ 
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2020 Intel Corporation
+
+# binary name
+APP = mp_crypto
+
+# all source are stored in SRCS-y
+SRCS-y := main.c mp_crypto_parser.c mp_crypto.c mp_crypto_ipc.c mp_crypto_vectors.c
+
+# Build using pkg-config variables if possible
+ifeq ($(shell pkg-config --exists libdpdk && echo 0),0)
+
+all: shared
+.PHONY: shared static
+shared: build/$(APP)-shared
+	ln -sf $(APP)-shared build/$(APP)
+static: build/$(APP)-static
+	ln -sf $(APP)-static build/$(APP)
+
+PKGCONF ?= pkg-config
+
+PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null)
+CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk)
+LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk)
+LDFLAGS_STATIC = -Wl,-Bstatic $(shell $(PKGCONF) --static --libs libdpdk)
+
+build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build
+	$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED)
+
+build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build
+	$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC)
+
+build:
+	@mkdir -p $@
+
+.PHONY: clean
+clean:
+	rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared
+	test -d build && rmdir -p build || true
+
+else # Build using legacy build system
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, detect a build directory, by looking for a path with a .config
+RTE_TARGET ?= $(notdir $(abspath $(dir $(firstword $(wildcard $(RTE_SDK)/*/.config)))))
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+CFLAGS += -I$(SRCDIR)
+CFLAGS += -O3 $(USER_FLAGS)
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+include $(RTE_SDK)/mk/rte.extapp.mk
+endif
diff --git a/examples/mp_crypto/main.c b/examples/mp_crypto/main.c
new file mode 100644
index 0000000..238b80d
--- /dev/null
+++ b/examples/mp_crypto/main.c
@@ -0,0 +1,1109 @@ 
+#include <rte_hexdump.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_cryptodev.h>
+#include <cmdline_parse.h>
+#include <rte_cycles.h>
+#include <rte_atomic.h>
+#include <signal.h>
+
+#include "mp_crypto_parser.h"
+#include "mp_crypto.h"
+
+int
+mp_crypto_init_devs(void) {
+	uint8_t valid_devs[RTE_CRYPTO_MAX_DEVS];
+	struct rte_cryptodev_config conf;
+	struct rte_cryptodev_info info;
+	int nb_devs = 0;
+	int i;
+
+	for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++)
+		mp_app_devs[i].id = -1;
+
+	if (mp_app_driver_id == -1) {
+		MP_APP_LOG(ERR, COL_RED, "No driver of type %s registered",
+				mp_app_params->devtype_name);
+		return -1;
+	}
+
+	nb_devs = rte_cryptodev_devices_get(mp_app_params->devtype_name,
+					valid_devs, RTE_CRYPTO_MAX_DEVS);
+
+	if (nb_devs < 1) {
+		MP_APP_LOG(ERR, COL_RED, "No %s devices found",
+				mp_app_params->devtype_name);
+		return -1;
+	}
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		mp_shared_data->devices_number = nb_devs;
+	} else {
+		if (mp_shared_data->devices_number != nb_devs) {
+			MP_APP_LOG(INFO, COL_RED,
+			"- Number of devices probed by primary process differs with current process config, number of devices = %d, number on primary = %d",
+					nb_devs,
+					mp_shared_data->devices_number);
+			return -1;
+		}
+	}
+
+	for (i = 0; i < nb_devs ; i++) {
+		rte_cryptodev_info_get(valid_devs[i], &info);
+		if (info.feature_flags & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) {
+			mp_app_devs[mp_app_devs_cnt].id = valid_devs[i];
+			mp_app_devs[mp_app_devs_cnt].probed = 1;
+			mp_app_devs[mp_app_devs_cnt++].max_queue_pairs =
+					info.max_nb_queue_pairs;
+
+			/* Last one is as good as first one */
+			mp_app_max_queues = info.max_nb_queue_pairs;
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+				strncpy(mp_shared_data->prim_dev_name[i].name,
+					info.device->name,
+					MP_APP_DEV_NAME_LEN);
+			} else {
+				if (strncmp(
+					mp_shared_data->prim_dev_name[i].name,
+						info.device->name,
+						MP_APP_DEV_NAME_LEN)) {
+					MP_APP_LOG(INFO, COL_RED,
+					"Wrong device: %s, are BDF passed to primary process the same?",
+						info.device->name);
+					return -1;
+				}
+			}
+		}
+	}
+	/* Pick one device to be used for session creation,
+	 * only valid when all devices of the same type.
+	 */
+	mp_app_device_id = mp_app_devs[0].id;
+
+	MP_APP_LOG(INFO, COL_GREEN,
+			"Configure devices according to mask: 0x%lX",
+			mp_app_params->dev_to_configure_mask);
+
+	uint64_t dev_mask_id;
+	int dev_id;
+
+	for (dev_mask_id = 1, dev_id = 0; dev_id <= MP_APP_MAX_DEVS;
+			dev_mask_id <<= 1, dev_id++) {
+		if (dev_mask_id & mp_app_params->dev_to_configure_mask) {
+			if (!mp_app_devs[dev_id].probed)
+				continue;
+
+			/* TODO check if already configured */
+
+			conf.nb_queue_pairs = info.max_nb_queue_pairs;
+			conf.socket_id = SOCKET_ID_ANY;
+			conf.ff_disable = RTE_CRYPTODEV_FF_SECURITY;
+
+			if (rte_cryptodev_configure(mp_app_devs[dev_id].id,
+				&conf) != 0) {
+				RTE_LOG(ERR, USER1,
+					"Error when configuring device number %d",
+						dev_id);
+				return -1;
+			}
+
+			mp_app_devs[dev_id].configured = 1;
+			MP_APP_LOG(INFO, COL_BLUE, "- Configure Device id %d",
+					mp_app_devs[dev_id].id);
+		}
+	}
+	return 0;
+}
+
+static int check_capabilities(int dev_id,
+			const struct mp_crypto_session_vector *vector)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+
+	cap_idx.type = vector->x_type;
+	if (vector->x_type == RTE_CRYPTO_SYM_XFORM_AEAD)
+		cap_idx.algo.aead = vector->aead_algo;
+
+	/* For now rescricted only to AEAD */
+
+	if (rte_cryptodev_sym_capability_get(dev_id, &cap_idx) == NULL)
+		return -ENOTSUP;
+
+	return 0;
+}
+
+int
+mp_crypto_init_sessions(void)
+{
+	uint64_t session_mask_id;
+	uint64_t session_id;
+	int i;
+	int capabiliy_checked = 1;
+	/* Check if all devices support vector 0 */
+	for (i = 0; i < MP_APP_MAX_VECTORS; i++) {
+		int dev_id = mp_app_devs[i].id;
+		/* TODO use proper vector(s), not hardcoded one */
+		if (dev_id < 0)
+			continue;
+
+		int k = 0;
+
+		while (mp_app_params->enq_param.vector_number[k] >= 0 &&
+				k < MP_APP_MAX_VECTORS) {
+			int vector_number =
+				mp_app_params->enq_param.vector_number[k];
+
+			if (vector_number >= (int)mp_app_numof_ops) {
+				MP_APP_LOG(ERR, COL_RED,
+					"Not recognized test vector %d",
+					vector_number);
+				return -1;
+			}
+			if (check_capabilities(dev_id,
+					&session_vectors[vectors[
+					vector_number].session])) {
+				MP_APP_LOG(ERR, COL_RED,
+					"Algorithm unsupported on dev %d",
+					dev_id);
+				capabiliy_checked = 0;
+			}
+			k++;
+		}
+	}
+	if (capabiliy_checked == 0)
+		return -1;
+
+	for (session_mask_id = 1, session_id = 0;
+			session_id <= mp_app_numof_sessions;
+			session_mask_id <<= 1, session_id++) {
+
+		if (session_mask_id & mp_app_params->session_mask) {
+			struct rte_cryptodev_sym_session *sess =
+				mp_app_create_session(mp_app_device_id,
+				&session_vectors[session_id]);
+			if (sess == NULL) {
+				MP_APP_LOG(ERR, COL_RED,
+					"Error when creating session = %p",
+					sess);
+				return -1;
+			}
+			rte_spinlock_lock(&mp_shared_data->sessions.lock);
+			int clear_session = 1;
+
+			if (mp_shared_data->sessions.sym_sessions[
+					session_id].session
+						== NULL) {
+				mp_shared_data->sessions.sym_sessions[
+					session_id].session
+					= sess;
+				clear_session = 0;
+				/* TODO Remove from spinlock */
+				MP_APP_LOG(INFO, COL_BLUE,
+					"Initialized session = %lu",
+					session_id);
+			} else {
+				/* Actually refcnt should be incremented
+				 * on demand mp_shared_data->sessions.
+				 * sym_sessions [session_id].refcnt++;
+				 */
+			}
+			rte_spinlock_unlock(&mp_shared_data->sessions.lock);
+			if (clear_session)
+				rte_cryptodev_sym_session_free(sess);
+		}
+	}
+	return 0;
+}
+
+int mp_crypto_setup_ops(void)
+{
+	int i;
+	int used_vectors = 0;
+	/* TODO check if device number and qp is correct */
+
+	int selected_vectors[MP_APP_MAX_VECTORS];
+	for (i = 0; i < MP_APP_MAX_VECTORS; i++)
+		selected_vectors[MP_APP_MAX_VECTORS] = -1;
+
+	i = 0;
+	while (mp_app_params->enq_param.vector_number[i] >= 0 &&
+			i < MP_APP_MAX_VECTORS)	{
+		int vector_number = mp_app_params->enq_param.vector_number[i];
+
+		if (mp_app_params->enq_param.vector_number[i] >=
+			(int)mp_app_numof_ops) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Crypto vector %d not defined, skipping",
+				mp_app_params->enq_param.vector_number[i]);
+				i++;
+				continue;
+		}
+		/* Aquire session */
+		int __run = 1;
+		int __continue = 0;
+		int session_id = vectors[vector_number].session;
+
+		while (__run) {
+			int was_session_set = 0;
+
+			rte_spinlock_lock(&mp_shared_data->sessions.lock);
+			if (mp_shared_data->sessions.sym_sessions[
+					session_id].session
+						!= NULL) {
+				mp_shared_data->sessions.sym_sessions[
+					session_id].refcnt++;
+				was_session_set = 1;
+			}
+			rte_spinlock_unlock(&mp_shared_data->sessions.lock);
+			if (was_session_set == 0) {
+				MP_APP_LOG(WARNING, COL_YEL,
+					"Session %d was not yet created, vector %d",
+					session_id, vector_number);
+				char c;
+
+				MP_APP_LOG(INFO, COL_NORM,
+					"Session %d not yet created.\n - Press 'w' to wait until other process will create it \n - Press 'n' to create local session",
+					vectors[session_id].session);
+				int __rte_unused r = scanf("%c", &c);
+
+				if (c == 'n') {
+					struct rte_cryptodev_sym_session *sess =
+						mp_app_create_session(
+							mp_app_device_id,
+						&session_vectors[session_id]);
+					mp_crypto_local_sessions[session_id] =
+						sess;
+					__run = 0;
+				} else if (c == 'w') {
+					int __timeout = 3;
+					int __counter = 1;
+
+					while (__counter <= __timeout) {
+						rte_delay_ms(1000);
+						MP_APP_LOG(INFO, COL_NORM,
+						"Waiting for %d out of %d seconds",
+						__counter++, __timeout);
+					}
+				}
+			} else
+				__run = 0;
+		}
+		if (__continue) {
+			i++;
+			continue;
+		}
+
+		MP_APP_LOG(INFO, COL_BLUE,
+						"Configuring vector %d, using session %d",
+						vector_number, session_id);
+
+		selected_vectors[used_vectors++] = vector_number;
+		i++;
+	}
+
+	if (used_vectors == 0)
+		return 0;
+
+	int curr_vector = 0;
+	/* Create vectors and attach to sessions */
+
+	for (i = 0; i < MP_CRYPTO_QP_DESC_NUM; i++)	{
+		int session_id =
+			vectors[selected_vectors[curr_vector]].session;
+		if (mp_crypto_local_sessions[session_id] != NULL) {
+			mp_crypto_create_op(mp_crypto_ops[i],
+					mp_crypto_mbufs[i],
+					selected_vectors[curr_vector],
+					mp_crypto_local_sessions[session_id]);
+		} else {
+			mp_crypto_create_op(mp_crypto_ops[i],
+					mp_crypto_mbufs[i],
+					selected_vectors[curr_vector],
+					mp_shared_data->sessions.sym_sessions
+					[session_id].session);
+		}
+	}
+	return 0;
+}
+
+int
+mp_crypto_setup_qps(void)
+{
+	int dev_id;
+	int qp_id;
+	int queue_count = 0;
+	int last_qp_on_device = mp_app_max_queues;
+
+	MP_APP_LOG_2(INFO, COL_NORM, "- Configuring queues:");
+	for (dev_id = 0; dev_id < MP_APP_MAX_DEVS; dev_id++) {
+		if (!mp_app_devs[dev_id].probed)
+			continue;
+		for (qp_id = 0; qp_id < mp_app_max_queues; qp_id++) {
+			if (mp_app_devs[dev_id].queue_pair_flag[qp_id]
+					!= QP_TO_CONFIGURE)
+				continue;
+
+			int __run = 1;
+			int __continue = 0;
+
+			while (__run) {
+				/* This could be is_dev_configured */
+				int ret  = rte_cryptodev_get_qp_status(
+					mp_app_devs[dev_id].id, qp_id);
+				if (ret == 1) {
+					mp_app_devs[dev_id].queue_pair_flag[
+						qp_id] = 0;
+					MP_APP_LOG_2(WARNING, COL_YEL,
+						"Queue was already configured by other process, skipping");
+					__run = 0;
+					__continue = 1;
+				} else if (ret < 0) {
+					mp_app_devs[dev_id].queue_pair_flag[
+						qp_id] = 0;
+					MP_APP_LOG_2(ERR, COL_RED,
+						"Error setting queues, was this device configured?");
+					printf(
+						"\n - Press 'w' to wait until other process will configure it");
+					printf("\n - Press 'x' to exit");
+					char c;
+					int __rte_unused r = scanf("%s", &c);
+
+					if (c == 'w') {
+						int __timeout = 3;
+						int __counter = 1;
+
+						while (__timeout <= __counter) {
+							rte_delay_ms(1000);
+							MP_APP_LOG(INFO,
+							COL_NORM,
+							"Waiting for %d out of %d seconds",
+							__counter++, 3);
+						}
+					} else if (c == 'x')
+						return -1;
+				} else if (ret == 0)
+					__run = 0;
+			}
+			if (__continue)
+				continue;
+
+			struct rte_cryptodev_qp_conf qp_conf;
+
+			qp_conf.nb_descriptors = MP_CRYPTO_QP_DESC_NUM;
+			qp_conf.mp_session = NULL;
+			qp_conf.mp_session_private = NULL;
+			if (rte_cryptodev_queue_pair_setup(
+					mp_app_devs[dev_id].id,
+					qp_id, &qp_conf,
+					rte_cryptodev_socket_id(
+					mp_app_devs[dev_id].id))) {
+				RTE_LOG(ERR, USER1,
+					"Error when setting up queue pair %d on dev %d",
+					qp_id, dev_id);
+				return -1;
+			}
+			MP_APP_LOG(INFO, COL_BLUE, "Created qp %d on dev %d",
+					qp_id, mp_app_devs[dev_id].id);
+			mp_app_devs[dev_id].queue_pair_flag[qp_id] = 1;
+			queue_count++;
+		}
+	}
+
+	for (dev_id = 0; dev_id < MP_APP_MAX_DEVS; dev_id++) {
+		if (!mp_app_devs[dev_id].probed)
+			continue;
+		for (qp_id = last_qp_on_device; qp_id < MP_APP_QUEUE_PAIRS_NUM;
+			qp_id++) {
+				if (mp_app_devs[dev_id].queue_pair_flag[qp_id]
+						== QP_TO_CONFIGURE) {
+					MP_APP_LOG(WARNING, COL_YEL,
+					"Cannot create qp %d on dev %d, maximum allowed by this device = %d (%d queue pairs)",
+					qp_id, mp_app_devs[dev_id].id,
+					mp_app_max_queues - 1,
+					mp_app_max_queues);
+				}
+		}
+	}
+
+	MP_APP_LOG(INFO, COL_GREEN, "- Configured %d queues.", queue_count);
+	return 0;
+}
+
+int mp_crypto_setup_mpool(void)
+{
+	int i;
+	char crypto_op_mpool_name[RTE_MEMZONE_NAMESIZE];
+	char mbuf_pool_name[RTE_MEMZONE_NAMESIZE];
+	char session_mpool_name_local[RTE_MEMZONE_NAMESIZE];
+	char session_priv_name_local[RTE_MEMZONE_NAMESIZE];
+
+	/* Op pool */
+	int n = snprintf(crypto_op_mpool_name, sizeof(crypto_op_mpool_name),
+				"%s_%hu", MP_APP_CRYPTO_OP_POOL_NAME,
+					mp_shared_data->proc_counter_total);
+
+	if (n >= (int)sizeof(crypto_op_mpool_name)) {
+		MP_APP_LOG_2(ERR, COL_RED, "Failed to create mpool name");
+		return -1;
+	}
+
+	/* mbuf pool */
+	n = snprintf(mbuf_pool_name, sizeof(mbuf_pool_name),
+				"%s_%hu", MP_APP_MBUFPOOL_NAME,
+				mp_shared_data->proc_counter_total);
+
+	if (n >= (int)sizeof(mbuf_pool_name)) {
+		RTE_LOG(ERR, USER1, "Failed to create mbuf pool name");
+		return -1;
+	}
+
+	/* Local session pool */
+	n = snprintf(session_mpool_name_local,
+			sizeof(session_mpool_name_local),
+			"%s_%hu", MP_APP_SESSION_POOL_NAME_LOC,
+			mp_shared_data->proc_counter_total);
+
+	if (n >= (int)sizeof(session_mpool_name_local)) {
+		MP_APP_LOG_2(ERR, COL_RED,
+			"Failed to local session mpool name");
+		return -1;
+	}
+
+	/* Local priv session pool */
+	n = snprintf(session_priv_name_local, sizeof(session_priv_name_local),
+				"%s_%hu", MP_APP_PRIV_SESSION_POOL_NAME_LOC,
+				mp_shared_data->proc_counter_total);
+
+	if (n >= (int)sizeof(session_priv_name_local)) {
+		MP_APP_LOG_2(ERR, COL_RED,
+		"Failed to local session private mpool name");
+		return -1;
+	}
+
+	/* Op pool */
+	mp_crypto_op_pool =
+		rte_mempool_lookup(crypto_op_mpool_name);
+
+	if (!mp_crypto_op_pool) {
+		mp_crypto_op_pool = rte_crypto_op_pool_create(
+			crypto_op_mpool_name,
+			RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+			MP_APP_NUM_MBUFS, MP_APP_MBUF_CACHE_SIZE,
+			MP_APP_DEFAULT_NUM_XFORMS *
+			sizeof(struct rte_crypto_sym_xform) +
+			MP_APP_MAXIMUM_IV_LENGTH,
+			rte_socket_id());
+	}
+
+	if (mp_crypto_op_pool == NULL) {
+		MP_APP_LOG_2(ERR, COL_RED, "Error in mempool creation for ops");
+		return -1;
+	}
+
+	/* Set session pools for this process */
+	mp_crypto_session_mempool_local =
+		rte_cryptodev_sym_session_pool_create(
+		session_mpool_name_local, MAX_NUM_OF_SESSIONS,
+		sizeof(struct rte_cryptodev_sym_session), 0, 0,
+		SOCKET_ID_ANY);
+
+	if (!mp_crypto_session_mempool_local) {
+		MP_APP_LOG_2(ERR, COL_RED,
+		"Failed to create local session mpool");
+		return -1;
+	}
+
+	/* Set private session pool for this process */
+	mp_crypto_priv_session_mp_local = rte_mempool_create(
+			session_priv_name_local,
+			MAX_NUM_OF_SESSIONS,
+			rte_cryptodev_sym_get_private_session_size(
+				mp_app_device_id),
+			0, 0, NULL, NULL, NULL,
+			NULL, SOCKET_ID_ANY,
+			0);
+	if (!mp_crypto_priv_session_mp_local) {
+		MP_APP_LOG_2(ERR, COL_RED,
+			"Failed to create local session priv mpool");
+		return -1;
+	}
+
+	int dev_id = mp_app_devs[0].id;
+	/* All devices use same driver so the same size of private data */
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		/* Set mempools for sessions */
+		mp_crypto_session_mempool =
+			rte_cryptodev_sym_session_pool_create(
+			MP_APP_SESSION_POOL_NAME, MAX_NUM_OF_SESSIONS,
+			sizeof(struct rte_cryptodev_sym_session), 0, 0,
+			SOCKET_ID_ANY);
+
+		if (!mp_crypto_session_mempool) {
+			MP_APP_LOG_2(ERR, COL_RED,
+				"Failed to create session mpool");
+			return -1;
+		}
+
+		mp_crypto_priv_session_mp = rte_mempool_create(
+			MP_APP_PRIV_SESSION_POOL_NAME,
+			MAX_NUM_OF_SESSIONS,
+			rte_cryptodev_sym_get_private_session_size(dev_id),
+			0, 0, NULL, NULL, NULL,
+			NULL, SOCKET_ID_ANY,
+			0);
+		if (!mp_crypto_priv_session_mp) {
+			MP_APP_LOG_2(ERR, COL_RED,
+			"Failed to create priv mpool");
+			return -1;
+		}
+		/* Set mempools for ops */
+
+	} else {
+		mp_crypto_session_mempool =
+			rte_mempool_lookup(MP_APP_SESSION_POOL_NAME);
+		if (!mp_crypto_session_mempool) {
+			MP_APP_LOG_2(ERR, COL_RED,
+			"Failed to get sess mpool, was it allocated?");
+			return -1;
+		}
+		mp_crypto_priv_session_mp =
+			rte_mempool_lookup(MP_APP_PRIV_SESSION_POOL_NAME);
+		if (!mp_crypto_session_mempool) {
+			MP_APP_LOG_2(ERR, COL_RED,
+			"Failed to get priv session mpool, was it allocated?");
+			return -1;
+		}
+	}
+
+	/* Mbuf pool */
+	mp_crypto_mbuf_pool =
+		rte_mempool_lookup(mbuf_pool_name);
+	if (mp_crypto_mbuf_pool == NULL) {
+		mp_crypto_mbuf_pool = rte_pktmbuf_pool_create(
+				mbuf_pool_name,
+				MP_APP_NUM_MBUFS, MP_APP_MBUF_CACHE_SIZE, 0,
+				MP_APP_MBUF_SIZE,
+				rte_socket_id());
+	}
+	if (mp_crypto_mbuf_pool == NULL) {
+		MP_APP_LOG_2(ERR, COL_RED,
+			"Error in pool creation for mbuf data");
+		return -1;
+	}
+
+	/* Create ops and mbufs */
+	for (i = 0; i < MP_CRYPTO_QP_DESC_NUM; i++)	{
+		mp_crypto_mbufs[i] = rte_pktmbuf_alloc(mp_crypto_mbuf_pool);
+		if (mp_crypto_mbufs[i] == NULL)	{
+			MP_APP_LOG_2(ERR, COL_RED, "Error allocating mbufs");
+			return -1;
+		}
+		memset(rte_pktmbuf_mtod(mp_crypto_mbufs[i], uint8_t *), 0,
+			rte_pktmbuf_data_len(mp_crypto_mbufs[i]));
+	}
+
+	for (i = 0; i < MP_CRYPTO_QP_DESC_NUM; i++) {
+		mp_crypto_ops[i] = rte_crypto_op_alloc(mp_crypto_op_pool,
+			RTE_CRYPTO_OP_TYPE_SYMMETRIC);
+		if (mp_crypto_ops[i] == NULL) {
+			MP_APP_LOG_2(ERR, COL_RED,
+				"Error allocating crypto op");
+			return -1;
+		}
+		mp_crypto_ops[i]->sym->m_src = mp_crypto_mbufs[i];
+	}
+
+	return 0;
+}
+
+static void sigkill_handler(int __rte_unused sig,
+				siginfo_t *siginfo __rte_unused,
+				void *context __rte_unused)
+{
+	mp_crypto_exit_flag = 1;
+	printf("\nInterrupted, finalizing...");
+}
+
+static int
+mp_app_init(int argc, char *argv[])
+{
+	/* init EAL */
+	int ret = rte_eal_init(argc, argv)
+;
+	if (ret < 0)
+		rte_exit(-1, "Invalid EAL arguments!\n");
+
+	argc -= ret;
+	argv += ret;
+
+	struct sigaction sigkill_action;
+
+	memset(&sigkill_action, 0, sizeof(sigkill_action));
+	sigkill_action.sa_sigaction = sigkill_handler;
+	sigkill_action.sa_flags = SA_SIGINFO;
+
+	if (sigaction(SIGINT, &sigkill_action, NULL) < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Cannot init sigation");
+		return -1;
+	}
+
+	if (get_options(argc, argv) != 0) {
+		MP_APP_LOG_2(ERR, COL_RED,
+			"Get cmdln options returned an error\n");
+		return -1;
+	};
+
+	/* Set driver id for this process */
+	mp_app_driver_id =
+		rte_cryptodev_driver_id_get(mp_app_params->devtype_name);
+	MP_APP_LOG(INFO, COL_BLUE, "- Setting driver %d for this process",
+		mp_app_driver_id);
+
+	/* Register IPC and allocate memzones */
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		MP_APP_LOG_2(INFO, COL_NORM, "- Starting PRIMARY process");
+		if (rte_mp_action_register(MP_APP_IPC_NAME,
+			mp_crypto_primary_handler)) {
+			RTE_LOG(ERR, USER1, "Cannot register IPC callback");
+			return -1;
+		}
+		/* Setup memzone for shared data */
+		mp_app_process_mz = rte_memzone_reserve(MP_APP_PROC_SHARED_NAME,
+				sizeof(struct mp_app_process_data), 0, 0);
+		if (mp_app_process_mz == NULL) {
+			RTE_LOG(ERR, USER1,
+				"mp_app_init: cannot create memzone for process");
+			return -1;
+		}
+		mp_shared_data = mp_app_process_mz->addr;
+		rte_spinlock_init(&mp_shared_data->sessions.lock);
+	} else {
+		MP_APP_LOG_2(INFO, COL_NORM, "- Starting SECONDARY process");
+		if (rte_mp_action_register(MP_APP_IPC_NAME,
+			mp_crypto_secondary_handler)) {
+			RTE_LOG(ERR, USER1, "Cannot register IPC callback");
+			return -1;
+		}
+		/* Setup memzone for shared data */
+		mp_app_process_mz =
+			rte_memzone_lookup(MP_APP_PROC_SHARED_NAME);
+		if (mp_app_process_mz == NULL) {
+			MP_APP_LOG(ERR, COL_RED,
+				"Cannot find memzone by name %s",
+			MP_APP_PROC_SHARED_NAME);
+			return -1;
+		}
+		mp_shared_data = mp_app_process_mz->addr;
+	}
+
+	mp_shared_data->proc_counter++;
+	mp_shared_data->proc_counter_total++;
+	MP_APP_LOG(INFO, COL_GREEN, "Number of processes = %d",
+		mp_shared_data->proc_counter);
+
+	return 0;
+}
+
+void mp_crypto_exit_app(void)
+{
+	const int timeout = 10;
+	int counter = 0;
+	struct rte_mp_msg icp_msg;
+
+	memset(&icp_msg, 0, sizeof(MP_APP_IPC_NAME));
+	mp_crypto_exit_flag = 1;
+	if (mp_shared_data == NULL)
+		return;
+
+/*	rte_mempool_free(mp_crypto_op_pool);
+ *	rte_mempool_free(mp_crypto_mbuf_pool);
+ */
+	rte_mempool_free(mp_crypto_session_mempool_local);
+	rte_mempool_free(mp_crypto_priv_session_mp_local);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		/* Inform of exit intention,
+		 * wait until all processes finish
+		 */
+
+		memcpy(icp_msg.name, MP_APP_IPC_NAME, sizeof(MP_APP_IPC_NAME));
+		memcpy(icp_msg.param, PRIMARY_PROC_EXIT,
+			sizeof(PRIMARY_PROC_EXIT));
+		icp_msg.len_param = sizeof(PRIMARY_PROC_EXIT);
+		icp_msg.num_fds = 0;
+		if (rte_mp_sendmsg(&icp_msg) < 0) {
+			MP_APP_LOG_2(ERR, COL_RED,
+			"Error when sending IPC to secondary processes");
+			return;
+		}
+		while (mp_shared_data->proc_counter > 1 && counter++
+				< timeout) {
+			rte_delay_ms(1000);
+			MP_APP_LOG(INFO, COL_NORM,
+			"Waiting for %d out of %d seconds", counter, timeout);
+		}
+		if (counter < timeout) {
+			MP_APP_LOG_2(INFO, COL_GREEN,
+			"All secondary processes exited normally");
+		} else {
+			MP_APP_LOG_2(ERR, COL_RED,
+			"One or more processes did not exit normally");
+		}
+		rte_mempool_free(mp_crypto_session_mempool);
+		rte_mempool_free(mp_crypto_priv_session_mp);
+
+		mp_shared_data->proc_counter = 0;
+	} else {
+		/* Inform primary of exiting */
+		mp_shared_data->proc_counter--;
+	}
+}
+
+static int check_for_queue(int dev_id, int qp_id)
+{
+	int ret = rte_cryptodev_get_qp_status(dev_id, qp_id);
+
+	if (ret <= 0) {
+		MP_APP_LOG(WARNING, COL_YEL,
+			"Queue %d on dev %d not initialized",
+			qp_id, dev_id);
+		printf(
+		"\n - Press 'w' to wait until other process will initialize it");
+		printf("\n - Press 'x' to exit");
+		char c;
+		int __rte_unused r = scanf("%s", &c);
+
+		if (c == 'w') {
+			int __timeout = 3;
+			int __counter = 1;
+
+			while (__counter <= __timeout) {
+				rte_delay_ms(1000);
+				MP_APP_LOG(INFO, COL_NORM,
+				"Waiting for %d out of %d seconds",
+					__counter++, 3);
+			}
+			return -1;
+		} else if (c == 'x')
+			return -2;
+		else
+			return -2;
+	}
+	return 0;
+}
+
+int mp_crypto_flow(void)
+{
+	int process_enq = 0, process_deq = 0;
+	uint64_t curr_offset_enq = 0;
+	uint64_t curr_offset_deq = 0;
+	uint64_t enqueued = 0;
+	uint64_t dequeued = 0;
+	uint64_t deq_threshold = 0;
+	char c = 0;
+	uint64_t pcks_to_enq = 0, pcks_to_deq = 0;
+
+	int enq_dev_id = mp_app_devs[mp_app_params->enq_param.dev_id].id;
+	int deq_dev_id = mp_app_devs[mp_app_params->deq_param.dev_id].id;
+	int enq_qp_id = mp_app_params->enq_param.qp_id;
+	int deq_qp_id = mp_app_params->deq_param.qp_id;
+	int enq_livesign = 0, deq_livesign = 0;
+	int64_t enq_stall_counter = 0, deq_stall_counter = 0;
+	int livesign_print_idx = 0;
+	int livesign_deq_print_idx = 0;
+
+	if (mp_app_params->enq_param.dev_id >= 0 &&
+			!mp_app_devs[mp_app_params->enq_param.dev_id].probed) {
+		MP_APP_LOG(ERR, COL_RED, "Incorrect enq device provided %d",
+				mp_app_params->enq_param.dev_id);
+	} else if (mp_app_params->enq_param.dev_id >= 0) {
+		MP_APP_LOG(INFO, COL_BLUE,
+			"Start enqueuing packets on dev %d qp %d",
+			mp_app_params->enq_param.dev_id,
+			mp_app_params->enq_param.qp_id);
+		pcks_to_enq = mp_app_params->enq_param.ops_no;
+		process_enq = 1;
+	}
+	if (mp_app_params->deq_param.dev_id >= 0 &&
+			!mp_app_devs[mp_app_params->deq_param.dev_id].probed) {
+		MP_APP_LOG(ERR, COL_RED, "Incorrect deq device provided %d",
+				mp_app_params->deq_param.dev_id);
+	} else if (mp_app_params->deq_param.dev_id >= 0) {
+		MP_APP_LOG(INFO, COL_BLUE,
+			"Start dequeuing packets on dev %d qp %d",
+			mp_app_params->deq_param.dev_id,
+			mp_app_params->deq_param.qp_id);
+		pcks_to_deq = mp_app_params->deq_param.ops_no;
+		process_deq = 1;
+	}
+
+	if (process_enq == 0 && process_deq == 0) {
+		MP_APP_LOG_2(WARNING, COL_YEL, "Nothing to process");
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+			while (c != 'k') {
+				printf("\nPress 'k' to exit: ");
+				int __rte_unused r = scanf("%c", &c);
+			}
+		}
+		return 0;
+	}
+
+	/* Check if enq queue was configured */
+	while (process_enq) {
+		int v = check_for_queue(enq_dev_id, enq_qp_id);
+
+		if (v == -1)
+			continue;
+		else if (v == 0)
+			break;
+		else
+			return -1;
+	}
+
+	/* Check if deq queue was configured */
+	while (process_deq) {
+		int v = check_for_queue(deq_dev_id, deq_qp_id);
+
+		if (v == -1)
+			continue;
+		else if (v == 0)
+			break;
+		else
+			return -1;
+	}
+
+	if (process_enq && !process_deq) {
+		MP_APP_LOG_NO_RET(INFO, COL_NORM, "Enqueuing %c",
+			livesign_print_char[livesign_print_idx]);
+	} else if (process_deq && !process_enq) {
+		MP_APP_LOG_NO_RET(INFO, COL_NORM, "Dequeuing %c",
+			livesign_print_char[livesign_deq_print_idx]);
+	} else if (process_enq && process_deq) {
+		MP_APP_LOG_NO_RET(INFO, COL_NORM, "Enqueuing %c Dequeueing %c",
+			livesign_print_char[livesign_print_idx],
+			livesign_print_char[livesign_deq_print_idx]);
+	}
+	while (1) {
+		if (process_enq && !mp_crypto_exit_flag) {
+			if (enqueued < pcks_to_enq || pcks_to_enq == 0) {
+				/* Consider clearing param above */
+				uint64_t __enq;
+				uint64_t to_enq = (MP_CRYPTO_QP_DESC_NUM -
+					curr_offset_enq) > MP_CRYPTO_BURST_NUM ?
+					MP_CRYPTO_BURST_NUM : MP_CRYPTO_QP_DESC_NUM
+					- curr_offset_enq;
+
+				if (pcks_to_enq && to_enq > pcks_to_enq - enqueued)
+					to_enq = pcks_to_enq - enqueued;
+				__enq = rte_cryptodev_enqueue_burst(enq_dev_id,
+					enq_qp_id, &mp_crypto_ops[curr_offset_enq],
+					to_enq);
+				enqueued += __enq;
+				enq_livesign += __enq;
+				curr_offset_enq = enqueued % MP_CRYPTO_QP_DESC_NUM;
+				if (enq_livesign > mp_app_params->enq_param.checkpoint) {
+					if (process_enq && !process_deq) {
+						MP_APP_LOG_NO_RET(INFO, COL_NORM,
+							"Enqueuing %c",
+							livesign_print_char[livesign_print_idx]);
+					}
+					if (process_enq && process_deq) {
+						MP_APP_LOG_NO_RET(INFO, COL_NORM,
+						"Enqueuing %c Dequeueing %c",
+						livesign_print_char[livesign_print_idx],
+						livesign_print_char[livesign_deq_print_idx]);
+					}
+					livesign_print_idx++;
+					livesign_print_idx %= 4;
+					enq_livesign = 0;
+				}
+				if (__enq == 0)
+					enq_stall_counter++;
+				else
+					enq_stall_counter = 0;
+			}
+		}
+
+		if (process_deq) {
+			if (dequeued < pcks_to_deq || pcks_to_deq == 0) {
+				uint64_t __deq;
+				uint64_t to_deq = (MP_CRYPTO_QP_DESC_NUM -
+					curr_offset_deq)
+					> MP_CRYPTO_BURST_NUM ?	MP_CRYPTO_BURST_NUM :
+					MP_CRYPTO_QP_DESC_NUM - curr_offset_deq;
+
+				if (pcks_to_deq && to_deq > pcks_to_deq - dequeued)
+					to_deq = pcks_to_deq - dequeued;
+				__deq = rte_cryptodev_dequeue_burst(deq_dev_id,
+					deq_qp_id, &mp_crypto_ops_ret[curr_offset_deq],
+					to_deq);
+				dequeued += __deq;
+				deq_livesign += __deq;
+				curr_offset_deq = dequeued % MP_CRYPTO_QP_DESC_NUM;
+				if (deq_livesign > mp_app_params->deq_param.checkpoint) {
+					if (process_deq && !process_enq) {
+						MP_APP_LOG_NO_RET(INFO, COL_NORM,
+						"Dequeueing %c",
+						livesign_print_char[livesign_deq_print_idx]);
+					}
+					if (process_enq && process_deq) {
+						MP_APP_LOG_NO_RET(INFO, COL_NORM,
+						"Enqueuing %c Dequeueing %c",
+						livesign_print_char[livesign_print_idx],
+						livesign_print_char[livesign_deq_print_idx]);
+					}
+					livesign_deq_print_idx++;
+					livesign_deq_print_idx %= 4;
+					deq_livesign = 0;
+				}
+				if (__deq == 0)
+					deq_stall_counter++;
+				else
+					deq_stall_counter = 0;
+				if (mp_crypto_exit_flag) {
+					deq_threshold += __deq;
+					if (deq_threshold > 100000)
+						break;
+					if (deq_stall_counter > 100000)
+						break;
+				}
+			}
+		}
+
+		if (((dequeued == pcks_to_deq && process_deq)) &&
+			 ((enqueued == pcks_to_enq && process_enq))) {
+			MP_APP_LOG(INFO, COL_GREEN,
+					"\nEnqueued %lu, dequeued %lu packets",
+					enqueued, dequeued);
+			break;
+		} else if (dequeued == pcks_to_deq && process_deq &&
+				!process_enq && pcks_to_deq)  {
+			MP_APP_LOG(INFO, COL_GREEN, "\nDequeued %lu packets",
+				dequeued);
+			break;
+		} else if (enqueued == pcks_to_enq && process_enq &&
+				!process_deq && process_enq)  {
+			MP_APP_LOG(INFO, COL_GREEN, "\nEnqueued %lu packets",
+				enqueued);
+			break;
+		}
+		if (mp_crypto_exit_flag && !process_deq)
+			break;
+	}
+
+	/* Verify if all packets are correct */
+	if (process_deq) {
+		uint64_t last_packet = pcks_to_deq > MP_CRYPTO_QP_DESC_NUM ?
+			MP_CRYPTO_QP_DESC_NUM : pcks_to_deq;
+		if (pcks_to_deq == 0)
+			last_packet = MP_CRYPTO_QP_DESC_NUM;
+		if (last_packet >= dequeued)
+			last_packet = dequeued;
+		uint64_t k;
+		int err = 0;
+
+		for (k = 0; k < last_packet; k++) {
+			if (mp_crypto_ops_ret[k]->status !=
+				RTE_CRYPTO_OP_STATUS_SUCCESS) {
+				MP_APP_LOG(ERR, COL_RED,
+					"error when checking status of %lu packet out of last %lu packets",
+					k, last_packet);
+					err = 1;
+					break;
+			}
+		}
+		if (err == 0) {
+			MP_APP_LOG(INFO, COL_GREEN,
+				"\nAll %lu last packets verified correctly",
+				last_packet);
+		}
+	}
+
+	if (mp_app_params->print_stats) {
+		struct rte_cryptodev_stats stats;
+
+		if (enq_qp_id >= 0) {
+			rte_cryptodev_stats_get(enq_dev_id, &stats);
+			MP_APP_LOG(INFO, COL_BLUE,
+				"STATS: Enqueued on dev %d, qp %d = %lu",
+				enq_dev_id, enq_qp_id, stats.enqueued_count);
+			MP_APP_LOG(INFO, COL_BLUE,
+				"STATS: Enqueue err count on dev %d, qp %d = %lu",
+				enq_dev_id, enq_qp_id,
+				stats.enqueue_err_count);
+		}
+		if (deq_qp_id >= 0) {
+			rte_cryptodev_stats_get(deq_dev_id, &stats);
+			MP_APP_LOG(INFO, COL_BLUE,
+				"STATS: Dequeued on dev %d, qp %d = %lu",
+				deq_dev_id, deq_qp_id, stats.dequeued_count);
+			MP_APP_LOG(INFO, COL_BLUE,
+				"STATS: Dequeue err count on dev %d, qp %d = %lu",
+				deq_dev_id, deq_qp_id, stats.dequeue_err_count);
+		}
+
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+			while (c != 'k') {
+				printf("\nPress 'k' to exit: ");
+				int __rte_unused r = scanf("%c", &c);
+			}
+		}
+	}
+
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	int ret = 0;
+
+	if (mp_app_init(argc, argv) < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Error when initializing");
+		goto err;
+	};
+
+	if (mp_crypto_init_devs() < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Devices cannot be initialized");
+		goto err;
+	};
+
+	ret = mp_crypto_setup_mpool();
+	if (ret < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Cannot create mempools");
+		goto err;
+	}
+
+	if (mp_crypto_setup_qps() < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Setup qps returned an error");
+		goto err;
+	};
+
+	ret = mp_crypto_init_sessions();
+	if (ret < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Cannot initialize sessions");
+		goto err;
+	}
+
+	ret = mp_crypto_setup_ops();
+	if (ret < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Cannot setup ops");
+		goto err;
+	}
+
+	ret = mp_crypto_flow();
+	if (ret < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Cannot enq/deq");
+		goto err;
+	}
+
+
+	mp_crypto_exit_app();
+	return 0;
+err:
+	mp_crypto_exit_app();
+
+	return 1;
+}
diff --git a/examples/mp_crypto/meson.build b/examples/mp_crypto/meson.build
new file mode 100644
index 0000000..08976ab
--- /dev/null
+++ b/examples/mp_crypto/meson.build
@@ -0,0 +1,18 @@ 
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2020 Intel Corporation
+
+# meson file, for building this example as part of a main DPDK build.
+#
+# To build this example as a standalone application with an already-installed
+# DPDK instance, use 'make'
+
+if not dpdk_conf.has('RTE_LIBRTE_CRYPTODEV')
+        build = false
+        subdir_done()
+endif
+
+allow_experimental_apis = true
+deps += ['cryptodev']
+sources = files(
+	'main.c', 'mp_crypto.c', 'mp_crypto_parser.c', 'mp_crypto_vectors.c', 'mp_crypto_ipc.c'
+)
diff --git a/examples/mp_crypto/mp_crypto.c b/examples/mp_crypto/mp_crypto.c
new file mode 100644
index 0000000..d041d35
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto.c
@@ -0,0 +1,139 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#include "mp_crypto_vectors.h"
+#include "mp_crypto.h"
+#include "mp_crypto_parser.h"
+
+int			mp_app_driver_id;
+/* Global driver id, one per mp_app */
+int			mp_app_device_id;
+/* For now we use only one device type, so for session
+ * init only one need to be provided
+ */
+struct mp_app_dev	mp_app_devs[MP_APP_MAX_DEVS];
+/* Global devices list */
+uint16_t		mp_app_devs_cnt;
+/* Global device counter */
+uint8_t			mp_app_max_queues;
+/* Per process queue counter */
+const struct rte_memzone *mp_app_process_mz;
+struct mp_app_process_data *mp_shared_data;
+/* Data shared across processes
+ * memzone name = MP_PROC_SHARED_MZ
+ */
+
+int mp_crypto_exit_flag;
+/* Global exit flag */
+
+struct rte_mempool *mp_crypto_session_mempool;
+/* Global crypto mempool used by all processes */
+struct rte_mempool *mp_crypto_session_mempool_local;
+/* Local crypto mempool used by this process */
+struct rte_mempool *mp_crypto_priv_session_mp;
+/* Global crypto private session mempool used by all processes */
+struct rte_mempool *mp_crypto_priv_session_mp_local;
+/* Local crypto private session mempool used by this process */
+struct rte_mempool *mp_crypto_op_pool;
+/* Per process op pool */
+struct rte_mempool *mp_crypto_mbuf_pool;
+/* Per process mbuf pool */
+
+struct rte_cryptodev_sym_session *mp_crypto_local_sessions[MAX_NUM_OF_SESSIONS];
+/* Array of private sessions */
+
+struct rte_crypto_op *mp_crypto_ops[MP_CRYPTO_OPS_NUM];
+/* Per process set of rte crypto ops */
+struct rte_crypto_op *mp_crypto_ops_ret[MP_CRYPTO_OPS_NUM];
+/* Per process set of return rte crypto ops */
+struct rte_mbuf *mp_crypto_mbufs[MP_CRYPTO_OPS_NUM];
+/* Per process set of rte mbufs */
+
+/* Function for creating sessions */
+struct rte_cryptodev_sym_session *mp_app_create_session
+	(int dev_id, const struct mp_crypto_session_vector *vector)
+{
+	if (vector->x_type == RTE_CRYPTO_SYM_XFORM_AEAD)
+		return mp_app_create_aead_session(dev_id, vector);
+	MP_APP_LOG_2(ERR, COL_RED, "Invalid xform type");
+	return NULL;
+}
+
+/* Create AEAD session */
+struct rte_cryptodev_sym_session*
+mp_app_create_aead_session(int dev_id,
+		const struct mp_crypto_session_vector *vector)
+{
+	struct rte_cryptodev_sym_session *session;
+	struct rte_crypto_sym_xform xform;
+
+	xform.next = NULL;
+	xform.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+	xform.aead.key.length = vector->crypto_key.len;
+	xform.aead.key.data = vector->crypto_key.data;
+	xform.aead.algo = vector->aead_algo;
+	xform.aead.digest_length = vector->digest_len;
+	xform.aead.iv.length = vector->iv_len;
+	xform.aead.iv.offset = IV_OFFSET;
+	xform.aead.aad_length = vector->aad_len;
+	xform.aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
+
+	session = rte_cryptodev_sym_session_create(mp_crypto_session_mempool);
+	if (session == NULL) {
+		MP_APP_LOG_2(ERR, COL_RED, "Failed to create session");
+		return NULL;
+	}
+	int status = rte_cryptodev_sym_session_init(dev_id, session,
+			&xform,	mp_crypto_priv_session_mp);
+	if (status < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Failed to init session");
+		return NULL;
+	}
+
+	return session;
+}
+
+int
+mp_crypto_create_op(struct rte_crypto_op *op, struct rte_mbuf *mbuf,
+			uint16_t vector_number,
+			struct rte_cryptodev_sym_session *sess)
+{
+	uint8_t *plaintext;
+	uint32_t aad_pad_len =
+		RTE_ALIGN_CEIL(session_vectors[vectors[vector_number].
+				session].aad_len, 16);
+
+	memset(rte_pktmbuf_mtod(mbuf, uint8_t *), 0,
+			rte_pktmbuf_tailroom(mbuf));
+	struct rte_crypto_sym_op *sym_op = op->sym;
+
+	sym_op->aead.aad.data = (uint8_t *)rte_pktmbuf_append(mbuf,
+			aad_pad_len);
+	sym_op->aead.aad.phys_addr =
+			rte_pktmbuf_iova(mbuf);
+	memcpy(sym_op->aead.aad.data, vectors[vector_number].aad.data,
+		session_vectors[vectors[vector_number].session].aad_len);
+	uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op,
+			uint8_t *, IV_OFFSET);
+	rte_memcpy(iv_ptr, vectors[vector_number].iv,
+		session_vectors[vectors[vector_number].session].iv_len);
+
+	plaintext = (uint8_t *)rte_pktmbuf_append(mbuf,
+			vectors[vector_number].plaintext.len);
+	rte_memcpy(plaintext, vectors[vector_number].plaintext.data,
+			vectors[vector_number].plaintext.len);
+
+	sym_op->aead.digest.phys_addr =
+		rte_pktmbuf_iova_offset(mbuf,
+			vectors[vector_number].plaintext.len);
+
+	sym_op->aead.digest.data = (uint8_t *)rte_pktmbuf_append(
+			mbuf, vectors[vector_number].digest.len);
+
+	sym_op->aead.data.length = vectors[vector_number].plaintext.len;
+	sym_op->aead.data.offset = 0;
+
+	if (rte_crypto_op_attach_sym_session(op, sess))
+		return -1;
+	return 0;
+}
diff --git a/examples/mp_crypto/mp_crypto.h b/examples/mp_crypto/mp_crypto.h
new file mode 100644
index 0000000..e708ee3
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto.h
@@ -0,0 +1,224 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#ifndef _MP_CRYPTO_SAMPLE_APP_
+#define _MP_CRYPTO_SAMPLE_APP_
+
+#include <stdint.h>
+#include <rte_hexdump.h>
+#include "mp_crypto_vectors.h"
+
+/* Intel QuickAssist Technology Symmetric service PMD name */
+#define CRYPTODEV_NAME_QAT_SYM_PMD	"crypto_qat"
+/* Maximum number of devices to configure with this app */
+#define MP_APP_MAX_DEVS			64
+/* Maximum number of queue pairs per device */
+#define MP_APP_QUEUE_PAIRS_NUM		8
+
+#define MP_APP_PROC_SHARED_NAME		"MP_PROC_SHARED_MZ"
+/* Memzone name for shared data across processes */
+#define MP_APP_IPC_NAME			"MP_APP_IPC_NAME"
+
+/* Session pool information */
+#define MP_APP_SESSION_POOL_NAME	"MP_APP_SESSION_POOL_NAME"
+#define MP_APP_PRIV_SESSION_POOL_NAME	"MP_APP_PRIV_SESSPOL_NAME"
+
+#define MP_APP_SESSION_POOL_NAME_LOC		"MP_APP_SESSP_NAME_LOC"
+#define MP_APP_PRIV_SESSION_POOL_NAME_LOC	"MP_APP_PRIV_SPOL_NLOC"
+
+#define MAX_NUM_OF_SESSIONS		(16)
+
+/* Crypto op information */
+#define MP_APP_CRYPTO_OP_POOL_NAME	"MP_APP_OP_NAME"
+/* Mbuf information */
+#define MP_APP_MBUFPOOL_NAME		"MP_APP_MBUF_NAME"
+
+extern int mp_crypto_exit_flag;
+/* Global exit flag */
+
+/*
+ * IPC COMMANDS
+ */
+#define PRIMARY_PROC_EXIT		"PRIMARY_EXIT"
+#define SECONDARY_PROC_EXIT		"SECONDARY_EXIT"
+
+#define MP_APP_DEV_NAME_LEN	64
+/* Max name length */
+
+/* Op pool constants */
+#define MP_APP_NUM_MBUFS					(4096)
+/* Same number as default/max ops */
+#define MP_APP_MBUF_CACHE_SIZE				(256)
+#define MP_APP_DEFAULT_NUM_XFORMS			(2)
+#define MP_APP_MAXIMUM_IV_LENGTH			(16)
+/* Mbuf constants */
+#define MP_APP_MBUF_SIZE			(sizeof(struct rte_mbuf) + \
+		RTE_PKTMBUF_HEADROOM + MBUF_DATAPAYLOAD_SIZE)
+/* qps constants */
+#define MP_CRYPTO_QP_DESC_NUM		(4096)
+#define NP_CRYPTO_OPS_TO_ENQ		(160000)
+#define NP_CRYPTO_OPS_TO_DEQ		(160000)
+/* Enqueue constants */
+#define MP_CRYPTO_BURST_NUM		(64)
+#define MP_CRYPTO_OPS_NUM		(MP_APP_NUM_MBUFS)
+/* Device information */
+#define MP_CRYPTO_MAX_DEVS		(64)
+
+extern struct rte_crypto_op *mp_crypto_ops[];
+/* Per process set of rte crypto ops */
+extern struct rte_crypto_op *mp_crypto_ops_ret[];
+/* Per process set of return rte crypto ops */
+extern struct rte_mbuf *mp_crypto_mbufs[];
+/* Per process set of rte mbufs */
+
+/* Name of the device */
+struct mp_app_dev_name {
+	char name[MP_APP_DEV_NAME_LEN];
+};
+
+extern struct rte_cryptodev_sym_session *mp_crypto_local_sessions[];
+/* Array of private sessions */
+
+/* Symmetric session + ref count*/
+struct mp_app_shared_sym_session {
+	struct rte_cryptodev_sym_session *session;
+	/* Pointer to symmetric session */
+	int refcnt;
+	/* Reference count, process that created this session
+	 * does not increment this value
+	 */
+};
+
+/* Data for session array to be shared */
+struct mp_app_session_array {
+	struct mp_app_shared_sym_session sym_sessions[MAX_NUM_OF_SESSIONS];
+	/* Array of pointers to sessions */
+	int sym_session_counter;
+	/* Counter of allocated sessions */
+	rte_spinlock_t lock;
+	/* Spinlock guarding this array */
+};
+
+/* Data to be shared across processes */
+struct mp_app_process_data {
+	uint16_t proc_counter;
+	/* Counter of processes */
+	uint16_t proc_counter_total;
+	/* Number of processes that joined, not decremented
+	 * can be used for naming in particular processes
+	 */
+	uint16_t devices_number;
+	/* Number of devices probed by primary process */
+	struct mp_app_dev_name prim_dev_name[MP_APP_MAX_DEVS];
+	/* Names of devices probed by primary process */
+	struct mp_app_session_array sessions;
+	/* Array of sessions to be visible by all processes */
+};
+
+extern const struct rte_memzone *mp_app_process_mz;
+extern struct mp_app_process_data *mp_shared_data;
+/* Data shared across processes
+ * memzone name = MP_PROC_SHARED_MZ
+ */
+
+extern struct rte_mempool *mp_crypto_session_mempool;
+/* Global crypto session mempool used by all processes */
+extern struct rte_mempool *mp_crypto_session_mempool_local;
+/* Local crypto mempool used by this process */
+extern struct rte_mempool *mp_crypto_priv_session_mp;
+/* Global crypto private session mempool used by all processes */
+extern struct rte_mempool *mp_crypto_priv_session_mp_local;
+/* Local crypto private session mempool used by this process */
+extern struct rte_mempool *mp_crypto_op_pool;
+/* Per process op pool */
+extern struct rte_mempool *mp_crypto_mbuf_pool;
+/* Per process mbuf pool */
+
+struct mp_app_dev {
+	int8_t id;
+	/* Cryptodev id of this dev */
+	int queue_pair_flag[MP_APP_QUEUE_PAIRS_NUM];
+	/* 1 means qp was configured for this device,
+	 * 0 not configured by this process, but still
+	 * could be initialized by another
+	 * -2 means this qp is to be configured
+	 */
+	uint16_t max_queue_pairs;
+	/* Per device info */
+	uint8_t probed;
+	/* If device was probed by EAL */
+	uint8_t configured;
+	/* Was this device configured */
+	const struct rte_memzone *shared_data;
+	/* This data is shared across processes
+	 * memzone name = MZ_DEV_SHARED_DATA_DEV_[ID]
+	 */
+};
+
+extern int			mp_app_driver_id;
+/* Global driver id, one per mp_app */
+extern int			mp_app_device_id;
+/* For now we use only one device type, so for session
+ * init only one need to be provided
+ */
+extern struct mp_app_dev	mp_app_devs[];
+/* Global devices list */
+extern uint16_t			mp_app_devs_cnt;
+/* Global device counter */
+extern uint8_t			mp_app_max_queues;
+/* Per process queue counter */
+
+void mp_crypto_exit_app(void);
+/* Exit function for both primary and secondary */
+
+int mp_crypto_setup_mpool(void);
+/* Function to set or lookup for mempools */
+
+int mp_crypto_flow(void);
+/* Flow function for enqueue dequeue */
+
+/*
+ * Primary process IPC handler
+ */
+int
+mp_crypto_primary_handler(const struct rte_mp_msg *mp_msg,
+		  const void *peer);
+int
+mp_crypto_secondary_handler(const struct rte_mp_msg *mp_msg,
+		  const void *peer);
+
+int mp_crypto_setup_qps(void);
+/* Function to setup queues according to input string */
+
+int mp_crypto_init_sessions(void);
+/* Function to setup session according to mask */
+
+int mp_crypto_init_devs(void);
+/* Function to setup devices according to mask */
+
+int mp_crypto_setup_ops(void);
+/* Function to setup opse according to input string enq=[] */
+
+/* Create and init symmetric session */
+struct rte_cryptodev_sym_session *mp_app_create_session
+		(int dev_id, const struct mp_crypto_session_vector *vector);
+
+/* Create AEAD session */
+struct rte_cryptodev_sym_session*
+		mp_app_create_aead_session(int dev_id,
+		const struct mp_crypto_session_vector *vector);
+
+/* Create op */
+int
+mp_crypto_create_op(struct rte_crypto_op *op, struct rte_mbuf *mbuf,
+					uint16_t vector_number,
+					struct rte_cryptodev_sym_session *sess);
+
+#define IV_OFFSET			(sizeof(struct rte_crypto_op) + \
+		sizeof(struct rte_crypto_sym_op) + DEFAULT_NUM_XFORMS * \
+		sizeof(struct rte_crypto_sym_xform))
+
+#define MBUF_DATAPAYLOAD_SIZE		(2048)
+#define DEFAULT_NUM_XFORMS			(2)
+
+#endif
diff --git a/examples/mp_crypto/mp_crypto_ipc.c b/examples/mp_crypto/mp_crypto_ipc.c
new file mode 100644
index 0000000..9d5a8cb
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto_ipc.c
@@ -0,0 +1,32 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#include "mp_crypto.h"
+
+/*
+ * Primary process IPC handler
+ */
+int
+mp_crypto_primary_handler(const struct rte_mp_msg *mp_msg,
+		  const void *peer)
+{
+	(void)peer;
+	if (!memcmp(SECONDARY_PROC_EXIT, (const char *)mp_msg->param,
+		sizeof(SECONDARY_PROC_EXIT))) {
+		RTE_LOG(ERR, USER1, "One of secondary processes exiting...");
+	}
+	return 0;
+}
+
+int
+mp_crypto_secondary_handler(const struct rte_mp_msg *mp_msg,
+		  const void *peer)
+{
+	(void)peer;
+	if (!memcmp(PRIMARY_PROC_EXIT, (const char *)mp_msg->param,
+		sizeof(PRIMARY_PROC_EXIT)))	{
+		RTE_LOG(ERR, USER1, "Primary process exiting...");
+		mp_crypto_exit_flag = 1;
+	}
+	return 0;
+}
diff --git a/examples/mp_crypto/mp_crypto_parser.c b/examples/mp_crypto/mp_crypto_parser.c
new file mode 100644
index 0000000..165f345
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto_parser.c
@@ -0,0 +1,511 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include <getopt.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <errno.h>
+
+#include <rte_string_fns.h>
+#include <rte_comp.h>
+
+#include "mp_crypto_parser.h"
+#include "mp_crypto.h"
+
+struct mp_crypto_app_parameters *mp_app_params;
+
+static void
+usage(char *progname)
+{
+	/* TODO, find better way of formatting columns... */
+	printf("%s [EAL options] -- [options]"
+	"\noptions:"
+	"\n  --devtype [device name]: \t\t\tdevice name, the same name need to be used"
+	" across all processes. \n\t\t\t\t\t\t--Example: --devtype=crypto_qat"
+	"\n  --config-dev [dev_id,]: \t\t\tid of device that should be"
+	" configured by this process. Note that order of ids depends on the"
+	" Cryptodev\n\t\t\t\t\t\tglobal array placement so BDF of smaller numbers will come"
+	" first. \n\t\t\t\t\t\t--Example: -w 03:01.2 -w 03:01.1 -w 03:01.3 --config-dev 0,2"
+	" will configure devices 03:01.1 and 03:01.3."
+	"\n  --qp-config=[dev_id]:[qp_id,];...: \t\tqueue_pairs qp_id's to be configured dev_id's"
+	"\n\t\t\t\t\t\t--Example: --qp-config=0:0,1;1:1;0:1; - will configure qp's 0,1 on device 0"
+	"' 1 on device 1, 0,1 on device 2.'"
+	"\n  --session-mask=[mask]\t\t\t\tsession to be shared for all processes, session list is in"
+	" mp_crypto_vectors.c file.\n\t\t\t\t\t\tIf session mask will not be set it still can be configured"
+	" interactively by user for certain process and the used by this process only"
+	"\n\t\t\t\t\t\t--Example --sesion-mask=0x3 will configure session 0 and 1."
+	"\n  --enq=[dev_id]:[qp_id]:[ops]:[vector_id]:\tEnqueue operation for this process"
+	"\n\t\t\t\t\t\t- dev_id: device selected the same way as in --config-dev option"
+	"\n\t\t\t\t\t\t- qp_id: queue pair to bu used for enqueue operation"
+	"\n\t\t\t\t\t\t- ops: 0 means it will run in infinite loop (ctrl-c will inform other processes),"
+	"other than that any positive number"
+	"\n\t\t\t\t\t\t- vector_id: vector id to be used, vector array can be found"
+	" in mp_crypto_vectors.c file. "
+	"\n\t\t\t\t\t\t- Only one can be specified by process"
+	"\n  --deq=[dev_id]:[qp_id]:[ops]:[vector_id]:\tDequeue operation for this process"
+	"\n\t\t\t\t\t\t- dev_id: device selected the same way as in --config-dev option"
+	"\n\t\t\t\t\t\t- qp_id: queue pair to bu used for dequeue operation"
+	"\n\t\t\t\t\t\t- ops: 0 means it will run in infinite loop (ctrl-c will inform other processes),"
+	"other than that any positive number"
+	"\n\t\t\t\t\t\t- vector_id: vector id to be used, vector array can be found"
+	" in mp_crypto_vectors.c file. "
+	"\n\t\t\t\t\t\t- Only one can be specified by process"
+	"\n  --print-stats: \t\t\t\tPrint stats at then end of program."
+	"\n",
+	progname);
+}
+
+static struct option lgopts[] = {
+	{ MP_DEV_CONFIGURE, required_argument, 0, 0 },
+	{ MP_QP_CONFIGURE, required_argument, 0, 0 },
+	{ MP_ENQ, required_argument, 0, 0 },
+	{ MP_DEQ, required_argument, 0, 0 },
+	{ MP_SESSION_MASK, required_argument, 0, 0 },
+	{ MP_PRINT_STATS, 0, 0, 0 },
+	{ MP_DEVTYPE_NAME, required_argument, 0, 0 },
+	{ NULL, 0, 0, 0 }
+};
+
+static void dump_test_data_options(struct mp_crypto_app_parameters
+	*test_data __rte_unused) {
+}
+
+int16_t
+get_options(int argc, char *argv[]) {
+	mp_app_params = rte_zmalloc_socket(NULL,
+					sizeof(struct mp_crypto_app_parameters),
+					0, rte_socket_id());
+
+	if (mp_app_params == NULL) {
+		RTE_LOG(ERR, USER1,
+			"Failed to allocate for test data\n");
+		return -1;
+	}
+
+	options_default(mp_app_params);
+
+	if (options_parse(mp_app_params, argc, argv) != 0) {
+		MP_APP_LOG_2(ERR, COL_RED,
+			"Parsing one or more user options failed");
+		return -1;
+	}
+
+	if (options_check(mp_app_params) != 0) {
+		RTE_LOG(ERR, USER1,
+			"Inconsistent user options.\n");
+		dump_test_data_options(mp_app_params);
+		return -1;
+	}
+
+	dump_test_data_options(mp_app_params);
+	return 0;
+}
+
+static int
+parse_config_dev(struct mp_crypto_app_parameters *mp_params,
+					const char *arg)
+{
+	char *end = NULL;
+	const char *start = arg;
+	uint64_t num;
+	char str[32];
+
+	while (1) {
+		memset(str, 0, sizeof(str));
+		end = strchr(start, ',');
+		if (end) {
+			memcpy(str, start, end - start);
+			errno = 0;
+			num = strtoull(str, NULL, 10);
+			if (errno) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Invalid device provided '%s'", str);
+				return -1;
+			}
+			if (num >= MP_CRYPTO_MAX_DEVS) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Device number not supported %lu", num);
+				return -1;
+			}
+			/* Sanity check, unfortunately c standard does not
+			 * force errno to be set when no conversion
+			 * can by performed (except for ERANGE)
+			 */
+			if (num == 0) {
+				if (start[0] != '0') {
+					MP_APP_LOG(ERR, COL_RED,
+					"Invalid device provided '%s'", str);
+					return -1;
+				}
+				if (start[1] != ',') {
+					MP_APP_LOG(ERR, COL_RED,
+					"Invalid device provided '%s'", str);
+					return -1;
+				}
+			}
+			mp_params->dev_to_configure_mask |= 1LU << (num);
+			start = end + 1;
+			if (*start == 0)
+				break;
+		} else {
+			end = strchr(start, '\0');
+			memcpy(str, start, end - start);
+			errno = 0;
+			num = strtoull(str, NULL, 10);
+			if (errno) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Invalid device provided '%s'", str);
+				return -1;
+			}
+			if (num >= 64) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Device number not supported %lu", num);
+				return -1;
+			}
+			/* Sanity check, unfortunately c standard does not force
+			 * errno to be set when no conversion can by performed
+			 * (except for ERANGE)
+			 */
+			if (num == 0) {
+				if (start[0] != '0') {
+					MP_APP_LOG(ERR, COL_RED,
+					"Invalid device provided '%s'", str);
+					return -1;
+				}
+				if (start[1] != '\0') {
+					MP_APP_LOG(ERR, COL_RED,
+					"Invalid device provided '%s'", str);
+					return -1;
+				}
+			}
+			mp_params->dev_to_configure_mask |= 1LU << (num);
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/* Veeeery simple parser */
+static int mp_parse_qps(const char *arg)
+{
+	char str[64] = { };
+	int dev_id = -1;
+	const char *start = arg;
+	const char *end;
+	int finish = 0;
+
+	while (1) {
+		end = strchr(start, ':');
+		if (end == NULL)
+			return 0;
+		memcpy(str, start, end - start);
+		dev_id = strtol(str, NULL, 10);
+		start = end + 1;
+		if (*start == '\0') {
+			MP_APP_LOG_2(ERR, COL_RED,
+				"Parsing queue pairs: error parsing");
+			return -1;
+		}
+		const char *curr = start;
+
+		while (1) {
+			memset(str, 0, sizeof(str));
+			if (*curr == ',' || *curr == ';' || *curr == '\0') {
+				memcpy(str, start, curr - start);
+				int qp_id = strtol(str, NULL, 10);
+
+				if (qp_id > (MP_APP_QUEUE_PAIRS_NUM - 1)) {
+					MP_APP_LOG(WARNING, COL_YEL, "Cannot create qp: %d, maximum qp number allowed %d (%d queues)",
+					qp_id, MP_APP_QUEUE_PAIRS_NUM - 1,
+					MP_APP_QUEUE_PAIRS_NUM);
+				}
+
+				mp_app_devs[dev_id].queue_pair_flag[qp_id] =
+						QP_TO_CONFIGURE;
+			}
+			if (*curr == ',') {
+				start = curr + 1;
+				curr++;
+				continue;
+			} else if (*curr == ';') {
+				start = curr + 1;
+				break;
+			} else if (*curr == '\0') {
+				finish = 1;
+				break;
+			}
+			curr++;
+		}
+		if (finish)
+			break;
+	}
+
+	return 0;
+}
+
+static int
+parse_qp_config(struct mp_crypto_app_parameters *mp_params, const char *arg)
+{
+	strncpy(mp_params->qp_config, arg, MP_APP_QP_PARAM_LEN);
+	if (mp_parse_qps(arg)) {
+		MP_APP_LOG_2(ERR, COL_RED, "- Parsing error, qpairs string");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+parse_enq(struct mp_crypto_app_parameters *mp_params, const char *arg)
+{
+	char str[64] = { };
+	const char *start = arg;
+	/* dev id */
+	char *end = strchr(start, ':');
+	int i = 0;
+
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->enq_param.dev_id = strtol(str, NULL, 10);
+	/* qp id */
+	memset(str, 0, sizeof(str));
+	start = end + 1;
+	end = strchr(start, ':');
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->enq_param.qp_id = strtol(str, NULL, 10);
+	/* ops no */
+	memset(str, 0, sizeof(str));
+	start = end + 1;
+	end = strchr(start, ':');
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->enq_param.ops_no = strtol(str, NULL, 10);
+	/* vector ids */
+	start = end + 1;
+	while ((end = strchr(start, ',')) != NULL) {
+		memset(str, 0, sizeof(str));
+		memcpy(str, start, end - start);
+		mp_params->enq_param.vector_number[i] = strtoul(str, NULL, 0);
+		start = end + 1;
+		i++;
+	}
+	if (i == 0)
+		goto err;
+
+	MP_APP_LOG(INFO, COL_BLUE, "Run enqueue on device %d",
+			mp_params->enq_param.dev_id);
+	MP_APP_LOG(INFO, COL_BLUE, "Run enqueue on qp %d",
+			mp_params->enq_param.qp_id);
+	i = 0;
+	while (mp_params->enq_param.vector_number[i] > 0 &&
+			i < MP_APP_MAX_VECTORS)	{
+		MP_APP_LOG(INFO, COL_BLUE, "Run enqueue vector %d",
+			mp_params->enq_param.vector_number[i]);
+		i++;
+	}
+
+	mp_params->enq_param.checkpoint = 1000000;
+
+	return 0;
+err:
+	MP_APP_LOG_2(ERR, COL_RED, "Error parsing enq");
+	return -1;
+}
+
+static int
+parse_deq(struct mp_crypto_app_parameters *mp_params, const char *arg)
+{
+	char str[64] = { };
+	const char *start = arg;
+	/* Dev id */
+	char *end = strchr(start, ':');
+	int i = 0;
+
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->deq_param.dev_id = strtol(str, NULL, 10);
+	/* qp id */
+	memset(str, 0, sizeof(str));
+	start = end + 1;
+	end = strchr(start, ':');
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->deq_param.qp_id = strtol(str, NULL, 10);
+	/* ops no */
+	memset(str, 0, sizeof(str));
+	start = end + 1;
+	end = strchr(start, ':');
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->deq_param.ops_no = strtol(str, NULL, 10);
+
+	/* vector no */
+	start = end + 1;
+	while ((end = strchr(start, ',')) != NULL) {
+		memset(str, 0, sizeof(str));
+		memcpy(str, start, end - start);
+		mp_params->deq_param.vector_number[i] = strtoul(str, NULL, 0);
+		start = end + 1;
+		i++;
+	}
+	if (i == 0)
+		goto err;
+
+	MP_APP_LOG(INFO, COL_BLUE, "Run dequeue on device %d",
+			mp_params->deq_param.dev_id);
+	MP_APP_LOG(INFO, COL_BLUE, "Run dequeue on qp %d",
+			mp_params->deq_param.qp_id);
+	i = 0;
+	while (mp_params->deq_param.vector_number[i] > 0 &&
+			i < MP_APP_MAX_VECTORS)	{
+		MP_APP_LOG(INFO, COL_BLUE, "Run dequeue vector %d",
+				mp_params->deq_param.vector_number[i]);
+		i++;
+	}
+
+	mp_params->deq_param.checkpoint = 1000000;
+
+	return 0;
+err:
+	MP_APP_LOG_2(ERR, COL_RED, "Error parsing deq");
+	return -1;
+}
+
+static int
+parse_print_stats(struct mp_crypto_app_parameters *mp_params,
+			const char *arg __rte_unused)
+{
+	mp_params->print_stats = 1;
+	return 0;
+}
+
+static int
+parse_session_mask(struct mp_crypto_app_parameters *mp_params,
+					const char *arg)
+{
+	char *end = NULL;
+
+	mp_params->session_mask = strtoull(arg, &end, 16);
+
+	return 0;
+}
+
+static int
+parse_devtype(struct mp_crypto_app_parameters *mp_params,
+					const char *arg)
+{
+	if (arg == NULL) {
+		RTE_LOG(ERR, USER1, "--%s param argument is null\n",
+			MP_DEVTYPE_NAME);
+	}
+
+	if (strlen(arg) > (sizeof(mp_params->devtype_name) - 1)) {
+		RTE_LOG(ERR, USER1, "--%s different lengths\n",
+			MP_DEVTYPE_NAME);
+		return 0;
+	}
+
+	strlcpy(mp_params->devtype_name, arg,
+			sizeof(mp_params->devtype_name));
+
+	return 0;
+};
+
+typedef int (*option_parser_t)(struct mp_crypto_app_parameters
+			*mp_params,	const char *arg);
+
+struct long_opt_parser {
+	const char *lgopt_name;
+	option_parser_t parser_fn;
+};
+
+static int
+opts_parse_long(int opt_idx, struct mp_crypto_app_parameters *mp_params)
+{
+	struct long_opt_parser parsermap[] = {
+		{ MP_DEV_CONFIGURE, parse_config_dev },
+		{ MP_QP_CONFIGURE, parse_qp_config },
+		{ MP_ENQ, parse_enq },
+		{ MP_DEQ, parse_deq },
+		{ MP_PRINT_STATS, parse_print_stats },
+		{ MP_SESSION_MASK, parse_session_mask },
+		{ MP_DEVTYPE_NAME, parse_devtype },
+	};
+	unsigned int i;
+
+	for (i = 0; i < RTE_DIM(parsermap); i++) {
+		if (strncmp(lgopts[opt_idx].name, parsermap[i].lgopt_name,
+				strlen(lgopts[opt_idx].name)) == 0) {
+			return parsermap[i].parser_fn(mp_params, optarg);
+		}
+	}
+
+	return 0;
+}
+
+int
+options_parse(struct mp_crypto_app_parameters *mp_params,
+					int argc, char **argv)
+{
+	int opt, retval;
+	int opt_idx;
+
+	while ((opt = getopt_long(argc, argv, "h", lgopts, &opt_idx))
+			!= EOF) {
+		switch (opt) {
+		case 'h':
+			usage(argv[0]);
+			rte_exit(0, "Select options as above.\n");
+			break;
+		case 0:
+			retval = opts_parse_long(opt_idx, mp_params);
+			if (retval != 0)
+				return retval;
+			break;
+		default:
+			RTE_LOG(ERR, USER1, "Parse error after %s\n",
+					lgopts[opt_idx].name);
+			usage(argv[0]);
+			return 0;
+		}
+	}
+
+	return 0;
+}
+
+void
+options_default(struct mp_crypto_app_parameters *mp_params)
+{
+	int i = 0;
+
+	for (i = 0; i < MP_APP_MAX_VECTORS; i++) {
+		mp_params->enq_param.dev_id = -1;
+		mp_params->enq_param.qp_id = -1;
+		mp_params->enq_param.vector_number[i] = -1;
+		mp_params->deq_param.dev_id = -1;
+		mp_params->deq_param.qp_id = -1;
+		mp_params->deq_param.vector_number[i] = -1;
+	}
+
+	mp_params->enq_param.ops_no = 0;
+	mp_params->deq_param.ops_no = 0;
+	mp_params->print_stats = 0;
+}
+
+int
+options_check(__rte_unused struct mp_crypto_app_parameters *mp_params)
+{
+	return 0;
+}
diff --git a/examples/mp_crypto/mp_crypto_parser.h b/examples/mp_crypto/mp_crypto_parser.h
new file mode 100644
index 0000000..dfbde82
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto_parser.h
@@ -0,0 +1,149 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#ifndef _MP_CRYPTO_SAMPLE_APP_PARSER_
+#define _MP_CRYPTO_SAMPLE_APP_PARSER_
+
+#include <rte_hexdump.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_cryptodev.h>
+#include <cmdline_parse.h>
+
+/* Make debug colorful! */
+#define COL_NORM	"\x1B[0m"
+#define COL_WHITE	"\x1B[37m"
+#define COL_RED		"\x1B[31m"
+#define COL_GREEN	"\x1B[32m"
+#define COL_YEL		"\x1B[33m"
+#define COL_BLUE	"\x1B[34m"
+#define COL_MAG		"\x1B[35m"
+
+#define MP_APP_LOG(level, color, str, args...) \
+	do {		\
+	printf("%s", color);			\
+	RTE_LOG(level, USER1, str, args);	\
+	printf("%s\n", COL_NORM);	\
+	} while (0)
+
+#define MP_APP_LOG_2(level, color, str) \
+	do {		\
+	printf("%s", color);			\
+	RTE_LOG(level, USER1, str);	\
+	printf("%s\n", COL_NORM);	\
+	} while (0)
+
+#define MP_APP_LOG_NO_RET(level, color, str, args...) \
+	do {		\
+	printf("\r%s", color);			\
+	RTE_LOG(level, USER1, str, args);	\
+	printf("%s", COL_NORM);	\
+	} while (0)
+
+#define MP_APP_QP_PARAM_LEN		(64 * 4)
+#define MP_APP_ENQ_PARAM_LEN	1024
+
+#define EMPTY_FLAGS		0
+
+#define MP_DEVTYPE_NAME		("devtype")
+#define MP_DEV_CONFIGURE	("config-dev")
+#define MP_QP_CONFIGURE		("qp-config")
+#define MP_ENQ				("enq")
+#define MP_DEQ				("deq")
+#define MP_SESSION_MASK		("session-mask")
+#define MP_PRINT_STATS		("print-stats")
+
+#define MP_APP_MAX_VECTORS	64
+
+extern const char *comp_perf_test_type_strs[];
+/* Command line parameters */
+extern struct mp_crypto_app_parameters *mp_app_params;
+/* Parser params */
+
+static const char livesign_print_char[4] = { '-', '\\', '|', '/'};
+
+int16_t
+get_options(int argc, char *argv[]);
+
+struct mp_crypto_app_enqdeq {
+	int dev_id;
+	int qp_id;
+	int vector_number[MP_APP_MAX_VECTORS];
+	int ops_no;
+	int checkpoint;
+};
+
+#define QP_TO_CONFIGURE		(-2)
+
+struct mp_crypto_app_parameters {
+	char devtype_name[RTE_DEV_NAME_MAX_LEN];
+	/* Driver to be used in this process */
+	char qp_config[MP_APP_QP_PARAM_LEN];
+	/* Queue Pairs configuration per device in process
+	 * in format q0,q1;q0,q1;, '-' means queue pair will not
+	 * be configured
+	 * Example: queue_pairs="0,1;0,-;-,1;" means that
+	 * device 0 will configure queue pairs 0 and 1,
+	 * device 1 will configure queue pairs 0
+	 * device 2 will configure queue pairs 1
+	 * Devices are order dependent
+	 */
+	char flow_config[MP_APP_ENQ_PARAM_LEN];
+	/* Enqueue configuration per process
+	 * Format "[dev_id]=qp_id:[op,]
+	 * Example: [0]=0:[enq, deq];[1]=0:[enq]
+	 * Mean that for this process qp 0 on device 0 will be
+	 * enqueuing and dequeuing in one queue pair,
+	 * meanwhile device 0 will only enqueue data on qpair 0.
+	 * Other process can then dequeue this data with
+	 * [1]=0:[deq]
+	 */
+	uint64_t dev_to_configure_mask;
+	/* Devices to configure, uint64 bitmask
+	 * 1 means dev 0, 2 dev 1, 4 dev... etc
+	 */
+	uint64_t session_mask;
+	/* Session to be created by this process,
+	 * if session was already created this step will be ommited.
+	 * Usage: session-mask=0x6 -> create session number 1 and 2.
+	 * Number of session refer to predefined array of sessions
+	 */
+	char enq[MP_APP_ENQ_PARAM_LEN];
+	struct mp_crypto_app_enqdeq enq_param;
+	char deq[MP_APP_ENQ_PARAM_LEN];
+	struct mp_crypto_app_enqdeq deq_param;
+	/* Enqueue/dequeue string used by this process.
+	 * Usage: [dev_id]:[qp_id]:[crypto_vector],[crypto_vector]...
+	 * Example 2:1:0,1,2, -> device no 2 on qp 1 enqueues ops from
+	 * vectors 0, 1, 2 .note ',' comma needs to be put after last arg
+	 */
+	int print_stats;
+	/* Print stats on the end on flow function */
+
+	uint16_t qp_id;
+	uint16_t waiting_qp_id;
+
+	int16_t configure_device;
+	int16_t setup_qp;
+	int16_t create_session_pool;
+	int16_t create_op_pool;
+	int16_t init_sessions;
+	int16_t build_ops;
+	int16_t dequeue;
+	int16_t enqueue;
+	int16_t dump_mempools;
+};
+
+int
+options_parse(struct mp_crypto_app_parameters *mp_params, int argc,
+			char **argv);
+void
+options_default(struct mp_crypto_app_parameters *mp_params);
+
+int
+options_check(struct mp_crypto_app_parameters *mp_params);
+
+#endif
diff --git a/examples/mp_crypto/mp_crypto_vectors.c b/examples/mp_crypto/mp_crypto_vectors.c
new file mode 100644
index 0000000..3a6b057
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto_vectors.c
@@ -0,0 +1,175 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#include "mp_crypto_vectors.h"
+
+const struct mp_crypto_session_vector session_vectors[] = {
+	{ /* Session 0 */
+		.aead_op = RTE_CRYPTO_AEAD_OP_ENCRYPT,
+		.x_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+		.aead_algo = RTE_CRYPTO_AEAD_AES_GCM,
+		.crypto_key = {
+			.data = {
+				0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c,
+				0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08
+			},
+			.len = 16,
+		},
+		.aad_len = 0,
+		.iv_len = 12,
+		.digest_len = 16,
+	},
+	{ /* Session 1 */
+		.aead_op = RTE_CRYPTO_AEAD_OP_ENCRYPT,
+		.x_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+		.aead_algo = RTE_CRYPTO_AEAD_AES_GCM,
+		.crypto_key = {
+			.data = {
+				0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C,
+				0x6D, 0x6A, 0x8F, 0x94, 0x67, 0x30, 0x83, 0x08,
+				0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C
+			},
+			.len = 24,
+		},
+		.aad_len = 0,
+		.iv_len = 12,
+		.digest_len = 16,
+	},
+};
+const uint64_t mp_app_numof_sessions =
+				RTE_DIM(session_vectors);
+/* Number of all sessions in array */
+
+const struct mp_crypto_vector vectors[] = {
+		{ /* Vector 0 AES128-GCM */
+			.session = 0,
+			.plaintext = {
+				.data = {
+					0xd9, 0x31, 0x32, 0x25,
+					0xf8, 0x84, 0x06, 0xe5,
+					0xa5, 0x59, 0x09, 0xc5,
+					0xaf, 0xf5, 0x26, 0x9a,
+					0x86, 0xa7, 0xa9, 0x53,
+					0x15, 0x34, 0xf7, 0xda,
+					0x2e, 0x4c, 0x30, 0x3d,
+					0x8a, 0x31, 0x8a, 0x72,
+					0x1c, 0x3c, 0x0c, 0x95,
+					0x95, 0x68, 0x09, 0x53,
+					0x2f, 0xcf, 0x0e, 0x24,
+					0x49, 0xa6, 0xb5, 0x25,
+					0xb1, 0x6a, 0xed, 0xf5,
+					0xaa, 0x0d, 0xe6, 0x57,
+					0xba, 0x63, 0x7b, 0x39,
+				},
+				.len = 60,
+			},
+			.iv = {
+					0xca, 0xfe, 0xba, 0xbe,
+					0xfa, 0xce, 0xdb, 0xad,
+					0xde, 0xca, 0xf8, 0x88
+			},
+			.ciphertext = {
+				.data = {
+					0x42, 0x83, 0x1e, 0xc2,
+					0x21, 0x77, 0x74, 0x24,
+					0x4b, 0x72, 0x21, 0xb7,
+					0x84, 0xd0, 0xd4, 0x9c,
+					0xe3, 0xaa, 0x21, 0x2f,
+					0x2c, 0x02, 0xa4, 0xe0,
+					0x35, 0xc1, 0x7e, 0x23,
+					0x29, 0xac, 0xa1, 0x2e,
+					0x21, 0xd5, 0x14, 0xb2,
+					0x54, 0x66, 0x93, 0x1c,
+					0x7d, 0x8f, 0x6a, 0x5a,
+					0xac, 0x84, 0xaa, 0x05,
+					0x1b, 0xa3, 0x0b, 0x39,
+					0x6a, 0x0a, 0xac, 0x97,
+					0x3d, 0x58, 0xe0, 0x91
+				},
+				.len = 60
+			},
+			.aad = {
+				.data = {
+
+				},
+			},
+			.digest = {
+				.data = {
+					0xA2, 0xA4, 0x35, 0x75,
+					0xDC, 0xB0, 0x57, 0x74,
+					0x07, 0x02, 0x30, 0xC2,
+					0xE7, 0x52, 0x02, 0x00},
+				.len = 16
+			},
+		},
+		{ /* Vector 1 AES192-GCM */
+			.session = 1,
+			.plaintext = {
+				.data = {
+					0xD9, 0x31, 0x32, 0x25,
+					0xF8, 0x84, 0x06, 0xE5,
+					0xA5, 0x59, 0x09, 0xC5,
+					0xAF, 0xF5, 0x26, 0x9A,
+					0x86, 0xA7, 0xA9, 0x53,
+					0x15, 0x34, 0xF7, 0xDA,
+					0x2E, 0x4C, 0x30, 0x3D,
+					0x8A, 0x31, 0x8A, 0x72,
+					0x1C, 0x3C, 0x0C, 0x95,
+					0x95, 0x68, 0x09, 0x53,
+					0x2F, 0xCF, 0x0E, 0x24,
+					0x49, 0xA6, 0xB5, 0x25,
+					0xB1, 0x6A, 0xED, 0xF5,
+					0xAA, 0x0D, 0xE6, 0x57,
+					0xBA, 0x63, 0x7B, 0x39,
+					0x1A, 0xAF, 0xD2, 0x55
+				},
+				.len = 60,
+			},
+			.iv = {
+				0xCA, 0xFE, 0xBA, 0xBE,
+				0xFA, 0xCE, 0xDB, 0xAD,
+				0xDE, 0xCA, 0xF8, 0x88
+			},
+			.ciphertext = {
+				.data = {
+					0x39, 0x80, 0xCA, 0x0B,
+					0x3C, 0x00, 0xE8, 0x41,
+					0xEB, 0x06, 0xFA, 0xC4,
+					0x87, 0x2A, 0x27, 0x57,
+					0x85, 0x9E, 0x1C, 0xEA,
+					0xA6, 0xEF, 0xD9, 0x84,
+					0x62, 0x85, 0x93, 0xB4,
+					0x0C, 0xA1, 0xE1, 0x9C,
+					0x7D, 0x77, 0x3D, 0x00,
+					0xC1, 0x44, 0xC5, 0x25,
+					0xAC, 0x61, 0x9D, 0x18,
+					0xC8, 0x4A, 0x3F, 0x47,
+					0x18, 0xE2, 0x44, 0x8B,
+					0x2F, 0xE3, 0x24, 0xD9,
+					0xCC, 0xDA, 0x27, 0x10,
+					0xAC, 0xAD, 0xE2, 0x56
+				},
+				.len = 60
+			},
+			.aad = {
+				.data = {
+
+				},
+			},
+			.digest = {
+				.data = {
+					0x99, 0x24, 0xA7, 0xC8,
+					0x58, 0x73, 0x36, 0xBF,
+					0xB1, 0x18, 0x02, 0x4D,
+					0xB8, 0x67, 0x4A, 0x14
+				},
+				.len = 16
+			},
+		},
+
+
+};
+
+const uint64_t mp_app_numof_ops =
+					RTE_DIM(vectors);
+/* Number of all operation instances */
diff --git a/examples/mp_crypto/mp_crypto_vectors.h b/examples/mp_crypto/mp_crypto_vectors.h
new file mode 100644
index 0000000..65386ad
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto_vectors.h
@@ -0,0 +1,66 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#ifndef _MP_CRYPTO_SAMPLE_APP_VECTORS_
+#define _MP_CRYPTO_SAMPLE_APP_VECTORS_
+
+#include <rte_cryptodev.h>
+
+struct mp_crypto_session_vector {
+	union {
+		enum rte_crypto_aead_algorithm aead_algo;
+		enum rte_crypto_auth_algorithm auth_algo;
+		enum rte_crypto_cipher_algorithm cipher_algo;
+	};
+	enum rte_crypto_sym_xform_type x_type;
+	union {
+		enum rte_crypto_aead_operation aead_op;
+		enum rte_crypto_cipher_operation cipher_op;
+		enum rte_crypto_auth_operation auth_op;
+	};
+	struct {
+		uint8_t data[64];
+		uint16_t len;
+	} crypto_key;
+	struct {
+		uint8_t data[64];
+		uint16_t len;
+	} auth_key;
+	uint16_t aad_len;
+	uint16_t iv_len;
+	uint16_t digest_len;
+	int chained;
+};
+
+struct mp_crypto_vector {
+	int session;
+	struct {
+		uint8_t data[2048];
+		int len;
+	} ciphertext;
+	struct {
+		uint8_t data[2048];
+		int len;
+	} plaintext;
+	struct {
+		uint8_t data[2048];
+		int len;
+	} digest;
+	struct {
+		uint8_t data[64];
+	} aad;
+	uint8_t iv[16];
+};
+
+/* Predefinced vectors */
+extern const struct mp_crypto_session_vector session_vectors[];
+/* Sessions vectors for this device */
+extern const uint64_t mp_app_numof_sessions;
+/* Number of all sessions in array */
+
+extern const struct mp_crypto_vector vectors[];
+/* Operation vectors for this device */
+const uint64_t mp_app_numof_ops;
+/* Total number of operation types */
+
+#endif