[v6,3/4] example/kni: add IOVA support for kni application
Checks
Commit Message
From: Vamsi Attunuru <vattunuru@marvell.com>
Current KNI implementation operates in IOVA = PA mode,
Patch adds support for IOVA = VA mode by addressing
the issues with page address translations(IOVA <==> KVA).
In this patch KNI application creates mempool with
"MEMPOOL_F_NO_PAGE_BOUND" flag to ensure all mbuf memory
is with in the page boundaries and subsequently kernel KNI
module uses iommu_iova_to_phys() and phys_to_virt() APIs
to get the kernel virtual addresses.
Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
Signed-off-by: Kiran Kumar K <kirankumark@marvell.com>
---
examples/kni/main.c | 53 ++++++++++++++++++++++-
lib/librte_eal/linux/eal/eal.c | 8 ----
lib/librte_eal/linux/eal/include/rte_kni_common.h | 1 +
lib/librte_kni/rte_kni.c | 2 +
4 files changed, 55 insertions(+), 9 deletions(-)
Comments
On 6/25/2019 4:56 AM, vattunuru@marvell.com wrote:
> From: Vamsi Attunuru <vattunuru@marvell.com>
>
> Current KNI implementation operates in IOVA = PA mode,
> Patch adds support for IOVA = VA mode by addressing
> the issues with page address translations(IOVA <==> KVA).
>
> In this patch KNI application creates mempool with
> "MEMPOOL_F_NO_PAGE_BOUND" flag to ensure all mbuf memory
> is with in the page boundaries and subsequently kernel KNI
> module uses iommu_iova_to_phys() and phys_to_virt() APIs
> to get the kernel virtual addresses.
>
> Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
> Signed-off-by: Kiran Kumar K <kirankumark@marvell.com>
> ---
> examples/kni/main.c | 53 ++++++++++++++++++++++-
> lib/librte_eal/linux/eal/eal.c | 8 ----
> lib/librte_eal/linux/eal/include/rte_kni_common.h | 1 +
> lib/librte_kni/rte_kni.c | 2 +
Need to document this new iova_mode and related behavior in KNI documentation.
<...>
> @@ -975,7 +1026,7 @@ main(int argc, char** argv)
> rte_exit(EXIT_FAILURE, "Could not parse input parameters\n");
>
> /* Create the mbuf pool */
> - pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF,
> + pktmbuf_pool = kni_packet_pool_create("mbuf_pool", NB_MBUF,
What about keeping the default mempool creation and use
"kni_packet_pool_create()" only when "iova_mode" is set. (Please read below
comment first J )
<...>
> diff --git a/lib/librte_kni/rte_kni.c b/lib/librte_kni/rte_kni.c
> index 99c4bf5..4263f21 100644
> --- a/lib/librte_kni/rte_kni.c
> +++ b/lib/librte_kni/rte_kni.c
> @@ -300,6 +300,8 @@ rte_kni_alloc(struct rte_mempool *pktmbuf_pool,
> kni->group_id = conf->group_id;
> kni->mbuf_size = conf->mbuf_size;
>
> + dev_info.iova_mode = (rte_eal_iova_mode() == RTE_IOVA_VA) ? 1 : 0;
> +
The application using the "iova_mode" requires a specific mempool requirements,
and just updating sample application won't solve the problem here.
What do you think making "iova_mode" user configurable instead of making it
automatically set in the library? So the application that can justify the
requirement can set the "iova_mode".
And make "iova_mode" disabled by default.
To be able to remove the 'kni' check in the eal, still need to check
'rte_eal_iova_mode()', what about following
a) iova_mode=pa && "rte_eal_iova_mode() == RTE_IOVA_PA" ==> Run KNI
b) iova_mode=pa && "rte_eal_iova_mode() == RTE_IOVA_VA" ==> Return error
c) iova_mode=va && "rte_eal_iova_mode() == RTE_IOVA_PA" ==> Run KNI
d) iova_mode=va && "rte_eal_iova_mode() == RTE_IOVA_VA" ==> Run KNI
For b) the workaround for the user want to run KNI can be user giving eal
parameter to force DPDK to IOVA=PA mode.
@@ -37,6 +37,7 @@
#include <rte_ethdev.h>
#include <rte_mempool.h>
#include <rte_mbuf.h>
+#include <rte_mbuf_pool_ops.h>
#include <rte_string_fns.h>
#include <rte_cycles.h>
#include <rte_malloc.h>
@@ -945,6 +946,56 @@ kni_free_kni(uint16_t port_id)
return 0;
}
+static struct rte_mempool *
+kni_packet_pool_create(const char *name, unsigned int n,
+ unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
+ int socket_id)
+{
+ struct rte_pktmbuf_pool_private mbp_priv;
+ const char *mp_ops_name;
+ struct rte_mempool *mp;
+ unsigned int elt_size;
+ int ret;
+
+ if (RTE_ALIGN(priv_size, RTE_MBUF_PRIV_ALIGN) != priv_size) {
+ RTE_LOG(ERR, MBUF, "mbuf priv_size=%u is not aligned\n",
+ priv_size);
+ rte_errno = EINVAL;
+ return NULL;
+ }
+ elt_size = sizeof(struct rte_mbuf) + (unsigned int)priv_size +
+ (unsigned int)data_room_size;
+ mbp_priv.mbuf_data_room_size = data_room_size;
+ mbp_priv.mbuf_priv_size = priv_size;
+
+ mp = rte_mempool_create_empty(name, n, elt_size, cache_size,
+ sizeof(struct rte_pktmbuf_pool_private), socket_id,
+ MEMPOOL_F_NO_PAGE_BOUND);
+ if (mp == NULL)
+ return NULL;
+
+ mp_ops_name = rte_mbuf_best_mempool_ops();
+ ret = rte_mempool_set_ops_byname(mp, mp_ops_name, NULL);
+ if (ret != 0) {
+ RTE_LOG(ERR, MBUF, "error setting mempool handler\n");
+ rte_mempool_free(mp);
+ rte_errno = -ret;
+ return NULL;
+ }
+ rte_pktmbuf_pool_init(mp, &mbp_priv);
+
+ ret = rte_mempool_populate_default(mp);
+ if (ret < 0) {
+ rte_mempool_free(mp);
+ rte_errno = -ret;
+ return NULL;
+ }
+
+ rte_mempool_obj_iter(mp, rte_pktmbuf_init, NULL);
+
+ return mp;
+}
+
/* Initialise ports/queues etc. and start main loop on each core */
int
main(int argc, char** argv)
@@ -975,7 +1026,7 @@ main(int argc, char** argv)
rte_exit(EXIT_FAILURE, "Could not parse input parameters\n");
/* Create the mbuf pool */
- pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF,
+ pktmbuf_pool = kni_packet_pool_create("mbuf_pool", NB_MBUF,
MEMPOOL_CACHE_SZ, 0, MBUF_DATA_SZ, rte_socket_id());
if (pktmbuf_pool == NULL) {
rte_exit(EXIT_FAILURE, "Could not initialise mbuf pool\n");
@@ -1041,14 +1041,6 @@ rte_eal_init(int argc, char **argv)
rte_eal_get_configuration()->iova_mode =
rte_bus_get_iommu_class();
- /* Workaround for KNI which requires physical address to work */
- if (rte_eal_get_configuration()->iova_mode == RTE_IOVA_VA &&
- rte_eal_check_module("rte_kni") == 1) {
- rte_eal_get_configuration()->iova_mode = RTE_IOVA_PA;
- RTE_LOG(WARNING, EAL,
- "Some devices want IOVA as VA but PA will be used because.. "
- "KNI module inserted\n");
- }
} else {
rte_eal_get_configuration()->iova_mode =
internal_config.iova_mode;
@@ -128,6 +128,7 @@ struct rte_kni_device_info {
unsigned mbuf_size;
unsigned int mtu;
uint8_t mac_addr[6];
+ uint8_t iova_mode;
};
#define KNI_DEVICE "kni"
@@ -300,6 +300,8 @@ rte_kni_alloc(struct rte_mempool *pktmbuf_pool,
kni->group_id = conf->group_id;
kni->mbuf_size = conf->mbuf_size;
+ dev_info.iova_mode = (rte_eal_iova_mode() == RTE_IOVA_VA) ? 1 : 0;
+
ret = ioctl(kni_fd, RTE_KNI_IOCTL_CREATE, &dev_info);
if (ret < 0)
goto ioctl_fail;