Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/123689/?format=api
{ "id": 123689, "url": "http://patches.dpdk.org/api/patches/123689/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20230210154828.397475-1-shibin.koikkara.reeny@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20230210154828.397475-1-shibin.koikkara.reeny@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20230210154828.397475-1-shibin.koikkara.reeny@intel.com", "date": "2023-02-10T15:48:28", "name": "[v5] net/af_xdp: support CNI Integration", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "6cf0f474395d9daa4b7da1398c2fe62437470821", "submitter": { "id": 2540, "url": "http://patches.dpdk.org/api/people/2540/?format=api", "name": "Koikkara Reeny, Shibin", "email": "shibin.koikkara.reeny@intel.com" }, "delegate": { "id": 319, "url": "http://patches.dpdk.org/api/users/319/?format=api", "username": "fyigit", "first_name": "Ferruh", "last_name": "Yigit", "email": "ferruh.yigit@amd.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20230210154828.397475-1-shibin.koikkara.reeny@intel.com/mbox/", "series": [ { "id": 26953, "url": "http://patches.dpdk.org/api/series/26953/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=26953", "date": "2023-02-10T15:48:28", "name": "[v5] net/af_xdp: support CNI Integration", "version": 5, "mbox": "http://patches.dpdk.org/series/26953/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/123689/comments/", "check": "warning", "checks": "http://patches.dpdk.org/api/patches/123689/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 9FBAF41C38;\n\tFri, 10 Feb 2023 16:48:46 +0100 (CET)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 7893A410EA;\n\tFri, 10 Feb 2023 16:48:46 +0100 (CET)", "from mga11.intel.com (mga11.intel.com [192.55.52.93])\n by mails.dpdk.org (Postfix) with ESMTP id 5B0D6410D3\n for <dev@dpdk.org>; Fri, 10 Feb 2023 16:48:44 +0100 (CET)", "from fmsmga007.fm.intel.com ([10.253.24.52])\n by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 10 Feb 2023 07:48:43 -0800", "from silpixa00400899.ir.intel.com ([10.243.22.108])\n by fmsmga007.fm.intel.com with ESMTP; 10 Feb 2023 07:48:41 -0800" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple;\n d=intel.com; i=@intel.com; q=dns/txt; s=Intel;\n t=1676044124; x=1707580124;\n h=from:to:cc:subject:date:message-id:in-reply-to:\n references:mime-version:content-transfer-encoding;\n bh=3x5ok0gd0ibn3A92SQD2U41B0yWzw764/G9V3b6luqI=;\n b=G9S+o6jCoxxn9j2gzlOzqQJax+N4/9UBf2K/Gyu8Egoe1Dfp3z+9WwmA\n sdfNwSGbieYGSgr1x2Nm8QGYyOjhbeep3Gi8m+tg37OzOCjEW6G7GpafQ\n 8+pA+mDzsnhOysFeVJfXH8axYRs9vGl7OBPI6KT6e3U+0wMFUQE7o3Zei\n 0V7y8JHvmTvt8Y1t2sXoZ2I6Sl4zCgNjbjF845wILpHhLHr1Ypi8zxtd8\n R5qp030qDwcQJstx7uJFO7ca+qGSak/0nzOez19g4brO92wEZs4ewErv3\n G61pcnk2L8rA0HqYfhJ+UYoCbVNaYO2ww9OgUJbLRx5gZYa32pYJN5kM9 A==;", "X-IronPort-AV": [ "E=McAfee;i=\"6500,9779,10617\"; a=\"328136038\"", "E=Sophos;i=\"5.97,287,1669104000\"; d=\"scan'208\";a=\"328136038\"", "E=McAfee;i=\"6500,9779,10617\"; a=\"670035629\"", "E=Sophos;i=\"5.97,287,1669104000\"; d=\"scan'208\";a=\"670035629\"" ], "X-ExtLoop1": "1", "From": "Shibin Koikkara Reeny <shibin.koikkara.reeny@intel.com>", "To": "dev@dpdk.org, qi.z.zhang@intel.com, anatoly.burakov@intel.com,\n bruce.richardson@intel.com, john.mcnamara@intel.com", "Cc": "ciara.loftus@intel.com,\n Shibin Koikkara Reeny <shibin.koikkara.reeny@intel.com>", "Subject": "[PATCH v5] net/af_xdp: support CNI Integration", "Date": "Fri, 10 Feb 2023 15:48:28 +0000", "Message-Id": "<20230210154828.397475-1-shibin.koikkara.reeny@intel.com>", "X-Mailer": "git-send-email 2.25.1", "In-Reply-To": "<20230209120549.388318-1-shibin.koikkara.reeny@intel.com>", "References": "<20230209120549.388318-1-shibin.koikkara.reeny@intel.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "Integrate support for the AF_XDP CNI and device plugin [1] so that the\nDPDK AF_XDP PMD can work in an unprivileged container environment.\nPart of the AF_XDP PMD initialization process involves loading\nan eBPF program onto the given netdev. This operation requires\nprivileges, which prevents the PMD from being able to work in an\nunprivileged container (without root access). The plugin CNI handles\nthe program loading. CNI open Unix Domain Socket (UDS) and waits\nlistening for a client to make requests over that UDS. The client(DPDK)\nconnects and a \"handshake\" occurs, then the File Descriptor which points\nto the XSKMAP associated with the loaded eBPF program is handed over\nto the client. The client can then proceed with creating an AF_XDP\nsocket and inserting the socket into the XSKMAP pointed to by the\nFD received on the UDS.\n\nA new vdev arg \"use_cni\" is created to indicate user wishes to run\nthe PMD in unprivileged mode and to receive the XSKMAP FD from the CNI.\nWhen this flag is set, the XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD libbpf flag\nshould be used when creating the socket, which tells libbpf not to load the\ndefault libbpf program on the netdev. We tell libbpf not to do this because\nthe loading is handled by the CNI in this scenario.\n\nPatch include howto doc explain how to configure AF_XDP CNI to\nworking with DPDK.\n\n[1]: https://github.com/intel/afxdp-plugins-for-kubernetes\n\nSigned-off-by: Shibin Koikkara Reeny <shibin.koikkara.reeny@intel.com>\nTested-by: Anatoly Burakov <anatoly.burakov@intel.com>\nReviewed-by: Qi Zhang <qi.z.zhang@intel.com>\n\n---\nv5: Updated patch title.\nv4: updated documentation and also updated the macro for pmdinfo.py.\nv3: add documentation.\nv2: corrected the typo.\n---\n---\n doc/guides/howto/af_xdp_cni.rst | 251 +++++++++++++++++++++\n doc/guides/howto/index.rst | 1 +\n doc/guides/nics/af_xdp.rst | 11 +\n drivers/net/af_xdp/rte_eth_af_xdp.c | 337 +++++++++++++++++++++++++++-\n 4 files changed, 589 insertions(+), 11 deletions(-)\n create mode 100644 doc/guides/howto/af_xdp_cni.rst", "diff": "diff --git a/doc/guides/howto/af_xdp_cni.rst b/doc/guides/howto/af_xdp_cni.rst\nnew file mode 100644\nindex 0000000000..72613a0dd7\n--- /dev/null\n+++ b/doc/guides/howto/af_xdp_cni.rst\n@@ -0,0 +1,251 @@\n+.. SPDX-License-Identifier: BSD-3-Clause\n+ Copyright(c) 2023 Intel Corporation.\n+\n+Using a CNI with the AF_XDP driver\n+==================================\n+\n+Introduction\n+------------\n+\n+CNI, the Container Network Interface, is a technology for configuring container\n+network interfaces and which can be used to setup Kubernetes networking. AF_XDP\n+is a Linux socket Address Family that enables an XDP program to redirect packets\n+to a memory buffer in userspace.\n+\n+This document explains how to enable the `AF_XDP Plugin for Kubernetes`_ within\n+a DPDK application using the `AF_XDP PMD`_ to connect and use these technologies.\n+\n+.. _AF_XDP Plugin for Kubernetes: https://github.com/intel/afxdp-plugins-for-kubernetes\n+\n+Background\n+----------\n+\n+The standard `AF_XDP PMD`_ initialization process involves loading an eBPF program\n+onto the kernel netdev to be used by the PMD. This operation requires root or\n+escalated Linux privileges and thus prevents the PMD from working in an\n+unprivileged container. The AF_XDP CNI plugin handles this situation by\n+providing a device plugin that performs the program loading.\n+\n+At a technical level the CNI opens a Unix Domain Socket and listens for a client\n+to make requests over that socket. A DPDK application acting as a client\n+connects and initiates a configuration \"handshake\". The client then receives a\n+file descriptor which points to the XSKMAP associated with the loaded eBPF\n+program. The XSKMAP is a BPF map of AF_XDP sockets (XSK). The client can then\n+proceed with creating an AF_XDP socket and inserting that socket into the XSKMAP\n+pointed to by the descriptor.\n+\n+The EAL vdev argument ``use_cni`` is used to indicate that the user wishes to\n+run the PMD in unprivileged mode and to receive the XSKMAP file descriptor from\n+the CNI. When this flag is set, the ``XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD``\n+libbpf flag should be used when creating the socket to instruct libbpf not to\n+load the default libbpf program on the netdev. Instead the loading is handled by\n+the CNI.\n+\n+.. _AF_XDP PMD: https://doc.dpdk.org/guides/nics/af_xdp.html\n+\n+.. Note::\n+\n+ The Unix Domain Socket file path appear in the end user is \"/tmp/afxdp.sock\".\n+\n+Prerequisites\n+-------------\n+\n+Docker and container prerequisites:\n+\n+* Set up the device plugin as described in the instructions for `AF_XDP Plugin\n+ for Kubernetes`_.\n+\n+* The Docker image should contain the libbpf and libxdp libraries, which\n+ are dependencies for AF_XDP, and should include support for the ``ethtool``\n+ command.\n+\n+* The Pod should have enabled the capabilities ``CAP_NET_RAW`` and ``CAP_BPF``\n+ for AF_XDP along with support for hugepages.\n+\n+* Increase locked memory limit so containers have enough memory for packet\n+ buffers.\n+ For example:\n+\n+ .. code-block:: console\n+\n+ cat << EOF | sudo tee /etc/systemd/system/containerd.service.d/limits.conf\n+ [Service]\n+ LimitMEMLOCK=infinity\n+ EOF\n+\n+* dpdk-testpmd application should have AF_XDP Feature enabled.\n+\n+ For further information see the docs for the: :doc:`../../nics/af_xdp`.\n+\n+\n+Example\n+-------\n+\n+Howto run dpdk-testpmd with CNI plugin:\n+\n+* Clone the CNI plugin\n+\n+ .. code-block:: console\n+\n+\t # git clone https://github.com/intel/afxdp-plugins-for-kubernetes.git\n+\n+* Build the CNI plugin\n+\n+ .. code-block:: console\n+\n+ # cd afxdp-plugins-for-kubernetes/\n+ # make build\n+\n+ .. Note::\n+\n+ CNI plugin has a dependence on the config.json.\n+\n+ Sample Config.json\n+\n+ .. code-block:: json\n+\n+ {\n+ \"logLevel\":\"debug\",\n+ \"logFile\":\"afxdp-dp-e2e.log\",\n+ \"pools\":[\n+ {\n+ \"name\":\"e2e\",\n+ \"mode\":\"primary\",\n+ \"timeout\":30,\n+ \"ethtoolCmds\" : [\"-L -device- combined 1\"],\n+ \"devices\":[\n+ {\n+ \"name\":\"ens785f0\"\n+ }\n+ ]\n+ }\n+ ]\n+ }\n+\n+ For further reference please use the `config.json`_\n+\n+ .. _config.json: https://github.com/intel/afxdp-plugins-for-kubernetes/blob/v0.0.2/test/e2e/config.json\n+\n+* Create the Network Attachment definition\n+\n+ .. code-block:: console\n+\n+ # kubectl create -f nad.yaml\n+\n+ Sample nad.yml\n+\n+ .. code-block:: yaml\n+\n+ apiVersion: \"k8s.cni.cncf.io/v1\"\n+ kind: NetworkAttachmentDefinition\n+ metadata:\n+ name: afxdp-e2e-test\n+ annotations:\n+ k8s.v1.cni.cncf.io/resourceName: afxdp/e2e\n+ spec:\n+ config: '{\n+ \"cniVersion\": \"0.3.0\",\n+ \"type\": \"afxdp\",\n+ \"mode\": \"cdq\",\n+ \"logFile\": \"afxdp-cni-e2e.log\",\n+ \"logLevel\": \"debug\",\n+ \"ipam\": {\n+ \"type\": \"host-local\",\n+ \"subnet\": \"192.168.1.0/24\",\n+ \"rangeStart\": \"192.168.1.200\",\n+ \"rangeEnd\": \"192.168.1.216\",\n+ \"routes\": [\n+ { \"dst\": \"0.0.0.0/0\" }\n+ ],\n+ \"gateway\": \"192.168.1.1\"\n+ }\n+ }'\n+\n+ For further reference please use the `nad.yaml`_\n+\n+ .. _nad.yaml: https://github.com/intel/afxdp-plugins-for-kubernetes/blob/v0.0.2/test/e2e/nad.yaml\n+\n+* Build the Docker image\n+\n+ .. code-block:: console\n+\n+ # docker build -t afxdp-e2e-test -f Dockerfile .\n+\n+ Sample Dockerfile:\n+\n+ .. code-block:: console\n+\n+\t FROM ubuntu:20.04\n+\t RUN apt-get update -y\n+\t RUN apt install build-essential libelf-dev -y\n+\t RUN apt-get install iproute2 acl -y\n+\t RUN apt install python3-pyelftools ethtool -y\n+\t RUN apt install libnuma-dev libjansson-dev libpcap-dev net-tools -y\n+\t RUN apt-get install clang llvm -y\n+\t COPY ./libbpf<version>.tar.gz /tmp\n+\t RUN cd /tmp && tar -xvmf libbpf<version>.tar.gz && cd libbpf/src && make install\n+\t COPY ./libxdp<version>.tar.gz /tmp\n+\t RUN cd /tmp && tar -xvmf libxdp<version>.tar.gz && cd libxdp && make install\n+\n+ .. Note::\n+\n+\t All the files that need to COPY-ed should be in the same directory as the Dockerfile\n+\n+* Run the Pod\n+\n+ .. code-block:: console\n+\n+ # kubectl create -f pod.yaml\n+\n+ Sample pod.yaml:\n+\n+ .. code-block:: yaml\n+\n+\t apiVersion: v1\n+\t kind: Pod\n+\t metadata:\n+\t name: afxdp-e2e-test\n+\t annotations:\n+\t k8s.v1.cni.cncf.io/networks: afxdp-e2e-test\n+\t spec:\n+\t containers:\n+\t - name: afxdp\n+\t image: afxdp-e2e-test:latest\n+\t imagePullPolicy: Never\n+\t env:\n+\t - name: LD_LIBRARY_PATH\n+\t value: /usr/lib64/:/usr/local/lib/\n+\t command: [\"tail\", \"-f\", \"/dev/null\"]\n+\t securityContext:\n+\t capabilities:\n+\t add:\n+\t - CAP_NET_RAW\n+\t - CAP_BPF\n+\t resources:\n+\t requests:\n+\t hugepages-2Mi: 2Gi\n+\t memory: 2Gi\n+\t afxdp/e2e: '1'\n+\t limits:\n+\t hugepages-2Mi: 2Gi\n+\t memory: 2Gi\n+\t afxdp/e2e: '1'\n+\n+ For further reference please use the `pod.yaml`_\n+\n+ .. _pod.yaml: https://github.com/intel/afxdp-plugins-for-kubernetes/blob/v0.0.2/test/e2e/pod-1c1d.yaml\n+\n+* Run DPDK with a command like the following:\n+\n+ .. code-block:: console\n+\n+\tkubectl exec -i <Pod name> --container <containers name> -- \\\n+ /<Path>/dpdk-testpmd -l 0,1 --no-pci \\\n+ --vdev=net_af_xdp0,use_cni=1,iface=<interface name> \\\n+ -- --no-mlockall --in-memory\n+\n+\n+For further reference please use the `e2e`_ test case in `AF_XDP Plugin for Kubernetes`_\n+\n+\n+ .. _e2e: https://github.com/intel/afxdp-plugins-for-kubernetes/tree/v0.0.2/test/e2e\ndiff --git a/doc/guides/howto/index.rst b/doc/guides/howto/index.rst\nindex bf6337d021..3349b383ac 100644\n--- a/doc/guides/howto/index.rst\n+++ b/doc/guides/howto/index.rst\n@@ -21,3 +21,4 @@ HowTo Guides\n debug_troubleshoot\n openwrt\n avx512\n+ af_xdp_cni\ndiff --git a/doc/guides/nics/af_xdp.rst b/doc/guides/nics/af_xdp.rst\nindex c99f750b44..4d80ff8437 100644\n--- a/doc/guides/nics/af_xdp.rst\n+++ b/doc/guides/nics/af_xdp.rst\n@@ -151,6 +151,17 @@ instead of zero copy mode (if available).\n \n --vdev net_af_xdp,iface=ens786f1,force_copy=1\n \n+use_cni\n+~~~~~~~\n+\n+The EAL vdev argument use_cni is used to indicate that the user wishes to\n+enable the `AF_XDP Plugin for Kubernetes`_ within a DPDK application.\n+\n+.. _AF_XDP Plugin for Kubernetes: https://github.com/intel/afxdp-plugins-for-kubernetes\n+\n+.. code-block:: console\n+\n+\t--vdev=net_af_xdp0,use_cni=1\n \n Limitations\n -----------\ndiff --git a/drivers/net/af_xdp/rte_eth_af_xdp.c b/drivers/net/af_xdp/rte_eth_af_xdp.c\nindex b6ec9bf490..07bebcf05e 100644\n--- a/drivers/net/af_xdp/rte_eth_af_xdp.c\n+++ b/drivers/net/af_xdp/rte_eth_af_xdp.c\n@@ -7,6 +7,7 @@\n #include <string.h>\n #include <netinet/in.h>\n #include <net/if.h>\n+#include <sys/un.h>\n #include <sys/socket.h>\n #include <sys/ioctl.h>\n #include <linux/if_ether.h>\n@@ -81,6 +82,23 @@ RTE_LOG_REGISTER_DEFAULT(af_xdp_logtype, NOTICE);\n \n #define ETH_AF_XDP_MP_KEY \"afxdp_mp_send_fds\"\n \n+#define MAX_LONG_OPT_SZ\t\t\t64\n+#define UDS_MAX_FD_NUM\t\t\t2\n+#define UDS_MAX_CMD_LEN\t\t\t64\n+#define UDS_MAX_CMD_RESP\t\t128\n+#define UDS_XSK_MAP_FD_MSG\t\t\"/xsk_map_fd\"\n+#define UDS_SOCK\t\t\t\"/tmp/afxdp.sock\"\n+#define UDS_CONNECT_MSG\t\t\t\"/connect\"\n+#define UDS_HOST_OK_MSG\t\t\t\"/host_ok\"\n+#define UDS_HOST_NAK_MSG\t\t\"/host_nak\"\n+#define UDS_VERSION_MSG\t\t\t\"/version\"\n+#define UDS_XSK_MAP_FD_MSG\t\t\"/xsk_map_fd\"\n+#define UDS_XSK_SOCKET_MSG\t\t\"/xsk_socket\"\n+#define UDS_FD_ACK_MSG\t\t\t\"/fd_ack\"\n+#define UDS_FD_NAK_MSG\t\t\t\"/fd_nak\"\n+#define UDS_FIN_MSG\t\t\t\"/fin\"\n+#define UDS_FIN_ACK_MSG\t\t\t\"/fin_ack\"\n+\n static int afxdp_dev_count;\n \n /* Message header to synchronize fds via IPC */\n@@ -151,6 +169,7 @@ struct pmd_internals {\n \tchar prog_path[PATH_MAX];\n \tbool custom_prog_configured;\n \tbool force_copy;\n+\tbool use_cni;\n \tstruct bpf_map *map;\n \n \tstruct rte_ether_addr eth_addr;\n@@ -170,6 +189,7 @@ struct pmd_process_private {\n #define ETH_AF_XDP_PROG_ARG\t\t\t\"xdp_prog\"\n #define ETH_AF_XDP_BUDGET_ARG\t\t\t\"busy_budget\"\n #define ETH_AF_XDP_FORCE_COPY_ARG\t\t\"force_copy\"\n+#define ETH_AF_XDP_USE_CNI_ARG\t\t\t\"use_cni\"\n \n static const char * const valid_arguments[] = {\n \tETH_AF_XDP_IFACE_ARG,\n@@ -179,6 +199,7 @@ static const char * const valid_arguments[] = {\n \tETH_AF_XDP_PROG_ARG,\n \tETH_AF_XDP_BUDGET_ARG,\n \tETH_AF_XDP_FORCE_COPY_ARG,\n+\tETH_AF_XDP_USE_CNI_ARG,\n \tNULL\n };\n \n@@ -1129,7 +1150,8 @@ xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals,\n \t\tret = xsk_umem__create(&umem->umem, base_addr, umem_size,\n \t\t\t\t&rxq->fq, &rxq->cq, &usr_config);\n \t\tif (ret) {\n-\t\t\tAF_XDP_LOG(ERR, \"Failed to create umem\\n\");\n+\t\t\tAF_XDP_LOG(ERR, \"Failed to create umem [%d]: [%s]\\n\",\n+\t\t\t\t errno, strerror(errno));\n \t\t\tgoto err;\n \t\t}\n \t\tumem->buffer = base_addr;\n@@ -1314,6 +1336,245 @@ configure_preferred_busy_poll(struct pkt_rx_queue *rxq)\n \treturn 0;\n }\n \n+static int\n+init_uds_sock(struct sockaddr_un *server)\n+{\n+\tint sock;\n+\n+\tsock = socket(AF_UNIX, SOCK_SEQPACKET, 0);\n+\tif (sock < 0) {\n+\t\tAF_XDP_LOG(ERR, \"Failed to opening stream socket\\n\");\n+\t\treturn -1;\n+\t}\n+\n+\tserver->sun_family = AF_UNIX;\n+\tstrlcpy(server->sun_path, UDS_SOCK, sizeof(server->sun_path));\n+\n+\tif (connect(sock, (struct sockaddr *)server, sizeof(struct sockaddr_un)) < 0) {\n+\t\tclose(sock);\n+\t\tAF_XDP_LOG(ERR, \"Error connecting stream socket errno = [%d]: [%s]\\n\",\n+\t\t\t errno, strerror(errno));\n+\t\treturn -1;\n+\t}\n+\n+\treturn sock;\n+}\n+\n+struct msg_internal {\n+\tchar response[UDS_MAX_CMD_RESP];\n+\tint len_param;\n+\tint num_fds;\n+\tint fds[UDS_MAX_FD_NUM];\n+};\n+\n+static int\n+send_msg(int sock, char *request, int *fd)\n+{\n+\tint snd;\n+\tstruct iovec iov;\n+\tstruct msghdr msgh;\n+\tstruct cmsghdr *cmsg;\n+\tstruct sockaddr_un dst;\n+\tchar control[CMSG_SPACE(sizeof(*fd))];\n+\n+\tmemset(&dst, 0, sizeof(dst));\n+\tdst.sun_family = AF_UNIX;\n+\tstrlcpy(dst.sun_path, UDS_SOCK, sizeof(dst.sun_path));\n+\n+\t/* Initialize message header structure */\n+\tmemset(&msgh, 0, sizeof(msgh));\n+\tmemset(control, 0, sizeof(control));\n+\tiov.iov_base = request;\n+\tiov.iov_len = strlen(request);\n+\n+\tmsgh.msg_name = &dst;\n+\tmsgh.msg_namelen = sizeof(dst);\n+\tmsgh.msg_iov = &iov;\n+\tmsgh.msg_iovlen = 1;\n+\tmsgh.msg_control = control;\n+\tmsgh.msg_controllen = sizeof(control);\n+\n+\t/* Translate the FD. */\n+\tcmsg = CMSG_FIRSTHDR(&msgh);\n+\tcmsg->cmsg_len = CMSG_LEN(sizeof(*fd));\n+\tcmsg->cmsg_level = SOL_SOCKET;\n+\tcmsg->cmsg_type = SCM_RIGHTS;\n+\tmemcpy(CMSG_DATA(cmsg), fd, sizeof(*fd));\n+\n+\t/* Send the request message. */\n+\tdo {\n+\t\tsnd = sendmsg(sock, &msgh, 0);\n+\t} while (snd < 0 && errno == EINTR);\n+\n+\treturn snd;\n+}\n+\n+static int\n+read_msg(int sock, char *response, struct sockaddr_un *s, int *fd)\n+{\n+\tint msglen;\n+\tstruct msghdr msgh;\n+\tstruct iovec iov;\n+\tchar control[CMSG_SPACE(sizeof(*fd))];\n+\tstruct cmsghdr *cmsg;\n+\n+\t/* Initialize message header structure */\n+\tmemset(&msgh, 0, sizeof(msgh));\n+\tiov.iov_base = response;\n+\tiov.iov_len = UDS_MAX_CMD_RESP;\n+\n+\tmsgh.msg_name = s;\n+\tmsgh.msg_namelen = sizeof(*s);\n+\tmsgh.msg_iov = &iov;\n+\tmsgh.msg_iovlen = 1;\n+\tmsgh.msg_control = control;\n+\tmsgh.msg_controllen = sizeof(control);\n+\n+\tmsglen = recvmsg(sock, &msgh, 0);\n+\n+\t/* zero length message means socket was closed */\n+\tif (msglen == 0)\n+\t\treturn 0;\n+\n+\tif (msglen < 0) {\n+\t\tAF_XDP_LOG(ERR, \"recvmsg failed, %s\\n\", strerror(errno));\n+\t\treturn -1;\n+\t}\n+\n+\t/* read auxiliary FDs if any */\n+\tfor (cmsg = CMSG_FIRSTHDR(&msgh); cmsg != NULL;\n+\t\t\tcmsg = CMSG_NXTHDR(&msgh, cmsg)) {\n+\t\tif (cmsg->cmsg_level == SOL_SOCKET &&\n+\t\t\t\tcmsg->cmsg_type == SCM_RIGHTS) {\n+\t\t\tmemcpy(fd, CMSG_DATA(cmsg), sizeof(*fd));\n+\t\t\tbreak;\n+\t\t}\n+\t}\n+\n+\tresponse[msglen] = '\\0';\n+\treturn msglen;\n+}\n+\n+static int\n+make_request_cni(int sock, struct sockaddr_un *server, char *request,\n+\t\t int *req_fd, char *response, int *out_fd)\n+{\n+\tint rval;\n+\n+\tAF_XDP_LOG(DEBUG, \"Request: [%s]\\n\", request);\n+\n+\t/* if no file descriptor to send then directly write to socket.\n+\t * else use sendmsg() to send the file descriptor.\n+\t */\n+\tif (req_fd == NULL)\n+\t\trval = write(sock, request, strlen(request));\n+\telse\n+\t\trval = send_msg(sock, request, req_fd);\n+\n+\tif (rval < 0) {\n+\t\tAF_XDP_LOG(ERR, \"Write error %s\\n\", strerror(errno));\n+\t\treturn -1;\n+\t}\n+\n+\trval = read_msg(sock, response, server, out_fd);\n+\tif (rval <= 0) {\n+\t\tAF_XDP_LOG(ERR, \"Read error %d\\n\", rval);\n+\t\treturn -1;\n+\t}\n+\tAF_XDP_LOG(DEBUG, \"Response: [%s]\\n\", request);\n+\n+\treturn 0;\n+}\n+\n+static int\n+check_response(char *response, char *exp_resp, long size)\n+{\n+\treturn strncmp(response, exp_resp, size);\n+}\n+\n+static int\n+get_cni_fd(char *if_name)\n+{\n+\tchar request[UDS_MAX_CMD_LEN], response[UDS_MAX_CMD_RESP];\n+\tchar hostname[MAX_LONG_OPT_SZ], exp_resp[UDS_MAX_CMD_RESP];\n+\tstruct sockaddr_un server;\n+\tint xsk_map_fd = -1, out_fd = 0;\n+\tint sock, err;\n+\n+\terr = gethostname(hostname, MAX_LONG_OPT_SZ - 1);\n+\tif (err)\n+\t\treturn -1;\n+\n+\tmemset(&server, 0, sizeof(server));\n+\tsock = init_uds_sock(&server);\n+\n+\t/* Initiates handshake to CNI send: /connect,hostname */\n+\tsnprintf(request, sizeof(request), \"%s,%s\", UDS_CONNECT_MSG, hostname);\n+\tmemset(response, 0, sizeof(response));\n+\tif (make_request_cni(sock, &server, request, NULL, response, &out_fd) < 0) {\n+\t\tAF_XDP_LOG(ERR, \"Error in processing cmd [%s]\\n\", request);\n+\t\tgoto err_close;\n+\t}\n+\n+\t/* Expect /host_ok */\n+\tstrlcpy(exp_resp, UDS_HOST_OK_MSG, UDS_MAX_CMD_LEN);\n+\tif (check_response(response, exp_resp, strlen(exp_resp)) < 0) {\n+\t\tAF_XDP_LOG(ERR, \"Unexpected response [%s]\\n\", response);\n+\t\tgoto err_close;\n+\t}\n+\t/* Request for \"/version\" */\n+\tstrlcpy(request, UDS_VERSION_MSG, UDS_MAX_CMD_LEN);\n+\tmemset(response, 0, sizeof(response));\n+\tif (make_request_cni(sock, &server, request, NULL, response, &out_fd) < 0) {\n+\t\tAF_XDP_LOG(ERR, \"Error in processing cmd [%s]\\n\", request);\n+\t\tgoto err_close;\n+\t}\n+\n+\t/* Request for file descriptor for netdev name*/\n+\tsnprintf(request, sizeof(request), \"%s,%s\", UDS_XSK_MAP_FD_MSG, if_name);\n+\tmemset(response, 0, sizeof(response));\n+\tif (make_request_cni(sock, &server, request, NULL, response, &out_fd) < 0) {\n+\t\tAF_XDP_LOG(ERR, \"Error in processing cmd [%s]\\n\", request);\n+\t\tgoto err_close;\n+\t}\n+\n+\tif (out_fd < 0) {\n+\t\tAF_XDP_LOG(ERR, \"Error in processing cmd [%s]\\n\", request);\n+\t\tgoto err_close;\n+\t}\n+\n+\txsk_map_fd = out_fd;\n+\n+\t/* Expect fd_ack with file descriptor */\n+\tstrlcpy(exp_resp, UDS_FD_ACK_MSG, UDS_MAX_CMD_LEN);\n+\tif (check_response(response, exp_resp, strlen(exp_resp)) < 0) {\n+\t\tAF_XDP_LOG(ERR, \"Unexpected response [%s]\\n\", response);\n+\t\tgoto err_close;\n+\t}\n+\n+\t/* Initiate close connection */\n+\tstrlcpy(request, UDS_FIN_MSG, UDS_MAX_CMD_LEN);\n+\tmemset(response, 0, sizeof(response));\n+\tif (make_request_cni(sock, &server, request, NULL, response, &out_fd) < 0) {\n+\t\tAF_XDP_LOG(ERR, \"Error in processing cmd [%s]\\n\", request);\n+\t\tgoto err_close;\n+\t}\n+\n+\t/* Connection close */\n+\tstrlcpy(exp_resp, UDS_FIN_ACK_MSG, UDS_MAX_CMD_LEN);\n+\tif (check_response(response, exp_resp, strlen(exp_resp)) < 0) {\n+\t\tAF_XDP_LOG(ERR, \"Unexpected response [%s]\\n\", response);\n+\t\tgoto err_close;\n+\t}\n+\tclose(sock);\n+\n+\treturn xsk_map_fd;\n+\n+err_close:\n+\tclose(sock);\n+\treturn -1;\n+}\n+\n static int\n xsk_configure(struct pmd_internals *internals, struct pkt_rx_queue *rxq,\n \t int ring_size)\n@@ -1362,6 +1623,10 @@ xsk_configure(struct pmd_internals *internals, struct pkt_rx_queue *rxq,\n \tcfg.bind_flags |= XDP_USE_NEED_WAKEUP;\n #endif\n \n+\t/* Disable libbpf from loading XDP program */\n+\tif (internals->use_cni)\n+\t\tcfg.libbpf_flags |= XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;\n+\n \tif (strnlen(internals->prog_path, PATH_MAX)) {\n \t\tif (!internals->custom_prog_configured) {\n \t\t\tret = load_custom_xdp_prog(internals->prog_path,\n@@ -1413,7 +1678,23 @@ xsk_configure(struct pmd_internals *internals, struct pkt_rx_queue *rxq,\n \t\t}\n \t}\n \n-\tif (rxq->busy_budget) {\n+\tif (internals->use_cni) {\n+\t\tint err, fd, map_fd;\n+\n+\t\t/* get socket fd from CNI plugin */\n+\t\tmap_fd = get_cni_fd(internals->if_name);\n+\t\tif (map_fd < 0) {\n+\t\t\tAF_XDP_LOG(ERR, \"Failed to receive CNI plugin fd\\n\");\n+\t\t\tgoto out_xsk;\n+\t\t}\n+\t\t/* get socket fd */\n+\t\tfd = xsk_socket__fd(rxq->xsk);\n+\t\terr = bpf_map_update_elem(map_fd, &rxq->xsk_queue_idx, &fd, 0);\n+\t\tif (err) {\n+\t\t\tAF_XDP_LOG(ERR, \"Failed to insert unprivileged xsk in map.\\n\");\n+\t\t\tgoto out_xsk;\n+\t\t}\n+\t} else if (rxq->busy_budget) {\n \t\tret = configure_preferred_busy_poll(rxq);\n \t\tif (ret) {\n \t\t\tAF_XDP_LOG(ERR, \"Failed configure busy polling.\\n\");\n@@ -1584,6 +1865,27 @@ static const struct eth_dev_ops ops = {\n \t.get_monitor_addr = eth_get_monitor_addr,\n };\n \n+/* CNI option works in unprivileged container environment\n+ * and ethernet device functionality will be reduced. So\n+ * additional customiszed eth_dev_ops struct is needed\n+ * for cni. Promiscuous enable and disable functionality\n+ * is removed.\n+ **/\n+static const struct eth_dev_ops ops_cni = {\n+\t.dev_start = eth_dev_start,\n+\t.dev_stop = eth_dev_stop,\n+\t.dev_close = eth_dev_close,\n+\t.dev_configure = eth_dev_configure,\n+\t.dev_infos_get = eth_dev_info,\n+\t.mtu_set = eth_dev_mtu_set,\n+\t.rx_queue_setup = eth_rx_queue_setup,\n+\t.tx_queue_setup = eth_tx_queue_setup,\n+\t.link_update = eth_link_update,\n+\t.stats_get = eth_stats_get,\n+\t.stats_reset = eth_stats_reset,\n+\t.get_monitor_addr = eth_get_monitor_addr,\n+};\n+\n /** parse busy_budget argument */\n static int\n parse_budget_arg(const char *key __rte_unused,\n@@ -1704,8 +2006,8 @@ xdp_get_channels_info(const char *if_name, int *max_queues,\n \n static int\n parse_parameters(struct rte_kvargs *kvlist, char *if_name, int *start_queue,\n-\t\t\tint *queue_cnt, int *shared_umem, char *prog_path,\n-\t\t\tint *busy_budget, int *force_copy)\n+\t\t int *queue_cnt, int *shared_umem, char *prog_path,\n+\t\t int *busy_budget, int *force_copy, int *use_cni)\n {\n \tint ret;\n \n@@ -1746,6 +2048,11 @@ parse_parameters(struct rte_kvargs *kvlist, char *if_name, int *start_queue,\n \tif (ret < 0)\n \t\tgoto free_kvlist;\n \n+\tret = rte_kvargs_process(kvlist, ETH_AF_XDP_USE_CNI_ARG,\n+\t\t\t\t &parse_integer_arg, use_cni);\n+\tif (ret < 0)\n+\t\tgoto free_kvlist;\n+\n free_kvlist:\n \trte_kvargs_free(kvlist);\n \treturn ret;\n@@ -1783,8 +2090,9 @@ get_iface_info(const char *if_name,\n \n static struct rte_eth_dev *\n init_internals(struct rte_vdev_device *dev, const char *if_name,\n-\t\tint start_queue_idx, int queue_cnt, int shared_umem,\n-\t\tconst char *prog_path, int busy_budget, int force_copy)\n+\t int start_queue_idx, int queue_cnt, int shared_umem,\n+\t const char *prog_path, int busy_budget, int force_copy,\n+\t int use_cni)\n {\n \tconst char *name = rte_vdev_device_name(dev);\n \tconst unsigned int numa_node = dev->device.numa_node;\n@@ -1813,6 +2121,7 @@ init_internals(struct rte_vdev_device *dev, const char *if_name,\n #endif\n \tinternals->shared_umem = shared_umem;\n \tinternals->force_copy = force_copy;\n+\tinternals->use_cni = use_cni;\n \n \tif (xdp_get_channels_info(if_name, &internals->max_queue_cnt,\n \t\t\t\t &internals->combined_queue_cnt)) {\n@@ -1871,7 +2180,11 @@ init_internals(struct rte_vdev_device *dev, const char *if_name,\n \teth_dev->data->dev_link = pmd_link;\n \teth_dev->data->mac_addrs = &internals->eth_addr;\n \teth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;\n-\teth_dev->dev_ops = &ops;\n+\tif (!internals->use_cni)\n+\t\teth_dev->dev_ops = &ops;\n+\telse\n+\t\teth_dev->dev_ops = &ops_cni;\n+\n \teth_dev->rx_pkt_burst = eth_af_xdp_rx;\n \teth_dev->tx_pkt_burst = eth_af_xdp_tx;\n \teth_dev->process_private = process_private;\n@@ -1998,6 +2311,7 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)\n \tchar prog_path[PATH_MAX] = {'\\0'};\n \tint busy_budget = -1, ret;\n \tint force_copy = 0;\n+\tint use_cni = 0;\n \tstruct rte_eth_dev *eth_dev = NULL;\n \tconst char *name = rte_vdev_device_name(dev);\n \n@@ -2043,7 +2357,7 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)\n \n \tif (parse_parameters(kvlist, if_name, &xsk_start_queue_idx,\n \t\t\t &xsk_queue_cnt, &shared_umem, prog_path,\n-\t\t\t &busy_budget, &force_copy) < 0) {\n+\t\t\t &busy_budget, &force_copy, &use_cni) < 0) {\n \t\tAF_XDP_LOG(ERR, \"Invalid kvargs value\\n\");\n \t\treturn -EINVAL;\n \t}\n@@ -2057,8 +2371,8 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)\n \t\t\t\t\tbusy_budget;\n \n \teth_dev = init_internals(dev, if_name, xsk_start_queue_idx,\n-\t\t\t\t\txsk_queue_cnt, shared_umem, prog_path,\n-\t\t\t\t\tbusy_budget, force_copy);\n+\t\t\t\t xsk_queue_cnt, shared_umem, prog_path,\n+\t\t\t\t busy_budget, force_copy, use_cni);\n \tif (eth_dev == NULL) {\n \t\tAF_XDP_LOG(ERR, \"Failed to init internals\\n\");\n \t\treturn -1;\n@@ -2118,4 +2432,5 @@ RTE_PMD_REGISTER_PARAM_STRING(net_af_xdp,\n \t\t\t \"shared_umem=<int> \"\n \t\t\t \"xdp_prog=<string> \"\n \t\t\t \"busy_budget=<int> \"\n-\t\t\t \"force_copy=<int> \");\n+\t\t\t \"force_copy=<int> \"\n+\t\t\t \"use_cni=<int> \");\n", "prefixes": [ "v5" ] }