Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/139173/?format=api
{ "id": 139173, "url": "http://patches.dpdk.org/api/patches/139173/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20240405211745.699697-7-stephen@networkplumber.org/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20240405211745.699697-7-stephen@networkplumber.org>", "list_archive_url": "https://inbox.dpdk.org/dev/20240405211745.699697-7-stephen@networkplumber.org", "date": "2024-04-05T21:14:58", "name": "[v6,6/8] net/tap: rewrite the RSS BPF program", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "4c39d24d45d346f2c60dc7bfbc5817a2bbde838f", "submitter": { "id": 27, "url": "http://patches.dpdk.org/api/people/27/?format=api", "name": "Stephen Hemminger", "email": "stephen@networkplumber.org" }, "delegate": { "id": 1, "url": "http://patches.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20240405211745.699697-7-stephen@networkplumber.org/mbox/", "series": [ { "id": 31694, "url": "http://patches.dpdk.org/api/series/31694/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=31694", "date": "2024-04-05T21:14:52", "name": "net/tap: cleanup and fix BPF flow support", "version": 6, "mbox": "http://patches.dpdk.org/series/31694/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/139173/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/139173/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id E80CB43E0E;\n\tFri, 5 Apr 2024 23:18:51 +0200 (CEST)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 4DD9B40A7D;\n\tFri, 5 Apr 2024 23:18:06 +0200 (CEST)", "from mail-pj1-f50.google.com (mail-pj1-f50.google.com\n [209.85.216.50]) by mails.dpdk.org (Postfix) with ESMTP id A1462406B7\n for <dev@dpdk.org>; Fri, 5 Apr 2024 23:18:02 +0200 (CEST)", "by mail-pj1-f50.google.com with SMTP id\n 98e67ed59e1d1-2a2d82537efso1447496a91.2\n for <dev@dpdk.org>; Fri, 05 Apr 2024 14:18:02 -0700 (PDT)", "from hermes.local (204-195-96-226.wavecable.com. [204.195.96.226])\n by smtp.gmail.com with ESMTPSA id\n r32-20020a17090a43a300b002a2f1e6f50bsm2137521pjg.1.2024.04.05.14.18.00\n (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n Fri, 05 Apr 2024 14:18:01 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=networkplumber-org.20230601.gappssmtp.com; s=20230601; t=1712351882;\n x=1712956682; darn=dpdk.org;\n h=content-transfer-encoding:mime-version:references:in-reply-to\n :message-id:date:subject:cc:to:from:from:to:cc:subject:date\n :message-id:reply-to;\n bh=38CZ4DWGHcOFCd2nCP00hPr47SsSpAChpDwZh8MAiuY=;\n b=SNFFkcWdQ1P21b0eEg/wSdDhFlw6iDJ88uaSOSHHicKG0RNz8rw8u0APmJMWD6YMh2\n zRuK6ZjWUF3YYqGxt+XlHvC1faplcMGoGSVvN541HNCgRsJKUif/JTpFCsf9hh1T0PGY\n asChrWilNEX+JNFWyNoz7v27x74YiTI7IAUCqI9GI29+GgOScf9Ut8w7hB7DSNeSY11X\n 4Egz2nUQwdNyTP9hs6l9t3UdH35L8ZvoWpTOz4aDa+I53++hTYjOuyxrSCpuPJf1Vnn9\n rnA2amxwth6iY7Eny9gdvIacoM8lDMPTl9oN2LCc66SCUBG2jwSwYCCTe9R+02V7yvd/\n fg6A==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=1e100.net; s=20230601; t=1712351882; x=1712956682;\n h=content-transfer-encoding:mime-version:references:in-reply-to\n :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc\n :subject:date:message-id:reply-to;\n bh=38CZ4DWGHcOFCd2nCP00hPr47SsSpAChpDwZh8MAiuY=;\n b=VNA0djVaAUCaGm2gl5E4XhrMMB3LwpH3jHNM/lUOTCD3OcLfJkjGrlkTw7L++JxJsD\n got6mYiHc0I2UKj4kCyXwmbWX6k6P8OI1DTXn8pv0ywBjxnHwDDmGv1xtnrkqkopaDmx\n vDoVypXNlgWOMb6tBja8Th649xcpyNTDmJhnxEniubmJyf9Jdl8uw96DnznnQ+o9+x/f\n GSn6/xlIEVUTXq1psjuI7x94CPFFUa1cFRoZYJZeGaohx4uKTfTdPZlEQuRAOqcE+Vgv\n clx5zse2jjQ+fbWWcwTpkU5Z+sQmT3zJU5mrzK36pWPABwYBHavdgXBYpHGd43Fj5Anp\n PT2g==", "X-Gm-Message-State": "AOJu0Yxm1Ci4M5dirWdE/H8oeAGHwYxvNqeq2sWy2/c5OD0IDRMINVh8\n pSvpt29setiXu19aSEYGfClAiDqxd1GBnhhIYRTUav++/dxLs6c4fyaN8mSFCWo/yQ7V8pEmXJC\n qjgkWfw==", "X-Google-Smtp-Source": "\n AGHT+IFkTwdjkJNOERQQgm0GzKyQ9JP+lT0r79tdKo6di0ZVPKZGI8Boo1okhD9uEA9Nz1EFft9mZg==", "X-Received": "by 2002:a17:90a:c291:b0:2a2:581f:c6de with SMTP id\n f17-20020a17090ac29100b002a2581fc6demr2653495pjt.3.1712351881638;\n Fri, 05 Apr 2024 14:18:01 -0700 (PDT)", "From": "Stephen Hemminger <stephen@networkplumber.org>", "To": "dev@dpdk.org", "Cc": "Stephen Hemminger <stephen@networkplumber.org>", "Subject": "[PATCH v6 6/8] net/tap: rewrite the RSS BPF program", "Date": "Fri, 5 Apr 2024 14:14:58 -0700", "Message-ID": "<20240405211745.699697-7-stephen@networkplumber.org>", "X-Mailer": "git-send-email 2.43.0", "In-Reply-To": "<20240405211745.699697-1-stephen@networkplumber.org>", "References": "<20240130034925.44869-1-stephen@networkplumber.org>\n <20240405211745.699697-1-stephen@networkplumber.org>", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=UTF-8", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "Rewrite the BPF program used to do queue based RSS.\nImportant changes:\n\t- uses newer BPF map format BTF\n\t- accepts key as parameter rather than constant default\n\t- can do L3 or L4 hashing\n\t- supports IPv4 options\n\t- supports IPv6 extension headers\n\t- restructured for readability\n\nThe usage of BPF is different as well:\n\t- the incoming configuration is looked up based on\n\t class parameters rather than patching the BPF.\n\t- the resulting queue is placed in skb rather\n\t than requiring a second pass through classifier step.\n\nNote: This version only works with later patch to enable it on\nthe DPDK driver side. It is submitted as an incremental patch\nto allow for easier review. Bisection still works because\nthe old instruction are still present for now.\n\nSigned-off-by: Stephen Hemminger <stephen@networkplumber.org>\n---\n .gitignore | 3 -\n drivers/net/tap/bpf/Makefile | 19 --\n drivers/net/tap/bpf/README | 38 ++++\n drivers/net/tap/bpf/bpf_api.h | 276 --------------------------\n drivers/net/tap/bpf/bpf_elf.h | 53 -----\n drivers/net/tap/bpf/bpf_extract.py | 85 --------\n drivers/net/tap/bpf/meson.build | 81 ++++++++\n drivers/net/tap/bpf/tap_bpf_program.c | 255 ------------------------\n drivers/net/tap/bpf/tap_rss.c | 264 ++++++++++++++++++++++++\n 9 files changed, 383 insertions(+), 691 deletions(-)\n delete mode 100644 drivers/net/tap/bpf/Makefile\n create mode 100644 drivers/net/tap/bpf/README\n delete mode 100644 drivers/net/tap/bpf/bpf_api.h\n delete mode 100644 drivers/net/tap/bpf/bpf_elf.h\n delete mode 100644 drivers/net/tap/bpf/bpf_extract.py\n create mode 100644 drivers/net/tap/bpf/meson.build\n delete mode 100644 drivers/net/tap/bpf/tap_bpf_program.c\n create mode 100644 drivers/net/tap/bpf/tap_rss.c", "diff": "diff --git a/.gitignore b/.gitignore\nindex 3f444dcace..01a47a7606 100644\n--- a/.gitignore\n+++ b/.gitignore\n@@ -36,9 +36,6 @@ TAGS\n # ignore python bytecode files\n *.pyc\n \n-# ignore BPF programs\n-drivers/net/tap/bpf/tap_bpf_program.o\n-\n # DTS results\n dts/output\n \ndiff --git a/drivers/net/tap/bpf/Makefile b/drivers/net/tap/bpf/Makefile\ndeleted file mode 100644\nindex 9efeeb1bc7..0000000000\n--- a/drivers/net/tap/bpf/Makefile\n+++ /dev/null\n@@ -1,19 +0,0 @@\n-# SPDX-License-Identifier: BSD-3-Clause\n-# This file is not built as part of normal DPDK build.\n-# It is used to generate the eBPF code for TAP RSS.\n-\n-CLANG=clang\n-CLANG_OPTS=-O2\n-TARGET=../tap_bpf_insns.h\n-\n-all: $(TARGET)\n-\n-clean:\n-\trm tap_bpf_program.o $(TARGET)\n-\n-tap_bpf_program.o: tap_bpf_program.c\n-\t$(CLANG) $(CLANG_OPTS) -emit-llvm -c $< -o - | \\\n-\tllc -march=bpf -filetype=obj -o $@\n-\n-$(TARGET): tap_bpf_program.o\n-\tpython3 bpf_extract.py -stap_bpf_program.c -o $@ $<\ndiff --git a/drivers/net/tap/bpf/README b/drivers/net/tap/bpf/README\nnew file mode 100644\nindex 0000000000..1d421ff42c\n--- /dev/null\n+++ b/drivers/net/tap/bpf/README\n@@ -0,0 +1,38 @@\n+This is the BPF program used to implement the RSS across queues flow action.\n+The program is loaded when first RSS flow rule is created and is never unloaded.\n+\n+Each flow rule creates a unique key (handle) and this is used as the key\n+for finding the RSS information for that flow rule.\n+\n+This version is built the BPF Compile Once — Run Everywhere (CO-RE)\n+framework and uses libbpf and bpftool.\n+\n+Limitations\n+-----------\n+- requires libbpf to run\n+- rebuilding the BPF requires Clang and bpftool.\n+ Some older versions of Ubuntu do not have working bpftool package.\n+ Need a version of Clang that can compile to BPF.\n+- only standard Toeplitz hash with standard 40 byte key is supported\n+- the number of flow rules using RSS is limited to 32\n+\n+Building\n+--------\n+During the DPDK build process the meson build file checks that\n+libbpf, bpftool, and clang are not available. If everything is\n+there then BPF RSS is enabled.\n+\n+1. Using clang to compile tap_rss.c the tap_rss.bpf.o file.\n+\n+2. Using bpftool generate a skeleton header file tap_rss.skel.h from tap_rss.bpf.o.\n+ This skeleton header is an large byte array which contains the\n+ BPF binary and wrappers to load and use it.\n+\n+3. The tap flow code then compiles that BPF byte array into the PMD object.\n+\n+4. When needed the BPF array is loaded by libbpf.\n+\n+References\n+----------\n+BPF and XDP reference guide\n+https://docs.cilium.io/en/latest/bpf/progtypes/\ndiff --git a/drivers/net/tap/bpf/bpf_api.h b/drivers/net/tap/bpf/bpf_api.h\ndeleted file mode 100644\nindex 4cd25fa593..0000000000\n--- a/drivers/net/tap/bpf/bpf_api.h\n+++ /dev/null\n@@ -1,276 +0,0 @@\n-/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */\n-\n-#ifndef __BPF_API__\n-#define __BPF_API__\n-\n-/* Note:\n- *\n- * This file can be included into eBPF kernel programs. It contains\n- * a couple of useful helper functions, map/section ABI (bpf_elf.h),\n- * misc macros and some eBPF specific LLVM built-ins.\n- */\n-\n-#include <stdint.h>\n-\n-#include <linux/pkt_cls.h>\n-#include <linux/bpf.h>\n-#include <linux/filter.h>\n-\n-#include <asm/byteorder.h>\n-\n-#include \"bpf_elf.h\"\n-\n-/** libbpf pin type. */\n-enum libbpf_pin_type {\n-\tLIBBPF_PIN_NONE,\n-\t/* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */\n-\tLIBBPF_PIN_BY_NAME,\n-};\n-\n-/** Type helper macros. */\n-\n-#define __uint(name, val) int (*name)[val]\n-#define __type(name, val) typeof(val) *name\n-#define __array(name, val) typeof(val) *name[]\n-\n-/** Misc macros. */\n-\n-#ifndef __stringify\n-# define __stringify(X)\t\t#X\n-#endif\n-\n-#ifndef __maybe_unused\n-# define __maybe_unused\t\t__attribute__((__unused__))\n-#endif\n-\n-#ifndef offsetof\n-# define offsetof(TYPE, MEMBER)\t__builtin_offsetof(TYPE, MEMBER)\n-#endif\n-\n-#ifndef likely\n-# define likely(X)\t\t__builtin_expect(!!(X), 1)\n-#endif\n-\n-#ifndef unlikely\n-# define unlikely(X)\t\t__builtin_expect(!!(X), 0)\n-#endif\n-\n-#ifndef htons\n-# define htons(X)\t\t__constant_htons((X))\n-#endif\n-\n-#ifndef ntohs\n-# define ntohs(X)\t\t__constant_ntohs((X))\n-#endif\n-\n-#ifndef htonl\n-# define htonl(X)\t\t__constant_htonl((X))\n-#endif\n-\n-#ifndef ntohl\n-# define ntohl(X)\t\t__constant_ntohl((X))\n-#endif\n-\n-#ifndef __inline__\n-# define __inline__\t\t__attribute__((always_inline))\n-#endif\n-\n-/** Section helper macros. */\n-\n-#ifndef __section\n-# define __section(NAME)\t\t\t\t\t\t\\\n-\t__attribute__((section(NAME), used))\n-#endif\n-\n-#ifndef __section_tail\n-# define __section_tail(ID, KEY)\t\t\t\t\t\\\n-\t__section(__stringify(ID) \"/\" __stringify(KEY))\n-#endif\n-\n-#ifndef __section_xdp_entry\n-# define __section_xdp_entry\t\t\t\t\t\t\\\n-\t__section(ELF_SECTION_PROG)\n-#endif\n-\n-#ifndef __section_cls_entry\n-# define __section_cls_entry\t\t\t\t\t\t\\\n-\t__section(ELF_SECTION_CLASSIFIER)\n-#endif\n-\n-#ifndef __section_act_entry\n-# define __section_act_entry\t\t\t\t\t\t\\\n-\t__section(ELF_SECTION_ACTION)\n-#endif\n-\n-#ifndef __section_lwt_entry\n-# define __section_lwt_entry\t\t\t\t\t\t\\\n-\t__section(ELF_SECTION_PROG)\n-#endif\n-\n-#ifndef __section_license\n-# define __section_license\t\t\t\t\t\t\\\n-\t__section(ELF_SECTION_LICENSE)\n-#endif\n-\n-#ifndef __section_maps\n-# define __section_maps\t\t\t\t\t\t\t\\\n-\t__section(ELF_SECTION_MAPS)\n-#endif\n-\n-/** Declaration helper macros. */\n-\n-#ifndef BPF_LICENSE\n-# define BPF_LICENSE(NAME)\t\t\t\t\t\t\\\n-\tchar ____license[] __section_license = NAME\n-#endif\n-\n-/** Classifier helper */\n-\n-#ifndef BPF_H_DEFAULT\n-# define BPF_H_DEFAULT\t-1\n-#endif\n-\n-/** BPF helper functions for tc. Individual flags are in linux/bpf.h */\n-\n-#ifndef __BPF_FUNC\n-# define __BPF_FUNC(NAME, ...)\t\t\t\t\t\t\\\n-\t(* NAME)(__VA_ARGS__) __maybe_unused\n-#endif\n-\n-#ifndef BPF_FUNC\n-# define BPF_FUNC(NAME, ...)\t\t\t\t\t\t\\\n-\t__BPF_FUNC(NAME, __VA_ARGS__) = (void *) BPF_FUNC_##NAME\n-#endif\n-\n-/* Map access/manipulation */\n-static void *BPF_FUNC(map_lookup_elem, void *map, const void *key);\n-static int BPF_FUNC(map_update_elem, void *map, const void *key,\n-\t\t const void *value, uint32_t flags);\n-static int BPF_FUNC(map_delete_elem, void *map, const void *key);\n-\n-/* Time access */\n-static uint64_t BPF_FUNC(ktime_get_ns);\n-\n-/* Debugging */\n-\n-/* FIXME: __attribute__ ((format(printf, 1, 3))) not possible unless\n- * llvm bug https://llvm.org/bugs/show_bug.cgi?id=26243 gets resolved.\n- * It would require ____fmt to be made const, which generates a reloc\n- * entry (non-map).\n- */\n-static void BPF_FUNC(trace_printk, const char *fmt, int fmt_size, ...);\n-\n-#ifndef printt\n-# define printt(fmt, ...)\t\t\t\t\t\t\\\n-\t__extension__ ({\t\t\t\t\t\t\\\n-\t\tchar ____fmt[] = fmt;\t\t\t\t\t\\\n-\t\ttrace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__);\t\\\n-\t})\n-#endif\n-\n-/* Random numbers */\n-static uint32_t BPF_FUNC(get_prandom_u32);\n-\n-/* Tail calls */\n-static void BPF_FUNC(tail_call, struct __sk_buff *skb, void *map,\n-\t\t uint32_t index);\n-\n-/* System helpers */\n-static uint32_t BPF_FUNC(get_smp_processor_id);\n-static uint32_t BPF_FUNC(get_numa_node_id);\n-\n-/* Packet misc meta data */\n-static uint32_t BPF_FUNC(get_cgroup_classid, struct __sk_buff *skb);\n-static int BPF_FUNC(skb_under_cgroup, void *map, uint32_t index);\n-\n-static uint32_t BPF_FUNC(get_route_realm, struct __sk_buff *skb);\n-static uint32_t BPF_FUNC(get_hash_recalc, struct __sk_buff *skb);\n-static uint32_t BPF_FUNC(set_hash_invalid, struct __sk_buff *skb);\n-\n-/* Packet redirection */\n-static int BPF_FUNC(redirect, int ifindex, uint32_t flags);\n-static int BPF_FUNC(clone_redirect, struct __sk_buff *skb, int ifindex,\n-\t\t uint32_t flags);\n-\n-/* Packet manipulation */\n-static int BPF_FUNC(skb_load_bytes, struct __sk_buff *skb, uint32_t off,\n-\t\t void *to, uint32_t len);\n-static int BPF_FUNC(skb_store_bytes, struct __sk_buff *skb, uint32_t off,\n-\t\t const void *from, uint32_t len, uint32_t flags);\n-\n-static int BPF_FUNC(l3_csum_replace, struct __sk_buff *skb, uint32_t off,\n-\t\t uint32_t from, uint32_t to, uint32_t flags);\n-static int BPF_FUNC(l4_csum_replace, struct __sk_buff *skb, uint32_t off,\n-\t\t uint32_t from, uint32_t to, uint32_t flags);\n-static int BPF_FUNC(csum_diff, const void *from, uint32_t from_size,\n-\t\t const void *to, uint32_t to_size, uint32_t seed);\n-static int BPF_FUNC(csum_update, struct __sk_buff *skb, uint32_t wsum);\n-\n-static int BPF_FUNC(skb_change_type, struct __sk_buff *skb, uint32_t type);\n-static int BPF_FUNC(skb_change_proto, struct __sk_buff *skb, uint32_t proto,\n-\t\t uint32_t flags);\n-static int BPF_FUNC(skb_change_tail, struct __sk_buff *skb, uint32_t nlen,\n-\t\t uint32_t flags);\n-\n-static int BPF_FUNC(skb_pull_data, struct __sk_buff *skb, uint32_t len);\n-\n-/* Event notification */\n-static int __BPF_FUNC(skb_event_output, struct __sk_buff *skb, void *map,\n-\t\t uint64_t index, const void *data, uint32_t size) =\n-\t\t (void *) BPF_FUNC_perf_event_output;\n-\n-/* Packet vlan encap/decap */\n-static int BPF_FUNC(skb_vlan_push, struct __sk_buff *skb, uint16_t proto,\n-\t\t uint16_t vlan_tci);\n-static int BPF_FUNC(skb_vlan_pop, struct __sk_buff *skb);\n-\n-/* Packet tunnel encap/decap */\n-static int BPF_FUNC(skb_get_tunnel_key, struct __sk_buff *skb,\n-\t\t struct bpf_tunnel_key *to, uint32_t size, uint32_t flags);\n-static int BPF_FUNC(skb_set_tunnel_key, struct __sk_buff *skb,\n-\t\t const struct bpf_tunnel_key *from, uint32_t size,\n-\t\t uint32_t flags);\n-\n-static int BPF_FUNC(skb_get_tunnel_opt, struct __sk_buff *skb,\n-\t\t void *to, uint32_t size);\n-static int BPF_FUNC(skb_set_tunnel_opt, struct __sk_buff *skb,\n-\t\t const void *from, uint32_t size);\n-\n-/** LLVM built-ins, mem*() routines work for constant size */\n-\n-#ifndef lock_xadd\n-# define lock_xadd(ptr, val)\t((void) __sync_fetch_and_add(ptr, val))\n-#endif\n-\n-#ifndef memset\n-# define memset(s, c, n)\t__builtin_memset((s), (c), (n))\n-#endif\n-\n-#ifndef memcpy\n-# define memcpy(d, s, n)\t__builtin_memcpy((d), (s), (n))\n-#endif\n-\n-#ifndef memmove\n-# define memmove(d, s, n)\t__builtin_memmove((d), (s), (n))\n-#endif\n-\n-/* FIXME: __builtin_memcmp() is not yet fully usable unless llvm bug\n- * https://llvm.org/bugs/show_bug.cgi?id=26218 gets resolved. Also\n- * this one would generate a reloc entry (non-map), otherwise.\n- */\n-#if 0\n-#ifndef memcmp\n-# define memcmp(a, b, n)\t__builtin_memcmp((a), (b), (n))\n-#endif\n-#endif\n-\n-unsigned long long load_byte(void *skb, unsigned long long off)\n-\tasm (\"llvm.bpf.load.byte\");\n-\n-unsigned long long load_half(void *skb, unsigned long long off)\n-\tasm (\"llvm.bpf.load.half\");\n-\n-unsigned long long load_word(void *skb, unsigned long long off)\n-\tasm (\"llvm.bpf.load.word\");\n-\n-#endif /* __BPF_API__ */\ndiff --git a/drivers/net/tap/bpf/bpf_elf.h b/drivers/net/tap/bpf/bpf_elf.h\ndeleted file mode 100644\nindex ea8a11c95c..0000000000\n--- a/drivers/net/tap/bpf/bpf_elf.h\n+++ /dev/null\n@@ -1,53 +0,0 @@\n-/* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */\n-#ifndef __BPF_ELF__\n-#define __BPF_ELF__\n-\n-#include <asm/types.h>\n-\n-/* Note:\n- *\n- * Below ELF section names and bpf_elf_map structure definition\n- * are not (!) kernel ABI. It's rather a \"contract\" between the\n- * application and the BPF loader in tc. For compatibility, the\n- * section names should stay as-is. Introduction of aliases, if\n- * needed, are a possibility, though.\n- */\n-\n-/* ELF section names, etc */\n-#define ELF_SECTION_LICENSE\t\"license\"\n-#define ELF_SECTION_MAPS\t\"maps\"\n-#define ELF_SECTION_PROG\t\"prog\"\n-#define ELF_SECTION_CLASSIFIER\t\"classifier\"\n-#define ELF_SECTION_ACTION\t\"action\"\n-\n-#define ELF_MAX_MAPS\t\t64\n-#define ELF_MAX_LICENSE_LEN\t128\n-\n-/* Object pinning settings */\n-#define PIN_NONE\t\t0\n-#define PIN_OBJECT_NS\t\t1\n-#define PIN_GLOBAL_NS\t\t2\n-\n-/* ELF map definition */\n-struct bpf_elf_map {\n-\t__u32 type;\n-\t__u32 size_key;\n-\t__u32 size_value;\n-\t__u32 max_elem;\n-\t__u32 flags;\n-\t__u32 id;\n-\t__u32 pinning;\n-\t__u32 inner_id;\n-\t__u32 inner_idx;\n-};\n-\n-#define BPF_ANNOTATE_KV_PAIR(name, type_key, type_val)\t\t\\\n-\tstruct ____btf_map_##name {\t\t\t\t\\\n-\t\ttype_key key;\t\t\t\t\t\\\n-\t\ttype_val value;\t\t\t\t\t\\\n-\t};\t\t\t\t\t\t\t\\\n-\tstruct ____btf_map_##name\t\t\t\t\\\n-\t __attribute__ ((section(\".maps.\" #name), used))\t\\\n-\t ____btf_map_##name = { }\n-\n-#endif /* __BPF_ELF__ */\ndiff --git a/drivers/net/tap/bpf/bpf_extract.py b/drivers/net/tap/bpf/bpf_extract.py\ndeleted file mode 100644\nindex 73c4dafe4e..0000000000\n--- a/drivers/net/tap/bpf/bpf_extract.py\n+++ /dev/null\n@@ -1,85 +0,0 @@\n-#!/usr/bin/env python3\n-# SPDX-License-Identifier: BSD-3-Clause\n-# Copyright (c) 2023 Stephen Hemminger <stephen@networkplumber.org>\n-\n-import argparse\n-import sys\n-import struct\n-from tempfile import TemporaryFile\n-from elftools.elf.elffile import ELFFile\n-\n-\n-def load_sections(elffile):\n- \"\"\"Get sections of interest from ELF\"\"\"\n- result = []\n- parts = [(\"cls_q\", \"cls_q_insns\"), (\"l3_l4\", \"l3_l4_hash_insns\")]\n- for name, tag in parts:\n- section = elffile.get_section_by_name(name)\n- if section:\n- insns = struct.iter_unpack('<BBhL', section.data())\n- result.append([tag, insns])\n- return result\n-\n-\n-def dump_section(name, insns, out):\n- \"\"\"Dump the array of BPF instructions\"\"\"\n- print(f'\\nstatic struct bpf_insn {name}[] = {{', file=out)\n- for bpf in insns:\n- code = bpf[0]\n- src = bpf[1] >> 4\n- dst = bpf[1] & 0xf\n- off = bpf[2]\n- imm = bpf[3]\n- print(f'\\t{{{code:#04x}, {dst:4d}, {src:4d}, {off:8d}, {imm:#010x}}},',\n- file=out)\n- print('};', file=out)\n-\n-\n-def parse_args():\n- \"\"\"Parse command line arguments\"\"\"\n- parser = argparse.ArgumentParser()\n- parser.add_argument('-s',\n- '--source',\n- type=str,\n- help=\"original source file\")\n- parser.add_argument('-o', '--out', type=str, help=\"output C file path\")\n- parser.add_argument(\"file\",\n- nargs='+',\n- help=\"object file path or '-' for stdin\")\n- return parser.parse_args()\n-\n-\n-def open_input(path):\n- \"\"\"Open the file or stdin\"\"\"\n- if path == \"-\":\n- temp = TemporaryFile()\n- temp.write(sys.stdin.buffer.read())\n- return temp\n- return open(path, 'rb')\n-\n-\n-def write_header(out, source):\n- \"\"\"Write file intro header\"\"\"\n- print(\"/* SPDX-License-Identifier: BSD-3-Clause\", file=out)\n- if source:\n- print(f' * Auto-generated from {source}', file=out)\n- print(\" * This not the original source file. Do NOT edit it.\", file=out)\n- print(\" */\\n\", file=out)\n-\n-\n-def main():\n- '''program main function'''\n- args = parse_args()\n-\n- with open(args.out, 'w',\n- encoding=\"utf-8\") if args.out else sys.stdout as out:\n- write_header(out, args.source)\n- for path in args.file:\n- elffile = ELFFile(open_input(path))\n- sections = load_sections(elffile)\n- for name, insns in sections:\n- dump_section(name, insns, out)\n-\n-\n-if __name__ == \"__main__\":\n- main()\ndiff --git a/drivers/net/tap/bpf/meson.build b/drivers/net/tap/bpf/meson.build\nnew file mode 100644\nindex 0000000000..f2c03a19fd\n--- /dev/null\n+++ b/drivers/net/tap/bpf/meson.build\n@@ -0,0 +1,81 @@\n+# SPDX-License-Identifier: BSD-3-Clause\n+# Copyright 2024 Stephen Hemminger <stephen@networkplumber.org>\n+\n+enable_tap_rss = false\n+\n+libbpf = dependency('libbpf', required: false, method: 'pkg-config')\n+if not libbpf.found()\n+ message('net/tap: no RSS support missing libbpf')\n+ subdir_done()\n+endif\n+\n+# Debian install this in /usr/sbin which is not in $PATH\n+bpftool = find_program('bpftool', '/usr/sbin/bpftool', required: false, version: '>= 5.6.0')\n+if not bpftool.found()\n+ message('net/tap: no RSS support missing bpftool')\n+ subdir_done()\n+endif\n+\n+clang_supports_bpf = false\n+clang = find_program('clang', required: false)\n+if clang.found()\n+ clang_supports_bpf = run_command(clang, '-target', 'bpf', '--print-supported-cpus',\n+ check: false).returncode() == 0\n+endif\n+\n+if not clang_supports_bpf\n+ message('net/tap: no RSS support missing clang BPF')\n+ subdir_done()\n+endif\n+\n+enable_tap_rss = true\n+\n+libbpf_include_dir = libbpf.get_variable(pkgconfig : 'includedir')\n+\n+# The include files <linux/bpf.h> and others include <asm/types.h>\n+# but <asm/types.h> is not defined for multi-lib environment target.\n+# Workaround by using include directoriy from the host build environment.\n+machine_name = run_command('uname', '-m').stdout().strip()\n+march_include_dir = '/usr/include/' + machine_name + '-linux-gnu'\n+\n+clang_flags = [\n+ '-O2',\n+ '-Wall',\n+ '-Wextra',\n+ '-target',\n+ 'bpf',\n+ '-g',\n+ '-c',\n+]\n+\n+bpf_o_cmd = [\n+ clang,\n+ clang_flags,\n+ '-idirafter',\n+ libbpf_include_dir,\n+ '-idirafter',\n+ march_include_dir,\n+ '@INPUT@',\n+ '-o',\n+ '@OUTPUT@'\n+]\n+\n+skel_h_cmd = [\n+ bpftool,\n+ 'gen',\n+ 'skeleton',\n+ '@INPUT@'\n+]\n+\n+tap_rss_o = custom_target(\n+ 'tap_rss.bpf.o',\n+ input: 'tap_rss.c',\n+ output: 'tap_rss.o',\n+ command: bpf_o_cmd)\n+\n+tap_rss_skel_h = custom_target(\n+ 'tap_rss.skel.h',\n+ input: tap_rss_o,\n+ output: 'tap_rss.skel.h',\n+ command: skel_h_cmd,\n+ capture: true)\ndiff --git a/drivers/net/tap/bpf/tap_bpf_program.c b/drivers/net/tap/bpf/tap_bpf_program.c\ndeleted file mode 100644\nindex f05aed021c..0000000000\n--- a/drivers/net/tap/bpf/tap_bpf_program.c\n+++ /dev/null\n@@ -1,255 +0,0 @@\n-/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0\n- * Copyright 2017 Mellanox Technologies, Ltd\n- */\n-\n-#include <stdint.h>\n-#include <stdbool.h>\n-#include <sys/types.h>\n-#include <sys/socket.h>\n-#include <asm/types.h>\n-#include <linux/in.h>\n-#include <linux/if.h>\n-#include <linux/if_ether.h>\n-#include <linux/ip.h>\n-#include <linux/ipv6.h>\n-#include <linux/if_tunnel.h>\n-#include <linux/filter.h>\n-\n-#include \"bpf_api.h\"\n-#include \"bpf_elf.h\"\n-#include \"../tap_rss.h\"\n-\n-/** Create IPv4 address */\n-#define IPv4(a, b, c, d) ((__u32)(((a) & 0xff) << 24) | \\\n-\t\t(((b) & 0xff) << 16) | \\\n-\t\t(((c) & 0xff) << 8) | \\\n-\t\t((d) & 0xff))\n-\n-#define PORT(a, b) ((__u16)(((a) & 0xff) << 8) | \\\n-\t\t((b) & 0xff))\n-\n-/*\n- * The queue number is offset by a unique QUEUE_OFFSET, to distinguish\n- * packets that have gone through this rule (skb->cb[1] != 0) from others.\n- */\n-#define QUEUE_OFFSET\t\t0x7cafe800\n-#define PIN_GLOBAL_NS\t\t2\n-\n-#define KEY_IDX\t\t\t0\n-#define BPF_MAP_ID_KEY\t1\n-\n-struct vlan_hdr {\n-\t__be16 proto;\n-\t__be16 tci;\n-};\n-\n-struct bpf_elf_map __attribute__((section(\"maps\"), used))\n-map_keys = {\n-\t.type = BPF_MAP_TYPE_HASH,\n-\t.id = BPF_MAP_ID_KEY,\n-\t.size_key = sizeof(__u32),\n-\t.size_value = sizeof(struct rss_key),\n-\t.max_elem = 256,\n-\t.pinning = PIN_GLOBAL_NS,\n-};\n-\n-__section(\"cls_q\") int\n-match_q(struct __sk_buff *skb)\n-{\n-\t__u32 queue = skb->cb[1];\n-\t/* queue is set by tap_flow_bpf_cls_q() before load */\n-\tvolatile __u32 q = 0xdeadbeef;\n-\t__u32 match_queue = QUEUE_OFFSET + q;\n-\n-\t/* printt(\"match_q$i() queue = %d\\n\", queue); */\n-\n-\tif (queue != match_queue)\n-\t\treturn TC_ACT_OK;\n-\n-\t/* queue match */\n-\tskb->cb[1] = 0;\n-\treturn TC_ACT_UNSPEC;\n-}\n-\n-\n-struct ipv4_l3_l4_tuple {\n-\t__u32 src_addr;\n-\t__u32 dst_addr;\n-\t__u16 dport;\n-\t__u16 sport;\n-} __attribute__((packed));\n-\n-struct ipv6_l3_l4_tuple {\n-\t__u8 src_addr[16];\n-\t__u8 dst_addr[16];\n-\t__u16 dport;\n-\t__u16 sport;\n-} __attribute__((packed));\n-\n-static const __u8 def_rss_key[TAP_RSS_HASH_KEY_SIZE] = {\n-\t0xd1, 0x81, 0xc6, 0x2c,\n-\t0xf7, 0xf4, 0xdb, 0x5b,\n-\t0x19, 0x83, 0xa2, 0xfc,\n-\t0x94, 0x3e, 0x1a, 0xdb,\n-\t0xd9, 0x38, 0x9e, 0x6b,\n-\t0xd1, 0x03, 0x9c, 0x2c,\n-\t0xa7, 0x44, 0x99, 0xad,\n-\t0x59, 0x3d, 0x56, 0xd9,\n-\t0xf3, 0x25, 0x3c, 0x06,\n-\t0x2a, 0xdc, 0x1f, 0xfc,\n-};\n-\n-static __u32 __attribute__((always_inline))\n-rte_softrss_be(const __u32 *input_tuple, const uint8_t *rss_key,\n-\t\t__u8 input_len)\n-{\n-\t__u32 i, j, hash = 0;\n-#pragma unroll\n-\tfor (j = 0; j < input_len; j++) {\n-#pragma unroll\n-\t\tfor (i = 0; i < 32; i++) {\n-\t\t\tif (input_tuple[j] & (1U << (31 - i))) {\n-\t\t\t\thash ^= ((const __u32 *)def_rss_key)[j] << i |\n-\t\t\t\t(__u32)((uint64_t)\n-\t\t\t\t(((const __u32 *)def_rss_key)[j + 1])\n-\t\t\t\t\t>> (32 - i));\n-\t\t\t}\n-\t\t}\n-\t}\n-\treturn hash;\n-}\n-\n-static int __attribute__((always_inline))\n-rss_l3_l4(struct __sk_buff *skb)\n-{\n-\tvoid *data_end = (void *)(long)skb->data_end;\n-\tvoid *data = (void *)(long)skb->data;\n-\t__u16 proto = (__u16)skb->protocol;\n-\t__u32 key_idx = 0xdeadbeef;\n-\t__u32 hash;\n-\tstruct rss_key *rsskey;\n-\t__u64 off = ETH_HLEN;\n-\tint j;\n-\t__u8 *key = 0;\n-\t__u32 len;\n-\t__u32 queue = 0;\n-\tbool mf = 0;\n-\t__u16 frag_off = 0;\n-\n-\trsskey = map_lookup_elem(&map_keys, &key_idx);\n-\tif (!rsskey) {\n-\t\tprintt(\"hash(): rss key is not configured\\n\");\n-\t\treturn TC_ACT_OK;\n-\t}\n-\tkey = (__u8 *)rsskey->key;\n-\n-\t/* Get correct proto for 802.1ad */\n-\tif (skb->vlan_present && skb->vlan_proto == htons(ETH_P_8021AD)) {\n-\t\tif (data + ETH_ALEN * 2 + sizeof(struct vlan_hdr) +\n-\t\t sizeof(proto) > data_end)\n-\t\t\treturn TC_ACT_OK;\n-\t\tproto = *(__u16 *)(data + ETH_ALEN * 2 +\n-\t\t\t\t sizeof(struct vlan_hdr));\n-\t\toff += sizeof(struct vlan_hdr);\n-\t}\n-\n-\tif (proto == htons(ETH_P_IP)) {\n-\t\tif (data + off + sizeof(struct iphdr) + sizeof(__u32)\n-\t\t\t> data_end)\n-\t\t\treturn TC_ACT_OK;\n-\n-\t\t__u8 *src_dst_addr = data + off + offsetof(struct iphdr, saddr);\n-\t\t__u8 *frag_off_addr = data + off + offsetof(struct iphdr, frag_off);\n-\t\t__u8 *prot_addr = data + off + offsetof(struct iphdr, protocol);\n-\t\t__u8 *src_dst_port = data + off + sizeof(struct iphdr);\n-\t\tstruct ipv4_l3_l4_tuple v4_tuple = {\n-\t\t\t.src_addr = IPv4(*(src_dst_addr + 0),\n-\t\t\t\t\t*(src_dst_addr + 1),\n-\t\t\t\t\t*(src_dst_addr + 2),\n-\t\t\t\t\t*(src_dst_addr + 3)),\n-\t\t\t.dst_addr = IPv4(*(src_dst_addr + 4),\n-\t\t\t\t\t*(src_dst_addr + 5),\n-\t\t\t\t\t*(src_dst_addr + 6),\n-\t\t\t\t\t*(src_dst_addr + 7)),\n-\t\t\t.sport = 0,\n-\t\t\t.dport = 0,\n-\t\t};\n-\t\t/** Fetch the L4-payer port numbers only in-case of TCP/UDP\n-\t\t ** and also if the packet is not fragmented. Since fragmented\n-\t\t ** chunks do not have L4 TCP/UDP header.\n-\t\t **/\n-\t\tif (*prot_addr == IPPROTO_UDP || *prot_addr == IPPROTO_TCP) {\n-\t\t\tfrag_off = PORT(*(frag_off_addr + 0),\n-\t\t\t\t\t*(frag_off_addr + 1));\n-\t\t\tmf = frag_off & 0x2000;\n-\t\t\tfrag_off = frag_off & 0x1fff;\n-\t\t\tif (mf == 0 && frag_off == 0) {\n-\t\t\t\tv4_tuple.sport = PORT(*(src_dst_port + 0),\n-\t\t\t\t\t\t*(src_dst_port + 1));\n-\t\t\t\tv4_tuple.dport = PORT(*(src_dst_port + 2),\n-\t\t\t\t\t\t*(src_dst_port + 3));\n-\t\t\t}\n-\t\t}\n-\t\t__u8 input_len = sizeof(v4_tuple) / sizeof(__u32);\n-\t\tif (rsskey->hash_fields & (1 << HASH_FIELD_IPV4_L3))\n-\t\t\tinput_len--;\n-\t\thash = rte_softrss_be((__u32 *)&v4_tuple, key, 3);\n-\t} else if (proto == htons(ETH_P_IPV6)) {\n-\t\tif (data + off + sizeof(struct ipv6hdr) +\n-\t\t\t\t\tsizeof(__u32) > data_end)\n-\t\t\treturn TC_ACT_OK;\n-\t\t__u8 *src_dst_addr = data + off +\n-\t\t\t\t\toffsetof(struct ipv6hdr, saddr);\n-\t\t__u8 *src_dst_port = data + off +\n-\t\t\t\t\tsizeof(struct ipv6hdr);\n-\t\t__u8 *next_hdr = data + off +\n-\t\t\t\t\toffsetof(struct ipv6hdr, nexthdr);\n-\n-\t\tstruct ipv6_l3_l4_tuple v6_tuple;\n-\t\tfor (j = 0; j < 4; j++)\n-\t\t\t*((uint32_t *)&v6_tuple.src_addr + j) =\n-\t\t\t\t__builtin_bswap32(*((uint32_t *)\n-\t\t\t\t\t\tsrc_dst_addr + j));\n-\t\tfor (j = 0; j < 4; j++)\n-\t\t\t*((uint32_t *)&v6_tuple.dst_addr + j) =\n-\t\t\t\t__builtin_bswap32(*((uint32_t *)\n-\t\t\t\t\t\tsrc_dst_addr + 4 + j));\n-\n-\t\t/** Fetch the L4 header port-numbers only if next-header\n-\t\t * is TCP/UDP **/\n-\t\tif (*next_hdr == IPPROTO_UDP || *next_hdr == IPPROTO_TCP) {\n-\t\t\tv6_tuple.sport = PORT(*(src_dst_port + 0),\n-\t\t\t\t *(src_dst_port + 1));\n-\t\t\tv6_tuple.dport = PORT(*(src_dst_port + 2),\n-\t\t\t\t *(src_dst_port + 3));\n-\t\t} else {\n-\t\t\tv6_tuple.sport = 0;\n-\t\t\tv6_tuple.dport = 0;\n-\t\t}\n-\n-\t\t__u8 input_len = sizeof(v6_tuple) / sizeof(__u32);\n-\t\tif (rsskey->hash_fields & (1 << HASH_FIELD_IPV6_L3))\n-\t\t\tinput_len--;\n-\t\thash = rte_softrss_be((__u32 *)&v6_tuple, key, 9);\n-\t} else {\n-\t\treturn TC_ACT_PIPE;\n-\t}\n-\n-\tqueue = rsskey->queues[(hash % rsskey->nb_queues) &\n-\t\t\t\t (TAP_MAX_QUEUES - 1)];\n-\tskb->cb[1] = QUEUE_OFFSET + queue;\n-\t/* printt(\">>>>> rss_l3_l4 hash=0x%x queue=%u\\n\", hash, queue); */\n-\n-\treturn TC_ACT_RECLASSIFY;\n-}\n-\n-#define RSS(L)\t\t\t\t\t\t\\\n-\t__section(#L) int\t\t\t\t\\\n-\t\tL ## _hash(struct __sk_buff *skb)\t\\\n-\t{\t\t\t\t\t\t\\\n-\t\treturn rss_ ## L (skb);\t\t\t\\\n-\t}\n-\n-RSS(l3_l4)\n-\n-BPF_LICENSE(\"Dual BSD/GPL\");\ndiff --git a/drivers/net/tap/bpf/tap_rss.c b/drivers/net/tap/bpf/tap_rss.c\nnew file mode 100644\nindex 0000000000..888b3bdc24\n--- /dev/null\n+++ b/drivers/net/tap/bpf/tap_rss.c\n@@ -0,0 +1,264 @@\n+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0\n+ * Copyright 2017 Mellanox Technologies, Ltd\n+ */\n+\n+#include <linux/in.h>\n+#include <linux/if_ether.h>\n+#include <linux/ip.h>\n+#include <linux/ipv6.h>\n+#include <linux/pkt_cls.h>\n+#include <linux/bpf.h>\n+\n+#include <bpf/bpf_helpers.h>\n+#include <bpf/bpf_endian.h>\n+\n+#include \"../tap_rss.h\"\n+\n+/*\n+ * This map provides configuration information about flows which need BPF RSS.\n+ *\n+ * The hash is indexed by the skb mark.\n+ */\n+struct {\n+\t__uint(type, BPF_MAP_TYPE_HASH);\n+\t__uint(key_size, sizeof(__u32));\n+\t__uint(value_size, sizeof(struct rss_key));\n+\t__uint(max_entries, TAP_RSS_MAX);\n+} rss_map SEC(\".maps\");\n+\n+#define IP_MF\t\t0x2000\t\t/** IP header Flags **/\n+#define IP_OFFSET\t0x1FFF\t\t/** IP header fragment offset **/\n+\n+/*\n+ * Compute Toeplitz hash over the input tuple.\n+ * This is same as rte_softrss_be in lib/hash\n+ * but loop needs to be setup to match BPF restrictions.\n+ */\n+static __u32 __attribute__((always_inline))\n+softrss_be(const __u32 *input_tuple, __u32 input_len, const __u32 *key)\n+{\n+\t__u32 i, j, hash = 0;\n+\n+#pragma unroll\n+\tfor (j = 0; j < input_len; j++) {\n+#pragma unroll\n+\t\tfor (i = 0; i < 32; i++) {\n+\t\t\tif (input_tuple[j] & (1U << (31 - i)))\n+\t\t\t\thash ^= key[j] << i | key[j + 1] >> (32 - i);\n+\t\t}\n+\t}\n+\treturn hash;\n+}\n+\n+/*\n+ * Compute RSS hash for IPv4 packet.\n+ * return in 0 if RSS not specified\n+ */\n+static __u32 __attribute__((always_inline))\n+parse_ipv4(const struct __sk_buff *skb, __u32 hash_type, const __u32 *key)\n+{\n+\tstruct iphdr iph;\n+\t__u32 off = 0;\n+\n+\tif (bpf_skb_load_bytes_relative(skb, off, &iph, sizeof(iph), BPF_HDR_START_NET))\n+\t\treturn 0;\t/* no IP header present */\n+\n+\tstruct {\n+\t\t__u32 src_addr;\n+\t\t__u32 dst_addr;\n+\t\t__u16 dport;\n+\t\t__u16 sport;\n+\t} v4_tuple = {\n+\t\t.src_addr = bpf_ntohl(iph.saddr),\n+\t\t.dst_addr = bpf_ntohl(iph.daddr),\n+\t};\n+\n+\t/* If only calculating L3 hash, do it now */\n+\tif (hash_type & (1 << HASH_FIELD_IPV4_L3))\n+\t\treturn softrss_be((__u32 *)&v4_tuple, sizeof(v4_tuple) / sizeof(__u32) - 1, key);\n+\n+\t/* If packet is fragmented then no L4 hash is possible */\n+\tif ((iph.frag_off & bpf_htons(IP_MF | IP_OFFSET)) != 0)\n+\t\treturn 0;\n+\n+\t/* Do RSS on UDP or TCP protocols */\n+\tif (iph.protocol == IPPROTO_UDP || iph.protocol == IPPROTO_TCP) {\n+\t\t__u16 src_dst_port[2];\n+\n+\t\toff += iph.ihl * 4;\n+\t\tif (bpf_skb_load_bytes_relative(skb, off, &src_dst_port, sizeof(src_dst_port),\n+\t\t\t\t\t\tBPF_HDR_START_NET))\n+\t\t\treturn 0; /* TCP or UDP header missing */\n+\n+\t\tv4_tuple.sport = bpf_ntohs(src_dst_port[0]);\n+\t\tv4_tuple.dport = bpf_ntohs(src_dst_port[1]);\n+\t\treturn softrss_be((__u32 *)&v4_tuple, sizeof(v4_tuple) / sizeof(__u32), key);\n+\t}\n+\n+\t/* Other protocol */\n+\treturn 0;\n+}\n+\n+/*\n+ * Parse Ipv6 extended headers, update offset and return next proto.\n+ * returns next proto on success, -1 on malformed header\n+ */\n+static int __attribute__((always_inline))\n+skip_ip6_ext(__u16 proto, const struct __sk_buff *skb, __u32 *off, int *frag)\n+{\n+\tstruct ext_hdr {\n+\t\t__u8 next_hdr;\n+\t\t__u8 len;\n+\t} xh;\n+\tunsigned int i;\n+\n+\t*frag = 0;\n+\n+#define MAX_EXT_HDRS 5\n+#pragma unroll\n+\tfor (i = 0; i < MAX_EXT_HDRS; i++) {\n+\t\tswitch (proto) {\n+\t\tcase IPPROTO_HOPOPTS:\n+\t\tcase IPPROTO_ROUTING:\n+\t\tcase IPPROTO_DSTOPTS:\n+\t\t\tif (bpf_skb_load_bytes_relative(skb, *off, &xh, sizeof(xh),\n+\t\t\t\t\t\t\tBPF_HDR_START_NET))\n+\t\t\t\treturn -1;\n+\n+\t\t\t*off += (xh.len + 1) * 8;\n+\t\t\tproto = xh.next_hdr;\n+\t\t\tbreak;\n+\t\tcase IPPROTO_FRAGMENT:\n+\t\t\tif (bpf_skb_load_bytes_relative(skb, *off, &xh, sizeof(xh),\n+\t\t\t\t\t\t\tBPF_HDR_START_NET))\n+\t\t\t\treturn -1;\n+\n+\t\t\t*off += 8;\n+\t\t\tproto = xh.next_hdr;\n+\t\t\t*frag = 1;\n+\t\t\treturn proto; /* this is always the last ext hdr */\n+\t\tdefault:\n+\t\t\treturn proto;\n+\t\t}\n+\t}\n+\n+\t/* too many extension headers give up */\n+\treturn -1;\n+}\n+\n+/*\n+ * Compute RSS hash for IPv6 packet.\n+ * return in 0 if RSS not specified\n+ */\n+static __u32 __attribute__((always_inline))\n+parse_ipv6(const struct __sk_buff *skb, __u32 hash_type, const __u32 *key)\n+{\n+\tstruct {\n+\t\t__u32 src_addr[4];\n+\t\t__u32 dst_addr[4];\n+\t\t__u16 dport;\n+\t\t__u16 sport;\n+\t} v6_tuple = { };\n+\tstruct ipv6hdr ip6h;\n+\t__u32 off = 0, j;\n+\tint proto, frag;\n+\n+\tif (bpf_skb_load_bytes_relative(skb, off, &ip6h, sizeof(ip6h), BPF_HDR_START_NET))\n+\t\treturn 0;\t/* missing IPv6 header */\n+\n+#pragma unroll\n+\tfor (j = 0; j < 4; j++) {\n+\t\tv6_tuple.src_addr[j] = bpf_ntohl(ip6h.saddr.in6_u.u6_addr32[j]);\n+\t\tv6_tuple.dst_addr[j] = bpf_ntohl(ip6h.daddr.in6_u.u6_addr32[j]);\n+\t}\n+\n+\t/* If only doing L3 hash, do it now */\n+\tif (hash_type & (1 << HASH_FIELD_IPV6_L3))\n+\t\treturn softrss_be((__u32 *)&v6_tuple, sizeof(v6_tuple) / sizeof(__u32) - 1, key);\n+\n+\t/* Skip extension headers if present */\n+\toff += sizeof(ip6h);\n+\tproto = skip_ip6_ext(ip6h.nexthdr, skb, &off, &frag);\n+\tif (proto < 0)\n+\t\treturn 0;\n+\n+\t/* If packet is a fragment then no L4 hash is possible */\n+\tif (frag)\n+\t\treturn 0;\n+\n+\t/* Do RSS on UDP or TCP */\n+\tif (proto == IPPROTO_UDP || proto == IPPROTO_TCP) {\n+\t\t__u16 src_dst_port[2];\n+\n+\t\tif (bpf_skb_load_bytes_relative(skb, off, &src_dst_port, sizeof(src_dst_port),\n+\t\t\t\t\t\tBPF_HDR_START_NET))\n+\t\t\treturn 0;\n+\n+\t\tv6_tuple.sport = bpf_ntohs(src_dst_port[0]);\n+\t\tv6_tuple.dport = bpf_ntohs(src_dst_port[1]);\n+\n+\t\treturn softrss_be((__u32 *)&v6_tuple, sizeof(v6_tuple) / sizeof(__u32), key);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/*\n+ * Compute RSS hash for packets.\n+ * Returns 0 if no hash is possible.\n+ */\n+static __u32 __attribute__((always_inline))\n+calculate_rss_hash(const struct __sk_buff *skb, const struct rss_key *rsskey)\n+{\n+\tconst __u32 *key = (const __u32 *)rsskey->key;\n+\n+\tif (skb->protocol == bpf_htons(ETH_P_IP))\n+\t\treturn parse_ipv4(skb, rsskey->hash_fields, key);\n+\telse if (skb->protocol == bpf_htons(ETH_P_IPV6))\n+\t\treturn parse_ipv6(skb, rsskey->hash_fields, key);\n+\telse\n+\t\treturn 0;\n+}\n+\n+/*\n+ * Scale value to be into range [0, n)\n+ * Assumes val is large (ie hash covers whole u32 range)\n+ */\n+static __u32 __attribute__((always_inline))\n+reciprocal_scale(__u32 val, __u32 n)\n+{\n+\treturn (__u32)(((__u64)val * n) >> 32);\n+}\n+\n+/*\n+ * When this BPF program is run by tc from the filter classifier,\n+ * it is able to read skb metadata and packet data.\n+ *\n+ * For packets where RSS is not possible, then just return TC_ACT_OK.\n+ * When RSS is desired, change the skb->queue_mapping and set TC_ACT_PIPE\n+ * to continue processing.\n+ *\n+ * This should be BPF_PROG_TYPE_SCHED_ACT so section needs to be \"action\"\n+ */\n+SEC(\"action\") int\n+rss_flow_action(struct __sk_buff *skb)\n+{\n+\tconst struct rss_key *rsskey;\n+\t__u32 mark = skb->mark;\n+\t__u32 hash;\n+\n+\t/* Lookup RSS configuration for that BPF class */\n+\trsskey = bpf_map_lookup_elem(&rss_map, &mark);\n+\tif (rsskey == NULL)\n+\t\treturn TC_ACT_OK;\n+\n+\thash = calculate_rss_hash(skb, rsskey);\n+\tif (!hash)\n+\t\treturn TC_ACT_OK;\n+\n+\t/* Fold hash to the number of queues configured */\n+\tskb->queue_mapping = reciprocal_scale(hash, rsskey->nb_queues);\n+\treturn TC_ACT_PIPE;\n+}\n+\n+char _license[] SEC(\"license\") = \"Dual BSD/GPL\";\n", "prefixes": [ "v6", "6/8" ] }