Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/59094/?format=api
{ "id": 59094, "url": "http://patches.dpdk.org/api/patches/59094/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/20190910095937.9731-3-hyonkim@cisco.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20190910095937.9731-3-hyonkim@cisco.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20190910095937.9731-3-hyonkim@cisco.com", "date": "2019-09-10T09:59:37", "name": "[2/2] net/enic: add flow implementation based on Flow Manager API", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "4ef432e1f0965daf0ecdd9d3e3f6f6dd1eacc108", "submitter": { "id": 948, "url": "http://patches.dpdk.org/api/people/948/?format=api", "name": "Hyong Youb Kim (hyonkim)", "email": "hyonkim@cisco.com" }, "delegate": { "id": 319, "url": "http://patches.dpdk.org/api/users/319/?format=api", "username": "fyigit", "first_name": "Ferruh", "last_name": "Yigit", "email": "ferruh.yigit@amd.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/20190910095937.9731-3-hyonkim@cisco.com/mbox/", "series": [ { "id": 6357, "url": "http://patches.dpdk.org/api/series/6357/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=6357", "date": "2019-09-10T09:59:35", "name": "net/enic: a couple new features", "version": 1, "mbox": "http://patches.dpdk.org/series/6357/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/59094/comments/", "check": "warning", "checks": "http://patches.dpdk.org/api/patches/59094/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 72B141ED84;\n\tTue, 10 Sep 2019 12:01:07 +0200 (CEST)", "from alln-iport-8.cisco.com (alln-iport-8.cisco.com\n\t[173.37.142.95]) by dpdk.org (Postfix) with ESMTP id 12AC71EBA6\n\tfor <dev@dpdk.org>; Tue, 10 Sep 2019 12:01:04 +0200 (CEST)", "from rcdn-core-9.cisco.com ([173.37.93.145])\n\tby alln-iport-8.cisco.com with ESMTP/TLS/DHE-RSA-SEED-SHA;\n\t10 Sep 2019 10:01:02 +0000", "from cisco.com (savbu-usnic-a.cisco.com [10.193.184.48])\n\tby rcdn-core-9.cisco.com (8.15.2/8.15.2) with ESMTP id x8AA12Xx023494;\n\tTue, 10 Sep 2019 10:01:02 GMT", "by cisco.com (Postfix, from userid 508933)\n\tid 1CEC720F2003; Tue, 10 Sep 2019 03:01:02 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple;\n\td=cisco.com; i=@cisco.com; l=96791; q=dns/txt;\n\ts=iport; t=1568109665; x=1569319265;\n\th=from:to:cc:subject:date:message-id:in-reply-to:\n\treferences:mime-version:content-transfer-encoding;\n\tbh=ss3PAW4rq7zSBqXHZfiDAyfd0YkjffV4QGBTKVLlZkM=;\n\tb=k9bAH+vEeNpIPUVSQFQ9QnXqHM9AoGdRQxmoWWLVRYTEx/toq9iK/A0d\n\t35R4fbaROmGsAU4eXq0LQM6a4fEF+62EZ0erFJuIx8vcD9GlIROmwbz7c\n\tmbExSEgLYYU04DoWO2O1namLnd+9gerVcxotH+BdYMN0Rq2VPPTKeOVB9 g=;", "X-IronPort-AV": "E=Sophos;i=\"5.64,489,1559520000\"; d=\"scan'208\";a=\"328864727\"", "From": "Hyong Youb Kim <hyonkim@cisco.com>", "To": "Ferruh Yigit <ferruh.yigit@intel.com>", "Cc": "dev@dpdk.org, John Daley <johndale@cisco.com>,\n\tHyong Youb Kim <hyonkim@cisco.com>", "Date": "Tue, 10 Sep 2019 02:59:37 -0700", "Message-Id": "<20190910095937.9731-3-hyonkim@cisco.com>", "X-Mailer": "git-send-email 2.22.0", "In-Reply-To": "<20190910095937.9731-1-hyonkim@cisco.com>", "References": "<20190910095937.9731-1-hyonkim@cisco.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "X-Outbound-SMTP-Client": "10.193.184.48, savbu-usnic-a.cisco.com", "X-Outbound-Node": "rcdn-core-9.cisco.com", "Subject": "[dpdk-dev] [PATCH 2/2] net/enic: add flow implementation based on\n\tFlow Manager API", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Flow Manager API is the new firmware interface that exposes match\naction capabilities in 1400 series VIC adapters. It is intended for\nvirtual switch offloads and enables more advanced features than the\nexisting filter API. For example, it supports VXLAN encap and decap\nactions, and exposes TCAM and exact match tables.\n\nAdd the new flow implementation using Flow Manager and use it when\navailable. When Flow Manager is not available, the driver will\ncontinue to use the old filter-based flow implementation.\n\nSigned-off-by: John Daley <johndale@cisco.com>\nSigned-off-by: Hyong Youb Kim <hyonkim@cisco.com>\n---\n doc/guides/nics/enic.rst | 9 +\n doc/guides/rel_notes/release_19_11.rst | 1 +\n drivers/net/enic/Makefile | 1 +\n drivers/net/enic/base/vnic_dev.c | 79 +-\n drivers/net/enic/base/vnic_dev.h | 1 +\n drivers/net/enic/base/vnic_devcmd.h | 12 +\n drivers/net/enic/base/vnic_flowman.h | 364 ++++\n drivers/net/enic/enic.h | 21 +-\n drivers/net/enic/enic_ethdev.c | 10 +-\n drivers/net/enic/enic_fm_flow.c | 2459 ++++++++++++++++++++++++\n drivers/net/enic/enic_main.c | 8 +-\n drivers/net/enic/enic_res.c | 9 +-\n drivers/net/enic/meson.build | 1 +\n 13 files changed, 2967 insertions(+), 8 deletions(-)\n create mode 100644 drivers/net/enic/base/vnic_flowman.h\n create mode 100644 drivers/net/enic/enic_fm_flow.c", "diff": "diff --git a/doc/guides/nics/enic.rst b/doc/guides/nics/enic.rst\nindex 2384cb5b8..b324d4506 100644\n--- a/doc/guides/nics/enic.rst\n+++ b/doc/guides/nics/enic.rst\n@@ -260,6 +260,15 @@ Generic Flow API is supported. The baseline support is:\n - Selectors: 'is', 'spec' and 'mask'. 'last' is not supported\n - In total, up to 64 bytes of mask is allowed across all headers\n \n+- **1400 and later series VICs with Flow Manager API enabled**\n+\n+ - Attributes: ingress, egress\n+ - Items: eth, vlan, ipv4, ipv6, sctp, udp, tcp, vxlan, raw, inner eth, vlan, ipv4, ipv6, sctp, udp, tcp\n+ - Ingress Actions: count, drop, flag, jump, mark, port_id, passthru, queue, rss, vxlan_decap, vxlan_encap, and void\n+ - Egress Actions: count, drop, jump, passthru, vxlan_encap, and void\n+ - Selectors: 'is', 'spec' and 'mask'. 'last' is not supported\n+ - In total, up to 64 bytes of mask is allowed across all headers\n+\n The VIC performs packet matching after applying VLAN strip. If VLAN\n stripping is enabled, EtherType in the ETH item corresponds to the\n stripped VLAN header's EtherType. Stripping does not affect the VLAN\ndiff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst\nindex afe71f8fa..82e38a904 100644\n--- a/doc/guides/rel_notes/release_19_11.rst\n+++ b/doc/guides/rel_notes/release_19_11.rst\n@@ -59,6 +59,7 @@ New Features\n * **Updated the enic driver.**\n \n * Added support for Geneve with options offload.\n+ * Added flow API implementation based on VIC Flow Manager API.\n \n \n Removed Items\ndiff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile\nindex 04bae35e3..10736a567 100644\n--- a/drivers/net/enic/Makefile\n+++ b/drivers/net/enic/Makefile\n@@ -32,6 +32,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rxtx.c\n SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c\n SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c\n SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_flow.c\n+SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_fm_flow.c\n SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c\n SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_wq.c\n SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_dev.c\ndiff --git a/drivers/net/enic/base/vnic_dev.c b/drivers/net/enic/base/vnic_dev.c\nindex 807d4096f..a52f7430b 100644\n--- a/drivers/net/enic/base/vnic_dev.c\n+++ b/drivers/net/enic/base/vnic_dev.c\n@@ -12,6 +12,7 @@\n #include \"vnic_devcmd.h\"\n #include \"vnic_nic.h\"\n #include \"vnic_stats.h\"\n+#include \"vnic_flowman.h\"\n \n \n enum vnic_proxy_type {\n@@ -47,6 +48,8 @@ struct vnic_dev {\n \tdma_addr_t stats_pa;\n \tstruct vnic_devcmd_fw_info *fw_info;\n \tdma_addr_t fw_info_pa;\n+\tstruct fm_info *flowman_info;\n+\tdma_addr_t flowman_info_pa;\n \tenum vnic_proxy_type proxy;\n \tu32 proxy_index;\n \tu64 args[VNIC_DEVCMD_NARGS];\n@@ -500,8 +503,74 @@ int vnic_dev_capable_adv_filters(struct vnic_dev *vdev)\n \treturn (a1 >= (u32)FILTER_DPDK_1);\n }\n \n-/* Determine the \"best\" filtering mode VIC is capaible of. Returns one of 3\n+int vnic_dev_flowman_cmd(struct vnic_dev *vdev, u64 *args, int nargs)\n+{\n+\tint wait = 1000;\n+\n+\treturn vnic_dev_cmd_args(vdev, CMD_FLOW_MANAGER_OP, args, nargs, wait);\n+}\n+\n+static int vnic_dev_flowman_enable(struct vnic_dev *vdev, u32 *mode,\n+\t\t\t\t u8 *filter_actions)\n+{\n+\tchar name[NAME_MAX];\n+\tu64 args[3];\n+\tu64 ops;\n+\tstatic u32 instance;\n+\n+\t/* flowman devcmd available? */\n+\tif (!vnic_dev_capable(vdev, CMD_FLOW_MANAGER_OP))\n+\t\treturn 0;\n+\t/* Have the version we are using? */\n+\targs[0] = FM_API_VERSION_QUERY;\n+\tif (vnic_dev_flowman_cmd(vdev, args, 1))\n+\t\treturn 0;\n+\tif ((args[0] & (1ULL << FM_VERSION)) == 0)\n+\t\treturn 0;\n+\t/* Select the version */\n+\targs[0] = FM_API_VERSION_SELECT;\n+\targs[1] = FM_VERSION;\n+\tif (vnic_dev_flowman_cmd(vdev, args, 2))\n+\t\treturn 0;\n+\t/* Can we get fm_info? */\n+\tif (!vdev->flowman_info) {\n+\t\tsnprintf((char *)name, sizeof(name), \"vnic_flowman_info-%u\",\n+\t\t\t instance++);\n+\t\tvdev->flowman_info = vdev->alloc_consistent(vdev->priv,\n+\t\t\tsizeof(struct fm_info),\n+\t\t\t&vdev->flowman_info_pa, (u8 *)name);\n+\t\tif (!vdev->flowman_info)\n+\t\t\treturn 0;\n+\t}\n+\targs[0] = FM_INFO_QUERY;\n+\targs[1] = vdev->flowman_info_pa;\n+\targs[2] = sizeof(struct fm_info);\n+\tif (vnic_dev_flowman_cmd(vdev, args, 3))\n+\t\treturn 0;\n+\t/* Have required operations? */\n+\tops = (1ULL << FMOP_END) |\n+\t\t(1ULL << FMOP_DROP) |\n+\t\t(1ULL << FMOP_RQ_STEER) |\n+\t\t(1ULL << FMOP_EXACT_MATCH) |\n+\t\t(1ULL << FMOP_MARK) |\n+\t\t(1ULL << FMOP_TAG) |\n+\t\t(1ULL << FMOP_EG_HAIRPIN) |\n+\t\t(1ULL << FMOP_ENCAP) |\n+\t\t(1ULL << FMOP_DECAP_NOSTRIP);\n+\tif ((vdev->flowman_info->fm_op_mask & ops) != ops)\n+\t\treturn 0;\n+\t/* Good to use flowman now */\n+\t*mode = FILTER_FLOWMAN;\n+\t*filter_actions = FILTER_ACTION_RQ_STEERING_FLAG |\n+\t\tFILTER_ACTION_FILTER_ID_FLAG |\n+\t\tFILTER_ACTION_COUNTER_FLAG |\n+\t\tFILTER_ACTION_DROP_FLAG;\n+\treturn 1;\n+}\n+\n+/* Determine the \"best\" filtering mode VIC is capaible of. Returns one of 4\n * value or 0 on error:\n+ *\tFILTER_FLOWMAN- flowman api capable\n *\tFILTER_DPDK_1- advanced filters availabile\n *\tFILTER_USNIC_IP_FLAG - advanced filters but with the restriction that\n *\t\tthe IP layer must explicitly specified. I.e. cannot have a UDP\n@@ -517,6 +586,10 @@ int vnic_dev_capable_filter_mode(struct vnic_dev *vdev, u32 *mode,\n \tint err;\n \tu32 max_level = 0;\n \n+\t/* If flowman is available, use it as it is the most capable API */\n+\tif (vnic_dev_flowman_enable(vdev, mode, filter_actions))\n+\t\treturn 0;\n+\n \terr = vnic_dev_advanced_filters_cap(vdev, args, 4);\n \n \t/* determine supported filter actions */\n@@ -977,6 +1050,10 @@ void vnic_dev_unregister(struct vnic_dev *vdev)\n \t\t\tvdev->free_consistent(vdev->priv,\n \t\t\t\tsizeof(struct vnic_stats),\n \t\t\t\tvdev->stats, vdev->stats_pa);\n+\t\tif (vdev->flowman_info)\n+\t\t\tvdev->free_consistent(vdev->priv,\n+\t\t\t\tsizeof(struct fm_info),\n+\t\t\t\tvdev->flowman_info, vdev->flowman_info_pa);\n \t\tif (vdev->fw_info)\n \t\t\tvdev->free_consistent(vdev->priv,\n \t\t\t\tsizeof(struct vnic_devcmd_fw_info),\ndiff --git a/drivers/net/enic/base/vnic_dev.h b/drivers/net/enic/base/vnic_dev.h\nindex 8a1363486..d84e0c773 100644\n--- a/drivers/net/enic/base/vnic_dev.h\n+++ b/drivers/net/enic/base/vnic_dev.h\n@@ -182,6 +182,7 @@ int vnic_dev_deinit_done(struct vnic_dev *vdev, int *status);\n int vnic_dev_set_mac_addr(struct vnic_dev *vdev, u8 *mac_addr);\n int vnic_dev_classifier(struct vnic_dev *vdev, u8 cmd, u16 *entry,\n \tstruct filter_v2 *data, struct filter_action_v2 *action_v2);\n+int vnic_dev_flowman_cmd(struct vnic_dev *vdev, u64 *args, int nargs);\n int vnic_dev_overlay_offload_ctrl(struct vnic_dev *vdev,\n \tu8 overlay, u8 config);\n int vnic_dev_overlay_offload_cfg(struct vnic_dev *vdev, u8 overlay,\ndiff --git a/drivers/net/enic/base/vnic_devcmd.h b/drivers/net/enic/base/vnic_devcmd.h\nindex da60be7b0..d85d40be2 100644\n--- a/drivers/net/enic/base/vnic_devcmd.h\n+++ b/drivers/net/enic/base/vnic_devcmd.h\n@@ -600,6 +600,16 @@ enum vnic_devcmd_cmd {\n \t * a3 = bitmask of supported actions\n \t */\n \tCMD_ADD_ADV_FILTER = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ENET, 77),\n+\n+\t/*\n+\t * Perform a Flow Manager Operation (see flowman_api.h)\n+\t * in:\t(u32) a0 = sub-command\n+\t *\t(u64) a1..15 = (sub-command specific)\n+\t *\n+\t * All arguments that have not been assigned a meaning should be\n+\t * initialized to 0 to allow for better driver forward compatibility.\n+\t */\n+\tCMD_FLOW_MANAGER_OP = _CMDC(_CMD_DIR_RW, _CMD_VTYPE_ENET, 88),\n };\n \n /* Modes for exchanging advanced filter capabilities. The modes supported by\n@@ -865,6 +875,7 @@ struct filter_action {\n #define FILTER_ACTION_RQ_STEERING_FLAG\t(1 << 0)\n #define FILTER_ACTION_FILTER_ID_FLAG\t(1 << 1)\n #define FILTER_ACTION_DROP_FLAG\t\t(1 << 2)\n+#define FILTER_ACTION_COUNTER_FLAG\t(1 << 3)\n #define FILTER_ACTION_V2_ALL\t\t(FILTER_ACTION_RQ_STEERING_FLAG \\\n \t\t\t\t\t | FILTER_ACTION_DROP_FLAG \\\n \t\t\t\t\t | FILTER_ACTION_FILTER_ID_FLAG)\n@@ -889,6 +900,7 @@ enum filter_type {\n \tFILTER_NVGRE_VMQ = 4,\n \tFILTER_USNIC_IP = 5,\n \tFILTER_DPDK_1 = 6,\n+\tFILTER_FLOWMAN = 7,\n \tFILTER_MAX\n };\n \ndiff --git a/drivers/net/enic/base/vnic_flowman.h b/drivers/net/enic/base/vnic_flowman.h\nnew file mode 100644\nindex 000000000..b586e8a0c\n--- /dev/null\n+++ b/drivers/net/enic/base/vnic_flowman.h\n@@ -0,0 +1,364 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright 2018-2019 Cisco Systems, Inc. All rights reserved.\n+ */\n+#ifndef _VNIC_FLOWMAN_H_\n+#define _VNIC_FLOWMAN_H_\n+\n+/* This file contains Flow Manager (FM) API of the firmware */\n+\n+/* Flow manager sub-ops */\n+enum {\n+\tFM_EXACT_TABLE_ALLOC,\n+\tFM_TCAM_TABLE_ALLOC,\n+\tFM_MATCH_TABLE_FREE,\n+\tFM_COUNTER_BRK,\n+\tFM_COUNTER_QUERY,\n+\tFM_COUNTER_CLEAR_ALL,\n+\tFM_COUNTER_DMA,\n+\tFM_ACTION_ALLOC,\n+\tFM_ACTION_FREE,\n+\tFM_EXACT_ENTRY_INSTALL,\n+\tFM_TCAM_ENTRY_INSTALL,\n+\tFM_MATCH_ENTRY_REMOVE,\n+\tFM_VNIC_FIND,\n+\tFM_API_VERSION_QUERY,\n+\tFM_API_VERSION_SELECT,\n+\tFM_INFO_QUERY\n+};\n+\n+/*\n+ * FKM (flow key metadata) flags used to match packet metadata\n+ * (e.g. packet is tcp)\n+ */\n+#define FKM_BITS\t\t\\\n+\tFBIT(FKM_QTAG)\t\t\\\n+\tFBIT(FKM_CMD)\t\t\\\n+\tFBIT(FKM_IPV4)\t\t\\\n+\tFBIT(FKM_IPV6)\t\t\\\n+\tFBIT(FKM_ROCE)\t\t\\\n+\tFBIT(FKM_UDP)\t\t\\\n+\tFBIT(FKM_TCP)\t\t\\\n+\tFBIT(FKM_TCPORUDP)\t\\\n+\tFBIT(FKM_IPFRAG)\t\\\n+\tFBIT(FKM_NVGRE)\t\t\\\n+\tFBIT(FKM_VXLAN)\t\t\\\n+\tFBIT(FKM_GENEVE)\t\\\n+\tFBIT(FKM_NSH)\t\t\\\n+\tFBIT(FKM_ROCEV2)\t\\\n+\tFBIT(FKM_VLAN_PRES)\t\\\n+\tFBIT(FKM_IPOK)\t\t\\\n+\tFBIT(FKM_L4OK)\t\t\\\n+\tFBIT(FKM_ROCEOK)\t\\\n+\tFBIT(FKM_FCSOK)\t\t\\\n+\tFBIT(FKM_EG_SPAN)\t\\\n+\tFBIT(FKM_IG_SPAN)\t\\\n+\tFBIT(FKM_EG_HAIRPINNED)\n+\n+/*\n+ * FKH (flow key header) flags.\n+ * This selects which headers are valid in the struct.\n+ * This is distinct from metadata in that metadata is requesting actual\n+ * selection criteria. If, for example, a TCAM match with metadata \"FKM_UDP\"\n+ * is feeding into an exact match table, there may be no need for the\n+ * exact match table to also specify FKM_UDP, so FKH_UDP is used to\n+ * specify that the UDP header fields should be used in the match.\n+ */\n+#define FKH_BITS\t\\\n+\tFBIT(FKH_ETHER)\t\\\n+\tFBIT(FKH_QTAG)\t\\\n+\tFBIT(FKH_L2RAW)\t\\\n+\tFBIT(FKH_IPV4)\t\\\n+\tFBIT(FKH_IPV6)\t\\\n+\tFBIT(FKH_L3RAW)\t\\\n+\tFBIT(FKH_UDP)\t\\\n+\tFBIT(FKH_TCP)\t\\\n+\tFBIT(FKH_ICMP)\t\\\n+\tFBIT(FKH_VXLAN)\t\\\n+\tFBIT(FKH_L4RAW)\n+\n+#define FBIT(X) X##_BIT,\n+enum {\n+\tFKM_BITS\n+\tFKM_BIT_COUNT\n+};\n+\n+enum {\n+\tFKH_BITS\n+\tFKH_BIT_COUNT\n+};\n+#undef FBIT\n+#define FBIT(X) X = (1 << X##_BIT),\n+enum {\n+\tFKM_BITS\n+};\n+enum {\n+\tFKH_BITS\n+};\n+#undef FBIT\n+\n+#define FM_ETH_ALEN 6\n+#define FM_LAYER_SIZE 64\n+\n+/* Header match pattern */\n+struct fm_header_set {\n+\tuint32_t fk_metadata; /* FKM flags */\n+\tuint32_t fk_header_select; /* FKH flags */\n+\tuint16_t fk_vlan;\n+\t/* L2: Ethernet Header (valid if FKH_ETHER) */\n+\tunion {\n+\t\tstruct {\n+\t\t\tuint8_t fk_dstmac[FM_ETH_ALEN];\n+\t\t\tuint8_t fk_srcmac[FM_ETH_ALEN];\n+\t\t\tuint16_t fk_ethtype;\n+\t\t} __rte_packed eth;\n+\t\tuint8_t rawdata[FM_LAYER_SIZE];\n+\t} __rte_packed l2;\n+\t/* L3: IPv4 or IPv6 (valid if FKH_IPV4,6) */\n+\tunion {\n+\t\t/* Valid if FKH_IPV4 */\n+\t\tstruct {\n+\t\t\tuint8_t fk_ihl_vers;\n+\t\t\tuint8_t fk_tos;\n+\t\t\tuint16_t fk_tot_len;\n+\t\t\tuint16_t fk_id;\n+\t\t\tuint16_t fk_frag_off;\n+\t\t\tuint8_t fk_ttl;\n+\t\t\tuint8_t fk_proto;\n+\t\t\tuint16_t fk_check;\n+\t\t\tuint32_t fk_saddr;\n+\t\t\tuint32_t fk_daddr;\n+\t\t} __rte_packed ip4;\n+\t\t/* Valid if FKH_IPV6 */\n+\t\tstruct {\n+\t\t\tunion {\n+\t\t\t\tstruct {\n+\t\t\t\t\tuint32_t fk_un1_flow;\n+\t\t\t\t\tuint16_t fk_un1_plen;\n+\t\t\t\t\tuint8_t fk_un1_nxt;\n+\t\t\t\t\tuint8_t fk_un1_hlim;\n+\t\t\t\t} unl;\n+\t\t\t\tuint8_t fk_un2_vfc;\n+\t\t\t} ctl;\n+\t\t\tuint8_t fk_srcip[16];\n+\t\t\tuint8_t fk_dstip[16];\n+\t\t} __rte_packed ip6;\n+\t\tuint8_t rawdata[FM_LAYER_SIZE];\n+\t} __rte_packed l3;\n+\t/* L4: UDP, TCP, or ICMP (valid if FKH_UDP,TCP,ICMP) */\n+\tunion {\n+\t\tstruct {\n+\t\t\tuint16_t fk_source;\n+\t\t\tuint16_t fk_dest;\n+\t\t\tuint16_t fk_len;\n+\t\t\tuint16_t fk_check;\n+\t\t} __rte_packed udp;\n+\t\tstruct {\n+\t\t\tuint16_t fk_source;\n+\t\t\tuint16_t fk_dest;\n+\t\t\tuint32_t fk_seq;\n+\t\t\tuint32_t fk_ack_seq;\n+\t\t\tuint16_t fk_flags;\n+\t\t\tuint16_t fk_window;\n+\t\t\tuint16_t fk_check;\n+\t\t\tuint16_t fk_urg_ptr;\n+\t\t} __rte_packed tcp;\n+\t\tstruct {\n+\t\t\tuint8_t fk_code;\n+\t\t\tuint8_t fk_type;\n+\t\t} __rte_packed icmp;\n+\t\tuint8_t rawdata[FM_LAYER_SIZE];\n+\t} __rte_packed l4;\n+\t/* VXLAN (valid if FKH_VXLAN) */\n+\tstruct {\n+\t\tuint8_t fkvx_flags;\n+\t\tuint8_t fkvx_res0[3];\n+\t\tuint8_t fkvx_vni[3];\n+\t\tuint8_t fkvx_res1;\n+\t} __rte_packed vxlan;\n+\t/* Payload or unknown inner-most protocol */\n+\tuint8_t fk_l5_data[64];\n+} __rte_packed;\n+\n+/*\n+ * FK (flow key) template.\n+ * fk_hdrset specifies a set of headers per layer of encapsulation.\n+ * Currently FM supports two header sets: outer (0) and inner(1)\n+ */\n+#define FM_HDRSET_MAX 2\n+\n+struct fm_key_template {\n+\tstruct fm_header_set fk_hdrset[FM_HDRSET_MAX];\n+\tuint32_t fk_flags;\n+\tuint16_t fk_packet_tag;\n+\tuint16_t fk_packet_size;\n+\tuint16_t fk_port_id;\n+\tuint32_t fk_wq_id; /* WQ index */\n+\tuint64_t fk_wq_vnic; /* VNIC handle for WQ index */\n+} __rte_packed;\n+\n+/* Action operation types */\n+enum {\n+\t/* End the action chain. */\n+\tFMOP_END,\n+\t/* Drop packet and end the action chain. */\n+\tFMOP_DROP,\n+\t/* Steer packet to an RQ. */\n+\tFMOP_RQ_STEER,\n+\t/*\n+\t * Jump to an exact match table.\n+\t * arg1: exact match table handle\n+\t */\n+\tFMOP_EXACT_MATCH,\n+\t/* Apply CQ-visible mark on packet. Mark is written to RSS HASH. */\n+\tFMOP_MARK,\n+\t/*\n+\t * Apply CQ-visible mark on packet. Mark is written to a field in\n+\t * extended CQ. RSS HASH is preserved.\n+\t */\n+\tFMOP_EXT_MARK,\n+\t/*\n+\t * Apply internal tag which can be matched in subsequent\n+\t * stages or hairpin.\n+\t */\n+\tFMOP_TAG,\n+\t/* Hairpin packet from EG -> IG */\n+\tFMOP_EG_HAIRPIN,\n+\t/* Hairpin packet from IG -> EG */\n+\tFMOP_IG_HAIRPIN,\n+\t/* Encap with VXLAN and inner VLAN from metadata. */\n+\tFMOP_ENCAP_IVLAN,\n+\t/* Encap, no inner VLAN. */\n+\tFMOP_ENCAP_NOIVLAN,\n+\t/* Encap, add inner VLAN if present. */\n+\tFMOP_ENCAP,\n+\t/* Set outer VLAN. */\n+\tFMOP_SET_OVLAN,\n+\t/* Decap when vlan_strip is off */\n+\tFMOP_DECAP_NOSTRIP,\n+\tFMOP_OP_MAX,\n+};\n+\n+/*\n+ * Action operation.\n+ * Complex actions are achieved by a series of \"transform operations\"\n+ * We can have complex transform operations like \"decap\" or \"vxlan\n+ * encap\" and also simple ops like insert this data, add PACKET_LEN to\n+ * this address, etc.\n+ */\n+struct fm_action_op {\n+\tuint32_t fa_op;\t\t/* FMOP flags */\n+\n+\tunion {\n+\t\tstruct {\n+\t\t\tuint8_t len1_offset;\n+\t\t\tuint8_t len1_delta;\n+\t\t\tuint8_t len2_offset;\n+\t\t\tuint8_t len2_delta;\n+\t\t\tuint16_t outer_vlan;\n+\t\t\tuint8_t template_offset;\n+\t\t\tuint8_t template_len;\n+\t\t} __rte_packed encap;\n+\t\tstruct {\n+\t\t\tuint32_t rq_index;\n+\t\t\tuint64_t vnic_handle;\n+\t\t} __rte_packed rq_steer;\n+\t\tstruct {\n+\t\t\tuint16_t vlan;\n+\t\t} __rte_packed ovlan;\n+\t\tstruct {\n+\t\t\tuint16_t mark;\n+\t\t} __rte_packed mark;\n+\t\tstruct {\n+\t\t\tuint32_t ext_mark;\n+\t\t} __rte_packed ext_mark;\n+\t\tstruct {\n+\t\t\tuint8_t tag;\n+\t\t} __rte_packed tag;\n+\t\tstruct {\n+\t\t\tuint64_t handle;\n+\t\t} __rte_packed exact;\n+\t} __rte_packed;\n+} __rte_packed;\n+\n+#define FM_ACTION_OP_MAX 64\n+#define FM_ACTION_DATA_MAX 96\n+\n+/*\n+ * Action is a series of action operations applied to matched\n+ * packet. FMA (flowman action).\n+ */\n+struct fm_action {\n+\tstruct fm_action_op fma_action_ops[FM_ACTION_OP_MAX];\n+\tuint8_t fma_data[FM_ACTION_DATA_MAX];\n+} __rte_packed;\n+\n+/* Match entry flags. FMEF (flow match entry flag) */\n+#define FMEF_COUNTER 0x0001 /* counter index is valid */\n+\n+/* FEM (flow exact match) entry */\n+struct fm_exact_match_entry {\n+\tstruct fm_key_template fem_data; /* Match data. Mask is per table */\n+\tuint32_t fem_flags; /* FMEF_xxx */\n+\tuint64_t fem_action; /* Action handle */\n+\tuint32_t fem_counter; /* Counter index */\n+} __rte_packed;\n+\n+/* FTM (flow TCAM match) entry */\n+struct fm_tcam_match_entry {\n+\tstruct fm_key_template ftm_mask; /* Key mask */\n+\tstruct fm_key_template ftm_data; /* Match data */\n+\tuint32_t ftm_flags; /* FMEF_xxx */\n+\tuint32_t ftm_position; /* Entry position */\n+\tuint64_t ftm_action; /* Action handle */\n+\tuint32_t ftm_counter; /* Counter index */\n+} __rte_packed;\n+\n+/* Match directions */\n+enum {\n+\tFM_INGRESS,\n+\tFM_EGRESS,\n+\tFM_DIR_CNT\n+};\n+\n+/* Last stage ID, independent of the number of stages in hardware */\n+#define FM_STAGE_LAST 0xff\n+\n+/* Hash based exact match table. FET (flow exact match table) */\n+struct fm_exact_match_table {\n+\tuint8_t fet_direction; /* FM_INGRESS or EGRESS*/\n+\tuint8_t fet_stage;\n+\tuint8_t pad[2];\n+\tuint32_t fet_max_entries;\n+\tuint64_t fet_dflt_action;\n+\tstruct fm_key_template fet_key;\n+} __rte_packed;\n+\n+/* TCAM based match table. FTT (flow TCAM match table) */\n+struct fm_tcam_match_table {\n+\tuint8_t ftt_direction;\n+\tuint8_t ftt_stage;\n+\tuint8_t pad[2];\n+\tuint32_t ftt_max_entries;\n+} __rte_packed;\n+\n+struct fm_counter_counts {\n+\tuint64_t fcc_packets;\n+\tuint64_t fcc_bytes;\n+} __rte_packed;\n+\n+/*\n+ * Return structure for FM_INFO_QUERY devcmd\n+ */\n+#define FM_VERSION 1\t\t/* This header file is for version 1 */\n+\n+struct fm_info {\n+\tuint64_t fm_op_mask;\t\t/* Bitmask of action supported ops */\n+\tuint64_t fm_current_ts;\t\t/* Current VIC timestamp */\n+\tuint64_t fm_clock_freq;\t\t/* Timestamp clock frequency */\n+\tuint16_t fm_max_ops;\t\t/* Max ops in an action */\n+\tuint8_t fm_stages;\t\t/* Number of match-action stages */\n+\tuint8_t pad[5];\n+\tuint32_t fm_counter_count;\t/* Number of allocated counters */\n+} __rte_packed;\n+\n+#endif /* _VNIC_FLOWMAN_H_ */\ndiff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h\nindex 87792deec..7413e9354 100644\n--- a/drivers/net/enic/enic.h\n+++ b/drivers/net/enic/enic.h\n@@ -8,6 +8,7 @@\n \n #include \"vnic_enet.h\"\n #include \"vnic_dev.h\"\n+#include \"vnic_flowman.h\"\n #include \"vnic_wq.h\"\n #include \"vnic_rq.h\"\n #include \"vnic_cq.h\"\n@@ -88,10 +89,17 @@ struct enic_memzone_entry {\n \tLIST_ENTRY(enic_memzone_entry) entries;\n };\n \n+/* Defined in enic_fm_flow.c */\n+struct enic_flowman;\n+struct enic_fm_flow;\n+\n struct rte_flow {\n \tLIST_ENTRY(rte_flow) next;\n-\tu16 enic_filter_id;\n+\t/* Data for filter API based flow (enic_flow.c) */\n+\tuint16_t enic_filter_id;\n \tstruct filter_v2 enic_filter;\n+\t/* Data for flow manager based flow (enic_fm_flow.c) */\n+\tstruct enic_fm_flow *fm;\n };\n \n /* Per-instance private data structure */\n@@ -195,6 +203,9 @@ struct enic {\n \t/* Multicast MAC addresses added to the NIC */\n \tuint32_t mc_count;\n \tstruct rte_ether_addr mc_addrs[ENIC_MULTICAST_PERFECT_FILTERS];\n+\n+\t/* Flow manager API */\n+\tstruct enic_flowman *fm;\n };\n \n /* Compute ethdev's max packet size from MTU */\n@@ -276,6 +287,7 @@ enic_ring_incr(uint32_t n_descriptors, uint32_t idx)\n \treturn idx;\n }\n \n+int dev_is_enic(struct rte_eth_dev *dev);\n void enic_fdir_stats_get(struct enic *enic,\n \t\t\t struct rte_eth_fdir_stats *stats);\n int enic_fdir_add_fltr(struct enic *enic,\n@@ -321,6 +333,12 @@ void enic_post_wq_index(struct vnic_wq *wq);\n int enic_probe(struct enic *enic);\n int enic_clsf_init(struct enic *enic);\n void enic_clsf_destroy(struct enic *enic);\n+int enic_fm_init(struct enic *enic);\n+void enic_fm_destroy(struct enic *enic);\n+void *enic_alloc_consistent(void *priv, size_t size, dma_addr_t *dma_handle,\n+\t\t\t u8 *name);\n+void enic_free_consistent(void *priv, size_t size, void *vaddr,\n+\t\t\t dma_addr_t dma_handle);\n uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,\n \t\t\tuint16_t nb_pkts);\n uint16_t enic_noscatter_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,\n@@ -340,4 +358,5 @@ bool enic_use_vector_rx_handler(struct enic *enic);\n void enic_fdir_info(struct enic *enic);\n void enic_fdir_info_get(struct enic *enic, struct rte_eth_fdir_info *stats);\n extern const struct rte_flow_ops enic_flow_ops;\n+extern const struct rte_flow_ops enic_fm_flow_ops;\n #endif /* _ENIC_H_ */\ndiff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c\nindex e1c6da3b3..e8afc2dcd 100644\n--- a/drivers/net/enic/enic_ethdev.c\n+++ b/drivers/net/enic/enic_ethdev.c\n@@ -143,7 +143,10 @@ enicpmd_dev_filter_ctrl(struct rte_eth_dev *dev,\n \tcase RTE_ETH_FILTER_GENERIC:\n \t\tif (filter_op != RTE_ETH_FILTER_GET)\n \t\t\treturn -EINVAL;\n-\t\t*(const void **)arg = &enic_flow_ops;\n+\t\tif (enic->flow_filter_mode == FILTER_FLOWMAN)\n+\t\t\t*(const void **)arg = &enic_fm_flow_ops;\n+\t\telse\n+\t\t\t*(const void **)arg = &enic_flow_ops;\n \t\tbreak;\n \tcase RTE_ETH_FILTER_FDIR:\n \t\tret = enicpmd_fdir_ctrl_func(dev, filter_op, arg);\n@@ -1260,6 +1263,11 @@ static struct rte_pci_driver rte_enic_pmd = {\n \t.remove = eth_enic_pci_remove,\n };\n \n+int dev_is_enic(struct rte_eth_dev *dev)\n+{\n+\treturn dev->device->driver == &rte_enic_pmd.driver;\n+}\n+\n RTE_PMD_REGISTER_PCI(net_enic, rte_enic_pmd);\n RTE_PMD_REGISTER_PCI_TABLE(net_enic, pci_id_enic_map);\n RTE_PMD_REGISTER_KMOD_DEP(net_enic, \"* igb_uio | uio_pci_generic | vfio-pci\");\ndiff --git a/drivers/net/enic/enic_fm_flow.c b/drivers/net/enic/enic_fm_flow.c\nnew file mode 100644\nindex 000000000..a1598b822\n--- /dev/null\n+++ b/drivers/net/enic/enic_fm_flow.c\n@@ -0,0 +1,2459 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright 2008-2019 Cisco Systems, Inc. All rights reserved.\n+ */\n+\n+#include <errno.h>\n+#include <stdint.h>\n+#include <rte_log.h>\n+#include <rte_ethdev_driver.h>\n+#include <rte_flow_driver.h>\n+#include <rte_ether.h>\n+#include <rte_ip.h>\n+#include <rte_udp.h>\n+\n+#include \"enic_compat.h\"\n+#include \"enic.h\"\n+#include \"vnic_dev.h\"\n+#include \"vnic_nic.h\"\n+\n+#define IP_DEFTTL 64 /* from RFC 1340. */\n+#define IP_VERSION 0x40\n+#define IP_HDRLEN 0x05 /* default IP header length == five 32-bits words. */\n+#define IP_VHL_DEF (IP_VERSION | IP_HDRLEN)\n+#define IP6_VTC_FLOW 0x60000000\n+\n+/* Highest Item type supported by Flowman */\n+#define FM_MAX_ITEM_TYPE RTE_FLOW_ITEM_TYPE_VXLAN\n+\n+/* Up to 1024 TCAM entries */\n+#define FM_MAX_TCAM_TABLE_SIZE 1024\n+\n+/* Up to 4096 entries per exact match table */\n+#define FM_MAX_EXACT_TABLE_SIZE 4096\n+\n+/* Number of counters to increase on for each increment */\n+#define FM_COUNTERS_EXPAND 100\n+\n+#define FM_INVALID_HANDLE 0\n+\n+/*\n+ * Flow exact match tables (FET) in the VIC and rte_flow groups.\n+ * Use a simple scheme to map groups to tables.\n+ * Group 0 uses the single TCAM tables, one for each direction.\n+ * Group 1, 2, ... uses its own exact match table.\n+ *\n+ * The TCAM tables are allocated upfront during init.\n+ *\n+ * Exact match tables are allocated on demand. 3 paths that lead allocations.\n+ *\n+ * 1. Add a flow that jumps from group 0 to group N.\n+ *\n+ * If N does not exist, we allocate an exact match table for it, using\n+ * a dummy key. A key is required for the table.\n+ *\n+ * 2. Add a flow that uses group N.\n+ *\n+ * If N does not exist, we allocate an exact match table for it, using\n+ * the flow's key. Subsequent flows to the same group all should have\n+ * the same key.\n+ *\n+ * Without a jump flow to N, N is not reachable in hardware. No packets\n+ * reach N and match.\n+ *\n+ * 3. Add a flow to an empty group N.\n+ *\n+ * N has been created via (1) and the dummy key. We free that table, allocate\n+ * a new table using the new flow's key. Also re-do the existing jump flow to\n+ * point to the new table.\n+ */\n+#define FM_TCAM_RTE_GROUP 0\n+\n+struct enic_fm_fet {\n+\tTAILQ_ENTRY(enic_fm_fet) list;\n+\tuint32_t group; /* rte_flow group ID */\n+\tuint64_t handle; /* Exact match table handle from flowman */\n+\tuint8_t ingress;\n+\tuint8_t default_key;\n+\tint ref; /* Reference count via get/put */\n+\tstruct fm_key_template key; /* Key associated with the table */\n+};\n+\n+struct enic_fm_counter {\n+\tSLIST_ENTRY(enic_fm_counter) next;\n+\tuint32_t handle;\n+};\n+\n+/* rte_flow.fm */\n+struct enic_fm_flow {\n+\tbool counter_valid;\n+\tuint64_t entry_handle;\n+\tuint64_t action_handle;\n+\tstruct enic_fm_counter *counter;\n+\tstruct enic_fm_fet *fet;\n+};\n+\n+struct enic_fm_jump_flow {\n+\tTAILQ_ENTRY(enic_fm_jump_flow) list;\n+\tstruct rte_flow *flow;\n+\tuint32_t group;\n+\tstruct fm_tcam_match_entry match;\n+\tstruct fm_action action;\n+};\n+\n+/*\n+ * Flowman uses host memory for commands. This structure is allocated\n+ * in DMA-able memory.\n+ */\n+union enic_flowman_cmd_mem {\n+\tstruct fm_tcam_match_table fm_tcam_match_table;\n+\tstruct fm_exact_match_table fm_exact_match_table;\n+\tstruct fm_tcam_match_entry fm_tcam_match_entry;\n+\tstruct fm_exact_match_entry fm_exact_match_entry;\n+\tstruct fm_action fm_action;\n+};\n+\n+struct enic_flowman {\n+\tstruct enic *enic;\n+\t/* Command buffer */\n+\tstruct {\n+\t\tunion enic_flowman_cmd_mem *va;\n+\t\tdma_addr_t pa;\n+\t} cmd;\n+\t/* TCAM tables allocated upfront, used for group 0 */\n+\tuint64_t ig_tcam_hndl;\n+\tuint64_t eg_tcam_hndl;\n+\t/* Counters */\n+\tSLIST_HEAD(enic_free_counters, enic_fm_counter) counters;\n+\tvoid *counter_stack;\n+\tuint32_t counters_alloced;\n+\t/* Exact match tables for groups != 0, dynamically allocated */\n+\tTAILQ_HEAD(fet_list, enic_fm_fet) fet_list;\n+\t/*\n+\t * Default exact match tables used for jump actions to\n+\t * non-existent groups.\n+\t */\n+\tstruct enic_fm_fet *default_eg_fet;\n+\tstruct enic_fm_fet *default_ig_fet;\n+\t/* Flows that jump to the default table above */\n+\tTAILQ_HEAD(jump_flow_list, enic_fm_jump_flow) jump_list;\n+\t/*\n+\t * Scratch data used during each invocation of flow_create\n+\t * and flow_validate.\n+\t */\n+\tstruct enic_fm_fet *fet;\n+\tstruct fm_tcam_match_entry tcam_entry;\n+\tstruct fm_action action;\n+\tstruct fm_action action_tmp; /* enic_fm_reorder_action_op */\n+\tint action_op_count;\n+};\n+\n+static int enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle);\n+\n+/*\n+ * Common arguments passed to copy_item functions. Use this structure\n+ * so we can easily add new arguments.\n+ * item: Item specification.\n+ * fm_tcam_entry: Flowman TCAM match entry.\n+ * header_level: 0 for outer header, 1 for inner header.\n+ */\n+struct copy_item_args {\n+\tconst struct rte_flow_item *item;\n+\tstruct fm_tcam_match_entry *fm_tcam_entry;\n+\tuint8_t header_level;\n+};\n+\n+/* functions for copying items into flowman match */\n+typedef int (enic_copy_item_fn)(struct copy_item_args *arg);\n+\n+/* Info about how to copy items into flowman match */\n+struct enic_fm_items {\n+\t/* Function for copying and validating an item. */\n+\tenic_copy_item_fn * const copy_item;\n+\t/* List of valid previous items. */\n+\tconst enum rte_flow_item_type * const prev_items;\n+\t/*\n+\t * True if it's OK for this item to be the first item. For some NIC\n+\t * versions, it's invalid to start the stack above layer 3.\n+\t */\n+\tconst uint8_t valid_start_item;\n+};\n+\n+static enic_copy_item_fn enic_fm_copy_item_eth;\n+static enic_copy_item_fn enic_fm_copy_item_ipv4;\n+static enic_copy_item_fn enic_fm_copy_item_ipv6;\n+static enic_copy_item_fn enic_fm_copy_item_raw;\n+static enic_copy_item_fn enic_fm_copy_item_sctp;\n+static enic_copy_item_fn enic_fm_copy_item_tcp;\n+static enic_copy_item_fn enic_fm_copy_item_udp;\n+static enic_copy_item_fn enic_fm_copy_item_vlan;\n+static enic_copy_item_fn enic_fm_copy_item_vxlan;\n+\n+/* Ingress actions */\n+static const enum rte_flow_action_type enic_fm_supported_ig_actions[] = {\n+\tRTE_FLOW_ACTION_TYPE_COUNT,\n+\tRTE_FLOW_ACTION_TYPE_DROP,\n+\tRTE_FLOW_ACTION_TYPE_FLAG,\n+\tRTE_FLOW_ACTION_TYPE_JUMP,\n+\tRTE_FLOW_ACTION_TYPE_MARK,\n+\tRTE_FLOW_ACTION_TYPE_PORT_ID,\n+\tRTE_FLOW_ACTION_TYPE_PASSTHRU,\n+\tRTE_FLOW_ACTION_TYPE_QUEUE,\n+\tRTE_FLOW_ACTION_TYPE_RSS,\n+\tRTE_FLOW_ACTION_TYPE_VOID,\n+\tRTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,\n+\tRTE_FLOW_ACTION_TYPE_VXLAN_DECAP,\n+\tRTE_FLOW_ACTION_TYPE_END, /* END must be the last entry */\n+};\n+\n+/* Egress actions */\n+static const enum rte_flow_action_type enic_fm_supported_eg_actions[] = {\n+\tRTE_FLOW_ACTION_TYPE_COUNT,\n+\tRTE_FLOW_ACTION_TYPE_DROP,\n+\tRTE_FLOW_ACTION_TYPE_JUMP,\n+\tRTE_FLOW_ACTION_TYPE_PASSTHRU,\n+\tRTE_FLOW_ACTION_TYPE_VOID,\n+\tRTE_FLOW_ACTION_TYPE_VXLAN_ENCAP,\n+\tRTE_FLOW_ACTION_TYPE_END,\n+};\n+\n+static const struct enic_fm_items enic_fm_items[] = {\n+\t[RTE_FLOW_ITEM_TYPE_RAW] = {\n+\t\t.copy_item = enic_fm_copy_item_raw,\n+\t\t.valid_start_item = 0,\n+\t\t.prev_items = (const enum rte_flow_item_type[]) {\n+\t\t\t RTE_FLOW_ITEM_TYPE_UDP,\n+\t\t\t RTE_FLOW_ITEM_TYPE_END,\n+\t\t},\n+\t},\n+\t[RTE_FLOW_ITEM_TYPE_ETH] = {\n+\t\t.copy_item = enic_fm_copy_item_eth,\n+\t\t.valid_start_item = 1,\n+\t\t.prev_items = (const enum rte_flow_item_type[]) {\n+\t\t\t RTE_FLOW_ITEM_TYPE_END,\n+\t\t},\n+\t},\n+\t[RTE_FLOW_ITEM_TYPE_VLAN] = {\n+\t\t.copy_item = enic_fm_copy_item_vlan,\n+\t\t.valid_start_item = 1,\n+\t\t.prev_items = (const enum rte_flow_item_type[]) {\n+\t\t\t RTE_FLOW_ITEM_TYPE_ETH,\n+\t\t\t RTE_FLOW_ITEM_TYPE_END,\n+\t\t},\n+\t},\n+\t[RTE_FLOW_ITEM_TYPE_IPV4] = {\n+\t\t.copy_item = enic_fm_copy_item_ipv4,\n+\t\t.valid_start_item = 1,\n+\t\t.prev_items = (const enum rte_flow_item_type[]) {\n+\t\t\t RTE_FLOW_ITEM_TYPE_ETH,\n+\t\t\t RTE_FLOW_ITEM_TYPE_VLAN,\n+\t\t\t RTE_FLOW_ITEM_TYPE_END,\n+\t\t},\n+\t},\n+\t[RTE_FLOW_ITEM_TYPE_IPV6] = {\n+\t\t.copy_item = enic_fm_copy_item_ipv6,\n+\t\t.valid_start_item = 1,\n+\t\t.prev_items = (const enum rte_flow_item_type[]) {\n+\t\t\t RTE_FLOW_ITEM_TYPE_ETH,\n+\t\t\t RTE_FLOW_ITEM_TYPE_VLAN,\n+\t\t\t RTE_FLOW_ITEM_TYPE_END,\n+\t\t},\n+\t},\n+\t[RTE_FLOW_ITEM_TYPE_UDP] = {\n+\t\t.copy_item = enic_fm_copy_item_udp,\n+\t\t.valid_start_item = 1,\n+\t\t.prev_items = (const enum rte_flow_item_type[]) {\n+\t\t\t RTE_FLOW_ITEM_TYPE_IPV4,\n+\t\t\t RTE_FLOW_ITEM_TYPE_IPV6,\n+\t\t\t RTE_FLOW_ITEM_TYPE_END,\n+\t\t},\n+\t},\n+\t[RTE_FLOW_ITEM_TYPE_TCP] = {\n+\t\t.copy_item = enic_fm_copy_item_tcp,\n+\t\t.valid_start_item = 1,\n+\t\t.prev_items = (const enum rte_flow_item_type[]) {\n+\t\t\t RTE_FLOW_ITEM_TYPE_IPV4,\n+\t\t\t RTE_FLOW_ITEM_TYPE_IPV6,\n+\t\t\t RTE_FLOW_ITEM_TYPE_END,\n+\t\t},\n+\t},\n+\t[RTE_FLOW_ITEM_TYPE_SCTP] = {\n+\t\t.copy_item = enic_fm_copy_item_sctp,\n+\t\t.valid_start_item = 0,\n+\t\t.prev_items = (const enum rte_flow_item_type[]) {\n+\t\t\t RTE_FLOW_ITEM_TYPE_IPV4,\n+\t\t\t RTE_FLOW_ITEM_TYPE_IPV6,\n+\t\t\t RTE_FLOW_ITEM_TYPE_END,\n+\t\t},\n+\t},\n+\t[RTE_FLOW_ITEM_TYPE_VXLAN] = {\n+\t\t.copy_item = enic_fm_copy_item_vxlan,\n+\t\t.valid_start_item = 1,\n+\t\t.prev_items = (const enum rte_flow_item_type[]) {\n+\t\t\t RTE_FLOW_ITEM_TYPE_UDP,\n+\t\t\t RTE_FLOW_ITEM_TYPE_END,\n+\t\t},\n+\t},\n+};\n+\n+static int\n+enic_fm_copy_item_eth(struct copy_item_args *arg)\n+{\n+\tconst struct rte_flow_item *item = arg->item;\n+\tconst struct rte_flow_item_eth *spec = item->spec;\n+\tconst struct rte_flow_item_eth *mask = item->mask;\n+\tconst uint8_t lvl = arg->header_level;\n+\tstruct fm_tcam_match_entry *entry = arg->fm_tcam_entry;\n+\tstruct fm_header_set *fm_data, *fm_mask;\n+\n+\tENICPMD_FUNC_TRACE();\n+\t/* Match all if no spec */\n+\tif (!spec)\n+\t\treturn 0;\n+\tif (!mask)\n+\t\tmask = &rte_flow_item_eth_mask;\n+\tfm_data = &entry->ftm_data.fk_hdrset[lvl];\n+\tfm_mask = &entry->ftm_mask.fk_hdrset[lvl];\n+\tfm_data->fk_header_select |= FKH_ETHER;\n+\tfm_mask->fk_header_select |= FKH_ETHER;\n+\tmemcpy(&fm_data->l2.eth, spec, sizeof(*spec));\n+\tmemcpy(&fm_mask->l2.eth, mask, sizeof(*mask));\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_copy_item_vlan(struct copy_item_args *arg)\n+{\n+\tconst struct rte_flow_item *item = arg->item;\n+\tconst struct rte_flow_item_vlan *spec = item->spec;\n+\tconst struct rte_flow_item_vlan *mask = item->mask;\n+\tconst uint8_t lvl = arg->header_level;\n+\tstruct fm_tcam_match_entry *entry = arg->fm_tcam_entry;\n+\tstruct fm_header_set *fm_data, *fm_mask;\n+\tstruct rte_ether_hdr *eth_mask;\n+\tstruct rte_ether_hdr *eth_val;\n+\tuint32_t meta;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tfm_data = &entry->ftm_data.fk_hdrset[lvl];\n+\tfm_mask = &entry->ftm_mask.fk_hdrset[lvl];\n+\t/* Outer and inner packet vlans need different flags */\n+\tmeta = FKM_VLAN_PRES;\n+\tif (lvl > 0)\n+\t\tmeta = FKM_QTAG;\n+\tfm_data->fk_metadata |= meta;\n+\tfm_mask->fk_metadata |= meta;\n+\n+\t/* Match all if no spec */\n+\tif (!spec)\n+\t\treturn 0;\n+\tif (!mask)\n+\t\tmask = &rte_flow_item_vlan_mask;\n+\n+\teth_mask = (void *)&fm_mask->l2.eth;\n+\teth_val = (void *)&fm_data->l2.eth;\n+\n+\t/* Outer TPID cannot be matched */\n+\tif (eth_mask->ether_type)\n+\t\treturn -ENOTSUP;\n+\n+\t/*\n+\t * When packet matching, the VIC always compares vlan-stripped\n+\t * L2, regardless of vlan stripping settings. So, the inner type\n+\t * from vlan becomes the ether type of the eth header.\n+\t */\n+\teth_mask->ether_type = mask->inner_type;\n+\teth_val->ether_type = spec->inner_type;\n+\tfm_data->fk_header_select |= FKH_ETHER | FKH_QTAG;\n+\tfm_mask->fk_header_select |= FKH_ETHER | FKH_QTAG;\n+\tfm_data->fk_vlan = rte_be_to_cpu_16(spec->tci);\n+\tfm_mask->fk_vlan = rte_be_to_cpu_16(mask->tci);\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_copy_item_ipv4(struct copy_item_args *arg)\n+{\n+\tconst struct rte_flow_item *item = arg->item;\n+\tconst struct rte_flow_item_ipv4 *spec = item->spec;\n+\tconst struct rte_flow_item_ipv4 *mask = item->mask;\n+\tconst uint8_t lvl = arg->header_level;\n+\tstruct fm_tcam_match_entry *entry = arg->fm_tcam_entry;\n+\tstruct fm_header_set *fm_data, *fm_mask;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tfm_data = &entry->ftm_data.fk_hdrset[lvl];\n+\tfm_mask = &entry->ftm_mask.fk_hdrset[lvl];\n+\tfm_data->fk_metadata |= FKM_IPV4;\n+\tfm_mask->fk_metadata |= FKM_IPV4;\n+\n+\tif (!spec)\n+\t\treturn 0;\n+\tif (!mask)\n+\t\tmask = &rte_flow_item_ipv4_mask;\n+\n+\tfm_data->fk_header_select |= FKH_IPV4;\n+\tfm_mask->fk_header_select |= FKH_IPV4;\n+\tmemcpy(&fm_data->l3.ip4, spec, sizeof(*spec));\n+\tmemcpy(&fm_mask->l3.ip4, mask, sizeof(*mask));\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_copy_item_ipv6(struct copy_item_args *arg)\n+{\n+\tconst struct rte_flow_item *item = arg->item;\n+\tconst struct rte_flow_item_ipv6 *spec = item->spec;\n+\tconst struct rte_flow_item_ipv6 *mask = item->mask;\n+\tconst uint8_t lvl = arg->header_level;\n+\tstruct fm_tcam_match_entry *entry = arg->fm_tcam_entry;\n+\tstruct fm_header_set *fm_data, *fm_mask;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tfm_data = &entry->ftm_data.fk_hdrset[lvl];\n+\tfm_mask = &entry->ftm_mask.fk_hdrset[lvl];\n+\tfm_data->fk_metadata |= FKM_IPV6;\n+\tfm_mask->fk_metadata |= FKM_IPV6;\n+\n+\tif (!spec)\n+\t\treturn 0;\n+\tif (!mask)\n+\t\tmask = &rte_flow_item_ipv6_mask;\n+\n+\tfm_data->fk_header_select |= FKH_IPV6;\n+\tfm_mask->fk_header_select |= FKH_IPV6;\n+\tmemcpy(&fm_data->l3.ip6, spec, sizeof(*spec));\n+\tmemcpy(&fm_mask->l3.ip6, mask, sizeof(*mask));\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_copy_item_udp(struct copy_item_args *arg)\n+{\n+\tconst struct rte_flow_item *item = arg->item;\n+\tconst struct rte_flow_item_udp *spec = item->spec;\n+\tconst struct rte_flow_item_udp *mask = item->mask;\n+\tconst uint8_t lvl = arg->header_level;\n+\tstruct fm_tcam_match_entry *entry = arg->fm_tcam_entry;\n+\tstruct fm_header_set *fm_data, *fm_mask;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tfm_data = &entry->ftm_data.fk_hdrset[lvl];\n+\tfm_mask = &entry->ftm_mask.fk_hdrset[lvl];\n+\tfm_data->fk_metadata |= FKM_UDP;\n+\tfm_mask->fk_metadata |= FKM_UDP;\n+\n+\tif (!spec)\n+\t\treturn 0;\n+\tif (!mask)\n+\t\tmask = &rte_flow_item_udp_mask;\n+\n+\tfm_data->fk_header_select |= FKH_UDP;\n+\tfm_mask->fk_header_select |= FKH_UDP;\n+\tmemcpy(&fm_data->l4.udp, spec, sizeof(*spec));\n+\tmemcpy(&fm_mask->l4.udp, mask, sizeof(*mask));\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_copy_item_tcp(struct copy_item_args *arg)\n+{\n+\tconst struct rte_flow_item *item = arg->item;\n+\tconst struct rte_flow_item_tcp *spec = item->spec;\n+\tconst struct rte_flow_item_tcp *mask = item->mask;\n+\tconst uint8_t lvl = arg->header_level;\n+\tstruct fm_tcam_match_entry *entry = arg->fm_tcam_entry;\n+\tstruct fm_header_set *fm_data, *fm_mask;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tfm_data = &entry->ftm_data.fk_hdrset[lvl];\n+\tfm_mask = &entry->ftm_mask.fk_hdrset[lvl];\n+\tfm_data->fk_metadata |= FKM_TCP;\n+\tfm_mask->fk_metadata |= FKM_TCP;\n+\n+\tif (!spec)\n+\t\treturn 0;\n+\tif (!mask)\n+\t\tmask = &rte_flow_item_tcp_mask;\n+\n+\tfm_data->fk_header_select |= FKH_TCP;\n+\tfm_mask->fk_header_select |= FKH_TCP;\n+\tmemcpy(&fm_data->l4.tcp, spec, sizeof(*spec));\n+\tmemcpy(&fm_mask->l4.tcp, mask, sizeof(*mask));\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_copy_item_sctp(struct copy_item_args *arg)\n+{\n+\tconst struct rte_flow_item *item = arg->item;\n+\tconst struct rte_flow_item_sctp *spec = item->spec;\n+\tconst struct rte_flow_item_sctp *mask = item->mask;\n+\tconst uint8_t lvl = arg->header_level;\n+\tstruct fm_tcam_match_entry *entry = arg->fm_tcam_entry;\n+\tstruct fm_header_set *fm_data, *fm_mask;\n+\tuint8_t *ip_proto_mask = NULL;\n+\tuint8_t *ip_proto = NULL;\n+\tuint32_t l3_fkh;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tfm_data = &entry->ftm_data.fk_hdrset[lvl];\n+\tfm_mask = &entry->ftm_mask.fk_hdrset[lvl];\n+\t/*\n+\t * The NIC filter API has no flags for \"match sctp\", so explicitly\n+\t * set the protocol number in the IP pattern.\n+\t */\n+\tif (fm_data->fk_metadata & FKM_IPV4) {\n+\t\tstruct rte_ipv4_hdr *ip;\n+\t\tip = (struct rte_ipv4_hdr *)&fm_mask->l3.ip4;\n+\t\tip_proto_mask = &ip->next_proto_id;\n+\t\tip = (struct rte_ipv4_hdr *)&fm_data->l3.ip4;\n+\t\tip_proto = &ip->next_proto_id;\n+\t\tl3_fkh = FKH_IPV4;\n+\t} else if (fm_data->fk_metadata & FKM_IPV6) {\n+\t\tstruct rte_ipv6_hdr *ip;\n+\t\tip = (struct rte_ipv6_hdr *)&fm_mask->l3.ip6;\n+\t\tip_proto_mask = &ip->proto;\n+\t\tip = (struct rte_ipv6_hdr *)&fm_data->l3.ip6;\n+\t\tip_proto = &ip->proto;\n+\t\tl3_fkh = FKH_IPV6;\n+\t} else {\n+\t\t/* Need IPv4/IPv6 pattern first */\n+\t\treturn -EINVAL;\n+\t}\n+\t*ip_proto = IPPROTO_SCTP;\n+\t*ip_proto_mask = 0xff;\n+\tfm_data->fk_header_select |= l3_fkh;\n+\tfm_mask->fk_header_select |= l3_fkh;\n+\n+\tif (!spec)\n+\t\treturn 0;\n+\tif (!mask)\n+\t\tmask = &rte_flow_item_sctp_mask;\n+\n+\tfm_data->fk_header_select |= FKH_L4RAW;\n+\tfm_mask->fk_header_select |= FKH_L4RAW;\n+\tmemcpy(fm_data->l4.rawdata, spec, sizeof(*spec));\n+\tmemcpy(fm_mask->l4.rawdata, mask, sizeof(*mask));\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_copy_item_vxlan(struct copy_item_args *arg)\n+{\n+\tconst struct rte_flow_item *item = arg->item;\n+\tconst struct rte_flow_item_vxlan *spec = item->spec;\n+\tconst struct rte_flow_item_vxlan *mask = item->mask;\n+\tstruct fm_tcam_match_entry *entry = arg->fm_tcam_entry;\n+\tstruct fm_header_set *fm_data, *fm_mask;\n+\n+\tENICPMD_FUNC_TRACE();\n+\t/* Only 2 header levels (outer and inner) allowed */\n+\tif (arg->header_level > 0)\n+\t\treturn -EINVAL;\n+\n+\tfm_data = &entry->ftm_data.fk_hdrset[0];\n+\tfm_mask = &entry->ftm_mask.fk_hdrset[0];\n+\tfm_data->fk_metadata |= FKM_VXLAN;\n+\tfm_mask->fk_metadata |= FKM_VXLAN;\n+\t/* items from here on out are inner header items */\n+\targ->header_level = 1;\n+\n+\t/* Match all if no spec */\n+\tif (!spec)\n+\t\treturn 0;\n+\tif (!mask)\n+\t\tmask = &rte_flow_item_vxlan_mask;\n+\n+\tfm_data->fk_header_select |= FKH_VXLAN;\n+\tfm_mask->fk_header_select |= FKH_VXLAN;\n+\tmemcpy(&fm_data->vxlan, spec, sizeof(*spec));\n+\tmemcpy(&fm_mask->vxlan, mask, sizeof(*mask));\n+\treturn 0;\n+}\n+\n+/*\n+ * Currently, raw pattern match is very limited. It is intended for matching\n+ * UDP tunnel header (e.g. vxlan or geneve).\n+ */\n+static int\n+enic_fm_copy_item_raw(struct copy_item_args *arg)\n+{\n+\tconst struct rte_flow_item *item = arg->item;\n+\tconst struct rte_flow_item_raw *spec = item->spec;\n+\tconst struct rte_flow_item_raw *mask = item->mask;\n+\tconst uint8_t lvl = arg->header_level;\n+\tstruct fm_tcam_match_entry *entry = arg->fm_tcam_entry;\n+\tstruct fm_header_set *fm_data, *fm_mask;\n+\n+\tENICPMD_FUNC_TRACE();\n+\t/* Cannot be used for inner packet */\n+\tif (lvl > 0)\n+\t\treturn -EINVAL;\n+\t/* Need both spec and mask */\n+\tif (!spec || !mask)\n+\t\treturn -EINVAL;\n+\t/* Only supports relative with offset 0 */\n+\tif (!spec->relative || spec->offset != 0 || spec->search ||\n+\t spec->limit)\n+\t\treturn -EINVAL;\n+\t/* Need non-null pattern that fits within the NIC's filter pattern */\n+\tif (spec->length == 0 ||\n+\t spec->length + sizeof(struct rte_udp_hdr) > FM_LAYER_SIZE ||\n+\t !spec->pattern || !mask->pattern)\n+\t\treturn -EINVAL;\n+\t/*\n+\t * Mask fields, including length, are often set to zero. Assume that\n+\t * means \"same as spec\" to avoid breaking existing apps. If length\n+\t * is not zero, then it should be >= spec length.\n+\t *\n+\t * No more pattern follows this, so append to the L4 layer instead of\n+\t * L5 to work with both recent and older VICs.\n+\t */\n+\tif (mask->length != 0 && mask->length < spec->length)\n+\t\treturn -EINVAL;\n+\n+\tfm_data = &entry->ftm_data.fk_hdrset[lvl];\n+\tfm_mask = &entry->ftm_mask.fk_hdrset[lvl];\n+\tfm_data->fk_header_select |= FKH_L4RAW;\n+\tfm_mask->fk_header_select |= FKH_L4RAW;\n+\tfm_data->fk_header_select &= ~FKH_UDP;\n+\tfm_mask->fk_header_select &= ~FKH_UDP;\n+\tmemcpy(fm_data->l4.rawdata + sizeof(struct rte_udp_hdr),\n+\t spec->pattern, spec->length);\n+\tmemcpy(fm_mask->l4.rawdata + sizeof(struct rte_udp_hdr),\n+\t mask->pattern, spec->length);\n+\treturn 0;\n+}\n+\n+static int\n+enic_fet_alloc(struct enic_flowman *fm, uint8_t ingress,\n+\t struct fm_key_template *key, int entries,\n+\t struct enic_fm_fet **fet_out)\n+{\n+\tstruct fm_exact_match_table *cmd;\n+\tstruct fm_header_set *hdr;\n+\tstruct enic_fm_fet *fet;\n+\tu64 args[3];\n+\tint ret;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tfet = calloc(1, sizeof(struct enic_fm_fet));\n+\tif (fet == NULL)\n+\t\treturn -ENOMEM;\n+\tcmd = &fm->cmd.va->fm_exact_match_table;\n+\tmemset(cmd, 0, sizeof(*cmd));\n+\tcmd->fet_direction = ingress ? FM_INGRESS : FM_EGRESS;\n+\tcmd->fet_stage = FM_STAGE_LAST;\n+\tcmd->fet_max_entries = entries ? entries : FM_MAX_EXACT_TABLE_SIZE;\n+\tif (key == NULL) {\n+\t\thdr = &cmd->fet_key.fk_hdrset[0];\n+\t\tmemset(hdr, 0, sizeof(*hdr));\n+\t\thdr->fk_header_select = FKH_IPV4 | FKH_UDP;\n+\t\thdr->l3.ip4.fk_saddr = 0xFFFFFFFF;\n+\t\thdr->l3.ip4.fk_daddr = 0xFFFFFFFF;\n+\t\thdr->l4.udp.fk_source = 0xFFFF;\n+\t\thdr->l4.udp.fk_dest = 0xFFFF;\n+\t\tfet->default_key = 1;\n+\t} else {\n+\t\tmemcpy(&cmd->fet_key, key, sizeof(*key));\n+\t\tmemcpy(&fet->key, key, sizeof(*key));\n+\t\tfet->default_key = 0;\n+\t}\n+\tcmd->fet_key.fk_packet_tag = 1;\n+\n+\targs[0] = FM_EXACT_TABLE_ALLOC;\n+\targs[1] = fm->cmd.pa;\n+\tret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);\n+\tif (ret) {\n+\t\tENICPMD_LOG(ERR, \"cannot alloc exact match table: rc=%d\", ret);\n+\t\tfree(fet);\n+\t\treturn ret;\n+\t}\n+\tfet->handle = args[0];\n+\tfet->ingress = ingress;\n+\tENICPMD_LOG(DEBUG, \"allocated exact match table: handle=0x%\" PRIx64,\n+\t\t fet->handle);\n+\t*fet_out = fet;\n+\treturn 0;\n+}\n+\n+static void\n+enic_fet_free(struct enic_flowman *fm, struct enic_fm_fet *fet)\n+{\n+\tENICPMD_FUNC_TRACE();\n+\tenic_fm_tbl_free(fm, fet->handle);\n+\tif (!fet->default_key)\n+\t\tTAILQ_REMOVE(&fm->fet_list, fet, list);\n+\tfree(fet);\n+}\n+\n+/*\n+ * Get the exact match table for the given combination of\n+ * <group, ingress, key>. Allocate one on the fly as necessary.\n+ */\n+static int\n+enic_fet_get(struct enic_flowman *fm,\n+\t uint32_t group,\n+\t uint8_t ingress,\n+\t struct fm_key_template *key,\n+\t struct enic_fm_fet **fet_out,\n+\t struct rte_flow_error *error)\n+{\n+\tstruct enic_fm_fet *fet;\n+\n+\tENICPMD_FUNC_TRACE();\n+\t/* See if we already have this table open */\n+\tTAILQ_FOREACH(fet, &fm->fet_list, list) {\n+\t\tif (fet->group == group && fet->ingress == ingress)\n+\t\t\tbreak;\n+\t}\n+\tif (fet == NULL) {\n+\t\t/* Jumping to a non-existing group? Use the default table */\n+\t\tif (key == NULL) {\n+\t\t\tfet = ingress ? fm->default_ig_fet : fm->default_eg_fet;\n+\t\t} else if (enic_fet_alloc(fm, ingress, key, 0, &fet)) {\n+\t\t\treturn rte_flow_error_set(error, EINVAL,\n+\t\t\t\tRTE_FLOW_ERROR_TYPE_UNSPECIFIED,\n+\t\t\t\tNULL, \"enic: cannot get exact match table\");\n+\t\t}\n+\t\tfet->group = group;\n+\t\t/* Default table is never on the open table list */\n+\t\tif (!fet->default_key)\n+\t\t\tTAILQ_INSERT_HEAD(&fm->fet_list, fet, list);\n+\t}\n+\tfet->ref++;\n+\t*fet_out = fet;\n+\tENICPMD_LOG(DEBUG, \"fet_get: %s %s group=%u ref=%u\",\n+\t\t fet->default_key ? \"default\" : \"\",\n+\t\t fet->ingress ? \"ingress\" : \"egress\",\n+\t\t fet->group, fet->ref);\n+\treturn 0;\n+}\n+\n+static void\n+enic_fet_put(struct enic_flowman *fm, struct enic_fm_fet *fet)\n+{\n+\tENICPMD_FUNC_TRACE();\n+\tRTE_ASSERT(fet->ref > 0);\n+\tfet->ref--;\n+\tENICPMD_LOG(DEBUG, \"fet_put: %s %s group=%u ref=%u\",\n+\t\t fet->default_key ? \"default\" : \"\",\n+\t\t fet->ingress ? \"ingress\" : \"egress\",\n+\t\t fet->group, fet->ref);\n+\tif (fet->ref == 0)\n+\t\tenic_fet_free(fm, fet);\n+}\n+\n+/* Return 1 if current item is valid on top of the previous one. */\n+static int\n+fm_item_stacking_valid(enum rte_flow_item_type prev_item,\n+\t\t const struct enic_fm_items *item_info,\n+\t\t uint8_t is_first_item)\n+{\n+\tenum rte_flow_item_type const *allowed_items = item_info->prev_items;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tfor (; *allowed_items != RTE_FLOW_ITEM_TYPE_END; allowed_items++) {\n+\t\tif (prev_item == *allowed_items)\n+\t\t\treturn 1;\n+\t}\n+\n+\t/* This is the first item in the stack. Check if that's cool */\n+\tif (is_first_item && item_info->valid_start_item)\n+\t\treturn 1;\n+\treturn 0;\n+}\n+\n+/*\n+ * Build the flow manager match entry structure from the provided pattern.\n+ * The pattern is validated as the items are copied.\n+ */\n+static int\n+enic_fm_copy_entry(struct enic_flowman *fm,\n+\t\t const struct rte_flow_item pattern[],\n+\t\t struct rte_flow_error *error)\n+{\n+\tconst struct enic_fm_items *item_info;\n+\tenum rte_flow_item_type prev_item;\n+\tconst struct rte_flow_item *item;\n+\tstruct copy_item_args args;\n+\tuint8_t prev_header_level;\n+\tuint8_t is_first_item;\n+\tint ret;\n+\n+\tENICPMD_FUNC_TRACE();\n+\titem = pattern;\n+\tis_first_item = 1;\n+\tprev_item = RTE_FLOW_ITEM_TYPE_END;\n+\n+\targs.fm_tcam_entry = &fm->tcam_entry;\n+\targs.header_level = 0;\n+\tprev_header_level = 0;\n+\tfor (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {\n+\t\t/*\n+\t\t * Get info about how to validate and copy the item. If NULL\n+\t\t * is returned the nic does not support the item.\n+\t\t */\n+\t\tif (item->type == RTE_FLOW_ITEM_TYPE_VOID)\n+\t\t\tcontinue;\n+\n+\t\titem_info = &enic_fm_items[item->type];\n+\n+\t\tif (item->type > FM_MAX_ITEM_TYPE ||\n+\t\t item_info->copy_item == NULL) {\n+\t\t\treturn rte_flow_error_set(error, ENOTSUP,\n+\t\t\t\tRTE_FLOW_ERROR_TYPE_ITEM,\n+\t\t\t\tNULL, \"enic: unsupported item\");\n+\t\t}\n+\n+\t\t/* check to see if item stacking is valid */\n+\t\tif (!fm_item_stacking_valid(prev_item, item_info,\n+\t\t\t\t\t is_first_item))\n+\t\t\tgoto stacking_error;\n+\n+\t\targs.item = item;\n+\t\tret = item_info->copy_item(&args);\n+\t\tif (ret)\n+\t\t\tgoto item_not_supported;\n+\t\t/* Going from outer to inner? Treat it as a new packet start */\n+\t\tif (prev_header_level != args.header_level) {\n+\t\t\tprev_item = RTE_FLOW_ITEM_TYPE_END;\n+\t\t\tis_first_item = 1;\n+\t\t} else {\n+\t\t\tprev_item = item->type;\n+\t\t\tis_first_item = 0;\n+\t\t}\n+\t\tprev_header_level = args.header_level;\n+\t}\n+\treturn 0;\n+\n+item_not_supported:\n+\treturn rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM,\n+\t\t\t\t NULL, \"enic: unsupported item type\");\n+\n+stacking_error:\n+\treturn rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,\n+\t\t\t\t item, \"enic: unsupported item stack\");\n+}\n+\n+static void\n+flow_item_skip_void(const struct rte_flow_item **item)\n+{\n+\tfor ( ; ; (*item)++)\n+\t\tif ((*item)->type != RTE_FLOW_ITEM_TYPE_VOID)\n+\t\t\treturn;\n+}\n+\n+static void\n+append_template(void **template, uint8_t *off, const void *data, int len)\n+{\n+\tmemcpy(*template, data, len);\n+\t*template = (char *)*template + len;\n+\t*off = *off + len;\n+}\n+\n+static int\n+enic_fm_append_action_op(struct enic_flowman *fm,\n+\t\t\t struct fm_action_op *fm_op,\n+\t\t\t struct rte_flow_error *error)\n+{\n+\tint count;\n+\n+\tcount = fm->action_op_count;\n+\tENICPMD_LOG(DEBUG, \"append action op: idx=%d op=%u\",\n+\t\t count, fm_op->fa_op);\n+\tif (count == FM_ACTION_OP_MAX) {\n+\t\treturn rte_flow_error_set(error, EINVAL,\n+\t\t\tRTE_FLOW_ERROR_TYPE_ACTION, NULL,\n+\t\t\t\"too many action operations\");\n+\t}\n+\tfm->action.fma_action_ops[count] = *fm_op;\n+\tfm->action_op_count = count + 1;\n+\treturn 0;\n+}\n+\n+/* Steer operations need to appear before other ops */\n+static void\n+enic_fm_reorder_action_op(struct enic_flowman *fm)\n+{\n+\tstruct fm_action_op *dst, *dst_head, *src, *src_head;\n+\n+\tENICPMD_FUNC_TRACE();\n+\t/* Move steer ops to the front. */\n+\tsrc = fm->action.fma_action_ops;\n+\tsrc_head = src;\n+\tdst = fm->action_tmp.fma_action_ops;\n+\tdst_head = dst;\n+\t/* Copy steer ops to tmp */\n+\twhile (src->fa_op != FMOP_END) {\n+\t\tif (src->fa_op == FMOP_RQ_STEER) {\n+\t\t\tENICPMD_LOG(DEBUG, \"move op: %ld -> dst %ld\",\n+\t\t\t\t (long)(src - src_head),\n+\t\t\t\t (long)(dst - dst_head));\n+\t\t\t*dst = *src;\n+\t\t\tdst++;\n+\t\t}\n+\t\tsrc++;\n+\t}\n+\t/* Then append non-steer ops */\n+\tsrc = src_head;\n+\twhile (src->fa_op != FMOP_END) {\n+\t\tif (src->fa_op != FMOP_RQ_STEER) {\n+\t\t\tENICPMD_LOG(DEBUG, \"move op: %ld -> dst %ld\",\n+\t\t\t\t (long)(src - src_head),\n+\t\t\t\t (long)(dst - dst_head));\n+\t\t\t*dst = *src;\n+\t\t\tdst++;\n+\t\t}\n+\t\tsrc++;\n+\t}\n+\t/* Copy END */\n+\t*dst = *src;\n+\t/* Finally replace the original action with the reordered one */\n+\tmemcpy(fm->action.fma_action_ops, fm->action_tmp.fma_action_ops,\n+\t sizeof(fm->action.fma_action_ops));\n+}\n+\n+/* VXLAN decap is done via flowman compound action */\n+static int\n+enic_fm_copy_vxlan_decap(struct enic_flowman *fm,\n+\t\t\t struct fm_tcam_match_entry *fmt,\n+\t\t\t const struct rte_flow_action *action,\n+\t\t\t struct rte_flow_error *error)\n+{\n+\tstruct fm_header_set *fm_data;\n+\tstruct fm_action_op fm_op;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tfm_data = &fmt->ftm_data.fk_hdrset[0];\n+\tif (!(fm_data->fk_metadata & FKM_VXLAN)) {\n+\t\treturn rte_flow_error_set(error, EINVAL,\n+\t\t\tRTE_FLOW_ERROR_TYPE_ACTION, action,\n+\t\t\t\"vxlan-decap: vxlan must be in pattern\");\n+\t}\n+\n+\tmemset(&fm_op, 0, sizeof(fm_op));\n+\tfm_op.fa_op = FMOP_DECAP_NOSTRIP;\n+\treturn enic_fm_append_action_op(fm, &fm_op, error);\n+}\n+\n+/* VXLAN encap is done via flowman compound action */\n+static int\n+enic_fm_copy_vxlan_encap(struct enic_flowman *fm,\n+\t\t\t const struct rte_flow_item *item,\n+\t\t\t struct rte_flow_error *error)\n+{\n+\tstruct fm_action_op fm_op;\n+\tstruct rte_ether_hdr *eth;\n+\tuint16_t *ethertype;\n+\tvoid *template;\n+\tuint8_t off;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tmemset(&fm_op, 0, sizeof(fm_op));\n+\tfm_op.fa_op = FMOP_ENCAP;\n+\ttemplate = fm->action.fma_data;\n+\toff = 0;\n+\t/*\n+\t * Copy flow items to the flowman template starting L2.\n+\t * L2 must be ethernet.\n+\t */\n+\tflow_item_skip_void(&item);\n+\tif (item->type != RTE_FLOW_ITEM_TYPE_ETH)\n+\t\treturn rte_flow_error_set(error, EINVAL,\n+\t\t\tRTE_FLOW_ERROR_TYPE_ITEM, item,\n+\t\t\t\"vxlan-encap: first item should be ethernet\");\n+\teth = (struct rte_ether_hdr *)template;\n+\tethertype = ð->ether_type;\n+\tappend_template(&template, &off, item->spec,\n+\t\t\tsizeof(struct rte_flow_item_eth));\n+\titem++;\n+\tflow_item_skip_void(&item);\n+\t/* Optional VLAN */\n+\tif (item->type == RTE_FLOW_ITEM_TYPE_VLAN) {\n+\t\tconst struct rte_flow_item_vlan *spec;\n+\n+\t\tENICPMD_LOG(DEBUG, \"vxlan-encap: vlan\");\n+\t\tspec = item->spec;\n+\t\tfm_op.encap.outer_vlan = rte_be_to_cpu_16(spec->tci);\n+\t\titem++;\n+\t\tflow_item_skip_void(&item);\n+\t}\n+\t/* L3 must be IPv4, IPv6 */\n+\tswitch (item->type) {\n+\tcase RTE_FLOW_ITEM_TYPE_IPV4:\n+\t{\n+\t\tstruct rte_ipv4_hdr *ip4;\n+\n+\t\tENICPMD_LOG(DEBUG, \"vxlan-encap: ipv4\");\n+\t\t*ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);\n+\t\tip4 = (struct rte_ipv4_hdr *)template;\n+\t\t/*\n+\t\t * Offset of IPv4 length field and its initial value\n+\t\t * (IP + UDP + VXLAN) are specified in the action. The NIC\n+\t\t * will add inner packet length.\n+\t\t */\n+\t\tfm_op.encap.len1_offset = off +\n+\t\t\toffsetof(struct rte_ipv4_hdr, total_length);\n+\t\tfm_op.encap.len1_delta = sizeof(struct rte_ipv4_hdr) +\n+\t\t\tsizeof(struct rte_udp_hdr) +\n+\t\t\tsizeof(struct rte_vxlan_hdr);\n+\t\tappend_template(&template, &off, item->spec,\n+\t\t\t\tsizeof(struct rte_ipv4_hdr));\n+\t\tip4->version_ihl = IP_VHL_DEF;\n+\t\tif (ip4->time_to_live == 0)\n+\t\t\tip4->time_to_live = IP_DEFTTL;\n+\t\tip4->next_proto_id = IPPROTO_UDP;\n+\t\tbreak;\n+\t}\n+\tcase RTE_FLOW_ITEM_TYPE_IPV6:\n+\t{\n+\t\tstruct rte_ipv6_hdr *ip6;\n+\n+\t\tENICPMD_LOG(DEBUG, \"vxlan-encap: ipv6\");\n+\t\t*ethertype = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);\n+\t\tip6 = (struct rte_ipv6_hdr *)template;\n+\t\tfm_op.encap.len1_offset = off +\n+\t\t\toffsetof(struct rte_ipv6_hdr, payload_len);\n+\t\tfm_op.encap.len1_delta = sizeof(struct rte_udp_hdr) +\n+\t\t\tsizeof(struct rte_vxlan_hdr);\n+\t\tappend_template(&template, &off, item->spec,\n+\t\t\t\tsizeof(struct rte_ipv6_hdr));\n+\t\tip6->vtc_flow |= rte_cpu_to_be_32(IP6_VTC_FLOW);\n+\t\tif (ip6->hop_limits == 0)\n+\t\t\tip6->hop_limits = IP_DEFTTL;\n+\t\tip6->proto = IPPROTO_UDP;\n+\t\tbreak;\n+\t}\n+\tdefault:\n+\t\treturn rte_flow_error_set(error,\n+\t\t\tEINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,\n+\t\t\t\"vxlan-encap: L3 must be IPv4/IPv6\");\n+\t}\n+\titem++;\n+\tflow_item_skip_void(&item);\n+\n+\t/* L4 is UDP */\n+\tif (item->type != RTE_FLOW_ITEM_TYPE_UDP)\n+\t\treturn rte_flow_error_set(error, EINVAL,\n+\t\t\tRTE_FLOW_ERROR_TYPE_ITEM, item,\n+\t\t\t\"vxlan-encap: UDP must follow IPv4/IPv6\");\n+\t/* UDP length = UDP + VXLAN. NIC will add inner packet length. */\n+\tfm_op.encap.len2_offset =\n+\t\toff + offsetof(struct rte_udp_hdr, dgram_len);\n+\tfm_op.encap.len2_delta =\n+\t\tsizeof(struct rte_udp_hdr) + sizeof(struct rte_vxlan_hdr);\n+\tappend_template(&template, &off, item->spec,\n+\t\t\tsizeof(struct rte_udp_hdr));\n+\titem++;\n+\tflow_item_skip_void(&item);\n+\n+\t/* Finally VXLAN */\n+\tif (item->type != RTE_FLOW_ITEM_TYPE_VXLAN)\n+\t\treturn rte_flow_error_set(error,\n+\t\t\tEINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item,\n+\t\t\t\"vxlan-encap: VXLAN must follow UDP\");\n+\tappend_template(&template, &off, item->spec,\n+\t\t\tsizeof(struct rte_flow_item_vxlan));\n+\n+\t/*\n+\t * Fill in the rest of the action structure.\n+\t * Indicate that we want to encap with vxlan at packet start.\n+\t */\n+\tfm_op.encap.template_offset = 0;\n+\tfm_op.encap.template_len = off;\n+\treturn enic_fm_append_action_op(fm, &fm_op, error);\n+}\n+\n+static int\n+enic_fm_find_vnic(struct enic *enic, const struct rte_pci_addr *addr,\n+\t\t uint64_t *handle)\n+{\n+\tuint32_t bdf;\n+\tu64 args[2];\n+\tint rc;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tENICPMD_LOG(DEBUG, \"bdf=%x:%x:%x\", addr->bus, addr->devid,\n+\t\t addr->function);\n+\tbdf = addr->bus << 8 | addr->devid << 3 | addr->function;\n+\targs[0] = FM_VNIC_FIND;\n+\targs[1] = bdf;\n+\trc = vnic_dev_flowman_cmd(enic->vdev, args, 2);\n+\tif (rc != 0) {\n+\t\tENICPMD_LOG(ERR, \"allocating counters rc=%d\", rc);\n+\t\treturn rc;\n+\t}\n+\t*handle = args[0];\n+\tENICPMD_LOG(DEBUG, \"found vnic: handle=0x%\" PRIx64, *handle);\n+\treturn 0;\n+}\n+\n+/* Translate flow actions to flowman TCAM entry actions */\n+static int\n+enic_fm_copy_action(struct enic_flowman *fm,\n+\t\t const struct rte_flow_action actions[],\n+\t\t uint8_t ingress,\n+\t\t struct rte_flow_error *error)\n+{\n+\tenum {\n+\t\tFATE = 1 << 0,\n+\t\tMARK = 1 << 1,\n+\t\tPASSTHRU = 1 << 2,\n+\t\tCOUNT = 1 << 3,\n+\t\tENCAP = 1 << 4,\n+\t};\n+\tstruct fm_tcam_match_entry *fmt;\n+\tstruct fm_action_op fm_op;\n+\tstruct enic *enic;\n+\tuint32_t overlap;\n+\tuint64_t vnic_h;\n+\tbool first_rq;\n+\tint ret;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tfmt = &fm->tcam_entry;\n+\tfirst_rq = true;\n+\tenic = fm->enic;\n+\toverlap = 0;\n+\tvnic_h = 0; /* 0 = current vNIC */\n+\tfor (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {\n+\t\tswitch (actions->type) {\n+\t\tcase RTE_FLOW_ACTION_TYPE_VOID:\n+\t\t\tcontinue;\n+\t\tcase RTE_FLOW_ACTION_TYPE_PASSTHRU: {\n+\t\t\tif (overlap & PASSTHRU)\n+\t\t\t\tgoto unsupported;\n+\t\t\toverlap |= PASSTHRU;\n+\t\t\tbreak;\n+\t\t}\n+\t\tcase RTE_FLOW_ACTION_TYPE_JUMP: {\n+\t\t\tconst struct rte_flow_action_jump *jump =\n+\t\t\t\tactions->conf;\n+\t\t\tstruct enic_fm_fet *fet;\n+\n+\t\t\tif (overlap & FATE)\n+\t\t\t\tgoto unsupported;\n+\t\t\tret = enic_fet_get(fm, jump->group, ingress, NULL,\n+\t\t\t\t\t &fet, error);\n+\t\t\tif (ret)\n+\t\t\t\treturn ret;\n+\t\t\toverlap |= FATE;\n+\t\t\tmemset(&fm_op, 0, sizeof(fm_op));\n+\t\t\tfm_op.fa_op = FMOP_EXACT_MATCH;\n+\t\t\tfm_op.exact.handle = fet->handle;\n+\t\t\tfm->fet = fet;\n+\t\t\tret = enic_fm_append_action_op(fm, &fm_op, error);\n+\t\t\tif (ret)\n+\t\t\t\treturn ret;\n+\t\t\tbreak;\n+\t\t}\n+\t\tcase RTE_FLOW_ACTION_TYPE_MARK: {\n+\t\t\tconst struct rte_flow_action_mark *mark =\n+\t\t\t\tactions->conf;\n+\n+\t\t\tif (overlap & MARK)\n+\t\t\t\tgoto unsupported;\n+\t\t\toverlap |= MARK;\n+\t\t\tif (mark->id >= ENIC_MAGIC_FILTER_ID - 1)\n+\t\t\t\treturn rte_flow_error_set(error, EINVAL,\n+\t\t\t\t\tRTE_FLOW_ERROR_TYPE_ACTION,\n+\t\t\t\t\tNULL, \"invalid mark id\");\n+\t\t\tmemset(&fm_op, 0, sizeof(fm_op));\n+\t\t\tfm_op.fa_op = FMOP_MARK;\n+\t\t\tfm_op.mark.mark = mark->id + 1;\n+\t\t\tret = enic_fm_append_action_op(fm, &fm_op, error);\n+\t\t\tif (ret)\n+\t\t\t\treturn ret;\n+\t\t\tbreak;\n+\t\t}\n+\t\tcase RTE_FLOW_ACTION_TYPE_FLAG: {\n+\t\t\tif (overlap & MARK)\n+\t\t\t\tgoto unsupported;\n+\t\t\toverlap |= MARK;\n+\t\t\t/* ENIC_MAGIC_FILTER_ID is reserved for flagging */\n+\t\t\tmemset(&fm_op, 0, sizeof(fm_op));\n+\t\t\tfm_op.fa_op = FMOP_MARK;\n+\t\t\tfm_op.mark.mark = ENIC_MAGIC_FILTER_ID;\n+\t\t\tret = enic_fm_append_action_op(fm, &fm_op, error);\n+\t\t\tif (ret)\n+\t\t\t\treturn ret;\n+\t\t\tbreak;\n+\t\t}\n+\t\tcase RTE_FLOW_ACTION_TYPE_QUEUE: {\n+\t\t\tconst struct rte_flow_action_queue *queue =\n+\t\t\t\tactions->conf;\n+\n+\t\t\t/*\n+\t\t\t * If other fate kind is set, fail. Multiple\n+\t\t\t * queue actions are ok.\n+\t\t\t */\n+\t\t\tif ((overlap & FATE) && first_rq)\n+\t\t\t\tgoto unsupported;\n+\t\t\tfirst_rq = false;\n+\t\t\toverlap |= FATE;\n+\t\t\tmemset(&fm_op, 0, sizeof(fm_op));\n+\t\t\tfm_op.fa_op = FMOP_RQ_STEER;\n+\t\t\tfm_op.rq_steer.rq_index =\n+\t\t\t\tenic_rte_rq_idx_to_sop_idx(queue->index);\n+\t\t\tfm_op.rq_steer.vnic_handle = vnic_h;\n+\t\t\tret = enic_fm_append_action_op(fm, &fm_op, error);\n+\t\t\tif (ret)\n+\t\t\t\treturn ret;\n+\t\t\tENICPMD_LOG(DEBUG, \"create QUEUE action rq: %u\",\n+\t\t\t\t fm_op.rq_steer.rq_index);\n+\t\t\tbreak;\n+\t\t}\n+\t\tcase RTE_FLOW_ACTION_TYPE_DROP: {\n+\t\t\tif (overlap & FATE)\n+\t\t\t\tgoto unsupported;\n+\t\t\toverlap |= FATE;\n+\t\t\tmemset(&fm_op, 0, sizeof(fm_op));\n+\t\t\tfm_op.fa_op = FMOP_DROP;\n+\t\t\tret = enic_fm_append_action_op(fm, &fm_op, error);\n+\t\t\tif (ret)\n+\t\t\t\treturn ret;\n+\t\t\tENICPMD_LOG(DEBUG, \"create DROP action\");\n+\t\t\tbreak;\n+\t\t}\n+\t\tcase RTE_FLOW_ACTION_TYPE_COUNT: {\n+\t\t\tif (overlap & COUNT)\n+\t\t\t\tgoto unsupported;\n+\t\t\toverlap |= COUNT;\n+\t\t\t/* Count is associated with entry not action on VIC. */\n+\t\t\tfmt->ftm_flags |= FMEF_COUNTER;\n+\t\t\tbreak;\n+\t\t}\n+\t\tcase RTE_FLOW_ACTION_TYPE_RSS: {\n+\t\t\tconst struct rte_flow_action_rss *rss = actions->conf;\n+\t\t\tbool allow;\n+\t\t\tuint16_t i;\n+\n+\t\t\t/*\n+\t\t\t * Hardware does not support general RSS actions, but\n+\t\t\t * we can still support the dummy one that is used to\n+\t\t\t * \"receive normally\".\n+\t\t\t */\n+\t\t\tallow = rss->func == RTE_ETH_HASH_FUNCTION_DEFAULT &&\n+\t\t\t\trss->level == 0 &&\n+\t\t\t\t(rss->types == 0 ||\n+\t\t\t\t rss->types == enic->rss_hf) &&\n+\t\t\t\trss->queue_num == enic->rq_count &&\n+\t\t\t\trss->key_len == 0;\n+\t\t\t/* Identity queue map is ok */\n+\t\t\tfor (i = 0; i < rss->queue_num; i++)\n+\t\t\t\tallow = allow && (i == rss->queue[i]);\n+\t\t\tif (!allow)\n+\t\t\t\tgoto unsupported;\n+\t\t\tif (overlap & FATE)\n+\t\t\t\tgoto unsupported;\n+\t\t\t/* Need MARK or FLAG */\n+\t\t\tif (!(overlap & MARK))\n+\t\t\t\tgoto unsupported;\n+\t\t\toverlap |= FATE;\n+\t\t\tbreak;\n+\t\t}\n+\t\tcase RTE_FLOW_ACTION_TYPE_PORT_ID: {\n+\t\t\tconst struct rte_flow_action_port_id *port;\n+\t\t\tstruct rte_pci_device *pdev;\n+\t\t\tstruct rte_eth_dev *dev;\n+\n+\t\t\tport = actions->conf;\n+\t\t\tif (port->original) {\n+\t\t\t\tvnic_h = 0; /* This port */\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t\tENICPMD_LOG(DEBUG, \"port id %u\", port->id);\n+\t\t\tif (!rte_eth_dev_is_valid_port(port->id)) {\n+\t\t\t\treturn rte_flow_error_set(error, EINVAL,\n+\t\t\t\t\tRTE_FLOW_ERROR_TYPE_ACTION,\n+\t\t\t\t\tNULL, \"invalid port_id\");\n+\t\t\t}\n+\t\t\tdev = &rte_eth_devices[port->id];\n+\t\t\tif (!dev_is_enic(dev)) {\n+\t\t\t\treturn rte_flow_error_set(error, EINVAL,\n+\t\t\t\t\tRTE_FLOW_ERROR_TYPE_ACTION,\n+\t\t\t\t\tNULL, \"port_id is not enic\");\n+\t\t\t}\n+\t\t\tpdev = RTE_ETH_DEV_TO_PCI(dev);\n+\t\t\tif (enic_fm_find_vnic(enic, &pdev->addr, &vnic_h)) {\n+\t\t\t\treturn rte_flow_error_set(error, EINVAL,\n+\t\t\t\t\tRTE_FLOW_ERROR_TYPE_ACTION,\n+\t\t\t\t\tNULL, \"port_id is not vnic\");\n+\t\t\t}\n+\t\t\tbreak;\n+\t\t}\n+\t\tcase RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: {\n+\t\t\tret = enic_fm_copy_vxlan_decap(fm, fmt, actions,\n+\t\t\t\terror);\n+\t\t\tif (ret != 0)\n+\t\t\t\treturn ret;\n+\t\t\tbreak;\n+\t\t}\n+\t\tcase RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP: {\n+\t\t\tconst struct rte_flow_action_vxlan_encap *encap;\n+\n+\t\t\tencap = actions->conf;\n+\t\t\tif (overlap & ENCAP)\n+\t\t\t\tgoto unsupported;\n+\t\t\toverlap |= ENCAP;\n+\t\t\tret = enic_fm_copy_vxlan_encap(fm, encap->definition,\n+\t\t\t\terror);\n+\t\t\tif (ret != 0)\n+\t\t\t\treturn ret;\n+\t\t\tbreak;\n+\t\t}\n+\t\tdefault:\n+\t\t\tgoto unsupported;\n+\t\t}\n+\t}\n+\n+\tif (!(overlap & (FATE | PASSTHRU | COUNT)))\n+\t\tgoto unsupported;\n+\tmemset(&fm_op, 0, sizeof(fm_op));\n+\tfm_op.fa_op = FMOP_END;\n+\tret = enic_fm_append_action_op(fm, &fm_op, error);\n+\tif (ret)\n+\t\treturn ret;\n+\tenic_fm_reorder_action_op(fm);\n+\treturn 0;\n+\n+unsupported:\n+\treturn rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,\n+\t\t\t\t NULL, \"enic: unsupported action\");\n+}\n+\n+/** Check if the action is supported */\n+static int\n+enic_fm_match_action(const struct rte_flow_action *action,\n+\t\t const enum rte_flow_action_type *supported_actions)\n+{\n+\tfor (; *supported_actions != RTE_FLOW_ACTION_TYPE_END;\n+\t supported_actions++) {\n+\t\tif (action->type == *supported_actions)\n+\t\t\treturn 1;\n+\t}\n+\treturn 0;\n+}\n+\n+/* Debug function to dump internal NIC action structure. */\n+static void\n+enic_fm_dump_tcam_actions(const struct fm_action *fm_action)\n+{\n+\t/* Manually keep in sync with FMOP commands */\n+\tconst char *fmop_str[FMOP_OP_MAX] = {\n+\t\t[FMOP_END] = \"end\",\n+\t\t[FMOP_DROP] = \"drop\",\n+\t\t[FMOP_RQ_STEER] = \"steer\",\n+\t\t[FMOP_EXACT_MATCH] = \"exmatch\",\n+\t\t[FMOP_MARK] = \"mark\",\n+\t\t[FMOP_EXT_MARK] = \"ext_mark\",\n+\t\t[FMOP_TAG] = \"tag\",\n+\t\t[FMOP_EG_HAIRPIN] = \"eg_hairpin\",\n+\t\t[FMOP_IG_HAIRPIN] = \"ig_hairpin\",\n+\t\t[FMOP_ENCAP_IVLAN] = \"encap_ivlan\",\n+\t\t[FMOP_ENCAP_NOIVLAN] = \"encap_noivlan\",\n+\t\t[FMOP_ENCAP] = \"encap\",\n+\t\t[FMOP_SET_OVLAN] = \"set_ovlan\",\n+\t\t[FMOP_DECAP_NOSTRIP] = \"decap_nostrip\",\n+\t};\n+\tconst struct fm_action_op *op = &fm_action->fma_action_ops[0];\n+\tchar buf[128], *bp = buf;\n+\tconst char *op_str;\n+\tint i, n, buf_len;\n+\n+\tbuf[0] = '\\0';\n+\tbuf_len = sizeof(buf);\n+\tfor (i = 0; i < FM_ACTION_OP_MAX; i++) {\n+\t\tif (op->fa_op == FMOP_END)\n+\t\t\tbreak;\n+\t\tif (op->fa_op >= FMOP_OP_MAX)\n+\t\t\top_str = \"unknown\";\n+\t\telse\n+\t\t\top_str = fmop_str[op->fa_op];\n+\t\tn = snprintf(bp, buf_len, \"%s,\", op_str);\n+\t\tif (n > 0 && n < buf_len) {\n+\t\t\tbp += n;\n+\t\t\tbuf_len -= n;\n+\t\t}\n+\t\top++;\n+\t}\n+\t/* Remove trailing comma */\n+\tif (buf[0])\n+\t\t*(bp - 1) = '\\0';\n+\tENICPMD_LOG(DEBUG, \" Acions: %s\", buf);\n+}\n+\n+static int\n+bits_to_str(uint32_t bits, const char *strings[], int max,\n+\t char *buf, int buf_len)\n+{\n+\tint i, n = 0, len = 0;\n+\n+\tfor (i = 0; i < max; i++) {\n+\t\tif (bits & (1 << i)) {\n+\t\t\tn = snprintf(buf, buf_len, \"%s,\", strings[i]);\n+\t\t\tif (n > 0 && n < buf_len) {\n+\t\t\t\tbuf += n;\n+\t\t\t\tbuf_len -= n;\n+\t\t\t\tlen += n;\n+\t\t\t}\n+\t\t}\n+\t}\n+\t/* Remove trailing comma */\n+\tif (len) {\n+\t\t*(buf - 1) = '\\0';\n+\t\tlen--;\n+\t}\n+\treturn len;\n+}\n+\n+/* Debug function to dump internal NIC filter structure. */\n+static void\n+__enic_fm_dump_tcam_match(const struct fm_header_set *fk_hdrset, char *buf,\n+\t\t\t int buf_len)\n+{\n+\t/* Manually keep in sync with FKM_BITS */\n+\tconst char *fm_fkm_str[FKM_BIT_COUNT] = {\n+\t\t[FKM_QTAG_BIT] = \"qtag\",\n+\t\t[FKM_CMD_BIT] = \"cmd\",\n+\t\t[FKM_IPV4_BIT] = \"ip4\",\n+\t\t[FKM_IPV6_BIT] = \"ip6\",\n+\t\t[FKM_ROCE_BIT] = \"roce\",\n+\t\t[FKM_UDP_BIT] = \"udp\",\n+\t\t[FKM_TCP_BIT] = \"tcp\",\n+\t\t[FKM_TCPORUDP_BIT] = \"tcpportudp\",\n+\t\t[FKM_IPFRAG_BIT] = \"ipfrag\",\n+\t\t[FKM_NVGRE_BIT] = \"nvgre\",\n+\t\t[FKM_VXLAN_BIT] = \"vxlan\",\n+\t\t[FKM_GENEVE_BIT] = \"geneve\",\n+\t\t[FKM_NSH_BIT] = \"nsh\",\n+\t\t[FKM_ROCEV2_BIT] = \"rocev2\",\n+\t\t[FKM_VLAN_PRES_BIT] = \"vlan_pres\",\n+\t\t[FKM_IPOK_BIT] = \"ipok\",\n+\t\t[FKM_L4OK_BIT] = \"l4ok\",\n+\t\t[FKM_ROCEOK_BIT] = \"roceok\",\n+\t\t[FKM_FCSOK_BIT] = \"fcsok\",\n+\t\t[FKM_EG_SPAN_BIT] = \"eg_span\",\n+\t\t[FKM_IG_SPAN_BIT] = \"ig_span\",\n+\t\t[FKM_EG_HAIRPINNED_BIT] = \"eg_hairpinned\",\n+\t};\n+\t/* Manually keep in sync with FKH_BITS */\n+\tconst char *fm_fkh_str[FKH_BIT_COUNT] = {\n+\t\t[FKH_ETHER_BIT] = \"eth\",\n+\t\t[FKH_QTAG_BIT] = \"qtag\",\n+\t\t[FKH_L2RAW_BIT] = \"l2raw\",\n+\t\t[FKH_IPV4_BIT] = \"ip4\",\n+\t\t[FKH_IPV6_BIT] = \"ip6\",\n+\t\t[FKH_L3RAW_BIT] = \"l3raw\",\n+\t\t[FKH_UDP_BIT] = \"udp\",\n+\t\t[FKH_TCP_BIT] = \"tcp\",\n+\t\t[FKH_ICMP_BIT] = \"icmp\",\n+\t\t[FKH_VXLAN_BIT] = \"vxlan\",\n+\t\t[FKH_L4RAW_BIT] = \"l4raw\",\n+\t};\n+\tuint32_t fkh_bits = fk_hdrset->fk_header_select;\n+\tuint32_t fkm_bits = fk_hdrset->fk_metadata;\n+\tint n;\n+\n+\tif (!fkm_bits && !fkh_bits)\n+\t\treturn;\n+\tn = snprintf(buf, buf_len, \"metadata(\");\n+\tif (n > 0 && n < buf_len) {\n+\t\tbuf += n;\n+\t\tbuf_len -= n;\n+\t}\n+\tn = bits_to_str(fkm_bits, fm_fkm_str, FKM_BIT_COUNT, buf, buf_len);\n+\tif (n > 0 && n < buf_len) {\n+\t\tbuf += n;\n+\t\tbuf_len -= n;\n+\t}\n+\tn = snprintf(buf, buf_len, \") valid hdr fields(\");\n+\tif (n > 0 && n < buf_len) {\n+\t\tbuf += n;\n+\t\tbuf_len -= n;\n+\t}\n+\tn = bits_to_str(fkh_bits, fm_fkh_str, FKH_BIT_COUNT, buf, buf_len);\n+\tif (n > 0 && n < buf_len) {\n+\t\tbuf += n;\n+\t\tbuf_len -= n;\n+\t}\n+\tsnprintf(buf, buf_len, \")\");\n+}\n+\n+static void\n+enic_fm_dump_tcam_match(const struct fm_tcam_match_entry *match,\n+\t\t\tuint8_t ingress)\n+{\n+\tchar buf[256];\n+\n+\tmemset(buf, 0, sizeof(buf));\n+\t__enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[0],\n+\t\t\t\t buf, sizeof(buf));\n+\tENICPMD_LOG(DEBUG, \" TCAM %s Outer: %s %scounter\",\n+\t\t (ingress) ? \"IG\" : \"EG\", buf,\n+\t\t (match->ftm_flags & FMEF_COUNTER) ? \"\" : \"no \");\n+\tmemset(buf, 0, sizeof(buf));\n+\t__enic_fm_dump_tcam_match(&match->ftm_mask.fk_hdrset[1],\n+\t\t\t\t buf, sizeof(buf));\n+\tif (buf[0])\n+\t\tENICPMD_LOG(DEBUG, \" Inner: %s\", buf);\n+}\n+\n+/* Debug function to dump internal NIC flow structures. */\n+static void\n+enic_fm_dump_tcam_entry(const struct fm_tcam_match_entry *fm_match,\n+\t\t\tconst struct fm_action *fm_action,\n+\t\t\tuint8_t ingress)\n+{\n+\tif (rte_log_get_level(enic_pmd_logtype) < (int)RTE_LOG_DEBUG)\n+\t\treturn;\n+\tenic_fm_dump_tcam_match(fm_match, ingress);\n+\tenic_fm_dump_tcam_actions(fm_action);\n+}\n+\n+static int\n+enic_fm_flow_parse(struct enic_flowman *fm,\n+\t\t const struct rte_flow_attr *attrs,\n+\t\t const struct rte_flow_item pattern[],\n+\t\t const struct rte_flow_action actions[],\n+\t\t struct rte_flow_error *error)\n+{\n+\tconst struct rte_flow_action *action;\n+\tunsigned int ret;\n+\tstatic const enum rte_flow_action_type *sa;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tret = 0;\n+\tif (!pattern) {\n+\t\trte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM_NUM,\n+\t\t\t\t NULL, \"no pattern specified\");\n+\t\treturn -rte_errno;\n+\t}\n+\n+\tif (!actions) {\n+\t\trte_flow_error_set(error, EINVAL,\n+\t\t\t\t RTE_FLOW_ERROR_TYPE_ACTION_NUM,\n+\t\t\t\t NULL, \"no action specified\");\n+\t\treturn -rte_errno;\n+\t}\n+\n+\tif (attrs) {\n+\t\tif (attrs->priority) {\n+\t\t\trte_flow_error_set(error, ENOTSUP,\n+\t\t\t\t\t RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,\n+\t\t\t\t\t NULL,\n+\t\t\t\t\t \"priorities are not supported\");\n+\t\t\treturn -rte_errno;\n+\t\t} else if (attrs->transfer) {\n+\t\t\trte_flow_error_set(error, ENOTSUP,\n+\t\t\t\t\t RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,\n+\t\t\t\t\t NULL,\n+\t\t\t\t\t \"transfer is not supported\");\n+\t\t\treturn -rte_errno;\n+\t\t} else if (attrs->ingress && attrs->egress) {\n+\t\t\trte_flow_error_set(error, ENOTSUP,\n+\t\t\t\t\t RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,\n+\t\t\t\t\t NULL,\n+\t\t\t\t\t \"bidirectional rules not supported\");\n+\t\t\treturn -rte_errno;\n+\t\t}\n+\n+\t} else {\n+\t\trte_flow_error_set(error, EINVAL,\n+\t\t\t\t RTE_FLOW_ERROR_TYPE_ATTR,\n+\t\t\t\t NULL, \"no attribute specified\");\n+\t\treturn -rte_errno;\n+\t}\n+\n+\t/* Verify Actions. */\n+\tsa = (attrs->ingress) ? enic_fm_supported_ig_actions :\n+\t enic_fm_supported_eg_actions;\n+\tfor (action = &actions[0]; action->type != RTE_FLOW_ACTION_TYPE_END;\n+\t action++) {\n+\t\tif (action->type == RTE_FLOW_ACTION_TYPE_VOID)\n+\t\t\tcontinue;\n+\t\telse if (!enic_fm_match_action(action, sa))\n+\t\t\tbreak;\n+\t}\n+\tif (action->type != RTE_FLOW_ACTION_TYPE_END) {\n+\t\trte_flow_error_set(error, EPERM, RTE_FLOW_ERROR_TYPE_ACTION,\n+\t\t\t\t action, \"invalid action\");\n+\t\treturn -rte_errno;\n+\t}\n+\tret = enic_fm_copy_entry(fm, pattern, error);\n+\tif (ret)\n+\t\treturn ret;\n+\tret = enic_fm_copy_action(fm, actions, attrs->ingress, error);\n+\treturn ret;\n+}\n+\n+static void\n+enic_fm_counter_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)\n+{\n+\tif (!fm_flow->counter_valid)\n+\t\treturn;\n+\tSLIST_INSERT_HEAD(&fm->counters, fm_flow->counter, next);\n+\tfm_flow->counter_valid = false;\n+}\n+\n+static int\n+enic_fm_more_counters(struct enic_flowman *fm)\n+{\n+\tstruct enic_fm_counter *new_stack;\n+\tstruct enic_fm_counter *ctrs;\n+\tstruct enic *enic;\n+\tint i, rc;\n+\tu64 args[2];\n+\n+\tENICPMD_FUNC_TRACE();\n+\tenic = fm->enic;\n+\tnew_stack = rte_realloc(fm->counter_stack, (fm->counters_alloced +\n+\t\t\t\tFM_COUNTERS_EXPAND) *\n+\t\t\t\tsizeof(struct enic_fm_counter), 0);\n+\tif (new_stack == NULL) {\n+\t\tENICPMD_LOG(ERR, \"cannot alloc counter memory\");\n+\t\treturn -ENOMEM;\n+\t}\n+\tfm->counter_stack = new_stack;\n+\n+\targs[0] = FM_COUNTER_BRK;\n+\targs[1] = fm->counters_alloced + FM_COUNTERS_EXPAND;\n+\trc = vnic_dev_flowman_cmd(enic->vdev, args, 2);\n+\tif (rc != 0) {\n+\t\tENICPMD_LOG(ERR, \"cannot alloc counters rc=%d\", rc);\n+\t\treturn rc;\n+\t}\n+\tctrs = (struct enic_fm_counter *)fm->counter_stack +\n+\t\tfm->counters_alloced;\n+\tfor (i = 0; i < FM_COUNTERS_EXPAND; i++, ctrs++) {\n+\t\tctrs->handle = fm->counters_alloced + i;\n+\t\tSLIST_INSERT_HEAD(&fm->counters, ctrs, next);\n+\t}\n+\tfm->counters_alloced += FM_COUNTERS_EXPAND;\n+\tENICPMD_LOG(DEBUG, \"%u counters allocated, total: %u\",\n+\t\t FM_COUNTERS_EXPAND, fm->counters_alloced);\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_counter_zero(struct enic_flowman *fm, struct enic_fm_counter *c)\n+{\n+\tstruct enic *enic;\n+\tu64 args[3];\n+\tint ret;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tenic = fm->enic;\n+\targs[0] = FM_COUNTER_QUERY;\n+\targs[1] = c->handle;\n+\targs[2] = 1; /* clear */\n+\tret = vnic_dev_flowman_cmd(enic->vdev, args, 3);\n+\tif (ret) {\n+\t\tENICPMD_LOG(ERR, \"counter init: rc=%d handle=0x%x\",\n+\t\t\t ret, c->handle);\n+\t\treturn ret;\n+\t}\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_counter_alloc(struct enic_flowman *fm, struct rte_flow_error *error,\n+\t\t struct enic_fm_counter **ctr)\n+{\n+\tstruct enic_fm_counter *c;\n+\tint ret;\n+\n+\tENICPMD_FUNC_TRACE();\n+\t*ctr = NULL;\n+\tif (SLIST_EMPTY(&fm->counters)) {\n+\t\tret = enic_fm_more_counters(fm);\n+\t\tif (ret)\n+\t\t\treturn rte_flow_error_set(error, -ret,\n+\t\t\t\tRTE_FLOW_ERROR_TYPE_UNSPECIFIED,\n+\t\t\t\tNULL, \"enic: out of counters\");\n+\t}\n+\tc = SLIST_FIRST(&fm->counters);\n+\tSLIST_REMOVE_HEAD(&fm->counters, next);\n+\t*ctr = c;\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_action_free(struct enic_flowman *fm, uint64_t handle)\n+{\n+\tu64 args[2];\n+\tint rc;\n+\n+\tENICPMD_FUNC_TRACE();\n+\targs[0] = FM_ACTION_FREE;\n+\targs[1] = handle;\n+\trc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);\n+\tif (rc)\n+\t\tENICPMD_LOG(ERR, \"cannot free action: rc=%d handle=0x%\" PRIx64,\n+\t\t\t rc, handle);\n+\treturn rc;\n+}\n+\n+static int\n+enic_fm_entry_free(struct enic_flowman *fm, uint64_t handle)\n+{\n+\tu64 args[2];\n+\tint rc;\n+\n+\tENICPMD_FUNC_TRACE();\n+\targs[0] = FM_MATCH_ENTRY_REMOVE;\n+\targs[1] = handle;\n+\trc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);\n+\tif (rc)\n+\t\tENICPMD_LOG(ERR, \"cannot free match entry: rc=%d\"\n+\t\t\t \" handle=0x%\" PRIx64, rc, handle);\n+\treturn rc;\n+}\n+\n+static struct enic_fm_jump_flow *\n+find_jump_flow(struct enic_flowman *fm, uint32_t group)\n+{\n+\tstruct enic_fm_jump_flow *j;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tTAILQ_FOREACH(j, &fm->jump_list, list) {\n+\t\tif (j->group == group)\n+\t\t\treturn j;\n+\t}\n+\treturn NULL;\n+}\n+\n+static void\n+remove_jump_flow(struct enic_flowman *fm, struct rte_flow *flow)\n+{\n+\tstruct enic_fm_jump_flow *j;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tTAILQ_FOREACH(j, &fm->jump_list, list) {\n+\t\tif (j->flow == flow) {\n+\t\t\tTAILQ_REMOVE(&fm->jump_list, j, list);\n+\t\t\tfree(j);\n+\t\t\treturn;\n+\t\t}\n+\t}\n+}\n+\n+static int\n+save_jump_flow(struct enic_flowman *fm,\n+\t struct rte_flow *flow,\n+\t uint32_t group,\n+\t struct fm_tcam_match_entry *match,\n+\t struct fm_action *action)\n+{\n+\tstruct enic_fm_jump_flow *j;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tj = calloc(1, sizeof(struct enic_fm_jump_flow));\n+\tif (j == NULL)\n+\t\treturn -ENOMEM;\n+\tj->flow = flow;\n+\tj->group = group;\n+\tj->match = *match;\n+\tj->action = *action;\n+\tTAILQ_INSERT_HEAD(&fm->jump_list, j, list);\n+\tENICPMD_LOG(DEBUG, \"saved jump flow: flow=%p group=%u\", flow, group);\n+\treturn 0;\n+}\n+\n+static void\n+__enic_fm_flow_free(struct enic_flowman *fm, struct enic_fm_flow *fm_flow)\n+{\n+\tif (fm_flow->entry_handle != FM_INVALID_HANDLE) {\n+\t\tenic_fm_entry_free(fm, fm_flow->entry_handle);\n+\t\tfm_flow->entry_handle = FM_INVALID_HANDLE;\n+\t}\n+\tif (fm_flow->action_handle != FM_INVALID_HANDLE) {\n+\t\tenic_fm_action_free(fm, fm_flow->action_handle);\n+\t\tfm_flow->action_handle = FM_INVALID_HANDLE;\n+\t}\n+\tenic_fm_counter_free(fm, fm_flow);\n+\tif (fm_flow->fet) {\n+\t\tenic_fet_put(fm, fm_flow->fet);\n+\t\tfm_flow->fet = NULL;\n+\t}\n+}\n+\n+static void\n+enic_fm_flow_free(struct enic_flowman *fm, struct rte_flow *flow)\n+{\n+\tif (flow->fm->fet && flow->fm->fet->default_key)\n+\t\tremove_jump_flow(fm, flow);\n+\t__enic_fm_flow_free(fm, flow->fm);\n+\tfree(flow->fm);\n+\tfree(flow);\n+}\n+\n+static int\n+enic_fm_add_tcam_entry(struct enic_flowman *fm,\n+\t\t struct fm_tcam_match_entry *match_in,\n+\t\t uint64_t *entry_handle,\n+\t\t uint8_t ingress,\n+\t\t struct rte_flow_error *error)\n+{\n+\tstruct fm_tcam_match_entry *ftm;\n+\tu64 args[3];\n+\tint ret;\n+\n+\tENICPMD_FUNC_TRACE();\n+\t/* Copy entry to the command buffer */\n+\tftm = &fm->cmd.va->fm_tcam_match_entry;\n+\tmemcpy(ftm, match_in, sizeof(*ftm));\n+\t/* Add TCAM entry */\n+\targs[0] = FM_TCAM_ENTRY_INSTALL;\n+\targs[1] = ingress ? fm->ig_tcam_hndl : fm->eg_tcam_hndl;\n+\targs[2] = fm->cmd.pa;\n+\tret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);\n+\tif (ret != 0) {\n+\t\tENICPMD_LOG(ERR, \"cannot add %s TCAM entry: rc=%d\",\n+\t\t\t ingress ? \"ingress\" : \"egress\", ret);\n+\t\trte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,\n+\t\t\tNULL, \"enic: devcmd(tcam-entry-install)\");\n+\t\treturn ret;\n+\t}\n+\tENICPMD_LOG(DEBUG, \"installed %s TCAM entry: handle=0x%\" PRIx64,\n+\t\t ingress ? \"ingress\" : \"egress\", (uint64_t)args[0]);\n+\t*entry_handle = args[0];\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_add_exact_entry(struct enic_flowman *fm,\n+\t\t\tstruct fm_tcam_match_entry *match_in,\n+\t\t\tuint64_t *entry_handle,\n+\t\t\tstruct enic_fm_fet *fet,\n+\t\t\tstruct rte_flow_error *error)\n+{\n+\tstruct fm_exact_match_entry *fem;\n+\tu64 args[3];\n+\tint ret;\n+\n+\tENICPMD_FUNC_TRACE();\n+\t/* The new entry must have the table's key */\n+\tif (memcmp(fet->key.fk_hdrset, match_in->ftm_mask.fk_hdrset,\n+\t\t sizeof(struct fm_header_set) * FM_HDRSET_MAX)) {\n+\t\treturn rte_flow_error_set(error, EINVAL,\n+\t\t\tRTE_FLOW_ERROR_TYPE_ITEM, NULL,\n+\t\t\t\"enic: key does not match group's key\");\n+\t}\n+\n+\t/* Copy entry to the command buffer */\n+\tfem = &fm->cmd.va->fm_exact_match_entry;\n+\t/*\n+\t * Translate TCAM entry to exact entry. As is only need to drop\n+\t * position and mask. The mask is part of the exact match table.\n+\t * Position (aka priority) is not supported in the exact match table.\n+\t */\n+\tfem->fem_data = match_in->ftm_data;\n+\tfem->fem_flags = match_in->ftm_flags;\n+\tfem->fem_action = match_in->ftm_action;\n+\tfem->fem_counter = match_in->ftm_counter;\n+\n+\t/* Add exact entry */\n+\targs[0] = FM_EXACT_ENTRY_INSTALL;\n+\targs[1] = fet->handle;\n+\targs[2] = fm->cmd.pa;\n+\tret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 3);\n+\tif (ret != 0) {\n+\t\tENICPMD_LOG(ERR, \"cannot add %s exact entry: group=%u\",\n+\t\t\t fet->ingress ? \"ingress\" : \"egress\", fet->group);\n+\t\trte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,\n+\t\t\tNULL, \"enic: devcmd(exact-entry-install)\");\n+\t\treturn ret;\n+\t}\n+\tENICPMD_LOG(DEBUG, \"installed %s exact entry: group=%u\"\n+\t\t \" handle=0x%\" PRIx64,\n+\t\t fet->ingress ? \"ingress\" : \"egress\", fet->group,\n+\t\t (uint64_t)args[0]);\n+\t*entry_handle = args[0];\n+\treturn 0;\n+}\n+\n+/* Push match-action to the NIC. */\n+static int\n+__enic_fm_flow_add_entry(struct enic_flowman *fm,\n+\t\t\t struct enic_fm_flow *fm_flow,\n+\t\t\t struct fm_tcam_match_entry *match_in,\n+\t\t\t struct fm_action *action_in,\n+\t\t\t uint32_t group,\n+\t\t\t uint8_t ingress,\n+\t\t\t struct rte_flow_error *error)\n+{\n+\tstruct enic_fm_counter *ctr;\n+\tstruct fm_action *fma;\n+\tuint64_t action_h;\n+\tuint64_t entry_h;\n+\tu64 args[3];\n+\tint ret;\n+\n+\tENICPMD_FUNC_TRACE();\n+\t/* Allocate action. */\n+\tfma = &fm->cmd.va->fm_action;\n+\tmemcpy(fma, action_in, sizeof(*fma));\n+\targs[0] = FM_ACTION_ALLOC;\n+\targs[1] = fm->cmd.pa;\n+\tret = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);\n+\tif (ret != 0) {\n+\t\tENICPMD_LOG(ERR, \"allocating TCAM table action rc=%d\", ret);\n+\t\trte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,\n+\t\t\tNULL, \"enic: devcmd(action-alloc)\");\n+\t\treturn ret;\n+\t}\n+\taction_h = args[0];\n+\tfm_flow->action_handle = action_h;\n+\tmatch_in->ftm_action = action_h;\n+\tENICPMD_LOG(DEBUG, \"action allocated: handle=0x%\" PRIx64, action_h);\n+\n+\t/* Allocate counter if requested. */\n+\tif (match_in->ftm_flags & FMEF_COUNTER) {\n+\t\tret = enic_fm_counter_alloc(fm, error, &ctr);\n+\t\tif (ret) /* error has been filled in */\n+\t\t\treturn ret;\n+\t\tfm_flow->counter_valid = true;\n+\t\tfm_flow->counter = ctr;\n+\t\tmatch_in->ftm_counter = ctr->handle;\n+\t}\n+\n+\t/*\n+\t * Get the group's table (either TCAM or exact match table) and\n+\t * add entry to it. If we use the exact match table, the handler\n+\t * will translate the TCAM entry (match_in) to the appropriate\n+\t * exact match entry and use that instead.\n+\t */\n+\tentry_h = FM_INVALID_HANDLE;\n+\tif (group == FM_TCAM_RTE_GROUP) {\n+\t\tret = enic_fm_add_tcam_entry(fm, match_in, &entry_h, ingress,\n+\t\t\t\t\t error);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t\t/* Jump action might have a ref to fet */\n+\t\tfm_flow->fet = fm->fet;\n+\t\tfm->fet = NULL;\n+\t} else {\n+\t\tstruct enic_fm_fet *fet = NULL;\n+\n+\t\tret = enic_fet_get(fm, group, ingress,\n+\t\t\t\t &match_in->ftm_mask, &fet, error);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t\tfm_flow->fet = fet;\n+\t\tret = enic_fm_add_exact_entry(fm, match_in, &entry_h, fet,\n+\t\t\t\t\t error);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\t/* Clear counter after adding entry, as it requires in-use counter */\n+\tif (fm_flow->counter_valid) {\n+\t\tret = enic_fm_counter_zero(fm, fm_flow->counter);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\tfm_flow->entry_handle = entry_h;\n+\treturn 0;\n+}\n+\n+/* Push match-action to the NIC. */\n+static struct rte_flow *\n+enic_fm_flow_add_entry(struct enic_flowman *fm,\n+\t\t struct fm_tcam_match_entry *match_in,\n+\t\t struct fm_action *action_in,\n+\t\t const struct rte_flow_attr *attrs,\n+\t\t struct rte_flow_error *error)\n+{\n+\tstruct enic_fm_flow *fm_flow;\n+\tstruct rte_flow *flow;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tenic_fm_dump_tcam_entry(match_in, action_in, attrs->ingress);\n+\tflow = calloc(1, sizeof(*flow));\n+\tfm_flow = calloc(1, sizeof(*fm_flow));\n+\tif (flow == NULL || fm_flow == NULL) {\n+\t\trte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,\n+\t\t\tNULL, \"enic: cannot allocate rte_flow\");\n+\t\tfree(flow);\n+\t\tfree(fm_flow);\n+\t\treturn NULL;\n+\t}\n+\tflow->fm = fm_flow;\n+\tfm_flow->action_handle = FM_INVALID_HANDLE;\n+\tfm_flow->entry_handle = FM_INVALID_HANDLE;\n+\tif (__enic_fm_flow_add_entry(fm, fm_flow, match_in, action_in,\n+\t\t\t\t attrs->group, attrs->ingress, error)) {\n+\t\tenic_fm_flow_free(fm, flow);\n+\t\treturn NULL;\n+\t}\n+\treturn flow;\n+}\n+\n+static void\n+convert_jump_flows(struct enic_flowman *fm, struct enic_fm_fet *fet,\n+\t\t struct rte_flow_error *error)\n+{\n+\tstruct enic_fm_flow *fm_flow;\n+\tstruct enic_fm_jump_flow *j;\n+\tstruct fm_action *fma;\n+\tuint32_t group;\n+\n+\tENICPMD_FUNC_TRACE();\n+\t/*\n+\t * Find the saved flows that should jump to the new table (fet).\n+\t * Then delete the old TCAM entry that jumps to the default table,\n+\t * and add a new one that jumps to the new table.\n+\t */\n+\tgroup = fet->group;\n+\tj = find_jump_flow(fm, group);\n+\twhile (j) {\n+\t\tENICPMD_LOG(DEBUG, \"convert jump flow: flow=%p group=%u\",\n+\t\t\t j->flow, group);\n+\t\t/* Delete old entry */\n+\t\tfm_flow = j->flow->fm;\n+\t\t__enic_fm_flow_free(fm, fm_flow);\n+\n+\t\t/* Add new entry */\n+\t\tfma = &j->action;\n+\t\tfma->fma_action_ops[0].exact.handle = fet->handle;\n+\t\tif (__enic_fm_flow_add_entry(fm, fm_flow, &j->match, fma,\n+\t\t\tFM_TCAM_RTE_GROUP, fet->ingress, error)) {\n+\t\t\t/* Cannot roll back changes at the moment */\n+\t\t\tENICPMD_LOG(ERR, \"cannot convert jump flow: flow=%p\",\n+\t\t\t\t j->flow);\n+\t\t} else {\n+\t\t\tfm_flow->fet = fet;\n+\t\t\tfet->ref++;\n+\t\t\tENICPMD_LOG(DEBUG, \"convert ok: group=%u ref=%u\",\n+\t\t\t\t fet->group, fet->ref);\n+\t\t}\n+\n+\t\tTAILQ_REMOVE(&fm->jump_list, j, list);\n+\t\tfree(j);\n+\t\tj = find_jump_flow(fm, group);\n+\t}\n+}\n+\n+static void\n+enic_fm_open_scratch(struct enic_flowman *fm)\n+{\n+\tfm->action_op_count = 0;\n+\tfm->fet = NULL;\n+\tmemset(&fm->tcam_entry, 0, sizeof(fm->tcam_entry));\n+\tmemset(&fm->action, 0, sizeof(fm->action));\n+}\n+\n+static void\n+enic_fm_close_scratch(struct enic_flowman *fm)\n+{\n+\tif (fm->fet) {\n+\t\tenic_fet_put(fm, fm->fet);\n+\t\tfm->fet = NULL;\n+\t}\n+\tfm->action_op_count = 0;\n+}\n+\n+static int\n+enic_fm_flow_validate(struct rte_eth_dev *dev,\n+\t\t const struct rte_flow_attr *attrs,\n+\t\t const struct rte_flow_item pattern[],\n+\t\t const struct rte_flow_action actions[],\n+\t\t struct rte_flow_error *error)\n+{\n+\tstruct fm_tcam_match_entry *fm_tcam_entry;\n+\tstruct fm_action *fm_action;\n+\tstruct enic_flowman *fm;\n+\tint ret;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tfm = pmd_priv(dev)->fm;\n+\tif (fm == NULL)\n+\t\treturn -ENOTSUP;\n+\tenic_fm_open_scratch(fm);\n+\tret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);\n+\tif (!ret) {\n+\t\tfm_tcam_entry = &fm->tcam_entry;\n+\t\tfm_action = &fm->action;\n+\t\tenic_fm_dump_tcam_entry(fm_tcam_entry, fm_action,\n+\t\t\t\t\tattrs->ingress);\n+\t}\n+\tenic_fm_close_scratch(fm);\n+\treturn ret;\n+}\n+\n+static int\n+enic_fm_flow_query_count(struct rte_eth_dev *dev,\n+\t\t\t struct rte_flow *flow, void *data,\n+\t\t\t struct rte_flow_error *error)\n+{\n+\tstruct rte_flow_query_count *query;\n+\tstruct enic_fm_flow *fm_flow;\n+\tstruct enic *enic;\n+\tu64 args[3];\n+\tint rc;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tenic = pmd_priv(dev);\n+\tquery = data;\n+\tfm_flow = flow->fm;\n+\tif (!fm_flow->counter_valid)\n+\t\treturn rte_flow_error_set(error, ENOTSUP,\n+\t\t\tRTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,\n+\t\t\t\"enic: flow does not have counter\");\n+\n+\targs[0] = FM_COUNTER_QUERY;\n+\targs[1] = fm_flow->counter->handle;\n+\targs[2] = query->reset;\n+\trc = vnic_dev_flowman_cmd(enic->vdev, args, 3);\n+\tif (rc) {\n+\t\tENICPMD_LOG(ERR, \"cannot query counter: rc=%d handle=0x%x\",\n+\t\t\t rc, fm_flow->counter->handle);\n+\t\treturn rc;\n+\t}\n+\tquery->hits_set = 1;\n+\tquery->hits = args[0];\n+\tquery->bytes_set = 1;\n+\tquery->bytes = args[1];\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_flow_query(struct rte_eth_dev *dev,\n+\t\t struct rte_flow *flow,\n+\t\t const struct rte_flow_action *actions,\n+\t\t void *data,\n+\t\t struct rte_flow_error *error)\n+{\n+\tint ret = 0;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tfor (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {\n+\t\tswitch (actions->type) {\n+\t\tcase RTE_FLOW_ACTION_TYPE_VOID:\n+\t\t\tbreak;\n+\t\tcase RTE_FLOW_ACTION_TYPE_COUNT:\n+\t\t\tret = enic_fm_flow_query_count(dev, flow, data, error);\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\treturn rte_flow_error_set(error, ENOTSUP,\n+\t\t\t\t\t\t RTE_FLOW_ERROR_TYPE_ACTION,\n+\t\t\t\t\t\t actions,\n+\t\t\t\t\t\t \"action not supported\");\n+\t\t}\n+\t\tif (ret < 0)\n+\t\t\treturn ret;\n+\t}\n+\treturn 0;\n+}\n+\n+static struct rte_flow *\n+enic_fm_flow_create(struct rte_eth_dev *dev,\n+\t\t const struct rte_flow_attr *attrs,\n+\t\t const struct rte_flow_item pattern[],\n+\t\t const struct rte_flow_action actions[],\n+\t\t struct rte_flow_error *error)\n+{\n+\tstruct fm_tcam_match_entry *fm_tcam_entry;\n+\tstruct fm_action *fm_action;\n+\tstruct enic_flowman *fm;\n+\tstruct enic_fm_fet *fet;\n+\tstruct rte_flow *flow;\n+\tstruct enic *enic;\n+\tint ret;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tenic = pmd_priv(dev);\n+\tfm = enic->fm;\n+\tif (fm == NULL) {\n+\t\trte_flow_error_set(error, ENOTSUP,\n+\t\t\tRTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,\n+\t\t\t\"flowman is not initialized\");\n+\t\treturn NULL;\n+\t}\n+\tenic_fm_open_scratch(fm);\n+\tflow = NULL;\n+\tret = enic_fm_flow_parse(fm, attrs, pattern, actions, error);\n+\tif (ret < 0)\n+\t\tgoto error_with_scratch;\n+\tfm_tcam_entry = &fm->tcam_entry;\n+\tfm_action = &fm->action;\n+\tflow = enic_fm_flow_add_entry(fm, fm_tcam_entry, fm_action,\n+\t\t\t\t attrs, error);\n+\tif (flow) {\n+\t\tLIST_INSERT_HEAD(&enic->flows, flow, next);\n+\t\tfet = flow->fm->fet;\n+\t\tif (fet && fet->default_key) {\n+\t\t\t/*\n+\t\t\t * Jump to non-existent group? Save the relevant info\n+\t\t\t * so we can convert this flow when that group\n+\t\t\t * materializes.\n+\t\t\t */\n+\t\t\tsave_jump_flow(fm, flow, fet->group,\n+\t\t\t\t fm_tcam_entry, fm_action);\n+\t\t} else if (fet && fet->ref == 1) {\n+\t\t\t/*\n+\t\t\t * A new table is created. Convert the saved flows\n+\t\t\t * that should jump to this group.\n+\t\t\t */\n+\t\t\tconvert_jump_flows(fm, fet, error);\n+\t\t}\n+\t}\n+\n+error_with_scratch:\n+\tenic_fm_close_scratch(fm);\n+\treturn flow;\n+}\n+\n+static int\n+enic_fm_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,\n+\t\t __rte_unused struct rte_flow_error *error)\n+{\n+\tstruct enic *enic = pmd_priv(dev);\n+\n+\tENICPMD_FUNC_TRACE();\n+\tif (enic->fm == NULL)\n+\t\treturn 0;\n+\tLIST_REMOVE(flow, next);\n+\tenic_fm_flow_free(enic->fm, flow);\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_flow_flush(struct rte_eth_dev *dev,\n+\t\t __rte_unused struct rte_flow_error *error)\n+{\n+\tstruct enic_fm_flow *fm_flow;\n+\tstruct enic_flowman *fm;\n+\tstruct rte_flow *flow;\n+\tstruct enic *enic = pmd_priv(dev);\n+\n+\tENICPMD_FUNC_TRACE();\n+\tif (enic->fm == NULL)\n+\t\treturn 0;\n+\tfm = enic->fm;\n+\twhile (!LIST_EMPTY(&enic->flows)) {\n+\t\tflow = LIST_FIRST(&enic->flows);\n+\t\tfm_flow = flow->fm;\n+\t\tLIST_REMOVE(flow, next);\n+\t\t/*\n+\t\t * If tables are null, then vNIC is closing, and the firmware\n+\t\t * has already cleaned up flowman state. So do not try to free\n+\t\t * resources, as it only causes errors.\n+\t\t */\n+\t\tif (fm->ig_tcam_hndl == FM_INVALID_HANDLE) {\n+\t\t\tfm_flow->entry_handle = FM_INVALID_HANDLE;\n+\t\t\tfm_flow->action_handle = FM_INVALID_HANDLE;\n+\t\t\tfm_flow->fet = NULL;\n+\t\t}\n+\t\tenic_fm_flow_free(fm, flow);\n+\t}\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_tbl_free(struct enic_flowman *fm, uint64_t handle)\n+{\n+\tu64 args[2];\n+\tint rc;\n+\n+\targs[0] = FM_MATCH_TABLE_FREE;\n+\targs[1] = handle;\n+\trc = vnic_dev_flowman_cmd(fm->enic->vdev, args, 2);\n+\tif (rc)\n+\t\tENICPMD_LOG(ERR, \"cannot free table: rc=%d handle=0x%\" PRIx64,\n+\t\t\t rc, handle);\n+\treturn rc;\n+}\n+\n+static int\n+enic_fm_tcam_tbl_alloc(struct enic_flowman *fm, uint32_t direction,\n+\t\t\tuint32_t max_entries, uint64_t *handle)\n+{\n+\tstruct fm_tcam_match_table *tcam_tbl;\n+\tstruct enic *enic;\n+\tu64 args[2];\n+\tint rc;\n+\n+\tENICPMD_FUNC_TRACE();\n+\tenic = fm->enic;\n+\ttcam_tbl = &fm->cmd.va->fm_tcam_match_table;\n+\ttcam_tbl->ftt_direction = direction;\n+\ttcam_tbl->ftt_stage = FM_STAGE_LAST;\n+\ttcam_tbl->ftt_max_entries = max_entries;\n+\targs[0] = FM_TCAM_TABLE_ALLOC;\n+\targs[1] = fm->cmd.pa;\n+\trc = vnic_dev_flowman_cmd(enic->vdev, args, 2);\n+\tif (rc) {\n+\t\tENICPMD_LOG(ERR, \"cannot alloc %s TCAM table: rc=%d\",\n+\t\t\t (direction == FM_INGRESS) ? \"IG\" : \"EG\", rc);\n+\t\treturn rc;\n+\t}\n+\t*handle = args[0];\n+\tENICPMD_LOG(DEBUG, \"%s TCAM table allocated, handle=0x%\" PRIx64,\n+\t\t (direction == FM_INGRESS) ? \"IG\" : \"EG\", *handle);\n+\treturn 0;\n+}\n+\n+static int\n+enic_fm_init_counters(struct enic_flowman *fm)\n+{\n+\tENICPMD_FUNC_TRACE();\n+\tSLIST_INIT(&fm->counters);\n+\treturn enic_fm_more_counters(fm);\n+}\n+\n+static void\n+enic_fm_free_all_counters(struct enic_flowman *fm)\n+{\n+\tstruct enic *enic;\n+\tu64 args[2];\n+\tint rc;\n+\n+\tenic = fm->enic;\n+\targs[0] = FM_COUNTER_BRK;\n+\targs[1] = 0;\n+\trc = vnic_dev_flowman_cmd(enic->vdev, args, 2);\n+\tif (rc != 0)\n+\t\tENICPMD_LOG(ERR, \"cannot free counters: rc=%d\", rc);\n+\trte_free(fm->counter_stack);\n+}\n+\n+static int\n+enic_fm_alloc_tcam_tables(struct enic_flowman *fm)\n+{\n+\tint rc;\n+\n+\tENICPMD_FUNC_TRACE();\n+\trc = enic_fm_tcam_tbl_alloc(fm, FM_INGRESS, FM_MAX_TCAM_TABLE_SIZE,\n+\t\t\t\t &fm->ig_tcam_hndl);\n+\tif (rc)\n+\t\treturn rc;\n+\trc = enic_fm_tcam_tbl_alloc(fm, FM_EGRESS, FM_MAX_TCAM_TABLE_SIZE,\n+\t\t\t\t &fm->eg_tcam_hndl);\n+\treturn rc;\n+}\n+\n+static void\n+enic_fm_free_tcam_tables(struct enic_flowman *fm)\n+{\n+\tENICPMD_FUNC_TRACE();\n+\tif (fm->ig_tcam_hndl) {\n+\t\tENICPMD_LOG(DEBUG, \"free IG TCAM table handle=0x%\" PRIx64,\n+\t\t\t fm->ig_tcam_hndl);\n+\t\tenic_fm_tbl_free(fm, fm->ig_tcam_hndl);\n+\t\tfm->ig_tcam_hndl = FM_INVALID_HANDLE;\n+\t}\n+\tif (fm->eg_tcam_hndl) {\n+\t\tENICPMD_LOG(DEBUG, \"free EG TCAM table handle=0x%\" PRIx64,\n+\t\t\t fm->eg_tcam_hndl);\n+\t\tenic_fm_tbl_free(fm, fm->eg_tcam_hndl);\n+\t\tfm->eg_tcam_hndl = FM_INVALID_HANDLE;\n+\t}\n+}\n+\n+int\n+enic_fm_init(struct enic *enic)\n+{\n+\tstruct enic_flowman *fm;\n+\tu8 name[NAME_MAX];\n+\tint rc;\n+\n+\tif (enic->flow_filter_mode != FILTER_FLOWMAN)\n+\t\treturn 0;\n+\tENICPMD_FUNC_TRACE();\n+\tfm = calloc(1, sizeof(*fm));\n+\tif (fm == NULL) {\n+\t\tENICPMD_LOG(ERR, \"cannot alloc flowman struct\");\n+\t\treturn -ENOMEM;\n+\t}\n+\tfm->enic = enic;\n+\tTAILQ_INIT(&fm->fet_list);\n+\tTAILQ_INIT(&fm->jump_list);\n+\t/* Allocate host memory for flowman commands */\n+\tsnprintf((char *)name, NAME_MAX, \"fm-cmd-%s\", enic->bdf_name);\n+\tfm->cmd.va = enic_alloc_consistent(enic,\n+\t\tsizeof(union enic_flowman_cmd_mem), &fm->cmd.pa, name);\n+\tif (!fm->cmd.va) {\n+\t\tENICPMD_LOG(ERR, \"cannot allocate flowman command memory\");\n+\t\trc = -ENOMEM;\n+\t\tgoto error_fm;\n+\t}\n+\t/* Allocate TCAM tables upfront as they are the main tables */\n+\trc = enic_fm_alloc_tcam_tables(fm);\n+\tif (rc) {\n+\t\tENICPMD_LOG(ERR, \"cannot alloc TCAM tables\");\n+\t\tgoto error_cmd;\n+\t}\n+\t/* Then a number of counters */\n+\trc = enic_fm_init_counters(fm);\n+\tif (rc) {\n+\t\tENICPMD_LOG(ERR, \"cannot alloc counters\");\n+\t\tgoto error_tables;\n+\t}\n+\t/*\n+\t * One default exact match table for each direction. We hold onto\n+\t * it until close.\n+\t */\n+\trc = enic_fet_alloc(fm, 1, NULL, 128, &fm->default_ig_fet);\n+\tif (rc) {\n+\t\tENICPMD_LOG(ERR, \"cannot alloc default IG exact match table\");\n+\t\tgoto error_counters;\n+\t}\n+\tfm->default_ig_fet->ref = 1;\n+\trc = enic_fet_alloc(fm, 0, NULL, 128, &fm->default_eg_fet);\n+\tif (rc) {\n+\t\tENICPMD_LOG(ERR, \"cannot alloc default EG exact match table\");\n+\t\tgoto error_ig_fet;\n+\t}\n+\tfm->default_eg_fet->ref = 1;\n+\tenic->fm = fm;\n+\treturn 0;\n+\n+error_ig_fet:\n+\tenic_fet_free(fm, fm->default_ig_fet);\n+error_counters:\n+\tenic_fm_free_all_counters(fm);\n+error_tables:\n+\tenic_fm_free_tcam_tables(fm);\n+error_cmd:\n+\tenic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),\n+\t\tfm->cmd.va, fm->cmd.pa);\n+error_fm:\n+\tfree(fm);\n+\treturn rc;\n+}\n+\n+void\n+enic_fm_destroy(struct enic *enic)\n+{\n+\tstruct enic_flowman *fm;\n+\tstruct enic_fm_fet *fet;\n+\n+\tif (enic->fm == NULL)\n+\t\treturn;\n+\tENICPMD_FUNC_TRACE();\n+\tfm = enic->fm;\n+\tenic_fet_free(fm, fm->default_eg_fet);\n+\tenic_fet_free(fm, fm->default_ig_fet);\n+\t/* Free all exact match tables still open */\n+\twhile (!TAILQ_EMPTY(&fm->fet_list)) {\n+\t\tfet = TAILQ_FIRST(&fm->fet_list);\n+\t\tenic_fet_free(fm, fet);\n+\t}\n+\tenic_fm_free_tcam_tables(fm);\n+\tenic_fm_free_all_counters(fm);\n+\tenic_free_consistent(enic, sizeof(union enic_flowman_cmd_mem),\n+\t\tfm->cmd.va, fm->cmd.pa);\n+\tfm->cmd.va = NULL;\n+\tfree(fm);\n+\tenic->fm = NULL;\n+}\n+\n+const struct rte_flow_ops enic_fm_flow_ops = {\n+\t.validate = enic_fm_flow_validate,\n+\t.create = enic_fm_flow_create,\n+\t.destroy = enic_fm_flow_destroy,\n+\t.flush = enic_fm_flow_flush,\n+\t.query = enic_fm_flow_query,\n+};\ndiff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c\nindex fa993c657..40bdd0963 100644\n--- a/drivers/net/enic/enic_main.c\n+++ b/drivers/net/enic/enic_main.c\n@@ -350,7 +350,7 @@ enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)\n \trq->need_initial_post = false;\n }\n \n-static void *\n+void *\n enic_alloc_consistent(void *priv, size_t size,\n \tdma_addr_t *dma_handle, u8 *name)\n {\n@@ -390,7 +390,7 @@ enic_alloc_consistent(void *priv, size_t size,\n \treturn vaddr;\n }\n \n-static void\n+void\n enic_free_consistent(void *priv,\n \t\t __rte_unused size_t size,\n \t\t void *vaddr,\n@@ -587,6 +587,9 @@ int enic_enable(struct enic *enic)\n \t\tdev_warning(enic, \"Init of hash table for clsf failed.\"\\\n \t\t\t\"Flow director feature will not work\\n\");\n \n+\tif (enic_fm_init(enic))\n+\t\tdev_warning(enic, \"Init of flowman failed.\\n\");\n+\n \tfor (index = 0; index < enic->rq_count; index++) {\n \t\terr = enic_alloc_rx_queue_mbufs(enic,\n \t\t\t&enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);\n@@ -1043,6 +1046,7 @@ int enic_disable(struct enic *enic)\n \tvnic_dev_disable(enic->vdev);\n \n \tenic_clsf_destroy(enic);\n+\tenic_fm_destroy(enic);\n \n \tif (!enic_is_sriov_vf(enic))\n \t\tvnic_dev_del_addr(enic->vdev, enic->mac_addr);\ndiff --git a/drivers/net/enic/enic_res.c b/drivers/net/enic/enic_res.c\nindex 742999cd0..f403a0b66 100644\n--- a/drivers/net/enic/enic_res.c\n+++ b/drivers/net/enic/enic_res.c\n@@ -85,17 +85,20 @@ int enic_get_vnic_config(struct enic *enic)\n \tvnic_dev_capable_udp_rss_weak(enic->vdev, &enic->nic_cfg_chk,\n \t\t\t\t &enic->udp_rss_weak);\n \n-\tdev_info(enic, \"Flow api filter mode: %s Actions: %s%s%s\\n\",\n+\tdev_info(enic, \"Flow api filter mode: %s Actions: %s%s%s%s\\n\",\n+\t\t((enic->flow_filter_mode == FILTER_FLOWMAN) ? \"FLOWMAN\" :\n \t\t((enic->flow_filter_mode == FILTER_DPDK_1) ? \"DPDK\" :\n \t\t((enic->flow_filter_mode == FILTER_USNIC_IP) ? \"USNIC\" :\n \t\t((enic->flow_filter_mode == FILTER_IPV4_5TUPLE) ? \"5TUPLE\" :\n-\t\t\"NONE\"))),\n+\t\t\"NONE\")))),\n \t\t((enic->filter_actions & FILTER_ACTION_RQ_STEERING_FLAG) ?\n \t\t \"steer \" : \"\"),\n \t\t((enic->filter_actions & FILTER_ACTION_FILTER_ID_FLAG) ?\n \t\t \"tag \" : \"\"),\n \t\t((enic->filter_actions & FILTER_ACTION_DROP_FLAG) ?\n-\t\t \"drop \" : \"\"));\n+\t\t \"drop \" : \"\"),\n+\t\t((enic->filter_actions & FILTER_ACTION_COUNTER_FLAG) ?\n+\t\t \"count \" : \"\"));\n \n \tc->wq_desc_count =\n \t\tmin_t(u32, ENIC_MAX_WQ_DESCS,\ndiff --git a/drivers/net/enic/meson.build b/drivers/net/enic/meson.build\nindex 6716cdaf3..1bd7cc7e1 100644\n--- a/drivers/net/enic/meson.build\n+++ b/drivers/net/enic/meson.build\n@@ -10,6 +10,7 @@ sources = files(\n \t'enic_clsf.c',\n \t'enic_ethdev.c',\n \t'enic_flow.c',\n+\t'enic_fm_flow.c',\n \t'enic_main.c',\n \t'enic_res.c',\n \t'enic_rxtx.c',\n", "prefixes": [ "2/2" ] }