[v6,08/14] net/nfp: add bond firmware creation initialization

diff --git a/drivers/net/nfp/flower/nfp_flower.c b/drivers/net/nfp/flower/nfp_flower.c
index 94b50611f0..195960e00d 100644
--- a/drivers/net/nfp/flower/nfp_flower.c
+++ b/drivers/net/nfp/flower/nfp_flower.c
@@ -16,6 +16,7 @@ 
 #include "../nfp_cpp_bridge.h"
 #include "../nfp_logs.h"
 #include "../nfp_mtr.h"
+#include "nfp_flower_bond.h"
 #include "nfp_flower_ctrl.h"
 #include "nfp_flower_representor.h"
 
@@ -228,6 +229,42 @@  nfp_flower_init_vnic_common(struct nfp_net_hw *hw,
 	return 0;
 }
 
+static int
+nfp_flower_bond_feature_init(struct nfp_app_fw_flower *app_fw_flower)
+{
+	int ret;
+
+	/* Notify hardware that driver supports hardware creation of bonding port */
+	ret = nfp_rtsym_write_le(app_fw_flower->pf_hw->pf_dev->sym_tbl,
+			"_abi_flower_balance_sync_enable", 1);
+	if (ret != 0) {
+		PMD_INIT_LOG(ERR, "Enable bonding port hardware creation failed.");
+		return ret;
+	}
+
+	ret = nfp_flower_bond_init(app_fw_flower);
+	if (ret != 0) {
+		PMD_INIT_LOG(ERR, "Initialize bonding structure failed.");
+		return ret;
+	}
+
+	app_fw_flower->flower_en_feats |= NFP_FL_ENABLE_BOND;
+
+	return 0;
+}
+
+static int
+nfp_flower_sync_feature_bits(struct nfp_app_fw_flower *app_fw_flower)
+{
+	int ret;
+
+	ret = nfp_flower_bond_feature_init(app_fw_flower);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
 static int
 nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw)
 {
@@ -538,6 +575,20 @@  nfp_flower_cleanup_ctrl_vnic(struct nfp_net_hw *hw)
 	rte_free(eth_dev);
 }
 
+static void
+nfp_flower_bond_feature_cleanup(struct nfp_app_fw_flower *app_fw_flower)
+{
+	nfp_flower_bond_cleanup(app_fw_flower);
+	app_fw_flower->flower_en_feats &= ~NFP_FL_ENABLE_BOND;
+}
+
+static void
+nfp_flower_sync_feature_cleanup(struct nfp_app_fw_flower *app_fw_flower)
+{
+	if (nfp_flower_support_bond_offload(app_fw_flower))
+		nfp_flower_bond_feature_cleanup(app_fw_flower);
+}
+
 static int
 nfp_flower_start_ctrl_vnic(struct nfp_net_hw *net_hw)
 {
@@ -768,22 +819,31 @@  nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev,
 		goto ctrl_vnic_cleanup;
 	}
 
+	/* Synchronize the features of driver and hardware */
+	ret = nfp_flower_sync_feature_bits(app_fw_flower);
+	if (ret != 0) {
+		PMD_INIT_LOG(ERR, "Sync feature bits failed");
+		goto ctrl_vnic_cleanup;
+	}
+
 	/* Start up flower services */
 	ret = nfp_flower_enable_services(app_fw_flower);
 	if (ret != 0) {
 		PMD_INIT_LOG(ERR, "Could not enable flower services");
 		ret = -ESRCH;
-		goto ctrl_vnic_cleanup;
+		goto sync_feature_cleanup;
 	}
 
 	ret = nfp_flower_repr_create(app_fw_flower);
 	if (ret != 0) {
 		PMD_INIT_LOG(ERR, "Could not create representor ports");
-		goto ctrl_vnic_cleanup;
+		goto sync_feature_cleanup;
 	}
 
 	return 0;
 
+sync_feature_cleanup:
+	nfp_flower_sync_feature_cleanup(app_fw_flower);
 ctrl_vnic_cleanup:
 	nfp_flower_cleanup_ctrl_vnic(app_fw_flower->ctrl_hw);
 ctrl_cpp_area_cleanup:
diff --git a/drivers/net/nfp/flower/nfp_flower.h b/drivers/net/nfp/flower/nfp_flower.h
index 8393de66c5..30443cd568 100644
--- a/drivers/net/nfp/flower/nfp_flower.h
+++ b/drivers/net/nfp/flower/nfp_flower.h
@@ -7,6 +7,7 @@ 
 #define __NFP_FLOWER_H__
 
 #include "../nfp_net_common.h"
+#include "nfp_flower_bond.h"
 
 /* Extra features bitmap. */
 #define NFP_FL_FEATS_GENEVE             RTE_BIT64(0)
@@ -61,6 +62,9 @@  struct nfp_app_fw_flower {
 	/** Number of phyport representors */
 	uint8_t num_phyport_reprs;
 
+	/** Bitmap of features enabled by HW */
+	uint8_t flower_en_feats;
+
 	/** Pointer to the PF vNIC */
 	struct nfp_net_hw *pf_hw;
 
@@ -96,6 +100,9 @@  struct nfp_app_fw_flower {
 
 	/** Function pointers for different NFD version */
 	struct nfp_flower_nfd_func nfd_func;
+
+	/** Link bond data block */
+	struct nfp_flower_bond *nfp_bond;
 };
 
 static inline bool
@@ -104,6 +111,12 @@  nfp_flower_support_decap_v2(const struct nfp_app_fw_flower *app_fw_flower)
 	return app_fw_flower->ext_features & NFP_FL_FEATS_DECAP_V2;
 }
 
+static inline bool
+nfp_flower_support_bond_offload(const struct nfp_app_fw_flower *app_fw_flower)
+{
+	return app_fw_flower->flower_en_feats & NFP_FL_ENABLE_BOND;
+}
+
 int nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev,
 		const struct nfp_dev_info *dev_info);
 void nfp_uninit_app_fw_flower(struct nfp_pf_dev *pf_dev);
diff --git a/drivers/net/nfp/flower/nfp_flower_bond.c b/drivers/net/nfp/flower/nfp_flower_bond.c
new file mode 100644
index 0000000000..bbd2818e68
--- /dev/null
+++ b/drivers/net/nfp/flower/nfp_flower_bond.c
@@ -0,0 +1,59 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 Corigine, Inc.
+ * All rights reserved.
+ */
+#include "nfp_flower_bond.h"
+
+#include <rte_malloc.h>
+
+#include "nfp_flower_representor.h"
+
+static void
+nfp_flower_bond_increment_version(struct nfp_flower_bond *nfp_bond)
+{
+	/* LSB is not considered by firmware so add 2 for each increment. */
+	nfp_bond->batch_ver += 2;
+	nfp_bond->batch_ver &= NFP_FL_BOND_VERSION_MASK;
+
+	/* Zero is reserved by firmware. */
+	if (nfp_bond->batch_ver == 0)
+		nfp_bond->batch_ver += 2;
+}
+
+int
+nfp_flower_bond_init(struct nfp_app_fw_flower *app_fw_flower)
+{
+	struct nfp_flower_bond *nfp_bond;
+
+	app_fw_flower->nfp_bond = rte_zmalloc("nfp_bond",
+			sizeof(struct nfp_flower_bond), RTE_CACHE_LINE_SIZE);
+	if (app_fw_flower->nfp_bond == NULL)
+		return -ENOMEM;
+
+	nfp_bond = app_fw_flower->nfp_bond;
+	pthread_mutex_init(&nfp_bond->mutex, NULL);
+	LIST_INIT(&nfp_bond->group_list);
+	nfp_flower_bond_increment_version(nfp_bond);
+	nfp_bond->app_fw_flower = app_fw_flower;
+
+	return 0;
+}
+
+void
+nfp_flower_bond_cleanup(struct nfp_app_fw_flower *app_fw_flower)
+{
+	struct nfp_bond_group *entry;
+	struct nfp_flower_bond *nfp_bond = app_fw_flower->nfp_bond;
+
+	pthread_mutex_lock(&nfp_bond->mutex);
+	LIST_FOREACH(entry, &nfp_bond->group_list, next) {
+		LIST_REMOVE(entry, next);
+		rte_free(entry);
+	}
+	pthread_mutex_unlock(&nfp_bond->mutex);
+
+	pthread_mutex_destroy(&nfp_bond->mutex);
+
+	rte_free(nfp_bond);
+	app_fw_flower->nfp_bond = NULL;
+}
diff --git a/drivers/net/nfp/flower/nfp_flower_bond.h b/drivers/net/nfp/flower/nfp_flower_bond.h
new file mode 100644
index 0000000000..be79764a23
--- /dev/null
+++ b/drivers/net/nfp/flower/nfp_flower_bond.h
@@ -0,0 +1,65 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Corigine, Inc.
+ * All rights reserved.
+ */
+
+#ifndef __NFP_FLOWER_BOND_H__
+#define __NFP_FLOWER_BOND_H__
+
+#include <pthread.h>
+#include <stdbool.h>
+#include <sys/queue.h>
+
+/* The batch version of bond offload packets between firmware and driver */
+#define NFP_FL_BOND_VERSION_MASK       0x007fffff    /* [0, 22] */
+
+#define NFP_FL_ENABLE_BOND             RTE_BIT32(1)
+
+/* ID 0 reserved and IDs 1 to 31 are valid */
+#define NFP_FL_BOND_GROUP_MIN          1
+#define NFP_FL_BOND_GROUP_MAX          32
+
+/* List entry for each bond group */
+struct nfp_bond_group {
+	/** List entry */
+	LIST_ENTRY(nfp_bond_group) next;
+	/** Marked if the group needs synced to HW */
+	bool dirty;
+	/** Marked if the group is currently offloaded to NIC */
+	bool offloaded;
+	/** Marked if the group should be removed from NIC */
+	bool to_remove;
+	/** Marked if the group should be removed from driver */
+	bool to_destroy;
+	/** Assigned group ID for host/kernel sync */
+	uint32_t group_id;
+	/** Number of members in group */
+	uint32_t member_cnt;
+	/** Group instance in case of ID reuse */
+	uint32_t group_inst;
+	/** Group main Netdev */
+	struct rte_eth_dev *main_dev;
+};
+
+/* Flower APP priv data for bond offload */
+struct nfp_flower_bond {
+	/** Marker to reset firmware bond config */
+	bool rst_cfg;
+	/** List of all main/member groups offloaded */
+	LIST_HEAD(, nfp_bond_group) group_list;
+	/** Lock to protect bond group_list */
+	pthread_mutex_t mutex;
+	/** Incremented for each batch of config packets */
+	uint32_t batch_ver;
+	/** Instance allocator for groups */
+	uint32_t global_inst;
+	/** Incremented for each config packet sent */
+	uint32_t pkt_num;
+	/** Pointer to the flower app */
+	struct nfp_app_fw_flower *app_fw_flower;
+};
+
+int nfp_flower_bond_init(struct nfp_app_fw_flower *app_fw_flower);
+void nfp_flower_bond_cleanup(struct nfp_app_fw_flower *app_fw_flower);
+
+#endif /* __NFP_FLOWER_BOND_H__ */
diff --git a/drivers/net/nfp/meson.build b/drivers/net/nfp/meson.build
index 46be6f60cd..f0d4bbcecb 100644
--- a/drivers/net/nfp/meson.build
+++ b/drivers/net/nfp/meson.build
@@ -13,6 +13,7 @@  sources = files(
         'flower/nfp_flower_ctrl.c',
         'flower/nfp_flower_flow.c',
         'flower/nfp_flower_representor.c',
+        'flower/nfp_flower_bond.c',
         'nfd3/nfp_nfd3_dp.c',
         'nfdk/nfp_nfdk_dp.c',
         'nfpcore/nfp_cppcore.c',