@@ -23,6 +23,7 @@
#include "ulp_mapper.h"
#include "ulp_port_db.h"
#include "ulp_tun.h"
+#include "ulp_ha_mgr.h"
/* Linked list of all TF sessions. */
STAILQ_HEAD(, bnxt_ulp_session_state) bnxt_ulp_session_list =
@@ -315,6 +316,9 @@ bnxt_ulp_cntxt_app_caps_init(struct bnxt_ulp_context *ulp_ctx,
if (info[i].flags & BNXT_ULP_APP_CAP_SHARED_EN)
ulp_ctx->cfg_data->ulp_flags |=
BNXT_ULP_SHARED_SESSION_ENABLED;
+ if (info[i].flags & BNXT_ULP_APP_CAP_HOT_UPGRADE_EN)
+ ulp_ctx->cfg_data->ulp_flags |=
+ BNXT_ULP_HIGH_AVAIL_ENABLED;
}
if (!found) {
BNXT_TF_DBG(ERR, "APP ID %d, Device ID: 0x%x not supported.\n",
@@ -1137,9 +1141,18 @@ static void
bnxt_ulp_deinit(struct bnxt *bp,
struct bnxt_ulp_session_state *session)
{
+ bool ha_enabled;
+
if (!bp->ulp_ctx || !bp->ulp_ctx->cfg_data)
return;
+ ha_enabled = bnxt_ulp_cntxt_ha_enabled(bp->ulp_ctx);
+ if (ha_enabled && session->session_opened) {
+ int32_t rc = ulp_ha_mgr_close(bp->ulp_ctx);
+ if (rc)
+ BNXT_TF_DBG(ERR, "Failed to close HA (%d)\n", rc);
+ }
+
/* clean up default flows */
bnxt_ulp_destroy_df_rules(bp, true);
@@ -1179,6 +1192,9 @@ bnxt_ulp_deinit(struct bnxt *bp,
/* free the flow db lock */
pthread_mutex_destroy(&bp->ulp_ctx->cfg_data->flow_db_lock);
+ if (ha_enabled)
+ ulp_ha_mgr_deinit(bp->ulp_ctx);
+
/* Delete the ulp context and tf session and free the ulp context */
ulp_ctx_deinit(bp, session);
BNXT_TF_DBG(DEBUG, "ulp ctx has been deinitialized\n");
@@ -1275,6 +1291,19 @@ bnxt_ulp_init(struct bnxt *bp,
BNXT_TF_DBG(ERR, "Failed to set tx global configuration\n");
goto jump_to_error;
}
+
+ if (bnxt_ulp_cntxt_ha_enabled(bp->ulp_ctx)) {
+ rc = ulp_ha_mgr_init(bp->ulp_ctx);
+ if (rc) {
+ BNXT_TF_DBG(ERR, "Failed to initialize HA %d\n", rc);
+ goto jump_to_error;
+ }
+ rc = ulp_ha_mgr_open(bp->ulp_ctx);
+ if (rc) {
+ BNXT_TF_DBG(ERR, "Failed to Process HA Open %d\n", rc);
+ goto jump_to_error;
+ }
+ }
BNXT_TF_DBG(DEBUG, "ulp ctx has been initialized\n");
return rc;
@@ -1828,3 +1857,33 @@ bnxt_ulp_cntxt_release_fdb_lock(struct bnxt_ulp_context *ulp_ctx)
pthread_mutex_unlock(&ulp_ctx->cfg_data->flow_db_lock);
}
+
+/* Function to set the ha info into the context */
+int32_t
+bnxt_ulp_cntxt_ptr2_ha_info_set(struct bnxt_ulp_context *ulp_ctx,
+ struct bnxt_ulp_ha_mgr_info *ulp_ha_info)
+{
+ if (ulp_ctx == NULL || ulp_ctx->cfg_data == NULL) {
+ BNXT_TF_DBG(ERR, "Invalid ulp context data\n");
+ return -EINVAL;
+ }
+ ulp_ctx->cfg_data->ha_info = ulp_ha_info;
+ return 0;
+}
+
+/* Function to retrieve the ha info from the context. */
+struct bnxt_ulp_ha_mgr_info *
+bnxt_ulp_cntxt_ptr2_ha_info_get(struct bnxt_ulp_context *ulp_ctx)
+{
+ if (ulp_ctx == NULL || ulp_ctx->cfg_data == NULL)
+ return NULL;
+ return ulp_ctx->cfg_data->ha_info;
+}
+
+bool
+bnxt_ulp_cntxt_ha_enabled(struct bnxt_ulp_context *ulp_ctx)
+{
+ if (ulp_ctx == NULL || ulp_ctx->cfg_data == NULL)
+ return false;
+ return !!ULP_HIGH_AVAIL_IS_ENABLED(ulp_ctx->cfg_data->ulp_flags);
+}
@@ -67,6 +67,7 @@ struct bnxt_ulp_data {
void *mapper_data;
struct bnxt_ulp_port_db *port_db;
struct bnxt_ulp_fc_info *fc_info;
+ struct bnxt_ulp_ha_mgr_info *ha_info;
uint32_t ulp_flags;
struct bnxt_ulp_df_rule_info df_rule_info[RTE_MAX_ETHPORTS];
struct bnxt_ulp_vfr_rule_info vfr_rule_info[RTE_MAX_ETHPORTS];
@@ -275,4 +276,13 @@ bnxt_ulp_cntxt_app_caps_init(struct bnxt_ulp_context *ulp_ctx,
struct bnxt_ulp_resource_resv_info *
bnxt_ulp_resource_resv_list_get(uint32_t *num_entries);
+int32_t
+bnxt_ulp_cntxt_ptr2_ha_info_set(struct bnxt_ulp_context *ulp_ctx,
+ struct bnxt_ulp_ha_mgr_info *ulp_ha_info);
+
+struct bnxt_ulp_ha_mgr_info *
+bnxt_ulp_cntxt_ptr2_ha_info_get(struct bnxt_ulp_context *ulp_ctx);
+
+bool
+bnxt_ulp_cntxt_ha_enabled(struct bnxt_ulp_context *ulp_ctx);
#endif /* _BNXT_ULP_H_ */
@@ -11,6 +11,7 @@
#include "ulp_mapper.h"
#include "ulp_fc_mgr.h"
#include "ulp_port_db.h"
+#include "ulp_ha_mgr.h"
#include <rte_malloc.h>
static int32_t
@@ -112,11 +113,17 @@ bnxt_ulp_init_mapper_params(struct bnxt_ulp_mapper_create_parms *mapper_cparms,
/* update the WC Priority flag */
if (!bnxt_ulp_cntxt_ptr2_ulp_flags_get(params->ulp_ctx, &ulp_flags) &&
ULP_HIGH_AVAIL_IS_ENABLED(ulp_flags)) {
- /* TBD: read the state and Set the WC priority */
- ULP_COMP_FLD_IDX_WR(params,
- BNXT_ULP_CF_IDX_WC_IS_HA_HIGH_REG, 1);
+ enum ulp_ha_mgr_region region = ULP_HA_REGION_LOW;
+ int32_t rc;
+
+ rc = ulp_ha_mgr_region_get(params->ulp_ctx, ®ion);
+ if (rc)
+ BNXT_TF_DBG(ERR, "Unable to get WC region\n");
+ if (region == ULP_HA_REGION_HI)
+ ULP_COMP_FLD_IDX_WR(params,
+ BNXT_ULP_CF_IDX_WC_IS_HA_HIGH_REG,
+ 1);
}
-
}
/* Function to create the rte flow. */
@@ -24,6 +24,7 @@ sources += files(
'ulp_tun.c',
'ulp_gen_tbl.c',
'ulp_gen_hash.c',
+ 'ulp_ha_mgr.c',
'ulp_rte_handler_tbl.c',
'ulp_template_db_wh_plus_act.c',
'ulp_template_db_wh_plus_class.c',
new file mode 100644
@@ -0,0 +1,551 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2021 Broadcom
+ * All rights reserved.
+ */
+
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_alarm.h>
+#include "bnxt.h"
+#include "bnxt_ulp.h"
+#include "bnxt_tf_common.h"
+#include "ulp_ha_mgr.h"
+#include "ulp_flow_db.h"
+
+/* Local only MACROs and defines that aren't exported */
+#define ULP_HA_TIMER_THREAD (1 << 0)
+#define ULP_HA_TIMER_IS_RUNNING(info) (!!((info)->flags & ULP_HA_TIMER_THREAD))
+#define ULP_HA_TIMER_SEC 1
+#define ULP_HA_WAIT_TIME (MS_PER_S / 10)
+#define ULP_HA_WAIT_TIMEOUT (MS_PER_S * 2)
+
+#define ULP_HA_IF_TBL_DIR TF_DIR_RX
+#define ULP_HA_IF_TBL_TYPE TF_IF_TBL_TYPE_PROF_PARIF_ERR_ACT_REC_PTR
+#define ULP_HA_IF_TBL_IDX 10
+
+static void ulp_ha_mgr_timer_cancel(struct bnxt_ulp_context *ulp_ctx);
+static int32_t ulp_ha_mgr_timer_start(struct bnxt_ulp_context *ulp_ctx);
+static void ulp_ha_mgr_timer_cb(void *arg);
+static int32_t ulp_ha_mgr_app_type_set(struct bnxt_ulp_context *ulp_ctx,
+ enum ulp_ha_mgr_app_type app_type);
+static int32_t
+ulp_ha_mgr_region_set(struct bnxt_ulp_context *ulp_ctx,
+ enum ulp_ha_mgr_region region);
+static int32_t
+ulp_ha_mgr_state_set(struct bnxt_ulp_context *ulp_ctx,
+ enum ulp_ha_mgr_state state);
+
+static int32_t
+ulp_ha_mgr_state_set(struct bnxt_ulp_context *ulp_ctx,
+ enum ulp_ha_mgr_state state)
+{
+ struct tf_set_if_tbl_entry_parms set_parms = { 0 };
+ struct tf *tfp;
+ uint32_t val = 0;
+ int32_t rc = 0;
+
+ if (ulp_ctx == NULL) {
+ BNXT_TF_DBG(ERR, "Invalid parms in state get.\n");
+ return -EINVAL;
+ }
+ tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx, BNXT_ULP_SHARED_SESSION_NO);
+ if (tfp == NULL) {
+ BNXT_TF_DBG(ERR, "Unable to get the TFP.\n");
+ return -EINVAL;
+ }
+
+ val = (uint32_t)state;
+
+ set_parms.dir = ULP_HA_IF_TBL_DIR;
+ set_parms.type = ULP_HA_IF_TBL_TYPE;
+ set_parms.data = (uint8_t *)&val;
+ set_parms.data_sz_in_bytes = sizeof(val);
+ set_parms.idx = ULP_HA_IF_TBL_IDX;
+
+ rc = tf_set_if_tbl_entry(tfp, &set_parms);
+ if (rc)
+ BNXT_TF_DBG(ERR, "Failed to write the HA state\n");
+
+ return rc;
+}
+
+static int32_t
+ulp_ha_mgr_region_set(struct bnxt_ulp_context *ulp_ctx,
+ enum ulp_ha_mgr_region region)
+{
+ struct bnxt_ulp_ha_mgr_info *ha_info;
+
+ if (ulp_ctx == NULL) {
+ BNXT_TF_DBG(ERR, "Invalid params in ha region get.\n");
+ return -EINVAL;
+ }
+
+ ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+ if (ha_info == NULL) {
+ BNXT_TF_DBG(ERR, "Unable to get ha info\n");
+ return -EINVAL;
+ }
+ ha_info->region = region;
+
+ return 0;
+}
+
+static int32_t
+ulp_ha_mgr_app_type_set(struct bnxt_ulp_context *ulp_ctx,
+ enum ulp_ha_mgr_app_type app_type)
+{
+ struct bnxt_ulp_ha_mgr_info *ha_info;
+
+ if (ulp_ctx == NULL) {
+ BNXT_TF_DBG(ERR, "Invalid Parms.\n");
+ return -EINVAL;
+ }
+
+ ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+ if (ha_info == NULL) {
+ BNXT_TF_DBG(ERR, "Unable to get the ha info.\n");
+ return -EINVAL;
+ }
+ ha_info->app_type = app_type;
+
+ return 0;
+}
+
+/*
+ * When a secondary opens, the timer is started and periodically checks for a
+ * close of the primary (state moved to SEC_TIMER_COPY).
+ * In SEC_TIMER_COPY:
+ * - The flow db must be locked to prevent flows from being added to the high
+ * region during a move.
+ * - Move the high entries to low
+ * - Set the region to low for subsequent flows
+ * - Switch our persona to Primary
+ * - Set the state to Primary Run
+ * - Release the flow db lock for flows to continue
+ */
+static void
+ulp_ha_mgr_timer_cb(void *arg)
+{
+ struct tf_move_tcam_shared_entries_parms mparms = { 0 };
+ struct bnxt_ulp_context *ulp_ctx;
+ enum ulp_ha_mgr_state curr_state;
+ struct tf *tfp;
+ int32_t rc;
+
+ ulp_ctx = (struct bnxt_ulp_context *)arg;
+ rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state);
+ if (rc) {
+ /*
+ * This shouldn't happen, if it does, resetart the timer
+ * and try again next time.
+ */
+ BNXT_TF_DBG(ERR, "On HA CB:Failed(%d) to get state.\n", rc);
+ goto cb_restart;
+ }
+ if (curr_state != ULP_HA_STATE_SEC_TIMER_COPY)
+ goto cb_restart;
+
+ /* Protect the flow database during the copy */
+ if (bnxt_ulp_cntxt_acquire_fdb_lock(ulp_ctx)) {
+ /* Should not fail, if we do, restart timer and try again */
+ BNXT_TF_DBG(ERR, "Flow db lock acquire failed\n");
+ goto cb_restart;
+ }
+ /* All paths after this point must release the fdb lock */
+
+ /* The Primary has issued a close and we are in the timer copy
+ * phase. Become the new Primary, Set state to Primary Run and
+ * move WC entries to Low Region.
+ */
+ BNXT_TF_DBG(INFO, "On HA CB: Moving entries HI to LOW\n");
+ mparms.dir = TF_DIR_RX;
+ mparms.tcam_tbl_type = TF_TCAM_TBL_TYPE_WC_TCAM_HIGH;
+ tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx, BNXT_ULP_SHARED_SESSION_YES);
+ if (tfp == NULL) {
+ BNXT_TF_DBG(ERR, "On HA CB: Unable to get the TFP.\n");
+ goto unlock;
+ }
+
+ rc = tf_move_tcam_shared_entries(tfp, &mparms);
+ if (rc) {
+ BNXT_TF_DBG(ERR, "On HA_CB: Failed to move entries\n");
+ goto unlock;
+ }
+
+ ulp_ha_mgr_region_set(ulp_ctx, ULP_HA_REGION_LOW);
+ ulp_ha_mgr_app_type_set(ulp_ctx, ULP_HA_APP_TYPE_PRIM);
+ ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_RUN);
+ BNXT_TF_DBG(INFO, "On HA CB: SEC[SEC_TIMER_COPY] => PRIM[PRIM_RUN]\n");
+unlock:
+ bnxt_ulp_cntxt_release_fdb_lock(ulp_ctx);
+ return;
+cb_restart:
+ ulp_ha_mgr_timer_start(ulp_ctx);
+}
+
+static int32_t
+ulp_ha_mgr_timer_start(struct bnxt_ulp_context *ulp_ctx)
+{
+ struct bnxt_ulp_ha_mgr_info *ha_info;
+
+ if (ulp_ctx == NULL) {
+ BNXT_TF_DBG(ERR, "Invalid parmsi for ha timer start.\n");
+ return -EINVAL;
+ }
+
+ ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+
+ if (ha_info == NULL) {
+ BNXT_TF_DBG(ERR, "Unable to get HA Info in timer start.\n");
+ return -EINVAL;
+ }
+ ha_info->flags |= ULP_HA_TIMER_THREAD;
+ rte_eal_alarm_set(US_PER_S * ULP_HA_TIMER_SEC,
+ ulp_ha_mgr_timer_cb,
+ (void *)ulp_ctx);
+ return 0;
+}
+
+static void
+ulp_ha_mgr_timer_cancel(struct bnxt_ulp_context *ulp_ctx)
+{
+ struct bnxt_ulp_ha_mgr_info *ha_info;
+
+ ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+ if (ha_info == NULL) {
+ BNXT_TF_DBG(ERR, "Unable to get ha info\n");
+ return;
+ }
+
+ ha_info->flags &= ~ULP_HA_TIMER_THREAD;
+ rte_eal_alarm_cancel(ulp_ha_mgr_timer_cb, (void *)ulp_ctx);
+}
+
+int32_t
+ulp_ha_mgr_init(struct bnxt_ulp_context *ulp_ctx)
+{
+ struct bnxt_ulp_ha_mgr_info *ha_info;
+ int32_t rc;
+ ha_info = rte_zmalloc("ulp_ha_mgr_info", sizeof(*ha_info), 0);
+ if (!ha_info)
+ return -ENOMEM;
+
+ /* Add the HA info tbl to the ulp context. */
+ bnxt_ulp_cntxt_ptr2_ha_info_set(ulp_ctx, ha_info);
+
+ rc = pthread_mutex_init(&ha_info->ha_lock, NULL);
+ if (rc) {
+ PMD_DRV_LOG(ERR, "Failed to initialize ha mutex\n");
+ goto cleanup;
+ }
+
+ return 0;
+cleanup:
+ if (ha_info != NULL)
+ ulp_ha_mgr_deinit(ulp_ctx);
+ return -ENOMEM;
+}
+
+void
+ulp_ha_mgr_deinit(struct bnxt_ulp_context *ulp_ctx)
+{
+ struct bnxt_ulp_ha_mgr_info *ha_info;
+
+ ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+ if (ha_info == NULL) {
+ BNXT_TF_DBG(ERR, "Unable to get HA Info for deinit.\n");
+ return;
+ }
+
+ pthread_mutex_destroy(&ha_info->ha_lock);
+ rte_free(ha_info);
+
+ bnxt_ulp_cntxt_ptr2_ha_info_set(ulp_ctx, NULL);
+}
+
+int32_t
+ulp_ha_mgr_app_type_get(struct bnxt_ulp_context *ulp_ctx,
+ enum ulp_ha_mgr_app_type *app_type)
+{
+ struct bnxt_ulp_ha_mgr_info *ha_info;
+
+ if (ulp_ctx == NULL || app_type == NULL) {
+ BNXT_TF_DBG(ERR, "Invalid Parms.\n");
+ return -EINVAL;
+ }
+
+ ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+ if (ha_info == NULL) {
+ BNXT_TF_DBG(ERR, "Unable to get the HA info.\n");
+ return -EINVAL;
+ }
+ *app_type = ha_info->app_type;
+
+ return 0;
+}
+
+int32_t
+ulp_ha_mgr_state_get(struct bnxt_ulp_context *ulp_ctx,
+ enum ulp_ha_mgr_state *state)
+{
+ struct tf_get_if_tbl_entry_parms get_parms = { 0 };
+ struct tf *tfp;
+ uint32_t val = 0;
+ int32_t rc = 0;
+
+ if (ulp_ctx == NULL || state == NULL) {
+ BNXT_TF_DBG(ERR, "Invalid parms in state get.\n");
+ return -EINVAL;
+ }
+ tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx, BNXT_ULP_SHARED_SESSION_NO);
+ if (tfp == NULL) {
+ BNXT_TF_DBG(ERR, "Unable to get the TFP.\n");
+ return -EINVAL;
+ }
+
+ get_parms.dir = ULP_HA_IF_TBL_DIR;
+ get_parms.type = ULP_HA_IF_TBL_TYPE;
+ get_parms.idx = ULP_HA_IF_TBL_IDX;
+ get_parms.data = (uint8_t *)&val;
+ get_parms.data_sz_in_bytes = sizeof(val);
+
+ rc = tf_get_if_tbl_entry(tfp, &get_parms);
+ if (rc)
+ BNXT_TF_DBG(ERR, "Failed to read the HA state\n");
+
+ *state = val;
+ return rc;
+}
+
+int32_t
+ulp_ha_mgr_open(struct bnxt_ulp_context *ulp_ctx)
+{
+ enum ulp_ha_mgr_state curr_state;
+ int32_t rc;
+
+ rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state);
+ if (rc) {
+ BNXT_TF_DBG(ERR, "Failed to get HA state on Open (%d)\n", rc);
+ return -EINVAL;
+ }
+
+ /*
+ * An Open can only occur during the Init and Primary Run states. During
+ * Init, the system attempting to Open will become the only system
+ * running. During Primary Run, the system attempting to Open will
+ * become the secondary system temporarily, and should eventually be
+ * transitioned to the primary system.
+ */
+ switch (curr_state) {
+ case ULP_HA_STATE_INIT:
+ /*
+ * No system is running, as we are the primary. Since no other
+ * system is running, we start writing into the low region. By
+ * writing into the low region, we save room for the secondary
+ * system to override our entries by using the high region.
+ */
+ ulp_ha_mgr_app_type_set(ulp_ctx, ULP_HA_APP_TYPE_PRIM);
+ ulp_ha_mgr_region_set(ulp_ctx, ULP_HA_REGION_LOW);
+ rc = ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_RUN);
+ if (rc) {
+ BNXT_TF_DBG(ERR, "On Open: Failed to set PRIM_RUN.\n");
+ return -EINVAL;
+ }
+
+ BNXT_TF_DBG(INFO, "On Open: [INIT] => PRIM[PRIM_RUN]\n");
+ break;
+ case ULP_HA_STATE_PRIM_RUN:
+ /*
+ * The secondary system is starting in order to take over.
+ * The current primary is expected to eventually close and pass
+ * full control to this system;however, until the primary closes
+ * both are operational.
+ *
+ * The timer is started in order to determine when the
+ * primary has closed.
+ */
+ ulp_ha_mgr_app_type_set(ulp_ctx, ULP_HA_APP_TYPE_SEC);
+ ulp_ha_mgr_region_set(ulp_ctx, ULP_HA_REGION_HI);
+
+ /*
+ * TODO:
+ * Clear the high region so the secondary can begin overriding
+ * the current entries.
+ */
+ rc = ulp_ha_mgr_timer_start(ulp_ctx);
+ if (rc) {
+ BNXT_TF_DBG(ERR, "Unable to start timer on HA Open.\n");
+ return -EINVAL;
+ }
+
+ rc = ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_SEC_RUN);
+ if (rc) {
+ BNXT_TF_DBG(ERR, "On Open: Failed to set PRIM_SEC_RUN\n");
+ return -EINVAL;
+ }
+ BNXT_TF_DBG(INFO, "On Open: [PRIM_RUN] => [PRIM_SEC_RUN]\n");
+ break;
+ default:
+ BNXT_TF_DBG(ERR, "On Open: Unknown state 0x%x\n", curr_state);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int32_t
+ulp_ha_mgr_close(struct bnxt_ulp_context *ulp_ctx)
+{
+ enum ulp_ha_mgr_state curr_state, next_state, poll_state;
+ enum ulp_ha_mgr_app_type app_type;
+ int32_t timeout;
+ int32_t rc;
+
+ rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state);
+ if (rc) {
+ BNXT_TF_DBG(ERR, "On Close: Failed(%d) to get HA state\n", rc);
+ return -EINVAL;
+ }
+
+ rc = ulp_ha_mgr_app_type_get(ulp_ctx, &app_type);
+ if (rc) {
+ BNXT_TF_DBG(ERR, "On Close: Failed to get the app type.\n");
+ return -EINVAL;
+ }
+
+ if (curr_state == ULP_HA_STATE_PRIM_RUN &&
+ app_type == ULP_HA_APP_TYPE_PRIM) {
+ /*
+ * Only the primary is running, so a close effectively moves the
+ * system back to INIT.
+ */
+ next_state = ULP_HA_STATE_INIT;
+ ulp_ha_mgr_state_set(ulp_ctx, next_state);
+ BNXT_TF_DBG(INFO, "On Close: PRIM[PRIM_RUN] => [INIT]\n");
+ } else if (curr_state == ULP_HA_STATE_PRIM_SEC_RUN &&
+ app_type == ULP_HA_APP_TYPE_PRIM) {
+ /*
+ * While both are running, the primary received a close.
+ * Cleanup the flows, set the COPY state, and wait for the
+ * secondary to become the Primary.
+ */
+ BNXT_TF_DBG(INFO,
+ "On Close: PRIM[PRIM_SEC_RUN] flushing flows.\n");
+
+ ulp_flow_db_flush_flows(ulp_ctx, BNXT_ULP_FDB_TYPE_REGULAR);
+ ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_SEC_TIMER_COPY);
+
+ /*
+ * TODO: This needs to be bounded in case the other system does
+ * not move to PRIM_RUN.
+ */
+ BNXT_TF_DBG(INFO,
+ "On Close: PRIM[PRIM_SEC_RUN] => [Copy], enter wait.\n");
+ timeout = ULP_HA_WAIT_TIMEOUT;
+ do {
+ rte_delay_ms(ULP_HA_WAIT_TIME);
+ rc = ulp_ha_mgr_state_get(ulp_ctx, &poll_state);
+ if (rc) {
+ BNXT_TF_DBG(ERR,
+ "Failed to get HA state on Close (%d)\n",
+ rc);
+ goto cleanup;
+ }
+ timeout -= ULP_HA_WAIT_TIME;
+ BNXT_TF_DBG(INFO,
+ "On Close: Waiting %d ms for PRIM_RUN\n",
+ timeout);
+ } while (poll_state != ULP_HA_STATE_PRIM_RUN && timeout > 0);
+
+ if (timeout <= 0) {
+ BNXT_TF_DBG(ERR, "On Close: SEC[COPY] Timed out\n");
+ goto cleanup;
+ }
+
+ BNXT_TF_DBG(INFO, "On Close: PRIM[PRIM_SEC_RUN] => [COPY]\n");
+ } else if (curr_state == ULP_HA_STATE_PRIM_SEC_RUN &&
+ app_type == ULP_HA_APP_TYPE_SEC) {
+ /*
+ * While both are running, the secondary unexpectedly received a
+ * close. Cancel the timer, set the state to Primary RUN since
+ * it is the only one running.
+ */
+ ulp_ha_mgr_timer_cancel(ulp_ctx);
+ ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_RUN);
+
+ BNXT_TF_DBG(INFO, "On Close: SEC[PRIM_SEC_RUN] => [PRIM_RUN]\n");
+ } else if (curr_state == ULP_HA_STATE_SEC_TIMER_COPY &&
+ app_type == ULP_HA_APP_TYPE_SEC) {
+ /*
+ * While both were running and the Secondary went into copy,
+ * secondary received a close. Wait until the former Primary
+ * clears the copy stage, close, and set to INIT.
+ */
+ BNXT_TF_DBG(INFO, "On Close: SEC[COPY] wait for PRIM_RUN\n");
+
+ timeout = ULP_HA_WAIT_TIMEOUT;
+ do {
+ rte_delay_ms(ULP_HA_WAIT_TIME);
+ rc = ulp_ha_mgr_state_get(ulp_ctx, &poll_state);
+ if (rc) {
+ BNXT_TF_DBG(ERR,
+ "Failed to get HA state on Close (%d)\n",
+ rc);
+ goto cleanup;
+ }
+
+ timeout -= ULP_HA_WAIT_TIME;
+ BNXT_TF_DBG(INFO,
+ "On Close: Waiting %d ms for PRIM_RUN\n",
+ timeout);
+ } while (poll_state != ULP_HA_STATE_PRIM_RUN &&
+ timeout >= 0);
+
+ if (timeout <= 0) {
+ BNXT_TF_DBG(ERR,
+ "On Close: SEC[COPY] Timed out\n");
+ goto cleanup;
+ }
+
+ next_state = ULP_HA_STATE_INIT;
+ rc = ulp_ha_mgr_state_set(ulp_ctx, next_state);
+ if (rc) {
+ BNXT_TF_DBG(ERR,
+ "On Close: Failed to set state to INIT(%x)\n",
+ rc);
+ goto cleanup;
+ }
+
+ BNXT_TF_DBG(INFO,
+ "On Close: SEC[COPY] => [INIT] after %d ms\n",
+ ULP_HA_WAIT_TIMEOUT - timeout);
+ } else {
+ BNXT_TF_DBG(ERR, "On Close: Invalid type/state %d/%d\n",
+ curr_state, app_type);
+ }
+cleanup:
+ return rc;
+}
+
+int32_t
+ulp_ha_mgr_region_get(struct bnxt_ulp_context *ulp_ctx,
+ enum ulp_ha_mgr_region *region)
+{
+ struct bnxt_ulp_ha_mgr_info *ha_info;
+
+ if (ulp_ctx == NULL || region == NULL) {
+ BNXT_TF_DBG(ERR, "Invalid params in ha region get.\n");
+ return -EINVAL;
+ }
+
+ ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+ if (ha_info == NULL) {
+ BNXT_TF_DBG(ERR, "Unable to get ha info\n");
+ return -EINVAL;
+ }
+ *region = ha_info->region;
+
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2021 Broadcom
+ * All rights reserved.
+ */
+
+#ifndef _ULP_HA_MGR_H_
+#define _ULP_HA_MGR_H_
+
+#include "bnxt_ulp.h"
+
+enum ulp_ha_mgr_state {
+ ULP_HA_STATE_INIT,
+ ULP_HA_STATE_PRIM_RUN,
+ ULP_HA_STATE_PRIM_SEC_RUN,
+ ULP_HA_STATE_SEC_TIMER_COPY,
+ ULP_HA_PRIM_CLOSE
+};
+
+enum ulp_ha_mgr_app_type {
+ ULP_HA_APP_TYPE_NONE,
+ ULP_HA_APP_TYPE_PRIM,
+ ULP_HA_APP_TYPE_SEC
+};
+
+enum ulp_ha_mgr_region {
+ ULP_HA_REGION_LOW,
+ ULP_HA_REGION_HI
+};
+
+struct bnxt_ulp_ha_mgr_info {
+ enum ulp_ha_mgr_app_type app_type;
+ enum ulp_ha_mgr_region region;
+ uint32_t flags;
+ pthread_mutex_t ha_lock;
+};
+
+bool
+ulp_ha_mgr_is_enabled(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_enable(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_init(struct bnxt_ulp_context *ulp_ctx);
+
+void
+ulp_ha_mgr_deinit(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_app_type_get(struct bnxt_ulp_context *ulp_ctx,
+ enum ulp_ha_mgr_app_type *app_type);
+
+int32_t
+ulp_ha_mgr_state_get(struct bnxt_ulp_context *ulp_ctx,
+ enum ulp_ha_mgr_state *state);
+
+int32_t
+ulp_ha_mgr_open(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_close(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_region_get(struct bnxt_ulp_context *ulp_ctx,
+ enum ulp_ha_mgr_region *region);
+
+#endif /* _ULP_HA_MGR_H_*/
@@ -19,6 +19,7 @@
#include "tf_util.h"
#include "ulp_template_db_tbl.h"
#include "ulp_port_db.h"
+#include "ulp_ha_mgr.h"
static uint8_t mapper_fld_zeros[16] = { 0 };
@@ -419,7 +420,7 @@ ulp_mapper_ident_fields_get(struct bnxt_ulp_mapper_parms *mparms,
}
static inline int32_t
-ulp_mapper_tcam_entry_free(struct bnxt_ulp_context *ulp __rte_unused,
+ulp_mapper_tcam_entry_free(struct bnxt_ulp_context *ulp,
struct tf *tfp,
struct ulp_flow_db_res_params *res)
{
@@ -429,6 +430,30 @@ ulp_mapper_tcam_entry_free(struct bnxt_ulp_context *ulp __rte_unused,
.idx = (uint16_t)res->resource_hndl
};
+ /* If HA is enabled, we may have to remap the TF Type */
+ if (bnxt_ulp_cntxt_ha_enabled(ulp)) {
+ enum ulp_ha_mgr_region region;
+ int32_t rc;
+
+ switch (res->resource_type) {
+ case TF_TCAM_TBL_TYPE_WC_TCAM_HIGH:
+ case TF_TCAM_TBL_TYPE_WC_TCAM_LOW:
+ rc = ulp_ha_mgr_region_get(ulp, ®ion);
+ if (rc)
+ /* Log this, but assume region is correct */
+ BNXT_TF_DBG(ERR,
+ "Unable to get HA region (%d)\n",
+ rc);
+ else
+ fparms.tcam_tbl_type =
+ (region == ULP_HA_REGION_LOW) ?
+ TF_TCAM_TBL_TYPE_WC_TCAM_LOW :
+ TF_TCAM_TBL_TYPE_WC_TCAM_HIGH;
+ break;
+ default:
+ break;
+ }
+ }
return tf_free_tcam_entry(tfp, &fparms);
}
@@ -2904,10 +2929,12 @@ static int32_t
ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx,
struct bnxt_ulp_mapper_data *mapper_data)
{
+ struct tf_get_shared_tbl_increment_parms iparms;
struct bnxt_ulp_glb_resource_info *glb_res;
struct tf_get_session_info_parms sparms;
uint32_t num_entries, i, dev_id, res;
struct tf_resource_info *res_info;
+ uint32_t addend;
uint64_t regval;
enum tf_dir dir;
int32_t rc = 0;
@@ -2915,13 +2942,11 @@ ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx,
uint8_t app_id;
memset(&sparms, 0, sizeof(sparms));
-
glb_res = bnxt_ulp_app_glb_resource_info_list_get(&num_entries);
if (!glb_res || !num_entries) {
BNXT_TF_DBG(ERR, "Invalid Arguments\n");
return -EINVAL;
}
-
tfp = bnxt_ulp_cntxt_shared_tfp_get(ulp_ctx);
if (!tfp) {
BNXT_TF_DBG(ERR, "Failed to get tfp for app global init");
@@ -2958,12 +2983,29 @@ ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx,
continue;
dir = glb_res[i].direction;
res = glb_res[i].resource_type;
+ addend = 1;
switch (glb_res[i].resource_func) {
case BNXT_ULP_RESOURCE_FUNC_IDENTIFIER:
res_info = &sparms.session_info.ident[dir].info[res];
break;
case BNXT_ULP_RESOURCE_FUNC_INDEX_TABLE:
+ /*
+ * Tables may have various strides for the allocations.
+ * Need to account.
+ */
+ memset(&iparms, 0, sizeof(iparms));
+ iparms.dir = dir;
+ iparms.type = res;
+ rc = tf_get_shared_tbl_increment(tfp, &iparms);
+ if (rc) {
+ BNXT_TF_DBG(ERR,
+ "Failed to get addend for %s[%s] rc=(%d)\n",
+ tf_tbl_type_2_str(res),
+ tf_dir_2_str(dir), rc);
+ return rc;
+ }
+ addend = iparms.increment_cnt;
res_info = &sparms.session_info.tbl[dir].info[res];
break;
case BNXT_ULP_RESOURCE_FUNC_TCAM_TABLE:
@@ -2977,10 +3019,8 @@ ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx,
glb_res[i].resource_func);
continue;
}
-
regval = tfp_cpu_to_be_64((uint64_t)res_info->start);
- res_info->start++;
-
+ res_info->start += addend;
/*
* All resources written to the global regfile are shared for
* this function.