new file mode 100644
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell International Ltd.
+ */
+
+#include "graph_private.h"
+#include <rte_graph_feature_arc_worker.h>
+#include <rte_malloc.h>
+
+#define __RTE_GRAPH_FEATURE_ARC_MAX 32
+
+#define rte_graph_uint_cast(x) ((unsigned int)x)
+
+rte_graph_feature_arc_main_t *__feature_arc_main;
+
+static int
+feature_lookup(struct rte_graph_feature_arc *dfl, const char *feat_name,
+ struct rte_graph_feature_node_list **ffinfo, uint32_t *slot)
+{
+ struct rte_graph_feature_node_list *finfo = NULL;
+ const char *name;
+
+ if (!feat_name)
+ return -1;
+
+ if (slot)
+ *slot = 0;
+
+ STAILQ_FOREACH(finfo, &dfl->all_features, next_feature) {
+ RTE_VERIFY(finfo->feature_arc == dfl);
+ name = rte_node_id_to_name(finfo->feature_node->id);
+ if (!strncmp(name, feat_name, RTE_GRAPH_NAMESIZE)) {
+ if (ffinfo)
+ *ffinfo = finfo;
+ return 0;
+ }
+ if (slot)
+ (*slot)++;
+ }
+ return -1;
+}
+
+static int
+feature_arc_lookup(rte_graph_feature_arc_t _dfl)
+{
+ struct rte_graph_feature_arc *dfl = rte_graph_feature_arc_get(_dfl);
+ rte_graph_feature_arc_main_t *dm = __feature_arc_main;
+ uint32_t iter;
+
+ if (!__feature_arc_main)
+ return -1;
+
+ for (iter = 0; iter < dm->max_feature_arcs; iter++) {
+ if (dm->feature_arcs[iter] == RTE_GRAPH_FEATURE_ARC_INITIALIZER)
+ continue;
+
+ if (dfl == (rte_graph_feature_arc_get(dm->feature_arcs[iter])))
+ return 0;
+ }
+ return -1;
+}
+
+static int
+get_existing_edge(const char *arc_name, struct rte_node_register *parent_node,
+ struct rte_node_register *child_node, rte_edge_t *_edge)
+{
+ char **next_edges = NULL;
+ uint32_t count, i;
+
+ RTE_SET_USED(arc_name);
+
+ count = rte_node_edge_get(parent_node->id, NULL);
+ next_edges = malloc(count);
+
+ if (!next_edges)
+ return -1;
+
+ count = rte_node_edge_get(parent_node->id, next_edges);
+ for (i = 0; i < count; i++) {
+ if (strstr(child_node->name, next_edges[i])) {
+ graph_dbg("%s: Edge exists [%s[%u]: \"%s\"]", arc_name,
+ parent_node->name, i, child_node->name);
+ if (_edge)
+ *_edge = (rte_edge_t)i;
+
+ free(next_edges);
+ return 0;
+ }
+ }
+ free(next_edges);
+
+ return -1;
+}
+
+static int
+connect_graph_nodes(struct rte_node_register *parent_node, struct rte_node_register *child_node,
+ rte_edge_t *_edge, char *arc_name)
+{
+ const char *next_node = NULL;
+ rte_edge_t edge;
+
+ if (!get_existing_edge(arc_name, parent_node, child_node, &edge)) {
+ graph_dbg("%s: add_feature: Edge reused [%s[%u]: \"%s\"]", arc_name,
+ parent_node->name, edge, child_node->name);
+
+ if (_edge)
+ *_edge = edge;
+
+ return 0;
+ }
+
+ /* Node to be added */
+ next_node = child_node->name;
+
+ edge = rte_node_edge_update(parent_node->id, RTE_EDGE_ID_INVALID, &next_node, 1);
+
+ if (edge == RTE_EDGE_ID_INVALID) {
+ graph_err("edge invalid");
+ return -1;
+ }
+ edge = rte_node_edge_count(parent_node->id) - 1;
+
+ graph_dbg("%s: add_feature: edge added [%s[%u]: \"%s\"]", arc_name, parent_node->name, edge,
+ child_node->name);
+
+ if (_edge)
+ *_edge = edge;
+
+ return 0;
+}
+
+static int
+feature_arc_init(rte_graph_feature_arc_main_t **pfl, uint32_t max_feature_arcs)
+{
+ rte_graph_feature_arc_main_t *pm = NULL;
+ uint32_t i;
+ size_t sz;
+
+ if (!pfl)
+ return -1;
+
+ sz = sizeof(rte_graph_feature_arc_main_t) +
+ (sizeof(pm->feature_arcs[0]) * max_feature_arcs);
+
+ pm = malloc(sz);
+ if (!pm)
+ return -1;
+
+ memset(pm, 0, sz);
+
+ for (i = 0; i < max_feature_arcs; i++)
+ pm->feature_arcs[i] = RTE_GRAPH_FEATURE_ARC_INITIALIZER;
+
+ pm->max_feature_arcs = max_feature_arcs;
+
+ *pfl = pm;
+
+ return 0;
+}
+
+int
+rte_graph_feature_arc_init(int max_feature_arcs)
+{
+ if (!max_feature_arcs)
+ return -1;
+
+ if (__feature_arc_main)
+ return -1;
+
+ return feature_arc_init(&__feature_arc_main, max_feature_arcs);
+}
+
+int
+rte_graph_feature_arc_create(const char *feature_arc_name, int max_features, int max_indexes,
+ struct rte_node_register *start_node, rte_graph_feature_arc_t *_dfl)
+{
+ char name[2 * RTE_GRAPH_FEATURE_ARC_NAMELEN];
+ rte_graph_feature_arc_main_t *dfm = NULL;
+ struct rte_graph_feature_arc *dfl = NULL;
+ struct rte_graph_feature_data *dfd = NULL;
+ struct rte_graph_feature *df = NULL;
+ uint32_t iter, j, arc_index;
+ size_t sz;
+
+ if (!_dfl)
+ return -1;
+
+ if (max_features < 1)
+ return -1;
+
+ if (!start_node)
+ return -1;
+
+ if (!feature_arc_name)
+ return -1;
+
+ if (max_features > RTE_GRAPH_FEATURE_MAX_PER_ARC) {
+ graph_err("Invalid max features: %u", max_features);
+ return -1;
+ }
+
+ /*
+ * Application hasn't called rte_graph_feature_arc_init(). Initialize with
+ * default values
+ */
+ if (!__feature_arc_main) {
+ if (rte_graph_feature_arc_init((int)__RTE_GRAPH_FEATURE_ARC_MAX) < 0) {
+ graph_err("rte_graph_feature_arc_init() failed");
+ return -1;
+ }
+ }
+
+ dfm = __feature_arc_main;
+
+ /* threshold check */
+ if (dfm->num_feature_arcs > (dfm->max_feature_arcs - 1)) {
+ graph_err("max threshold for num_feature_arcs: %d reached",
+ dfm->max_feature_arcs - 1);
+ return -1;
+ }
+ /* Find the free slot for feature arc */
+ for (iter = 0; iter < dfm->max_feature_arcs; iter++) {
+ if (dfm->feature_arcs[iter] == RTE_GRAPH_FEATURE_ARC_INITIALIZER)
+ break;
+ }
+ arc_index = iter;
+
+ if (arc_index >= dfm->max_feature_arcs) {
+ graph_err("No free slot found for num_feature_arc");
+ return -1;
+ }
+
+ /* This should not happen */
+ RTE_VERIFY(dfm->feature_arcs[arc_index] == RTE_GRAPH_FEATURE_ARC_INITIALIZER);
+
+ sz = sizeof(*dfl) + (sizeof(uint64_t) * max_indexes);
+
+ dfl = rte_malloc(feature_arc_name, sz, RTE_CACHE_LINE_SIZE);
+
+ if (!dfl) {
+ graph_err("malloc failed for feature_arc_create()");
+ return -1;
+ }
+
+ memset(dfl, 0, sz);
+
+ snprintf(name, sizeof(name), "%s-%s", feature_arc_name, "feat");
+
+ dfl->features_by_index =
+ rte_malloc(name, sizeof(struct rte_graph_feature) * max_indexes,
+ RTE_CACHE_LINE_SIZE);
+
+ if (!dfl->features_by_index) {
+ rte_free(dfl);
+ graph_err("rte_malloc failed for allocating features_by_index()");
+ return -ENOMEM;
+ }
+ memset(dfl->features_by_index, 0, sizeof(rte_graph_feature_t) * max_indexes);
+
+ /* Initialize rte_graph port group fixed variables */
+ STAILQ_INIT(&dfl->all_features);
+ strncpy(dfl->feature_arc_name, feature_arc_name, RTE_GRAPH_FEATURE_ARC_NAMELEN - 1);
+ dfl->feature_arc_main = (void *)dfm;
+ dfl->start_node = start_node;
+ dfl->max_features = max_features;
+ dfl->max_indexes = max_indexes;
+
+ for (iter = 0; iter < dfl->max_indexes; iter++) {
+ df = rte_graph_feature_get(dfl, iter);
+ for (j = 0; j < dfl->max_features; j++) {
+ dfd = rte_graph_feature_data_get(df, j);
+ dfd->feature_data_index = RTE_GRAPH_FEATURE_INVALID_VALUE;
+ }
+ }
+ dfl->feature_arc_index = arc_index;
+ dfm->feature_arcs[dfl->feature_arc_index] = (rte_graph_feature_arc_t)dfl;
+ dfm->num_feature_arcs++;
+
+ if (_dfl)
+ *_dfl = (rte_graph_feature_arc_t)dfl;
+
+ return 0;
+}
+
+int
+rte_graph_feature_add(rte_graph_feature_arc_t _dfl, struct rte_node_register *feature_node,
+ const char *after_feature, const char *before_feature)
+{
+ struct rte_graph_feature_node_list *after_finfo = NULL, *before_finfo = NULL;
+ struct rte_graph_feature_node_list *temp = NULL, *finfo = NULL;
+ struct rte_graph_feature_arc *dfl = rte_graph_feature_arc_get(_dfl);
+ uint32_t slot, add_flag;
+ rte_edge_t edge = -1;
+
+ RTE_VERIFY(dfl->feature_arc_main == __feature_arc_main);
+
+ if (feature_arc_lookup(_dfl)) {
+ graph_err("invalid feature arc: 0x%016" PRIx64, (uint64_t)_dfl);
+ return -1;
+ }
+
+ if (dfl->feature_enable_started) {
+ graph_err("adding features after enabling any one of them is not supported");
+ return -1;
+ }
+
+ if ((after_feature != NULL) && (before_feature != NULL) &&
+ (after_feature == before_feature)) {
+ graph_err("after_feature and before_feature are same '%s:%s]", after_feature,
+ before_feature);
+ return -1;
+ }
+
+ if (!feature_node) {
+ graph_err("feature_node: %p invalid", feature_node);
+ return -1;
+ }
+
+ dfl = rte_graph_feature_arc_get(_dfl);
+
+ if (feature_node->id == RTE_NODE_ID_INVALID) {
+ graph_err("Invalid node: %s", feature_node->name);
+ return -1;
+ }
+
+ if (!feature_lookup(dfl, feature_node->name, &finfo, &slot)) {
+ graph_err("%s feature already added", feature_node->name);
+ return -1;
+ }
+
+ if (slot >= RTE_GRAPH_FEATURE_MAX_PER_ARC) {
+ graph_err("Max slot %u reached for feature addition", slot);
+ return -1;
+ }
+
+ if (strstr(feature_node->name, dfl->start_node->name)) {
+ graph_err("Feature %s cannot point to itself: %s", feature_node->name,
+ dfl->start_node->name);
+ return -1;
+ }
+
+ if (connect_graph_nodes(dfl->start_node, feature_node, &edge, dfl->feature_arc_name)) {
+ graph_err("unable to connect %s -> %s", dfl->start_node->name, feature_node->name);
+ return -1;
+ }
+
+ finfo = malloc(sizeof(*finfo));
+ if (!finfo)
+ return -1;
+
+ memset(finfo, 0, sizeof(*finfo));
+
+ finfo->feature_arc = (void *)dfl;
+ finfo->feature_node = feature_node;
+ finfo->edge_to_this_feature = edge;
+
+ /* Check for before and after constraints */
+ if (before_feature) {
+ /* before_feature sanity */
+ if (feature_lookup(dfl, before_feature, &before_finfo, NULL))
+ SET_ERR_JMP(EINVAL, finfo_free,
+ "Invalid before feature name: %s", before_feature);
+
+ if (!before_finfo)
+ SET_ERR_JMP(EINVAL, finfo_free,
+ "before_feature %s does not exist", before_feature);
+
+ /*
+ * Starting from 0 to before_feature, continue connecting edges
+ */
+ add_flag = 1;
+ STAILQ_FOREACH(temp, &dfl->all_features, next_feature) {
+ /*
+ * As soon as we see before_feature. stop adding edges
+ */
+ if (!strncmp(temp->feature_node->name, before_feature,
+ RTE_GRAPH_NAMESIZE))
+ if (!connect_graph_nodes(finfo->feature_node, temp->feature_node,
+ &edge, dfl->feature_arc_name))
+ add_flag = 0;
+
+ if (add_flag)
+ connect_graph_nodes(temp->feature_node, finfo->feature_node, NULL,
+ dfl->feature_arc_name);
+ }
+ }
+
+ if (after_feature) {
+ if (feature_lookup(dfl, after_feature, &after_finfo, NULL))
+ SET_ERR_JMP(EINVAL, finfo_free,
+ "Invalid after feature_name %s", after_feature);
+
+ if (!after_finfo)
+ SET_ERR_JMP(EINVAL, finfo_free,
+ "after_feature %s does not exist", after_feature);
+
+ /* Starting from after_feature to end continue connecting edges */
+ add_flag = 0;
+ STAILQ_FOREACH(temp, &dfl->all_features, next_feature) {
+ /* We have already seen after_feature now */
+ if (add_flag)
+ /* Add all features as next node to current feature*/
+ connect_graph_nodes(finfo->feature_node, temp->feature_node, NULL,
+ dfl->feature_arc_name);
+
+ /* as soon as we see after_feature. start adding edges
+ * from next iteration
+ */
+ if (!strncmp(temp->feature_node->name, after_feature, RTE_GRAPH_NAMESIZE))
+ /* connect after_feature to this feature */
+ if (!connect_graph_nodes(temp->feature_node, finfo->feature_node,
+ &edge, dfl->feature_arc_name))
+ add_flag = 1;
+ }
+
+ /* add feature next to after_feature */
+ STAILQ_INSERT_AFTER(&dfl->all_features, after_finfo, finfo, next_feature);
+ } else {
+ if (before_finfo) {
+ after_finfo = NULL;
+ STAILQ_FOREACH(temp, &dfl->all_features, next_feature) {
+ if (before_finfo == temp) {
+ if (after_finfo)
+ STAILQ_INSERT_AFTER(&dfl->all_features, after_finfo,
+ finfo, next_feature);
+ else
+ STAILQ_INSERT_HEAD(&dfl->all_features, finfo,
+ next_feature);
+
+ return 0;
+ }
+ after_finfo = temp;
+ }
+ } else {
+ STAILQ_INSERT_TAIL(&dfl->all_features, finfo, next_feature);
+ }
+ }
+
+ return 0;
+
+finfo_free:
+ free(finfo);
+
+ return -1;
+}
+
+int
+rte_graph_feature_destroy(rte_graph_feature_arc_t _dfl, const char *feature_name)
+{
+ RTE_SET_USED(_dfl);
+ RTE_SET_USED(feature_name);
+ return 0;
+}
+
+int
+rte_graph_feature_validate(rte_graph_feature_arc_t _dfl, uint32_t index, const char *feature_name,
+ int is_enable_disable)
+{
+ struct rte_graph_feature_arc *dfl = rte_graph_feature_arc_get(_dfl);
+ struct rte_graph_feature_node_list *finfo = NULL;
+ struct rte_graph_feature_data *dfd = NULL;
+ struct rte_graph_feature *df = NULL;
+ uint32_t slot;
+
+ /* validate _dfl */
+ if (dfl->feature_arc_main != __feature_arc_main) {
+ graph_err("invalid feature arc: 0x%016" PRIx64, (uint64_t)_dfl);
+ return -EINVAL;
+ }
+
+ /* validate index */
+ if (index >= dfl->max_indexes) {
+ graph_err("%s: Invalid provided index: %u >= %u configured", dfl->feature_arc_name,
+ index, dfl->max_indexes);
+ return -1;
+ }
+
+ /* validate feature_name is already added or not */
+ if (feature_lookup(dfl, feature_name, &finfo, &slot)) {
+ graph_err("%s: No feature %s added", dfl->feature_arc_name, feature_name);
+ return -EINVAL;
+ }
+
+ if (!finfo) {
+ graph_err("%s: No feature: %s found", dfl->feature_arc_name, feature_name);
+ return -EINVAL;
+ }
+
+ /* slot should be in valid range */
+ if (slot >= dfl->max_features) {
+ graph_err("%s/%s: Invalid free slot %u(max=%u) for feature", dfl->feature_arc_name,
+ feature_name, slot, dfl->max_features);
+ return -EINVAL;
+ }
+
+ df = rte_graph_feature_get(dfl, index);
+
+ /* Exceeded all enabled features for index */
+ if (is_enable_disable && (df->num_enabled_features >= dfl->max_features)) {
+ graph_err("%s: Index: %u has already enabled all features(%d/%d)",
+ dfl->feature_arc_name, index, df->num_enabled_features, dfl->max_features);
+ return -EINVAL;
+ }
+
+ dfd = rte_graph_feature_data_get(df, slot);
+
+ /* validate via bitmask if asked feature is already enabled on index */
+ if (is_enable_disable && (dfl->feature_bit_mask_by_index[index] &
+ RTE_BIT64(slot))) {
+ graph_err("%s: %s already enabled on index: %u",
+ dfl->feature_arc_name, feature_name, index);
+ return -1;
+ }
+
+ if (!is_enable_disable && !(dfl->feature_bit_mask_by_index[index] & RTE_BIT64(slot))) {
+ graph_err("%s: %s not enabled in bitmask for index: %u", dfl->feature_arc_name,
+ feature_name, index);
+ return -1;
+ }
+
+ /* validate via feature data that feature_data not in use */
+ if (is_enable_disable && (dfd->feature_data_index !=
+ RTE_GRAPH_FEATURE_INVALID_VALUE)) {
+ graph_err("%s/%s: slot: %u already in use by %s",
+ dfl->feature_arc_name, feature_name, slot,
+ dfd->node_info->feature_node->name);
+ return -1;
+ }
+
+ if (!is_enable_disable && (dfd->feature_data_index == RTE_GRAPH_FEATURE_INVALID_VALUE)) {
+ graph_err("%s/%s: feature data slot: %u not in use ", dfl->feature_arc_name,
+ feature_name, slot);
+ return -1;
+ }
+ return 0;
+}
+
+int
+rte_graph_feature_enable(rte_graph_feature_arc_t _dfl, uint32_t index, const
+ char *feature_name, int64_t data)
+{
+ struct rte_graph_feature_data *dfd = NULL, *prev_dfd = NULL, *next_dfd = NULL;
+ uint64_t original_mask, lower_feature_mask, upper_feature_mask;
+ struct rte_graph_feature_arc *dfl = rte_graph_feature_arc_get(_dfl);
+ struct rte_graph_feature_node_list *finfo = NULL;
+ uint32_t slot, prev_feature, next_feature;
+ struct rte_graph_feature *df = NULL;
+ rte_edge_t edge = 0;
+ int rc = 0;
+
+ if (rte_graph_feature_validate(_dfl, index, feature_name, 1))
+ return -1;
+
+ if (feature_lookup(dfl, feature_name, &finfo, &slot))
+ return -1;
+
+ df = rte_graph_feature_get(dfl, index);
+ dfd = rte_graph_feature_data_get(df, slot);
+
+ graph_dbg("%s: Enabling feature %s in index: %u at slot %u", dfl->feature_arc_name,
+ feature_name, index, slot);
+
+ memset(dfd, 0, sizeof(*dfd));
+
+ /* app data */
+ dfd->data = data;
+ /* First fill invalid value until everything succeeds */
+ dfd->feature_data_index = RTE_GRAPH_FEATURE_INVALID_VALUE;
+ dfd->node_info = finfo;
+
+ /* edge from base feature arc node to this feature */
+ dfd->edge_to_this_feature = finfo->edge_to_this_feature;
+ dfd->edge_to_next_feature = RTE_GRAPH_FEATURE_INVALID_VALUE;
+
+ /* This should be the case */
+ RTE_VERIFY(slot == (dfd - df->feature_data));
+
+ /* Adjust next edge for previous enabled feature and next enabled
+ * feature for this index
+ */
+ original_mask = dfl->feature_bit_mask_by_index[index];
+
+ /* If slot == 0, no lower feature is enabled
+ * if slot = 1, lower_feature_mask = 0x1,
+ * if slot = 2, lower_feature_mask = 0x3,
+ * if slot = 3, lower_feature_mask = 0x7,
+ */
+ lower_feature_mask = (slot) ? (RTE_BIT64(slot) - 1) : 0;
+
+ /*
+ * If slot =0, upper_feature_mask = (0xff ff ff ff ff ff ff ff) & ~lower_feature_mask
+ * If slot =1, upper_feature_mask = (0xff ff ff ff ff ff ff fe) & ~lower_feature_mask
+ * If slot =2, upper_feature_mask = (0xff ff ff ff ff ff ff fc) & ~lower_feature_mask
+ * If slot =3, upper_feature_mask = (0xff ff ff ff ff ff ff f8) & ~lower_feature_mask
+ * If slot =4, upper_feature_mask = (0xff ff ff ff ff ff ff f0) & ~lower_feature_mask
+ */
+ upper_feature_mask = ~(RTE_BIT64(slot)) & (~lower_feature_mask);
+
+ /* And with original bit mask */
+ upper_feature_mask &= original_mask;
+
+ /* set bits lesser than slot */
+ lower_feature_mask &= original_mask;
+
+ /* immediate lower enabled feature wrt slot is most significant bit in
+ * lower_feature_mask
+ */
+ prev_feature = rte_fls_u64(lower_feature_mask);
+
+ if (prev_feature) {
+ /* for us slot starts from 0 instead of 1 */
+ prev_feature--;
+ prev_dfd = rte_graph_feature_data_get(df, prev_feature);
+
+ graph_dbg("%s: enabling for index: %u, %s[] = %s", dfl->feature_arc_name, index,
+ prev_dfd->node_info->feature_node->name,
+ dfd->node_info->feature_node->name);
+ RTE_VERIFY(prev_dfd->feature_data_index != RTE_GRAPH_FEATURE_INVALID_VALUE);
+ if (get_existing_edge(dfl->feature_arc_name, prev_dfd->node_info->feature_node,
+ dfd->node_info->feature_node, &edge)) {
+ graph_err("%s: index: %u, Could not add next edge from %s to %s",
+ dfl->feature_arc_name, index,
+ prev_dfd->node_info->feature_node->name,
+ dfd->node_info->feature_node->name);
+ rc = -1;
+ } else {
+ graph_dbg("%s: enabled for index: %u, slot %u, %s[%u] = %s",
+ dfl->feature_arc_name, index, slot,
+ prev_dfd->node_info->feature_node->name, edge,
+ dfd->node_info->feature_node->name);
+ prev_dfd->edge_to_next_feature = edge;
+ }
+ if (rc < 0)
+ return -1;
+ }
+
+ /* immediate next upper feature wrt slot is least significant bit in
+ * upper_feature_mask
+ */
+ rc = 0;
+ if (rte_bsf64_safe(upper_feature_mask, &next_feature)) {
+ next_dfd = rte_graph_feature_data_get(df, next_feature);
+
+ graph_dbg("%s: enabling for index: %u, %s[] = %s ", dfl->feature_arc_name, index,
+ dfd->node_info->feature_node->name,
+ next_dfd->node_info->feature_node->name);
+ RTE_VERIFY(next_dfd->feature_data_index != RTE_GRAPH_FEATURE_INVALID_VALUE);
+ if (get_existing_edge(dfl->feature_arc_name, dfd->node_info->feature_node,
+ next_dfd->node_info->feature_node, &edge)) {
+ graph_err("%s: index: %u, Could not add next edge from %s to %s",
+ dfl->feature_arc_name, index, dfd->node_info->feature_node->name,
+ next_dfd->node_info->feature_node->name);
+ rc = -1;
+ } else {
+ graph_dbg("%s: enabled for index: %u, slot %u, %s[%u] = %s",
+ dfl->feature_arc_name, index, slot,
+ dfd->node_info->feature_node->name, edge,
+ next_dfd->node_info->feature_node->name);
+ dfd->edge_to_next_feature = edge;
+ }
+ if (rc < 0)
+ return -1;
+ }
+
+ graph_dbg("%s: enabled for index: %u, slot %u, %s[%u] = %s", dfl->feature_arc_name, index,
+ slot, dfl->start_node->name, dfd->edge_to_this_feature,
+ dfd->node_info->feature_node->name);
+
+ /* Make dfd valid now */
+ dfd->feature_data_index = dfd - df->feature_data;
+
+ /* Increase feature node info reference count */
+ finfo->ref_count++;
+
+ /* Increment number of enabled feature on this index */
+ df->num_enabled_features++;
+
+ /* Make rte_graph_feature_add() disable for this feature arc now */
+ dfl->feature_enable_started++;
+
+ /* Update bitmask feature arc bit mask */
+ rte_bit_relaxed_set64(rte_graph_uint_cast(slot), &dfl->feature_bit_mask_by_index[index]);
+
+ /* Make sure changes made into affect */
+ RTE_VERIFY(dfl->feature_bit_mask_by_index[index] & RTE_BIT64(slot));
+
+ return 0;
+}
+
+int
+rte_graph_feature_disable(rte_graph_feature_arc_t _dfl, uint32_t index, const char *feature_name)
+{
+ struct rte_graph_feature_data *dfd = NULL, *prev_dfd = NULL, *next_dfd = NULL;
+ uint64_t original_mask, lower_feature_mask, upper_feature_mask;
+ struct rte_graph_feature_arc *dfl = rte_graph_feature_arc_get(_dfl);
+ struct rte_graph_feature_node_list *finfo = NULL;
+ uint32_t slot, prev_feature, next_feature;
+ struct rte_graph_feature *df = NULL;
+ rte_edge_t edge = 0;
+ int rc = 0;
+
+ if (rte_graph_feature_validate(_dfl, index, feature_name, 0))
+ return -1;
+
+ if (feature_lookup(dfl, feature_name, &finfo, &slot))
+ return -1;
+
+ df = rte_graph_feature_get(dfl, index);
+ dfd = rte_graph_feature_data_get(df, slot);
+
+ /* This should be the case */
+ RTE_VERIFY(slot == (dfd - df->feature_data));
+
+ graph_dbg("%s: Disbling feature %s in index: %u at slot %u", dfl->feature_arc_name,
+ feature_name, index, slot);
+
+ /* Adjust next edge for previous enabled feature and next enabled
+ * feature for this index
+ */
+ original_mask = dfl->feature_bit_mask_by_index[index];
+
+ lower_feature_mask = (slot) ? (RTE_BIT64(slot) - 1) : 0;
+ upper_feature_mask = ~(RTE_BIT64(slot)) & (~lower_feature_mask);
+ upper_feature_mask &= original_mask;
+ lower_feature_mask &= original_mask;
+
+ /* immediate lower enabled feature wrt slot is most significant bit in
+ * lower_feature_mask
+ */
+ prev_feature = rte_fls_u64(lower_feature_mask);
+
+ if (prev_feature) {
+ /* for us slot starts from 0 instead of 1 */
+ prev_feature--;
+ prev_dfd = rte_graph_feature_data_get(df, prev_feature);
+
+ /* Adjust later to next enabled feature below */
+ prev_dfd->edge_to_next_feature = RTE_GRAPH_FEATURE_INVALID_VALUE;
+
+ /* If we also have next enable feature */
+ if (rte_bsf64_safe(upper_feature_mask, &next_feature)) {
+ next_dfd = rte_graph_feature_data_get(df, next_feature);
+
+ graph_dbg("%s: index: %u updating next enabled feature for %s to %s ",
+ dfl->feature_arc_name, index,
+ prev_dfd->node_info->feature_node->name,
+ next_dfd->node_info->feature_node->name);
+ if (get_existing_edge(dfl->feature_arc_name,
+ prev_dfd->node_info->feature_node,
+ next_dfd->node_info->feature_node, &edge)) {
+ graph_err("%s: index: %u, Could not get next edge from %s to %s",
+ dfl->feature_arc_name, index,
+ prev_dfd->node_info->feature_node->name,
+ next_dfd->node_info->feature_node->name);
+ rc = -1;
+ } else {
+ graph_dbg("%s: index: %u updated next enable feature for %s to %s at edge %u",
+ dfl->feature_arc_name, index,
+ prev_dfd->node_info->feature_node->name,
+ next_dfd->node_info->feature_node->name, edge);
+ prev_dfd->edge_to_next_feature = edge;
+ }
+ if (rc < 9)
+ return -1;
+ }
+ }
+
+ /* First fill invalid value until everything succeeds */
+ dfd->feature_data_index = RTE_GRAPH_FEATURE_INVALID_VALUE;
+ dfd->edge_to_this_feature = RTE_GRAPH_FEATURE_INVALID_VALUE;
+ dfd->edge_to_next_feature = RTE_GRAPH_FEATURE_INVALID_VALUE;
+
+ /* Decrease feature node info reference count */
+ finfo->ref_count--;
+
+ /* Decrement number of enabled feature on this index */
+ df->num_enabled_features++;
+
+ /* Update bitmask feature arc bit mask */
+ rte_bit_relaxed_clear64(rte_graph_uint_cast(slot), &dfl->feature_bit_mask_by_index[index]);
+
+ return 0;
+}
+
+int
+rte_graph_feature_arc_destroy(rte_graph_feature_arc_t epg)
+{
+ RTE_SET_USED(epg);
+ return 0;
+}
+
+int
+rte_graph_feature_arc_cleanup(void)
+{
+ rte_graph_feature_arc_main_t *dm = __feature_arc_main;
+ uint32_t iter;
+
+ if (!__feature_arc_main)
+ return -1;
+
+ for (iter = 0; iter < dm->max_feature_arcs; iter++) {
+ if (dm->feature_arcs[iter] == RTE_GRAPH_FEATURE_ARC_INITIALIZER)
+ continue;
+
+ rte_graph_feature_arc_destroy((rte_graph_feature_arc_t)dm->feature_arcs[iter]);
+ }
+
+ return 0;
+}
+
+int
+rte_graph_feature_arc_lookup_by_name(const char *arc_name, rte_graph_feature_arc_t *_dfl)
+{
+ rte_graph_feature_arc_main_t *dm = __feature_arc_main;
+ struct rte_graph_feature_arc *dfl = NULL;
+ uint32_t iter;
+
+ if (!__feature_arc_main)
+ return -1;
+
+ for (iter = 0; iter < dm->max_feature_arcs; iter++) {
+ if (dm->feature_arcs[iter] == RTE_GRAPH_FEATURE_ARC_INITIALIZER)
+ continue;
+
+ dfl = rte_graph_feature_arc_get(dm->feature_arcs[iter]);
+
+ if (strstr(arc_name, dfl->feature_arc_name)) {
+ if (_dfl)
+ *_dfl = (rte_graph_feature_arc_t)dfl;
+ return 0;
+ }
+ }
+
+ return -1;
+}
@@ -14,11 +14,13 @@ sources = files(
'graph_debug.c',
'graph_stats.c',
'graph_populate.c',
+ 'graph_feature_arc.c',
'graph_pcap.c',
'rte_graph_worker.c',
'rte_graph_model_mcore_dispatch.c',
)
headers = files('rte_graph.h', 'rte_graph_worker.h')
+headers += files('rte_graph_feature_arc.h', 'rte_graph_feature_arc_worker.h')
indirect_headers += files(
'rte_graph_model_mcore_dispatch.h',
'rte_graph_model_rtc.h',
new file mode 100644
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell International Ltd.
+ */
+
+#ifndef _RTE_GRAPH_FEATURE_ARC_H_
+#define _RTE_GRAPH_FEATURE_ARC_H_
+
+#include <assert.h>
+#include <errno.h>
+#include <signal.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <rte_common.h>
+#include <rte_compat.h>
+#include <rte_debug.h>
+#include <rte_graph.h>
+#include <rte_graph_worker.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @file
+ *
+ * rte_graph_feature_arc.h
+ *
+ * Define APIs and structures/variables with respect to
+ *
+ * - Feature arc(s)
+ * - Feature(s)
+ *
+ * A feature arc represents an ordered list of features/protocols at a given
+ * networking layer. Feature arc provides a high level abstraction to connect
+ * various rte_graph nodes, designated as *feature nodes*, and allowing
+ * steering of packets across these feature nodes fast path processing in a
+ * generic manner. In a typical network stack, often a protocol or feature must
+ * be first enabled on a given interface, before any packet received on that
+ * interface is steered to feature processing. For eg: incoming IPv4 packets
+ * are sent to routing sub-system only after a valid IPv4 address is assigned
+ * to the received interface. In other words, often packets needs to be steered
+ * across features not based on the packet content but based on whether feature
+ * is enable or disable on a given incoming/outgoing interface. Feature arc
+ * provides mechanism to enable/disable feature(s) on each interface and
+ * allowing seamless packet steering across enabled feature nodes in fast path.
+ *
+ * Feature arc also provides a way to steer packets from standard nodes to
+ * custom/user-defined *feature nodes* without any change in standard node's
+ * fast path functions
+ *
+ * On a given interface multiple feature(s) might be enabled in a particular
+ * feature arc. For instance, both "ipv4-output" and "IPsec policy output"
+ * features may be enabled on "eth0" interface in "L3-output" feature arc.
+ * Similarly, "ipv6-output" and "ipsec-output" may be enabled on "eth1"
+ * interface in same "L3-output" feature arc.
+ *
+ * When multiple features are present in a given feature arc, its imperative
+ * to allow each feature processing in a particular sequential order. For
+ * instance, in "L3-input" feature arc it may be required to run "IPsec
+ * input" feature first, for packet decryption, before "ip-lookup". So a
+ * sequential order must be maintained among features present in a feature arc.
+ *
+ * Features are enabled/disabled multiple times at runtime to some or all
+ * available interfaces present in the system. Features can be enabled/disabled
+ * even after @b rte_graph_create() is called. Enable/disabling features on one
+ * interface is independent of other interface.
+ *
+ * A given feature might consume packet (if it's configured to consume) or may
+ * forward it to next enabled feature. For instance, "IPsec input" feature may
+ * consume/drop all packets with "Protect" policy action while all packets with
+ * policy action as "Bypass" may be forwarded to next enabled feature (with in
+ * same feature arc)
+ *
+ * This library facilitates rte graph based applications to steer packets in
+ * fast path to different feature nodes with-in a feature arc and support all
+ * functionalities described above
+ *
+ * In order to use feature-arc APIs, applications needs to do following in
+ * control path:
+ * - Initialize feature arc library via rte_graph_feature_arc_init()
+ * - Create feature arc via rte_graph_feature_arc_create()
+ * - Before calling rte_graph_create(), features must be added to feature-arc
+ * via rte_graph_feature_add(). rte_graph_feature_add() allows adding
+ * features in a sequential order with "runs_after" and "runs_before"
+ * constraints.
+ * - Post rte_graph_create(), features can be enabled/disabled at runtime on
+ * any interface via rte_graph_feature_enable()/rte_graph_feature_disable()
+ *
+ * In fast path, nodes uses
+ * - rte_graph_feature_arc_has_feature() and
+ * rte_graph_feature_arc_feature_data_get() APIs to steer packets across
+ * feature nodes
+ *
+ * rte_graph_feature_enable()/rte_graph_feature_disable() APIs are not
+ * thread-safe hence must be called by single core while other cores are not
+ * using any fast path feature arc APIs.
+ */
+
+/**< Initializer value for rte_graph_feature_arc_t */
+#define RTE_GRAPH_FEATURE_ARC_INITIALIZER ((rte_graph_feature_arc_t)UINT64_MAX)
+
+/**< Initializer value for rte_graph_feature_arc_t */
+#define RTE_GRAPH_FEATURE_INVALID_VALUE UINT16_MAX
+
+/** Max number of features supported in a given feature arc */
+#define RTE_GRAPH_FEATURE_MAX_PER_ARC 64
+
+/** Length of feature arc name */
+#define RTE_GRAPH_FEATURE_ARC_NAMELEN RTE_NODE_NAMESIZE
+
+/** @internal */
+#define rte_graph_feature_cast(x) ((rte_graph_feature_t)x)
+
+/** rte_graph feature arc object */
+typedef uint64_t rte_graph_feature_arc_t;
+
+/** rte_graph feature object */
+typedef uint32_t rte_graph_feature_t;
+
+/**
+ * Initialize feature arc subsystem
+ *
+ * @param max_feature_arcs
+ * Maximum number of feature arcs required to be supported
+ *
+ * @return
+ * 0: Success
+ * <0: Failure
+ */
+__rte_experimental
+int rte_graph_feature_arc_init(int max_feature_arcs);
+
+/**
+ * Create a feature arc
+ *
+ * @param feature_arc_name
+ * Feature arc name with max length of @ref RTE_GRAPH_FEATURE_ARC_NAMELEN
+ * @param max_features
+ * Maximum number of features to be supported in this feature arc
+ * @param max_indexes
+ * Maximum number of interfaces/ports/indexes to be supported
+ * @param start_node
+ * Base node where this feature arc's features are checked in fast path
+ * @param[out] _dfl
+ * Feature arc object
+ *
+ * @return
+ * 0: Success
+ * <0: Failure
+ */
+__rte_experimental
+int rte_graph_feature_arc_create(const char *feature_arc_name, int max_features, int max_indexes,
+ struct rte_node_register *start_node, rte_graph_feature_arc_t *_dfl);
+
+/**
+ * Get feature arc object with name
+ *
+ * @param arc_name
+ * Feature arc name provided to successful @ref rte_graph_feature_arc_create
+ * @param[out] _dfl
+ * Feature arc object returned
+ *
+ * @return
+ * 0: Success
+ * <0: Failure
+ */
+__rte_experimental
+int rte_graph_feature_arc_lookup_by_name(const char *arc_name, rte_graph_feature_arc_t *_dfl);
+
+/**
+ * Add a feature to already created feature arc
+ *
+ * @param _dfl
+ * Feature arc handle returned from @ref rte_graph_feature_arc_create()
+ * @param feature_node
+ * Graph node representing feature. On success, feature_node is next_node of
+ * feature_arc->start_node
+ * @param runs_after
+ * Add this feature_node after already added "runs_after". Creates
+ * start_node -> runs_after -> this_feature sequence
+ * @param runs_before
+ * Add this feature_node before already added "runs_before". Creates
+ * start_node -> this_feature -> runs_before sequence
+ *
+ * <I> Must be called before rte_graph_create </I>
+ *
+ * @return
+ * 0: Success
+ * <0: Failure
+ */
+__rte_experimental
+int rte_graph_feature_add(rte_graph_feature_arc_t _dfl, struct rte_node_register *feature_node,
+ const char *runs_after, const char *runs_before);
+
+/**
+ * Enable feature within a feature arc
+ *
+ * Must be called after @b rte_graph_create(). API is NOT Thread-safe
+ *
+ * @param _dfl
+ * Feature arc object returned by @ref rte_graph_feature_arc_create or @ref
+ * rte_graph_feature_arc_lookup_by_name
+ * @param index
+ * Application specific index. Can be corresponding to interface_id/port_id etc
+ * @param feature_name
+ * Name of the node which is already added via @ref rte_graph_feature_add
+ * @param data
+ * Application specific data which is retrieved in fast path
+ *
+ * @return
+ * 0: Success
+ * <0: Failure
+ */
+__rte_experimental
+int rte_graph_feature_enable(rte_graph_feature_arc_t _dfl, uint32_t index, const char *feature_name,
+ int64_t data);
+
+/**
+ * Validate whether subsequent enable/disable feature would succeed or not
+ * API is thread-safe
+ *
+ * @param _dfl
+ * Feature arc object returned by @ref rte_graph_feature_arc_create or @ref
+ * rte_graph_feature_arc_lookup_by_name
+ * @param index
+ * Application specific index. Can be corresponding to interface_id/port_id etc
+ * @param feature_name
+ * Name of the node which is already added via @ref rte_graph_feature_add
+ * @param is_enable_disable
+ * If 1, validate whether subsequent @ref rte_graph_feature_enable would pass or not
+ * If 0, validate whether subsequent @ref rte_graph_feature_disable would pass or not
+ *
+ * @return
+ * 0: Subsequent enable/disable API would pass
+ * <0: Subsequent enable/disable API would not pass
+ */
+__rte_experimental
+int rte_graph_feature_validate(rte_graph_feature_arc_t _dfl, uint32_t index,
+ const char *feature_name, int is_enable_disable);
+
+/**
+ * Disable already enabled feature within a feature arc
+ *
+ * Must be called after @b rte_graph_create(). API is NOT Thread-safe
+ *
+ * @param _dfl
+ * Feature arc object returned by @ref rte_graph_feature_arc_create or @ref
+ * rte_graph_feature_arc_lookup_by_name
+ * @param index
+ * Application specific index. Can be corresponding to interface_id/port_id etc
+ * @param feature_name
+ * Name of the node which is already added via @ref rte_graph_feature_add
+ *
+ * @return
+ * 0: Success
+ * <0: Failure
+ */
+__rte_experimental
+int rte_graph_feature_disable(rte_graph_feature_arc_t _dfl, uint32_t index,
+ const char *feature_name);
+
+/**
+ * Destroy Feature
+ *
+ * @param _dfl
+ * Feature arc object returned by @ref rte_graph_feature_arc_create or @ref
+ * rte_graph_feature_arc_lookup_by_name
+ * @param feature_name
+ * Feature name provided to @ref rte_graph_feature_add
+ *
+ * @return
+ * 0: Success
+ * <0: Failure
+ */
+__rte_experimental
+int rte_graph_feature_destroy(rte_graph_feature_arc_t _dfl, const char *feature_name);
+
+/**
+ * Delete feature_arc object
+ *
+ * @param _dfl
+ * Feature arc object returned by @ref rte_graph_feature_arc_create or @ref
+ * rte_graph_feature_arc_lookup_by_name
+ *
+ * @return
+ * 0: Success
+ * <0: Failure
+ */
+__rte_experimental
+int rte_graph_feature_arc_destroy(rte_graph_feature_arc_t _dfl);
+
+/**
+ * Cleanup all feature arcs
+ *
+ * @return
+ * 0: Success
+ * <0: Failure
+ */
+__rte_experimental
+int rte_graph_feature_arc_cleanup(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
new file mode 100644
@@ -0,0 +1,483 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell International Ltd.
+ */
+
+#ifndef _RTE_GRAPH_FEATURE_ARC_WORKER_H_
+#define _RTE_GRAPH_FEATURE_ARC_WORKER_H_
+
+#include <rte_graph_feature_arc.h>
+#include <rte_bitops.h>
+
+/**
+ * @file
+ *
+ * rte_graph_feature_arc_worker.h
+ *
+ * Defines fast path structure
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** @internal
+ *
+ * Slow path feature node info list
+ */
+struct __rte_cache_aligned rte_graph_feature_node_list {
+ /** Next feature */
+ STAILQ_ENTRY(rte_graph_feature_node_list) next_feature;
+
+ /** node representing feature */
+ struct rte_node_register *feature_node;
+
+ /** How many indexes/interfaces using this feature */
+ int32_t ref_count;
+
+ /** Back pointer to feature arc */
+ void *feature_arc;
+
+ /** rte_edge_t to this feature node from feature_arc->start_node */
+ rte_edge_t edge_to_this_feature;
+};
+
+/**
+ * RTE_GRAPH feature data representing a fast path feature object on an interface/index
+ */
+typedef struct rte_graph_feature_data {
+ /** Data provided by application during @ref rte_graph_feature_enable on interface */
+ int64_t data;
+
+ /** this feature data index */
+ uint32_t feature_data_index;
+
+ /** Edge to this feature node from feature_arc->start_node */
+ rte_edge_t edge_to_this_feature;
+
+ /**
+ * Edge to next enabled feature on a given interface/index. This field
+ * keeps on changing as @ref rte_graph_feature_enable()/@ref
+ * rte_graph_feature_disable() are called on a given interface/index
+ */
+ rte_edge_t edge_to_next_feature;
+
+ /** Slow path node_info object */
+ struct rte_graph_feature_node_list *node_info;
+} rte_graph_feature_data_t;
+
+/**
+ * RTE_GRAPH Feature object
+ *
+ * Holds all feature related data of a given feature on *all* interfaces
+ */
+struct __rte_cache_aligned rte_graph_feature {
+ /**
+ * Slow path node_info
+ * 1st DWORD
+ */
+ struct rte_graph_feature_node_list *node_info;
+
+ /** Feature arc back pointer
+ * 2nd DWORD
+ */
+ void *feature_arc;
+
+ /**
+ * Number of enabled features in this feature_arc
+ * 3rd WORD
+ */
+ uint32_t num_enabled_features;
+
+ /* uint32_t reserved; */
+
+ /**
+ * Array of feature_data by index/interface
+ *
+ */
+ struct rte_graph_feature_data feature_data[RTE_GRAPH_FEATURE_MAX_PER_ARC];
+};
+
+/**
+ * RTE_GRAPH Feature arc object
+ *
+ * Representing a feature arc holding all features which are enabled/disabled on any interfaces
+ */
+struct __rte_cache_aligned rte_graph_feature_arc {
+ /** All feature lists */
+ STAILQ_HEAD(, rte_graph_feature_node_list) all_features;
+
+ /** feature arc name */
+ char feature_arc_name[RTE_GRAPH_FEATURE_ARC_NAMELEN];
+
+ /** this feature group index in feature_arc_main */
+ uint32_t feature_arc_index;
+
+ /** Back pointer to feature_arc_main */
+ void *feature_arc_main;
+
+ /**
+ * Start_node or Base node where this feature arc is checked for any feature
+ */
+ struct rte_node_register *start_node;
+
+ /** Max features supported in this arc */
+ uint32_t max_features;
+
+ /** Boolean indicating @ref rte_graph_feature_enable has started and not
+ * further addition is allowed
+ */
+ int feature_enable_started;
+
+ /* Fast path stuff*/
+ alignas(RTE_CACHE_LINE_SIZE) RTE_MARKER c0;
+
+ /** RTE_GRAPH feature by interface */
+ struct rte_graph_feature *features_by_index;
+
+ /** Max interfaces supported */
+ uint32_t max_indexes;
+
+ /** Bitmask by interface. Set bit indicates feature is enabled on interface */
+ uint64_t feature_bit_mask_by_index[];
+};
+
+/** Feature arc main */
+typedef struct feature_arc_main {
+ /** number of feature arcs created by application */
+ uint32_t num_feature_arcs;
+
+ /** max features arcs allowed */
+ uint32_t max_feature_arcs;
+
+ /** feature arcs */
+ rte_graph_feature_arc_t feature_arcs[];
+} rte_graph_feature_arc_main_t;
+
+/** @internal Get feature arc pointer from object */
+#define rte_graph_feature_arc_get(dfl) ((struct rte_graph_feature_arc *)dfl)
+
+extern rte_graph_feature_arc_main_t *__feature_arc_main;
+
+/**
+ * Get rte_graph feature data object for a index in feature
+ *
+ * @param df
+ * Feature pointer
+ * @param feature_index
+ * Index of feature maintained in slow path linked list
+ *
+ * @return
+ * Valid feature data
+ */
+static inline struct rte_graph_feature_data *
+rte_graph_feature_data_get(struct rte_graph_feature *df, uint32_t feature_index)
+{
+ return (df->feature_data + feature_index);
+}
+
+/**
+ * Get rte_graph_feature object for a given interface/index from feature arc
+ *
+ * @param dfl
+ * Feature arc pointer
+ * @param index
+ * Interface index
+ *
+ * @return
+ * Valid feature pointer
+ */
+static inline struct rte_graph_feature *
+rte_graph_feature_get(struct rte_graph_feature_arc *dfl, uint32_t index)
+{
+ return (dfl->features_by_index + index);
+}
+
+/**
+ * Fast path API to check if first feature enabled on a feature arc
+ *
+ * Must be called in feature_arc->start_node processing
+ *
+ * @param dfl
+ * Feature arc object
+ * @param index
+ * Interface/Index
+ * @param[out] feature
+ * Pointer to rte_graph_feature_t. Valid if API returns 1
+ *
+ * @return
+ * 1: If feature is enabled
+ * 0: If feature is not enabled
+ *
+ */
+static inline int
+rte_graph_feature_arc_has_first_feature(struct rte_graph_feature_arc *dfl,
+ uint32_t index, rte_graph_feature_t *feature)
+{
+ return rte_bsf64_safe(dfl->feature_bit_mask_by_index[index], feature);
+}
+
+/**
+ * Fast path API to get next feature when current node is already on an feature
+ * arc and not consuming packet. This feature must forward the packet to next
+ * enabled feature by passing returned rte_graph_feature_t to
+ * rte_graph_feature_arc_next_feature_data_get()
+ *
+ * @param dfl
+ * Feature arc object
+ * @param index
+ * Interface/Index
+ * @param[out] feature
+ * Pointer to rte_graph_feature_t. Valid if API returns 1
+ *
+ * @return
+ * 1: If next feature is enabled
+ * 0: If next feature is not enabled
+ */
+static inline int
+rte_graph_feature_arc_has_next_feature(struct rte_graph_feature_arc *dfl,
+ uint32_t index, rte_graph_feature_t *feature)
+{
+ uint32_t next_feature;
+ uint64_t bitmask;
+
+#ifdef RTE_GRAPH_FEATURE_ARC_DEBUG
+ struct rte_graph_feature *df = rte_graph_feature_get(dfl, index);
+ struct rte_graph_feature_data *dfd = NULL;
+
+ dfd = rte_graph_feature_data_get(df, *feature);
+ /** Check feature sanity */
+ if (unlikely(dfd->feature_data_index != *feature))
+ return 0;
+#endif
+
+ /* Create bitmask where current feature is cleared to get next feature
+ * bit set
+ */
+ next_feature = (uint32_t)*feature;
+ bitmask = UINT64_MAX << (next_feature + 1);
+ bitmask = dfl->feature_bit_mask_by_index[index] & bitmask;
+
+ return rte_bsf64_safe(bitmask, feature);
+}
+
+/**
+ * Fast path API to check if any feature enabled on a feature arc
+ *
+ * @param _dfl
+ * Feature arc object
+ * @param index
+ * Interface/Index
+ * @param[out] feature
+ * Pointer to rte_graph_feature_t. Valid if API returns 1
+ *
+ * @return
+ * 1: If feature is enabled
+ * 0: If feature is not enabled
+ *
+ */
+static inline int
+rte_graph_feature_arc_has_feature(rte_graph_feature_arc_t _dfl, uint32_t index,
+ rte_graph_feature_t *feature)
+{
+ struct rte_graph_feature_arc *dfl = rte_graph_feature_arc_get(_dfl);
+
+#ifdef RTE_GRAPH_FEATURE_ARC_DEBUG
+ if (unlikely(dfl->max_indexes < index))
+ return 0;
+
+ if (unlikely(!feature))
+ return 0;
+#endif
+ /* Look for first feature */
+ if (*feature == RTE_GRAPH_FEATURE_INVALID_VALUE)
+ return rte_graph_feature_arc_has_first_feature(dfl, index, feature);
+ else
+ return rte_graph_feature_arc_has_next_feature(dfl, index, feature);
+}
+
+
+/**
+ * Prefetch feature data upfront
+ *
+ * @param _dfl
+ * RTE_GRAPH feature arc object
+ * @param index
+ * Interface/index
+ * @param feature
+ * Pointer to feature object returned from @ref
+ * rte_graph_feature_arc_has_feature() or @ref
+ * rte_graph_feature_arc_first_feature_data_get()
+ */
+static inline void
+__rte_graph_prefetch_data_prefetch(rte_graph_feature_arc_t _dfl, int index,
+ rte_graph_feature_t feature)
+{
+ struct rte_graph_feature_arc *dfl = rte_graph_feature_arc_get(_dfl);
+ struct rte_graph_feature *df = rte_graph_feature_get(dfl, index);
+
+ rte_prefetch0((void *)rte_graph_feature_data_get(df, feature));
+}
+
+/**
+ * Prefetch feature data upfront. Perform sanity
+ *
+ * @param _dfl
+ * RTE_GRAPH feature arc object
+ * @param index
+ * Interface/index
+ * @param feature
+ * Pointer to feature object returned from @ref
+ * rte_graph_feature_arc_has_feature() or @ref
+ * rte_graph_feature_arc_first_feature_data_get()
+ */
+static inline void
+rte_graph_feature_data_prefetch(rte_graph_feature_arc_t _dfl, uint32_t index,
+ rte_graph_feature_t feature)
+{
+#ifdef RTE_GRAPH_FEATURE_ARC_DEBUG
+ struct rte_graph_feature_arc *dfl = rte_graph_feature_arc_get(_dfl);
+
+ if (unlikely(index >= dfl->max_indexes))
+ return;
+
+ if (unlikely(feature >= rte_graph_feature_cast(dfl->max_features)))
+ return;
+#endif
+
+ if (feature != RTE_GRAPH_FEATURE_INVALID_VALUE)
+ __rte_graph_prefetch_data_prefetch(_dfl, index, feature);
+}
+
+/**
+ * Fast path API to get first feature data aka {edge, int32_t data}
+ *
+ * Must be called in feature_arc->start_node processing
+ *
+ * @param _dfl
+ * Feature arc object
+ * @param feature
+ * returned from rte_graph_feature_arc_has_feature()
+ * @param index
+ * Interface/Index
+ * @param[out] edge
+ * Pointer to rte_node edge. Valid if API returns Success
+ * @param[out] data
+ * Pointer to int64_t data set via rte_graph_feature_enable(). Valid if API returns
+ * Success
+ *
+ * @return
+ * 0: Success
+ * <0: Failure
+ */
+static inline int
+rte_graph_feature_arc_first_feature_data_get(struct rte_graph_feature_arc *dfl,
+ rte_graph_feature_t feature,
+ uint32_t index, rte_edge_t *edge,
+ int64_t *data)
+{
+ struct rte_graph_feature *df = rte_graph_feature_get(dfl, index);
+ struct rte_graph_feature_data *dfd = NULL;
+
+ dfd = rte_graph_feature_data_get(df, feature);
+
+#ifdef RTE_GRAPH_FEATURE_ARC_DEBUG
+ /** Check feature sanity */
+ if (unlikely(dfd->feature_data_index != feature))
+ return -1;
+
+ if (unlikely(!edge && !data))
+ return -1;
+#endif
+
+ *edge = dfd->edge_to_this_feature;
+ *data = dfd->data;
+
+ return 0;
+}
+
+/**
+ * Fast path API to get next feature data aka {edge, int32_t data}
+ *
+ * Must NOT be called in feature_arc->start_node processing instead must be
+ * called in intermediate feature nodes on a featur-arc.
+ *
+ * @param _dfl
+ * Feature arc object
+ * @param feature
+ * returned from rte_graph_feature_arc_has_next_feature()
+ * @param index
+ * Interface/Index
+ * @param[out] edge
+ * Pointer to rte_node edge. Valid if API returns Success
+ * @param[out] data
+ * Pointer to int64_t data set via rte_graph_feature_enable(). Valid if API returns
+ * Success
+ *
+ * @return
+ * 0: Success
+ * <0: Failure
+ */
+static inline int
+rte_graph_feature_arc_next_feature_data_get(struct rte_graph_feature_arc *dfl,
+ rte_graph_feature_t feature,
+ uint32_t index, rte_edge_t *edge,
+ int64_t *data)
+{
+ struct rte_graph_feature *df = rte_graph_feature_get(dfl, index);
+ struct rte_graph_feature_data *dfd = NULL;
+
+ dfd = rte_graph_feature_data_get(df, feature);
+
+#ifdef RTE_GRAPH_FEATURE_ARC_DEBUG
+ /** Check feature sanity */
+ if (unlikely(dfd->feature_data_index != feature))
+ return -1;
+
+ if (unlikely(!edge && !data))
+ return -1;
+#endif
+
+ *edge = dfd->edge_to_next_feature;
+ *data = dfd->data;
+
+ return 0;
+}
+
+/**
+ * Fast path API to get next feature data aka {edge, int32_t data}
+ *
+ * @param _dfl
+ * Feature arc object
+ * @param feature
+ * returned from rte_graph_feature_arc_has_feature()
+ * @param index
+ * Interface/Index
+ * @param[out] edge
+ * Pointer to rte_node edge. Valid if API returns Success
+ * @param[out] data
+ * Pointer to int64_t data set via rte_graph_feature_enable(). Valid if API returns
+ * Success
+ *
+ * @return
+ * 0: Success
+ * <0: Failure
+ */
+
+static inline int
+rte_graph_feature_arc_feature_data_get(rte_graph_feature_arc_t _dfl,
+ rte_graph_feature_t feature, uint32_t
+ index, rte_edge_t *edge, int64_t *data)
+{
+ struct rte_graph_feature_arc *dfl = rte_graph_feature_arc_get(_dfl);
+
+ if (feature == RTE_GRAPH_FEATURE_INVALID_VALUE)
+ return rte_graph_feature_arc_first_feature_data_get(dfl, feature, index, edge,
+ data);
+ else
+ return rte_graph_feature_arc_next_feature_data_get(dfl, feature, index, edge, data);
+}
+
+#ifdef __cplusplus
+}
+#endif
+#endif
@@ -52,3 +52,19 @@ DPDK_24 {
local: *;
};
+
+EXPERIMENTAL {
+ global:
+
+ # added in 24.07
+ rte_graph_feature_arc_init;
+ rte_graph_feature_arc_create;
+ rte_graph_feature_arc_lookup_by_name;
+ rte_graph_feature_add;
+ rte_graph_feature_enable;
+ rte_graph_feature_validate;
+ rte_graph_feature_disable;
+ rte_graph_feature_destroy;
+ rte_graph_feature_arc_destroy;
+ rte_graph_feature_arc_cleanup;
+};