@@ -241,6 +241,17 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
.free = mlx5_free,
.type = "mlx5_jump_ipool",
},
+ {
+ .size = sizeof(struct mlx5_flow_dv_sample_resource),
+ .trunk_size = 64,
+ .grow_trunk = 3,
+ .grow_shift = 2,
+ .need_lock = 0,
+ .release_mem_en = 1,
+ .malloc = mlx5_malloc,
+ .free = mlx5_free,
+ .type = "mlx5_sample_ipool",
+ },
#endif
{
.size = sizeof(struct mlx5_flow_meter),
@@ -39,6 +39,7 @@ enum mlx5_ipool_index {
MLX5_IPOOL_TAG, /* Pool for tag resource. */
MLX5_IPOOL_PORT_ID, /* Pool for port id resource. */
MLX5_IPOOL_JUMP, /* Pool for jump resource. */
+ MLX5_IPOOL_SAMPLE, /* Pool for sample resource. */
#endif
MLX5_IPOOL_MTR, /* Pool for meter resource. */
MLX5_IPOOL_MCP, /* Pool for metadata resource. */
@@ -512,6 +513,7 @@ struct mlx5_flow_tbl_resource {
/* Tables for metering splits should be added here. */
#define MLX5_MAX_TABLES_EXTERNAL (MLX5_MAX_TABLES - 3)
#define MLX5_MAX_TABLES_FDB UINT16_MAX
+#define MLX5_FLOW_TABLE_FACTOR 10
/* ID generation structure. */
struct mlx5_flow_id_pool {
@@ -640,6 +642,7 @@ struct mlx5_dev_ctx_shared {
struct mlx5_hlist *tag_table;
uint32_t port_id_action_list; /* List of port ID actions. */
uint32_t push_vlan_action_list; /* List of push VLAN actions. */
+ uint32_t sample_action_list; /* List of sample actions. */
struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
struct mlx5_flow_default_miss_resource default_miss;
/* Default miss action resource structure. */
@@ -3543,6 +3543,8 @@ struct mlx5_flow_tunnel_info {
* Pointer to return the created subflow, may be NULL.
* @param[in] prefix_layers
* Prefix subflow layers, may be 0.
+ * @param[in] prefix_mark
+ * Prefix subflow mark flag, may be 0.
* @param[in] attr
* Flow rule attributes.
* @param[in] items
@@ -3563,6 +3565,7 @@ struct mlx5_flow_tunnel_info {
struct rte_flow *flow,
struct mlx5_flow **sub_flow,
uint64_t prefix_layers,
+ uint32_t prefix_mark,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
@@ -3582,10 +3585,13 @@ struct mlx5_flow_tunnel_info {
dev_flow->handle, next);
/*
* If dev_flow is as one of the suffix flow, some actions in suffix
- * flow may need some user defined item layer flags.
+ * flow may need some user defined item layer flags, and pass the
+ * Metadate rxq mark flag to suffix flow as well.
*/
if (prefix_layers)
dev_flow->handle->layers = prefix_layers;
+ if (prefix_mark)
+ dev_flow->handle->mark = 1;
if (sub_flow)
*sub_flow = dev_flow;
return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
@@ -3915,6 +3921,143 @@ struct mlx5_flow_tunnel_info {
return 0;
}
+
+/**
+ * Check the match action from the action list.
+ *
+ * @param[in] actions
+ * Pointer to the list of actions.
+ * @param[in] action
+ * The action to be check if exist.
+ *
+ * @return
+ * > 0 the total number of actions.
+ * 0 if not found match action in action list.
+ */
+static int
+flow_check_match_action(const struct rte_flow_action actions[],
+ enum rte_flow_action_type action)
+{
+ int actions_n = 0;
+ int flag = 0;
+
+ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+ if (actions->type == action)
+ flag = 1;
+ actions_n++;
+ }
+ /* Count RTE_FLOW_ACTION_TYPE_END. */
+ return flag ? actions_n + 1 : 0;
+}
+
+/**
+ * Split the sample flow.
+ *
+ * As sample flow will split to two sub flow, sample flow with
+ * sample action, the other actions will move to new suffix flow.
+ *
+ * Also add unique tag id with tag action in the sample flow,
+ * the same tag id will be as match in the suffix flow.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[in] attr
+ * Flow rule attributes.
+ * @param[out] sfx_items
+ * Suffix flow match items (list terminated by the END pattern item).
+ * @param[in] actions
+ * Associated actions (list terminated by the END action).
+ * @param[out] actions_sfx
+ * Suffix flow actions.
+ * @param[out] actions_pre
+ * Prefix flow actions.
+ *
+ * @return
+ * 0 on success, or unique flow_id, a negative errno value
+ * otherwise and rte_errno is set.
+ */
+static int
+flow_sample_split_prep(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ struct rte_flow_item sfx_items[],
+ const struct rte_flow_action actions[],
+ struct rte_flow_action actions_sfx[],
+ struct rte_flow_action actions_pre[])
+{
+ struct mlx5_rte_flow_action_set_tag *set_tag;
+ struct mlx5_rte_flow_item_tag *tag_spec;
+ struct mlx5_rte_flow_item_tag *tag_mask;
+ struct rte_flow_item *tag_item;
+ struct rte_flow_action *tag_action = NULL;
+ bool pre_sample = true;
+ struct rte_flow_error error;
+ uint32_t tag_id = 0;
+ int ret;
+
+ /* Prepare the actions for prefix and suffix flow. */
+ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+ struct rte_flow_action **action_cur = NULL;
+
+ switch (actions->type) {
+ case RTE_FLOW_ACTION_TYPE_SAMPLE:
+ if (!attr->transfer) {
+ /* Add the extra tag action first for NIC-RX. */
+ tag_action = actions_pre;
+ tag_action->type = (enum rte_flow_action_type)
+ MLX5_RTE_FLOW_ACTION_TYPE_TAG;
+ actions_pre++;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_JUMP:
+ case RTE_FLOW_ACTION_TYPE_METER:
+ case RTE_FLOW_ACTION_TYPE_QUEUE:
+ action_cur = &actions_sfx;
+ break;
+ default:
+ break;
+ }
+ if (pre_sample && !action_cur)
+ action_cur = &actions_pre;
+ else
+ action_cur = &actions_sfx;
+ memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
+ (*action_cur)++;
+ if (actions->type == RTE_FLOW_ACTION_TYPE_SAMPLE)
+ pre_sample = false;
+ }
+ /* Add end action to the actions. */
+ actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
+ actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
+ if (!attr->transfer) {
+ actions_pre++;
+ /* Set the tag. */
+ set_tag = (void *)actions_pre;
+ ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, &error);
+ if (ret < 0)
+ return ret;
+ set_tag->id = ret;
+ tag_id = flow_qrss_get_id(dev);
+ set_tag->data = tag_id;
+ assert(tag_action);
+ tag_action->conf = set_tag;
+ /* Prepare the suffix subflow items. */
+ tag_item = sfx_items++;
+ sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
+ sfx_items++;
+ tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
+ tag_spec->data = tag_id;
+ tag_spec->id = set_tag->id;
+ tag_mask = tag_spec + 1;
+ tag_mask->data = UINT32_MAX;
+ tag_item->type = (enum rte_flow_item_type)
+ MLX5_RTE_FLOW_ITEM_TYPE_TAG;
+ tag_item->spec = tag_spec;
+ tag_item->last = NULL;
+ tag_item->mask = tag_mask;
+ }
+ return tag_id;
+}
+
/**
* The splitting for metadata feature.
*
@@ -3931,6 +4074,8 @@ struct mlx5_flow_tunnel_info {
* Parent flow structure pointer.
* @param[in] prefix_layers
* Prefix flow layer flags.
+ * @param[in] prefix_mark
+ * Prefix subflow mark flag, may be 0.
* @param[in] attr
* Flow rule attributes.
* @param[in] items
@@ -3950,6 +4095,7 @@ struct mlx5_flow_tunnel_info {
flow_create_split_metadata(struct rte_eth_dev *dev,
struct rte_flow *flow,
uint64_t prefix_layers,
+ uint32_t prefix_mark,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
@@ -3973,8 +4119,9 @@ struct mlx5_flow_tunnel_info {
config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
!mlx5_flow_ext_mreg_supported(dev))
return flow_create_split_inner(dev, flow, NULL, prefix_layers,
- attr, items, actions, external,
- flow_idx, error);
+ prefix_mark, attr, items,
+ actions, external, flow_idx,
+ error);
actions_n = flow_parse_metadata_split_actions_info(actions, &qrss,
&encap_idx);
if (qrss) {
@@ -4059,7 +4206,8 @@ struct mlx5_flow_tunnel_info {
goto exit;
}
/* Add the unmodified original or prefix subflow. */
- ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers, attr,
+ ret = flow_create_split_inner(dev, flow, &dev_flow, prefix_layers,
+ prefix_mark, attr,
items, ext_actions ? ext_actions :
actions, external, flow_idx, error);
if (ret < 0)
@@ -4122,7 +4270,7 @@ struct mlx5_flow_tunnel_info {
}
dev_flow = NULL;
/* Add suffix subflow to execute Q/RSS. */
- ret = flow_create_split_inner(dev, flow, &dev_flow, layers,
+ ret = flow_create_split_inner(dev, flow, &dev_flow, layers, 0,
&q_attr, mtr_sfx ? items :
q_items, q_actions,
external, flow_idx, error);
@@ -4158,6 +4306,10 @@ struct mlx5_flow_tunnel_info {
* Pointer to Ethernet device.
* @param[in] flow
* Parent flow structure pointer.
+ * @param[in] prefix_layers
+ * Prefix subflow layers, may be 0.
+ * @param[in] prefix_mark
+ * Prefix subflow mark flag, may be 0.
* @param[in] attr
* Flow rule attributes.
* @param[in] items
@@ -4175,12 +4327,14 @@ struct mlx5_flow_tunnel_info {
*/
static int
flow_create_split_meter(struct rte_eth_dev *dev,
- struct rte_flow *flow,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- const struct rte_flow_action actions[],
- bool external, uint32_t flow_idx,
- struct rte_flow_error *error)
+ struct rte_flow *flow,
+ uint64_t prefix_layers,
+ uint32_t prefix_mark,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external, uint32_t flow_idx,
+ struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_action *sfx_actions = NULL;
@@ -4223,8 +4377,10 @@ struct mlx5_flow_tunnel_info {
goto exit;
}
/* Add the prefix subflow. */
- ret = flow_create_split_inner(dev, flow, &dev_flow, 0, attr,
- items, pre_actions, external,
+ ret = flow_create_split_inner(dev, flow, &dev_flow,
+ prefix_layers, 0,
+ attr, items,
+ pre_actions, external,
flow_idx, error);
if (ret) {
ret = -rte_errno;
@@ -4239,8 +4395,10 @@ struct mlx5_flow_tunnel_info {
/* Add the prefix subflow. */
ret = flow_create_split_metadata(dev, flow, dev_flow ?
flow_get_prefix_layer_flags(dev_flow) :
- 0, &sfx_attr,
- sfx_items ? sfx_items : items,
+ prefix_layers, dev_flow ?
+ dev_flow->handle->mark : prefix_mark,
+ &sfx_attr, sfx_items ?
+ sfx_items : items,
sfx_actions ? sfx_actions : actions,
external, flow_idx, error);
exit:
@@ -4250,6 +4408,122 @@ struct mlx5_flow_tunnel_info {
}
/**
+ * The splitting for sample feature.
+ *
+ * The sample flow will be split to two flows as prefix and
+ * suffix flow.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[in] flow
+ * Parent flow structure pointer.
+ * @param[in] attr
+ * Flow rule attributes.
+ * @param[in] items
+ * Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ * Associated actions (list terminated by the END action).
+ * @param[in] external
+ * This flow rule is created by request external to PMD.
+ * @param[in] flow_idx
+ * This memory pool index to the flow.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ * @return
+ * 0 on success, negative value otherwise
+ */
+static int
+flow_create_split_sample(struct rte_eth_dev *dev,
+ struct rte_flow *flow,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external, uint32_t flow_idx,
+ struct rte_flow_error *error)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct rte_flow_action *sfx_actions = NULL;
+ struct rte_flow_action *pre_actions = NULL;
+ struct rte_flow_item *sfx_items = NULL;
+ struct mlx5_flow *dev_flow = NULL;
+ struct rte_flow_attr sfx_attr = *attr;
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ struct mlx5_flow_dv_sample_resource *sample_res;
+ struct mlx5_flow_tbl_data_entry *sfx_tbl_data;
+ struct mlx5_flow_tbl_resource *sfx_tbl;
+ union mlx5_flow_tbl_key sfx_table_key;
+#endif
+ size_t act_size;
+ size_t item_size;
+ int32_t tag_id = 0;
+ int actions_n = 0;
+ int ret = 0;
+
+ if (priv->sampler_en)
+ actions_n = flow_check_match_action(actions,
+ RTE_FLOW_ACTION_TYPE_SAMPLE);
+ if (actions_n) {
+ /* The prefix actions must includes sample, tag, end. */
+ act_size = sizeof(struct rte_flow_action) * (actions_n * 2 + 1)
+ + sizeof(struct mlx5_rte_flow_action_set_tag);
+ /* tag, end. */
+#define SAMPLE_SUFFIX_ITEM 2
+ item_size = sizeof(struct rte_flow_item) * SAMPLE_SUFFIX_ITEM +
+ sizeof(struct mlx5_rte_flow_item_tag) * 2;
+ sfx_actions = rte_zmalloc(__func__, (act_size + item_size), 0);
+ if (!sfx_actions)
+ return rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ NULL, "no memory to split "
+ "sample flow");
+ if (!attr->transfer)
+ sfx_items = (struct rte_flow_item *)((char *)sfx_actions
+ + act_size);
+ pre_actions = sfx_actions + actions_n;
+ tag_id = flow_sample_split_prep(dev, attr, sfx_items,
+ actions, sfx_actions,
+ pre_actions);
+ if (tag_id < 0 || (!attr->transfer && !tag_id)) {
+ ret = -rte_errno;
+ goto exit;
+ }
+ /* Add the prefix subflow. */
+ ret = flow_create_split_inner(dev, flow, &dev_flow, 0, 0, attr,
+ items, pre_actions, external,
+ flow_idx, error);
+ if (ret) {
+ ret = -rte_errno;
+ goto exit;
+ }
+ dev_flow->handle->split_flow_id = tag_id;
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ /* Set the sfx group attr. */
+ sample_res = (struct mlx5_flow_dv_sample_resource *)
+ dev_flow->dv.sample_res;
+ sfx_tbl = (struct mlx5_flow_tbl_resource *)
+ sample_res->normal_path_tbl;
+ sfx_tbl_data = container_of(sfx_tbl,
+ struct mlx5_flow_tbl_data_entry, tbl);
+ sfx_table_key.v64 = sfx_tbl_data->entry.key;
+ sfx_attr.group = sfx_attr.transfer ?
+ (sfx_table_key.table_id - 1) :
+ sfx_table_key.table_id;
+#endif
+ }
+ /* Add the suffix subflow. */
+ ret = flow_create_split_meter(dev, flow, dev_flow ?
+ flow_get_prefix_layer_flags(dev_flow) : 0,
+ dev_flow ? dev_flow->handle->mark : 0,
+ &sfx_attr, sfx_items ? sfx_items : items,
+ sfx_actions ? sfx_actions : actions,
+ external, flow_idx, error);
+exit:
+ if (sfx_actions)
+ rte_free(sfx_actions);
+ return ret;
+}
+
+/**
* Split the flow to subflow set. The splitters might be linked
* in the chain, like this:
* flow_create_split_outer() calls:
@@ -4297,7 +4571,7 @@ struct mlx5_flow_tunnel_info {
{
int ret;
- ret = flow_create_split_meter(dev, flow, attr, items,
+ ret = flow_create_split_sample(dev, flow, attr, items,
actions, external, flow_idx, error);
MLX5_ASSERT(ret <= 0);
return ret;
@@ -507,6 +507,39 @@ struct mlx5_flow_tbl_data_entry {
uint32_t idx; /**< index for the indexed mempool. */
};
+/* Sub rdma-core actions list. */
+struct mlx5_flow_sub_actions_list {
+ uint32_t actions_num; /**< Number of sample actions. */
+ uint64_t action_flags;
+ void *dr_queue_action;
+ void *dr_tag_action;
+ void *dr_cnt_action;
+};
+
+/* Sample sub-actions resource list. */
+struct mlx5_flow_sub_actions_idx {
+ uint32_t rix_hrxq; /**< Hash Rx queue object index. */
+ uint32_t rix_tag; /**< Index to the tag action. */
+ uint32_t cnt;
+};
+
+/* Sample action resource structure. */
+struct mlx5_flow_dv_sample_resource {
+ ILIST_ENTRY(uint32_t)next; /**< Pointer to next element. */
+ rte_atomic32_t refcnt; /**< Reference counter. */
+ void *verbs_action; /**< Verbs sample action object. */
+ uint8_t ft_type; /** Flow Table Type */
+ uint32_t ft_id; /** Flow Table Level */
+ uint32_t ratio; /** Sample Ratio */
+ uint64_t set_action; /** Restore reg_c0 value */
+ void *normal_path_tbl; /** Flow Table pointer */
+ void *default_miss; /** default_miss dr_action. */
+ struct mlx5_flow_sub_actions_idx sample_idx;
+ /**< Action index resources. */
+ struct mlx5_flow_sub_actions_list sample_act;
+ /**< Action resources. */
+};
+
/* Verbs specification header. */
struct ibv_spec_header {
enum ibv_flow_spec_type type;
@@ -539,6 +572,8 @@ struct mlx5_flow_handle_dv {
/**< Index to push VLAN action resource in cache. */
uint32_t rix_tag;
/**< Index to the tag action. */
+ uint32_t rix_sample;
+ /**< Index to sample action resource in cache. */
} __rte_packed;
/** Device flow handle structure: used both for creating & destroying. */
@@ -604,6 +639,8 @@ struct mlx5_flow_dv_workspace {
/**< Pointer to the jump action resource. */
struct mlx5_flow_dv_match_params value;
/**< Holds the value that the packet is compared to. */
+ struct mlx5_flow_dv_sample_resource *sample_res;
+ /**< Pointer to the sample action resource. */
};
/*