@@ -26,6 +26,7 @@
#include <rte_dev.h>
#include <rte_tailq.h>
#include <rte_hash_crc.h>
+#include <rte_bitmap.h>
#include "i40e_logs.h"
#include "base/i40e_prototype.h"
@@ -1045,8 +1046,16 @@ static int
i40e_init_fdir_filter_list(struct rte_eth_dev *dev)
{
struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
+ struct i40e_hw *hw = I40E_PF_TO_HW(pf);
struct i40e_fdir_info *fdir_info = &pf->fdir;
char fdir_hash_name[RTE_HASH_NAMESIZE];
+ uint32_t alloc = hw->func_caps.fd_filters_guaranteed;
+ uint32_t best = hw->func_caps.fd_filters_best_effort;
+ uint32_t glqf_ctl_reg = 0;
+ struct rte_bitmap *bmp = NULL;
+ uint32_t bmp_size;
+ void *mem = NULL;
+ uint32_t i = 0;
int ret;
struct rte_hash_parameters fdir_hash_params = {
@@ -1067,6 +1076,7 @@ i40e_init_fdir_filter_list(struct rte_eth_dev *dev)
PMD_INIT_LOG(ERR, "Failed to create fdir hash table!");
return -EINVAL;
}
+
fdir_info->hash_map = rte_zmalloc("i40e_fdir_hash_map",
sizeof(struct i40e_fdir_filter *) *
I40E_MAX_FDIR_FILTER_NUM,
@@ -1077,8 +1087,93 @@ i40e_init_fdir_filter_list(struct rte_eth_dev *dev)
ret = -ENOMEM;
goto err_fdir_hash_map_alloc;
}
+
+ fdir_info->fdir_filter_array = rte_zmalloc("fdir_filter",
+ sizeof(struct i40e_fdir_filter) *
+ I40E_MAX_FDIR_FILTER_NUM,
+ 0);
+
+ if (!fdir_info->fdir_filter_array) {
+ PMD_INIT_LOG(ERR,
+ "Failed to allocate memory for fdir filter array!");
+ ret = -ENOMEM;
+ goto err_fdir_filter_array_alloc;
+ }
+
+ glqf_ctl_reg = i40e_read_rx_ctl(hw, I40E_GLQF_CTL);
+ if (!pf->support_multi_driver) {
+ fdir_info->fdir_invalprio = 1;
+ glqf_ctl_reg |= I40E_GLQF_CTL_INVALPRIO_MASK;
+ PMD_DRV_LOG(INFO, "FDIR INVALPRIO set to guaranteed first");
+ } else {
+ if (glqf_ctl_reg | I40E_GLQF_CTL_INVALPRIO_MASK) {
+ fdir_info->fdir_invalprio = 1;
+ PMD_DRV_LOG(INFO, "FDIR INVALPRIO is: guaranteed first");
+ } else {
+ fdir_info->fdir_invalprio = 0;
+ PMD_DRV_LOG(INFO, "FDIR INVALPRIO is: shared first");
+ }
+ }
+
+ i40e_write_rx_ctl(hw, I40E_GLQF_CTL, glqf_ctl_reg);
+ PMD_DRV_LOG(INFO, "FDIR guarantee space: %u, best_effort space %u.", alloc, best);
+
+ fdir_info->fdir_space_size = alloc + best;
+ fdir_info->fdir_actual_cnt = 0;
+ fdir_info->fdir_guarantee_available_space = alloc;
+ fdir_info->fdir_guarantee_free_space =
+ fdir_info->fdir_guarantee_available_space;
+
+ fdir_info->fdir_flow_pool.pool =
+ rte_zmalloc("i40e_fdir_entry",
+ sizeof(struct i40e_fdir_entry) *
+ fdir_info->fdir_space_size,
+ 0);
+
+ if (!fdir_info->fdir_flow_pool.pool) {
+ PMD_INIT_LOG(ERR,
+ "Failed to allocate memory for bitmap flow!");
+ ret = -ENOMEM;
+ goto err_fdir_bitmap_flow_alloc;
+ }
+
+ for (i = 0; i < fdir_info->fdir_space_size; i++)
+ fdir_info->fdir_flow_pool.pool[i].idx = i;
+
+ bmp_size =
+ rte_bitmap_get_memory_footprint(fdir_info->fdir_space_size);
+
+ mem = rte_zmalloc("fdir_bmap", bmp_size, RTE_CACHE_LINE_SIZE);
+ if (mem == NULL) {
+ PMD_INIT_LOG(ERR,
+ "Failed to allocate memory for fdir bitmap!");
+ ret = -ENOMEM;
+ goto err_fdir_mem_alloc;
+ }
+
+ bmp = rte_bitmap_init(fdir_info->fdir_space_size, mem, bmp_size);
+ if (bmp == NULL) {
+ PMD_INIT_LOG(ERR,
+ "Failed to initialization fdir bitmap!");
+ ret = -ENOMEM;
+ goto err_fdir_bmp_alloc;
+ }
+
+ for (i = 0; i < fdir_info->fdir_space_size; i++)
+ rte_bitmap_set(bmp, i);
+
+ fdir_info->fdir_flow_pool.b = bmp;
+
return 0;
+err_fdir_bmp_alloc:
+ rte_free(mem);
+err_fdir_mem_alloc:
+ rte_free(fdir_info->fdir_flow_pool.pool);
+err_fdir_bitmap_flow_alloc:
+ rte_free(fdir_info->fdir_filter_array);
+err_fdir_filter_array_alloc:
+ rte_free(fdir_info->hash_map);
err_fdir_hash_map_alloc:
rte_hash_free(fdir_info->hash_table);
@@ -1749,18 +1844,34 @@ i40e_rm_fdir_filter_list(struct i40e_pf *pf)
struct i40e_fdir_info *fdir_info;
fdir_info = &pf->fdir;
- /* Remove all flow director rules and hash */
+
+ /* Remove all flow director rules */
+ while ((p_fdir = TAILQ_FIRST(&fdir_info->fdir_list)))
+ TAILQ_REMOVE(&fdir_info->fdir_list, p_fdir, rules);
+}
+
+static void
+i40e_fdir_memory_cleanup(struct i40e_pf *pf)
+{
+ struct i40e_fdir_info *fdir_info;
+
+ fdir_info = &pf->fdir;
+
+ /* flow director memory cleanup */
if (fdir_info->hash_map)
rte_free(fdir_info->hash_map);
if (fdir_info->hash_table)
rte_hash_free(fdir_info->hash_table);
+ if (fdir_info->fdir_flow_pool.b)
+ rte_bitmap_free(fdir_info->fdir_flow_pool.b);
+ if (fdir_info->fdir_flow_pool.pool)
+ rte_free(fdir_info->fdir_flow_pool.pool);
+ if (fdir_info->fdir_filter_array)
+ rte_free(fdir_info->fdir_filter_array);
- while ((p_fdir = TAILQ_FIRST(&fdir_info->fdir_list))) {
- TAILQ_REMOVE(&fdir_info->fdir_list, p_fdir, rules);
- rte_free(p_fdir);
- }
}
+
void i40e_flex_payload_reg_set_default(struct i40e_hw *hw)
{
/*
@@ -2618,9 +2729,14 @@ i40e_dev_close(struct rte_eth_dev *dev)
/* Remove all flows */
while ((p_flow = TAILQ_FIRST(&pf->flow_list))) {
TAILQ_REMOVE(&pf->flow_list, p_flow, node);
- rte_free(p_flow);
+ /* Do not free FDIR flows since they are static allocated */
+ if (p_flow->filter_type != RTE_ETH_FILTER_FDIR)
+ rte_free(p_flow);
}
+ /* release the fdir static allocated memory */
+ i40e_fdir_memory_cleanup(pf);
+
/* Remove all Traffic Manager configuration */
i40e_tm_conf_uninit(dev);
@@ -711,8 +711,8 @@ struct i40e_fdir_info {
uint16_t match_counter_index; /* Statistic counter index used for fdir*/
struct i40e_tx_queue *txq;
struct i40e_rx_queue *rxq;
- void *prg_pkt; /* memory for fdir program packet */
- uint64_t dma_addr; /* physic address of packet memory*/
+ void *prg_pkt[PRG_PKT_CNT]; /* memory for fdir program packet */
+ uint64_t dma_addr[PRG_PKT_CNT]; /* physic address of packet memory*/
/* input set bits for each pctype */
uint64_t input_set[I40E_FILTER_PCTYPE_MAX];
@@ -1386,7 +1386,7 @@ int i40e_add_del_fdir_filter(struct rte_eth_dev *dev,
bool add);
int i40e_flow_add_del_fdir_filter(struct rte_eth_dev *dev,
const struct i40e_fdir_filter_conf *filter,
- bool add);
+ bool add, bool wait_status);
int i40e_dev_tunnel_filter_set(struct i40e_pf *pf,
struct rte_eth_tunnel_filter_conf *tunnel_filter,
uint8_t add);
@@ -99,7 +99,7 @@ static int
i40e_flow_fdir_filter_programming(struct i40e_pf *pf,
enum i40e_filter_pctype pctype,
const struct i40e_fdir_filter_conf *filter,
- bool add);
+ bool add, bool wait_status);
static int
i40e_fdir_rx_queue_init(struct i40e_rx_queue *rxq)
@@ -163,6 +163,7 @@ i40e_fdir_setup(struct i40e_pf *pf)
char z_name[RTE_MEMZONE_NAMESIZE];
const struct rte_memzone *mz = NULL;
struct rte_eth_dev *eth_dev = pf->adapter->eth_dev;
+ uint16_t i;
if ((pf->flags & I40E_FLAG_FDIR) == 0) {
PMD_INIT_LOG(ERR, "HW doesn't support FDIR");
@@ -179,6 +180,7 @@ i40e_fdir_setup(struct i40e_pf *pf)
PMD_DRV_LOG(INFO, "FDIR initialization has been done.");
return I40E_SUCCESS;
}
+
/* make new FDIR VSI */
vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, pf->main_vsi, 0);
if (!vsi) {
@@ -233,17 +235,27 @@ i40e_fdir_setup(struct i40e_pf *pf)
eth_dev->device->driver->name,
I40E_FDIR_MZ_NAME,
eth_dev->data->port_id);
- mz = i40e_memzone_reserve(z_name, I40E_FDIR_PKT_LEN, SOCKET_ID_ANY);
+ mz = i40e_memzone_reserve(z_name, I40E_FDIR_PKT_LEN * PRG_PKT_CNT,
+ SOCKET_ID_ANY);
if (!mz) {
PMD_DRV_LOG(ERR, "Cannot init memzone for "
"flow director program packet.");
err = I40E_ERR_NO_MEMORY;
goto fail_mem;
}
- pf->fdir.prg_pkt = mz->addr;
- pf->fdir.dma_addr = mz->iova;
+
+ for (i = 0; i < PRG_PKT_CNT; i++) {
+ pf->fdir.prg_pkt[i] = (uint8_t *)mz->addr +
+ I40E_FDIR_PKT_LEN * i;
+ pf->fdir.dma_addr[i] = mz->iova +
+ I40E_FDIR_PKT_LEN * i;
+ }
pf->fdir.match_counter_index = I40E_COUNTER_INDEX_FDIR(hw->pf_id);
+ pf->fdir.fdir_actual_cnt = 0;
+ pf->fdir.fdir_guarantee_free_space =
+ pf->fdir.fdir_guarantee_available_space;
+
PMD_DRV_LOG(INFO, "FDIR setup successfully, with programming queue %u.",
vsi->base_queue);
return I40E_SUCCESS;
@@ -1560,11 +1572,11 @@ i40e_sw_fdir_filter_lookup(struct i40e_fdir_info *fdir_info,
return fdir_info->hash_map[ret];
}
-/* Add a flow director filter into the SW list */
static int
i40e_sw_fdir_filter_insert(struct i40e_pf *pf, struct i40e_fdir_filter *filter)
{
struct i40e_fdir_info *fdir_info = &pf->fdir;
+ struct i40e_fdir_filter *hash_filter;
int ret;
if (filter->fdir.input.flow_ext.pkt_template)
@@ -1580,9 +1592,14 @@ i40e_sw_fdir_filter_insert(struct i40e_pf *pf, struct i40e_fdir_filter *filter)
ret);
return ret;
}
- fdir_info->hash_map[ret] = filter;
- TAILQ_INSERT_TAIL(&fdir_info->fdir_list, filter, rules);
+ if (fdir_info->hash_map[ret])
+ return -1;
+
+ hash_filter = &fdir_info->fdir_filter_array[ret];
+ rte_memcpy(hash_filter, filter, sizeof(*filter));
+ fdir_info->hash_map[ret] = hash_filter;
+ TAILQ_INSERT_TAIL(&fdir_info->fdir_list, hash_filter, rules);
return 0;
}
@@ -1611,7 +1628,6 @@ i40e_sw_fdir_filter_del(struct i40e_pf *pf, struct i40e_fdir_input *input)
fdir_info->hash_map[ret] = NULL;
TAILQ_REMOVE(&fdir_info->fdir_list, filter, rules);
- rte_free(filter);
return 0;
}
@@ -1629,7 +1645,7 @@ i40e_add_del_fdir_filter(struct rte_eth_dev *dev,
{
struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
- unsigned char *pkt = (unsigned char *)pf->fdir.prg_pkt;
+ unsigned char *pkt = (unsigned char *)pf->fdir.prg_pkt[0];
enum i40e_filter_pctype pctype;
int ret = 0;
@@ -1678,23 +1694,50 @@ i40e_add_del_fdir_filter(struct rte_eth_dev *dev,
return ret;
}
+static inline unsigned char *
+i40e_find_available_buffer(struct rte_eth_dev *dev)
+{
+ struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
+ struct i40e_fdir_info *fdir_info = &pf->fdir;
+ struct i40e_tx_queue *txq = pf->fdir.txq;
+ volatile struct i40e_tx_desc *txdp = &txq->tx_ring[txq->tx_tail + 1];
+ uint32_t i;
+
+ /* wait until the tx descriptor is ready */
+ for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) {
+ if ((txdp->cmd_type_offset_bsz &
+ rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) ==
+ rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
+ break;
+ rte_delay_us(1);
+ }
+ if (i >= I40E_FDIR_MAX_WAIT_US) {
+ PMD_DRV_LOG(ERR,
+ "Failed to program FDIR filter: time out to get DD on tx queue.");
+ return NULL;
+ }
+
+ return (unsigned char *)fdir_info->prg_pkt[txq->tx_tail >> 1];
+}
+
/**
* i40e_flow_add_del_fdir_filter - add or remove a flow director filter.
* @pf: board private structure
* @filter: fdir filter entry
* @add: 0 - delete, 1 - add
*/
+
int
i40e_flow_add_del_fdir_filter(struct rte_eth_dev *dev,
const struct i40e_fdir_filter_conf *filter,
- bool add)
+ bool add, bool wait_status)
{
struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
- unsigned char *pkt = (unsigned char *)pf->fdir.prg_pkt;
+ unsigned char *pkt = NULL;
enum i40e_filter_pctype pctype;
struct i40e_fdir_info *fdir_info = &pf->fdir;
- struct i40e_fdir_filter *fdir_filter, *node;
+ struct i40e_fdir_filter *node;
struct i40e_fdir_filter check_filter; /* Check if the filter exists */
int ret = 0;
@@ -1727,25 +1770,41 @@ i40e_flow_add_del_fdir_filter(struct rte_eth_dev *dev,
/* Check if there is the filter in SW list */
memset(&check_filter, 0, sizeof(check_filter));
i40e_fdir_filter_convert(filter, &check_filter);
- node = i40e_sw_fdir_filter_lookup(fdir_info, &check_filter.fdir.input);
- if (add && node) {
- PMD_DRV_LOG(ERR,
- "Conflict with existing flow director rules!");
- return -EINVAL;
- }
- if (!add && !node) {
- PMD_DRV_LOG(ERR,
- "There's no corresponding flow firector filter!");
- return -EINVAL;
+ if (add) {
+ ret = i40e_sw_fdir_filter_insert(pf, &check_filter);
+ if (ret < 0) {
+ PMD_DRV_LOG(ERR,
+ "Conflict with existing flow director rules!");
+ return -EINVAL;
+ }
+ } else {
+ node = i40e_sw_fdir_filter_lookup(fdir_info,
+ &check_filter.fdir.input);
+ if (!node) {
+ PMD_DRV_LOG(ERR,
+ "There's no corresponding flow firector filter!");
+ return -EINVAL;
+ }
+
+ ret = i40e_sw_fdir_filter_del(pf, &node->fdir.input);
+ if (ret < 0) {
+ PMD_DRV_LOG(ERR,
+ "Error deleting fdir rule from hash table!");
+ return -EINVAL;
+ }
}
- memset(pkt, 0, I40E_FDIR_PKT_LEN);
+ /* find a buffer to store the pkt */
+ pkt = i40e_find_available_buffer(dev);
+ if (pkt == NULL)
+ goto error_op;
+ memset(pkt, 0, I40E_FDIR_PKT_LEN);
ret = i40e_flow_fdir_construct_pkt(pf, &filter->input, pkt);
if (ret < 0) {
PMD_DRV_LOG(ERR, "construct packet for fdir fails.");
- return ret;
+ goto error_op;
}
if (hw->mac.type == I40E_MAC_X722) {
@@ -1754,28 +1813,22 @@ i40e_flow_add_del_fdir_filter(struct rte_eth_dev *dev,
hw, I40E_GLQF_FD_PCTYPES((int)pctype));
}
- ret = i40e_flow_fdir_filter_programming(pf, pctype, filter, add);
+ ret = i40e_flow_fdir_filter_programming(pf, pctype, filter, add,
+ wait_status);
if (ret < 0) {
PMD_DRV_LOG(ERR, "fdir programming fails for PCTYPE(%u).",
pctype);
- return ret;
+ goto error_op;
}
- if (add) {
- fdir_filter = rte_zmalloc("fdir_filter",
- sizeof(*fdir_filter), 0);
- if (fdir_filter == NULL) {
- PMD_DRV_LOG(ERR, "Failed to alloc memory.");
- return -ENOMEM;
- }
+ return ret;
- rte_memcpy(fdir_filter, &check_filter, sizeof(check_filter));
- ret = i40e_sw_fdir_filter_insert(pf, fdir_filter);
- if (ret < 0)
- rte_free(fdir_filter);
- } else {
- ret = i40e_sw_fdir_filter_del(pf, &node->fdir.input);
- }
+error_op:
+ /* roll back */
+ if (add)
+ i40e_sw_fdir_filter_del(pf, &check_filter.fdir.input);
+ else
+ i40e_sw_fdir_filter_insert(pf, &check_filter);
return ret;
}
@@ -1878,7 +1931,7 @@ i40e_fdir_filter_programming(struct i40e_pf *pf,
PMD_DRV_LOG(INFO, "filling transmit descriptor.");
txdp = &(txq->tx_ring[txq->tx_tail + 1]);
- txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr);
+ txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr[0]);
td_cmd = I40E_TX_DESC_CMD_EOP |
I40E_TX_DESC_CMD_RS |
I40E_TX_DESC_CMD_DUMMY;
@@ -1928,7 +1981,7 @@ static int
i40e_flow_fdir_filter_programming(struct i40e_pf *pf,
enum i40e_filter_pctype pctype,
const struct i40e_fdir_filter_conf *filter,
- bool add)
+ bool add, bool wait_status)
{
struct i40e_tx_queue *txq = pf->fdir.txq;
struct i40e_rx_queue *rxq = pf->fdir.rxq;
@@ -1936,8 +1989,10 @@ i40e_flow_fdir_filter_programming(struct i40e_pf *pf,
volatile struct i40e_tx_desc *txdp;
volatile struct i40e_filter_program_desc *fdirdp;
uint32_t td_cmd;
- uint16_t vsi_id, i;
+ uint16_t vsi_id;
uint8_t dest;
+ uint32_t i;
+ uint8_t retry_count = 0;
PMD_DRV_LOG(INFO, "filling filter programming descriptor.");
fdirdp = (volatile struct i40e_filter_program_desc *)
@@ -2012,7 +2067,8 @@ i40e_flow_fdir_filter_programming(struct i40e_pf *pf,
PMD_DRV_LOG(INFO, "filling transmit descriptor.");
txdp = &txq->tx_ring[txq->tx_tail + 1];
- txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr);
+ txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr[txq->tx_tail / 2]);
+
td_cmd = I40E_TX_DESC_CMD_EOP |
I40E_TX_DESC_CMD_RS |
I40E_TX_DESC_CMD_DUMMY;
@@ -2025,25 +2081,34 @@ i40e_flow_fdir_filter_programming(struct i40e_pf *pf,
txq->tx_tail = 0;
/* Update the tx tail register */
rte_wmb();
+
+ /* capture the previous error report(if any) from rx ring */
+ while ((i40e_check_fdir_programming_status(rxq) < 0) &&
+ (++retry_count < 100))
+ PMD_DRV_LOG(INFO, "previous error report captured.");
+
I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
- for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) {
- if ((txdp->cmd_type_offset_bsz &
- rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) ==
- rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
- break;
- rte_delay_us(1);
- }
- if (i >= I40E_FDIR_MAX_WAIT_US) {
- PMD_DRV_LOG(ERR,
- "Failed to program FDIR filter: time out to get DD on tx queue.");
- return -ETIMEDOUT;
- }
- /* totally delay 10 ms to check programming status*/
- rte_delay_us(I40E_FDIR_MAX_WAIT_US);
- if (i40e_check_fdir_programming_status(rxq) < 0) {
- PMD_DRV_LOG(ERR,
- "Failed to program FDIR filter: programming status reported.");
- return -ETIMEDOUT;
+
+ if (wait_status) {
+ for (i = 0; i < I40E_FDIR_MAX_WAIT_US; i++) {
+ if ((txdp->cmd_type_offset_bsz &
+ rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) ==
+ rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
+ break;
+ rte_delay_us(1);
+ }
+ if (i >= I40E_FDIR_MAX_WAIT_US) {
+ PMD_DRV_LOG(ERR,
+ "Failed to program FDIR filter: time out to get DD on tx queue.");
+ return -ETIMEDOUT;
+ }
+ /* totally delay 10 ms to check programming status*/
+ rte_delay_us(I40E_FDIR_MAX_WAIT_US);
+ if (i40e_check_fdir_programming_status(rxq) < 0) {
+ PMD_DRV_LOG(ERR,
+ "Failed to program FDIR filter: programming status reported.");
+ return -ETIMEDOUT;
+ }
}
return 0;
@@ -2327,7 +2392,7 @@ i40e_fdir_filter_restore(struct i40e_pf *pf)
uint32_t best_cnt; /**< Number of filters in best effort spaces. */
TAILQ_FOREACH(f, fdir_list, rules)
- i40e_flow_add_del_fdir_filter(dev, &f->fdir, TRUE);
+ i40e_flow_add_del_fdir_filter(dev, &f->fdir, TRUE, TRUE);
fdstat = I40E_READ_REG(hw, I40E_PFQF_FDSTAT);
guarant_cnt =
@@ -17,6 +17,7 @@
#include <rte_malloc.h>
#include <rte_tailq.h>
#include <rte_flow_driver.h>
+#include <rte_bitmap.h>
#include "i40e_logs.h"
#include "base/i40e_type.h"
@@ -144,6 +145,8 @@ const struct rte_flow_ops i40e_flow_ops = {
static union i40e_filter_t cons_filter;
static enum rte_filter_type cons_filter_type = RTE_ETH_FILTER_NONE;
+/* internal pattern w/o VOID items */
+struct rte_flow_item g_items[32];
/* Pattern matched ethertype filter */
static enum rte_flow_item_type pattern_ethertype[] = {
@@ -2044,9 +2047,6 @@ i40e_flow_parse_ethertype_pattern(struct rte_eth_dev *dev,
const struct rte_flow_item_eth *eth_spec;
const struct rte_flow_item_eth *eth_mask;
enum rte_flow_item_type item_type;
- uint16_t outer_tpid;
-
- outer_tpid = i40e_get_outer_vlan(dev);
for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
if (item->last) {
@@ -2106,7 +2106,7 @@ i40e_flow_parse_ethertype_pattern(struct rte_eth_dev *dev,
if (filter->ether_type == RTE_ETHER_TYPE_IPV4 ||
filter->ether_type == RTE_ETHER_TYPE_IPV6 ||
filter->ether_type == RTE_ETHER_TYPE_LLDP ||
- filter->ether_type == outer_tpid) {
+ filter->ether_type == i40e_get_outer_vlan(dev)) {
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ITEM,
item,
@@ -2349,6 +2349,7 @@ i40e_flow_set_fdir_flex_pit(struct i40e_pf *pf,
field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i;
flx_pit = MK_FLX_PIT(min_next_off, NONUSE_FLX_PIT_FSIZE,
NONUSE_FLX_PIT_DEST_OFF);
+
I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(field_idx), flx_pit);
min_next_off++;
}
@@ -2608,7 +2609,6 @@ i40e_flow_parse_fdir_pattern(struct rte_eth_dev *dev,
uint16_t flex_size;
bool cfg_flex_pit = true;
bool cfg_flex_msk = true;
- uint16_t outer_tpid;
uint16_t ether_type;
uint32_t vtc_flow_cpu;
bool outer_ip = true;
@@ -2617,7 +2617,6 @@ i40e_flow_parse_fdir_pattern(struct rte_eth_dev *dev,
memset(off_arr, 0, sizeof(off_arr));
memset(len_arr, 0, sizeof(len_arr));
memset(flex_mask, 0, I40E_FDIR_MAX_FLEX_LEN);
- outer_tpid = i40e_get_outer_vlan(dev);
filter->input.flow_ext.customized_pctype = false;
for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
if (item->last) {
@@ -2685,7 +2684,7 @@ i40e_flow_parse_fdir_pattern(struct rte_eth_dev *dev,
if (next_type == RTE_FLOW_ITEM_TYPE_VLAN ||
ether_type == RTE_ETHER_TYPE_IPV4 ||
ether_type == RTE_ETHER_TYPE_IPV6 ||
- ether_type == outer_tpid) {
+ ether_type == i40e_get_outer_vlan(dev)) {
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ITEM,
item,
@@ -2729,7 +2728,7 @@ i40e_flow_parse_fdir_pattern(struct rte_eth_dev *dev,
if (ether_type == RTE_ETHER_TYPE_IPV4 ||
ether_type == RTE_ETHER_TYPE_IPV6 ||
- ether_type == outer_tpid) {
+ ether_type == i40e_get_outer_vlan(dev)) {
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ITEM,
item,
@@ -5263,7 +5262,6 @@ i40e_flow_validate(struct rte_eth_dev *dev,
NULL, "NULL attribute.");
return -rte_errno;
}
-
memset(&cons_filter, 0, sizeof(cons_filter));
/* Get the non-void item of action */
@@ -5285,12 +5283,18 @@ i40e_flow_validate(struct rte_eth_dev *dev,
}
item_num++;
- items = rte_zmalloc("i40e_pattern",
- item_num * sizeof(struct rte_flow_item), 0);
- if (!items) {
- rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_ITEM_NUM,
- NULL, "No memory for PMD internal items.");
- return -ENOMEM;
+ if (item_num <= ARRAY_SIZE(g_items)) {
+ items = g_items;
+ } else {
+ items = rte_zmalloc("i40e_pattern",
+ item_num * sizeof(struct rte_flow_item), 0);
+ if (!items) {
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_ITEM_NUM,
+ NULL,
+ "No memory for PMD internal items.");
+ return -ENOMEM;
+ }
}
i40e_pattern_skip_void_item(items, pattern);
@@ -5298,20 +5302,26 @@ i40e_flow_validate(struct rte_eth_dev *dev,
i = 0;
do {
parse_filter = i40e_find_parse_filter_func(items, &i);
+
if (!parse_filter && !flag) {
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ITEM,
pattern, "Unsupported pattern");
- rte_free(items);
+
+ if (items != g_items)
+ rte_free(items);
return -rte_errno;
}
+
if (parse_filter)
ret = parse_filter(dev, attr, items, actions,
error, &cons_filter);
+
flag = true;
} while ((ret < 0) && (i < RTE_DIM(i40e_supported_patterns)));
- rte_free(items);
+ if (items != g_items)
+ rte_free(items);
return ret;
}
@@ -5324,21 +5334,67 @@ i40e_flow_create(struct rte_eth_dev *dev,
struct rte_flow_error *error)
{
struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
- struct rte_flow *flow;
+ struct rte_flow *flow = NULL;
+ struct i40e_fdir_info *fdir_info = &pf->fdir;
+ uint32_t i = 0;
+ uint32_t pos = 0;
+ uint64_t slab = 0;
+ bool wait_for_status = true;
int ret;
- flow = rte_zmalloc("i40e_flow", sizeof(struct rte_flow), 0);
- if (!flow) {
- rte_flow_error_set(error, ENOMEM,
- RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
- "Failed to allocate memory");
- return flow;
- }
-
ret = i40e_flow_validate(dev, attr, pattern, actions, error);
if (ret < 0)
return NULL;
+ if (cons_filter_type == RTE_ETH_FILTER_FDIR) {
+ if (fdir_info->fdir_actual_cnt >=
+ fdir_info->fdir_space_size) {
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Fdir space full");
+ return NULL;
+ }
+
+ ret = rte_bitmap_scan(fdir_info->fdir_flow_pool.b, &pos,
+ &slab);
+
+ /* normally this won't happen as the fdir_actual_cnt should be
+ * same with the number of the set bits in fdir_flow_pool,
+ * but anyway handle this error condition here for safe
+ */
+ if (ret == 0) {
+ PMD_DRV_LOG(ERR, "fdir_actual_cnt out of sync");
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Fdir space full");
+ return NULL;
+ }
+
+ i = rte_bsf64(slab);
+ pos += i;
+ rte_bitmap_clear(fdir_info->fdir_flow_pool.b, pos);
+ flow = &fdir_info->fdir_flow_pool.pool[pos].flow;
+ memset(flow, 0, sizeof(struct rte_flow));
+
+ if (fdir_info->fdir_invalprio == 1) {
+ if (fdir_info->fdir_guarantee_free_space > 0) {
+ fdir_info->fdir_guarantee_free_space--;
+ wait_for_status = false;
+ }
+ }
+
+ fdir_info->fdir_actual_cnt++;
+
+ } else {
+ flow = rte_zmalloc("i40e_flow", sizeof(struct rte_flow), 0);
+ if (!flow) {
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
+ "Failed to allocate memory");
+ return flow;
+ }
+ }
+
switch (cons_filter_type) {
case RTE_ETH_FILTER_ETHERTYPE:
ret = i40e_ethertype_filter_set(pf,
@@ -5350,9 +5406,17 @@ i40e_flow_create(struct rte_eth_dev *dev,
break;
case RTE_ETH_FILTER_FDIR:
ret = i40e_flow_add_del_fdir_filter(dev,
- &cons_filter.fdir_filter, 1);
- if (ret)
+ &cons_filter.fdir_filter, 1,
+ wait_for_status);
+ if (ret) {
+ rte_bitmap_set(fdir_info->fdir_flow_pool.b, pos);
+ fdir_info->fdir_actual_cnt--;
+ if (fdir_info->fdir_invalprio == 1)
+ fdir_info->fdir_guarantee_free_space++;
+
goto free_flow;
+ }
+
flow->rule = TAILQ_LAST(&pf->fdir.fdir_list,
i40e_fdir_filter_list);
break;
@@ -5384,7 +5448,10 @@ i40e_flow_create(struct rte_eth_dev *dev,
rte_flow_error_set(error, -ret,
RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
"Failed to create flow.");
- rte_free(flow);
+
+ if (cons_filter_type != RTE_ETH_FILTER_FDIR)
+ rte_free(flow);
+
return NULL;
}
@@ -5395,6 +5462,8 @@ i40e_flow_destroy(struct rte_eth_dev *dev,
{
struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
enum rte_filter_type filter_type = flow->filter_type;
+ struct i40e_fdir_info *fdir_info = &pf->fdir;
+ struct i40e_fdir_entry *f;
int ret = 0;
switch (filter_type) {
@@ -5408,7 +5477,8 @@ i40e_flow_destroy(struct rte_eth_dev *dev,
break;
case RTE_ETH_FILTER_FDIR:
ret = i40e_flow_add_del_fdir_filter(dev,
- &((struct i40e_fdir_filter *)flow->rule)->fdir, 0);
+ &((struct i40e_fdir_filter *)flow->rule)->fdir,
+ 0, false);
/* If the last flow is destroyed, disable fdir. */
if (!ret && TAILQ_EMPTY(&pf->fdir.fdir_list)) {
@@ -5428,11 +5498,27 @@ i40e_flow_destroy(struct rte_eth_dev *dev,
if (!ret) {
TAILQ_REMOVE(&pf->flow_list, flow, node);
- rte_free(flow);
- } else
+ if (filter_type == RTE_ETH_FILTER_FDIR) {
+ f = FLOW_TO_FLOW_BITMAP(flow);
+ rte_bitmap_set(fdir_info->fdir_flow_pool.b, f->idx);
+ fdir_info->fdir_actual_cnt--;
+
+ if (fdir_info->fdir_invalprio == 1)
+ /* check if the budget will be gained back to
+ * guaranteed space
+ */
+ if (fdir_info->fdir_guarantee_free_space <
+ fdir_info->fdir_guarantee_available_space)
+ fdir_info->fdir_guarantee_free_space++;
+ } else {
+ rte_free(flow);
+ }
+
+ } else {
rte_flow_error_set(error, -ret,
RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
"Failed to destroy flow.");
+ }
return ret;
}
@@ -5582,6 +5668,7 @@ i40e_flow_flush_fdir_filter(struct i40e_pf *pf)
struct rte_flow *flow;
void *temp;
int ret;
+ uint32_t i = 0;
ret = i40e_fdir_flush(dev);
if (!ret) {
@@ -5597,10 +5684,24 @@ i40e_flow_flush_fdir_filter(struct i40e_pf *pf)
TAILQ_FOREACH_SAFE(flow, &pf->flow_list, node, temp) {
if (flow->filter_type == RTE_ETH_FILTER_FDIR) {
TAILQ_REMOVE(&pf->flow_list, flow, node);
- rte_free(flow);
}
}
+ /* clear bitmap */
+ rte_bitmap_reset(fdir_info->fdir_flow_pool.b);
+ for (i = 0; i < fdir_info->fdir_space_size; i++) {
+ fdir_info->fdir_flow_pool.pool[i].idx = i;
+ rte_bitmap_set(fdir_info->fdir_flow_pool.b, i);
+ }
+
+ fdir_info->fdir_actual_cnt = 0;
+ fdir_info->fdir_guarantee_free_space =
+ fdir_info->fdir_guarantee_available_space;
+ memset(fdir_info->fdir_filter_array,
+ 0,
+ sizeof(struct i40e_fdir_filter) *
+ I40E_MAX_FDIR_FILTER_NUM);
+
for (pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
pctype <= I40E_FILTER_PCTYPE_L2_PAYLOAD; pctype++)
pf->fdir.inset_flag[pctype] = 0;
@@ -2940,16 +2940,17 @@ i40e_dev_free_queues(struct rte_eth_dev *dev)
}
}
-#define I40E_FDIR_NUM_TX_DESC I40E_MIN_RING_DESC
-#define I40E_FDIR_NUM_RX_DESC I40E_MIN_RING_DESC
+#define I40E_FDIR_NUM_TX_DESC 256
+#define I40E_FDIR_NUM_RX_DESC 256
enum i40e_status_code
i40e_fdir_setup_tx_resources(struct i40e_pf *pf)
{
struct i40e_tx_queue *txq;
const struct rte_memzone *tz = NULL;
- uint32_t ring_size;
+ volatile struct i40e_tx_desc *txdp;
struct rte_eth_dev *dev;
+ uint32_t ring_size, i;
if (!pf) {
PMD_DRV_LOG(ERR, "PF is not available");
@@ -2989,6 +2990,14 @@ i40e_fdir_setup_tx_resources(struct i40e_pf *pf)
txq->tx_ring_phys_addr = tz->iova;
txq->tx_ring = (struct i40e_tx_desc *)tz->addr;
+
+ /* Set all the DD flags to 1 */
+ for (i = 0; i < I40E_FDIR_NUM_TX_DESC; i += 2) {
+ txdp = &txq->tx_ring[i + 1];
+ txdp->cmd_type_offset_bsz |= I40E_TX_DESC_DTYPE_DESC_DONE;
+ txdp->buffer_addr = rte_cpu_to_le_64(pf->fdir.dma_addr[i / 2]);
+ }
+
/*
* don't need to allocate software ring and reset for the fdir
* program queue just set the queue has been configured.
@@ -3060,7 +3060,7 @@ int rte_pmd_i40e_flow_add_del_packet_template(
(enum i40e_fdir_status)conf->action.report_status;
filter_conf.action.flex_off = conf->action.flex_off;
- return i40e_flow_add_del_fdir_filter(dev, &filter_conf, add);
+ return i40e_flow_add_del_fdir_filter(dev, &filter_conf, add, true);
}
int