From patchwork Sun Jul 5 09:23:40 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ori Kam X-Patchwork-Id: 73125 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id DD4D1A00C5; Sun, 5 Jul 2020 11:26:35 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 6617D1DB8F; Sun, 5 Jul 2020 11:25:00 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id F228B1DB52 for ; Sun, 5 Jul 2020 11:24:57 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from orika@mellanox.com) with SMTP; 5 Jul 2020 12:24:54 +0300 Received: from pegasus04.mtr.labs.mlnx. (pegasus04.mtr.labs.mlnx [10.210.16.126]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 0659OC3x028191; Sun, 5 Jul 2020 12:24:53 +0300 From: Ori Kam To: jerinj@marvell.com, xiang.w.wang@intel.com, matan@mellanox.com, viacheslavo@mellanox.com Cc: guyk@marvell.com, dev@dpdk.org, pbhagavatula@marvell.com, shahafs@mellanox.com, hemant.agrawal@nxp.com, opher@mellanox.com, alexr@mellanox.com, dovrat@marvell.com, pkapoor@marvell.com, nipun.gupta@nxp.com, bruce.richardson@intel.com, yang.a.hong@intel.com, harry.chang@intel.com, gu.jian1@zte.com.cn, shanjiangh@chinatelecom.cn, zhangy.yun@chinatelecom.cn, lixingfu@huachentel.com, wushuai@inspur.com, yuyingxia@yxlink.com, fanchenggang@sunyainfo.com, davidfgao@tencent.com, liuzhong1@chinaunicom.cn, zhaoyong11@huawei.com, oc@yunify.com, jim@netgate.com, hongjun.ni@intel.com, deri@ntop.org, fc@napatech.com, arthur.su@lionic.com, thomas@monjalon.net, orika@mellanox.com, rasland@mellanox.com, Francis Kelly Date: Sun, 5 Jul 2020 09:23:40 +0000 Message-Id: <1593941027-86651-14-git-send-email-orika@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1593941027-86651-1-git-send-email-orika@mellanox.com> References: <1593941027-86651-1-git-send-email-orika@mellanox.com> Subject: [dpdk-dev] [PATCH 13/20] regex/mlx5: add program rules support X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Francis Kelly This commit introduce the ability to program rules to the RegEx engine. Signed-off-by: Francis Kelly Acked-by: Ori Kam --- drivers/regex/mlx5/mlx5_regex.c | 33 ++ drivers/regex/mlx5/mlx5_regex.h | 55 ++- drivers/regex/mlx5/mlx5_rxp.c | 1039 +++++++++++++++++++++++++++++++++++++-- drivers/regex/mlx5/mlx5_rxp.h | 138 ++++++ 4 files changed, 1216 insertions(+), 49 deletions(-) create mode 100644 drivers/regex/mlx5/mlx5_rxp.h diff --git a/drivers/regex/mlx5/mlx5_regex.c b/drivers/regex/mlx5/mlx5_regex.c index 94e4352..8801cd7 100644 --- a/drivers/regex/mlx5/mlx5_regex.c +++ b/drivers/regex/mlx5/mlx5_regex.c @@ -24,6 +24,7 @@ const struct rte_regexdev_ops mlx5_regexdev_ops = { .dev_info_get = mlx5_regex_info_get, .dev_configure = mlx5_regex_configure, + .dev_db_import = mlx5_regex_rules_db_import, }; @@ -149,6 +150,8 @@ goto error; } priv->ctx = ctx; + /* Default RXP programming mode to Shared. */ + priv->prog_mode = MLX5_RXP_SHARED_PROG_MODE; mlx5_regex_get_name(name, pci_dev); priv->regexdev = rte_regexdev_register(name); if (priv->regexdev == NULL) { @@ -156,6 +159,24 @@ rte_errno = rte_errno ? rte_errno : EINVAL; goto error; } + ret = mlx5_glue->devx_query_eqn(ctx, 0, &priv->eqn); + if (ret) { + DRV_LOG(ERR, "can't query event queue number."); + rte_errno = ENOMEM; + goto error; + } + priv->uar = mlx5_glue->devx_alloc_uar(ctx, 0); + if (!priv->uar) { + DRV_LOG(ERR, "can't allocate uar."); + rte_errno = ENOMEM; + goto error; + } + priv->pd = mlx5_glue->alloc_pd(ctx); + if (!priv->pd) { + DRV_LOG(ERR, "can't allocate pd."); + rte_errno = ENOMEM; + goto error; + } priv->regexdev->dev_ops = &mlx5_regexdev_ops; priv->regexdev->device = (struct rte_device *)pci_dev; priv->regexdev->data->dev_private = priv; @@ -163,6 +184,12 @@ return 0; error: + if (priv->pd) + mlx5_glue->dealloc_pd(priv->pd); + if (priv->uar) + mlx5_glue->devx_free_uar(priv->uar); + if (priv->regexdev) + rte_regexdev_unregister(priv->regexdev); if (ctx) mlx5_glue->close_device(ctx); if (priv) @@ -194,6 +221,12 @@ return 0; priv = dev->data->dev_private; if (priv) { + if (priv->pd) + mlx5_glue->dealloc_pd(priv->pd); + if (priv->uar) + mlx5_glue->devx_free_uar(priv->uar); + if (priv->regexdev) + rte_regexdev_unregister(priv->regexdev); if (priv->ctx) mlx5_glue->close_device(priv->ctx); if (priv->regexdev) diff --git a/drivers/regex/mlx5/mlx5_regex.h b/drivers/regex/mlx5/mlx5_regex.h index 5238f24..fca9ebb 100644 --- a/drivers/regex/mlx5/mlx5_regex.h +++ b/drivers/regex/mlx5/mlx5_regex.h @@ -5,14 +5,53 @@ #ifndef MLX5_REGEX_H #define MLX5_REGEX_H +#ifdef PEDANTIC +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#include +#include +#ifdef PEDANTIC +#pragma GCC diagnostic error "-Wpedantic" +#endif + +#include + +#include "mlx5_rxp.h" + struct mlx5_regex_sq { - uint32_t nb_desc; /* Number of desc for this object. */ + uint16_t log_nb_desc; /* Log 2 number of desc for this object. */ + struct mlx5_devx_obj *obj; /* The SQ DevX object. */ + int64_t dbr_offset; /* Door bell record offset. */ + uint32_t dbr_umem; /* Door bell record umem id. */ + volatile struct mlx5_cqe *wqe; /* The SQ ring buffer. */ + struct mlx5dv_devx_umem *wqe_umem; /* SQ buffer umem. */ +}; + +struct mlx5_regex_cq { + uint32_t log_nb_desc; /* Log 2 number of desc for this object. */ + struct mlx5_devx_obj *obj; /* The CQ DevX object. */ + int64_t dbr_offset; /* Door bell record offset. */ + uint32_t dbr_umem; /* Door bell record umem id. */ + volatile struct mlx5_cqe *cqe; /* The CQ ring buffer. */ + struct mlx5dv_devx_umem *cqe_umem; /* CQ buffer umem. */ }; struct mlx5_regex_qp { uint32_t flags; /* QP user flags. */ - uint32_t nb_desc; /* Total number of desc for thsi qp. */ + uint16_t nb_desc; /* Total number of desc for this qp. */ struct mlx5_regex_sq *sqs; /* Pointer to sq array. */ + uint16_t nb_obj; /* Number of sq objects. */ + struct mlx5_regex_cq cq; /* CQ struct. */ +}; + +struct mlx5_regex_db { + void *ptr; /* Pointer to the db memory. */ + uint32_t len; /* The memory len. */ + bool active; /* Active flag. */ + uint8_t db_assigned_to_eng_num; + /**< To which engine the db is connected. */ + struct mlx5_regex_umem umem; + /**< The umem struct. */ }; struct mlx5_regex_priv { @@ -23,6 +62,13 @@ struct mlx5_regex_priv { uint16_t nb_queues; /* Number of queues. */ struct mlx5_regex_qp *qps; /* Pointer to the qp array. */ uint16_t nb_max_matches; /* Max number of matches. */ + enum mlx5_rxp_program_mode prog_mode; + struct mlx5_regex_db db[MLX5_RXP_MAX_ENGINES + + MLX5_RXP_EM_COUNT]; + struct mlx5_dbr_page_list dbrpgs; /* Door-bell pages. */ + uint32_t eqn; /* EQ number. */ + struct mlx5dv_devx_uar *uar; /* UAR object. */ + struct ibv_pd *pd; }; /* mlx5_rxp.c */ @@ -30,5 +76,8 @@ int mlx5_regex_info_get(struct rte_regexdev *dev, struct rte_regexdev_info *info); int mlx5_regex_configure(struct rte_regexdev *dev, const struct rte_regexdev_config *cfg); - +int mlx5_regex_qp_setup(struct rte_regexdev *dev, uint16_t qp_ind, + const struct rte_regexdev_qp_conf *cfg); +int mlx5_regex_rules_db_import(struct rte_regexdev *dev, + const char *rule_db, uint32_t rule_db_len); #endif /* MLX5_REGEX_H */ diff --git a/drivers/regex/mlx5/mlx5_rxp.c b/drivers/regex/mlx5/mlx5_rxp.c index 60a4640..2bb3229 100644 --- a/drivers/regex/mlx5/mlx5_rxp.c +++ b/drivers/regex/mlx5/mlx5_rxp.c @@ -3,6 +3,7 @@ */ #include +#include #include #include @@ -14,15 +15,113 @@ #include #include #include +#include #include "mlx5_regex.h" #include "mlx5_regex_utils.h" #include "mlx5_rxp_csrs.h" +#include "mlx5_rxp.h" -#define MLX5_REGEX_MAX_MATCHES 255 -#define MLX5_REGEX_MAX_PAYLOAD_SIZE UINT16_MAX -#define MLX5_REGEX_MAX_RULES_PER_GROUP UINT16_MAX -#define MLX5_REGEX_MAX_GROUPS UINT16_MAX +/* TODO: get actual VIPER version capability bit when available! */ +#define VIPER_BF2 1 + +#define MLX5_REGEX_MAX_MATCHES MLX5_RXP_MAX_MATCHES +#define MLX5_REGEX_MAX_PAYLOAD_SIZE MLX5_RXP_MAX_JOB_LENGTH +#define MLX5_REGEX_MAX_RULES_PER_GROUP UINT32_MAX +#define MLX5_REGEX_MAX_GROUPS MLX5_RXP_MAX_SUBSETS + +/* Private Declarations */ +static int +rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value, + uint32_t address, uint32_t expected_value, + uint32_t expected_mask, uint32_t timeout_ms, uint8_t id); +static int +mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use); +static int +mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id); +static int +mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id); +static int +program_rxp_rules(struct mlx5_regex_priv *priv, + struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id); +static int +rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id); +static int +write_private_rules(struct mlx5_regex_priv *priv, + struct mlx5_rxp_ctl_rules_pgm *rules, + uint8_t id); +static int +write_shared_rules(struct mlx5_regex_priv *priv, + struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count, + uint8_t db_to_program); +static int +rxp_db_setup(struct mlx5_regex_priv *priv); +static void +rxp_dump_csrs(struct ibv_context *ctx, uint8_t id); +static int +rxp_write_rules_via_cp(struct ibv_context *ctx, + struct mlx5_rxp_rof_entry *rules, + int count, uint8_t id); +static int +rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules, + int count, uint8_t id); +static int +rxp_start_engine(struct ibv_context *ctx, uint8_t id); +static int +rxp_stop_engine(struct ibv_context *ctx, uint8_t id); + +/** + * RXP Register display function + * + * @param ctx + * The IBV context. + * @param id + * The selected engine to read back registers. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static void __rte_unused +rxp_dump_csrs(struct ibv_context *ctx __rte_unused, uint8_t id __rte_unused) +{ + uint32_t reg, i; + + /* Main CSRs*/ + for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) { + if (mlx5_devx_regex_register_read(ctx, id, + (MLX5_RXP_CSR_WIDTH * i) + + MLX5_RXP_CSR_BASE_ADDRESS, + ®)) { + DRV_LOG(ERR, "Failed to read Main CSRs Engine %d!", id); + return; + } + DRV_LOG(DEBUG, "RXP Main CSRs (Eng%d) register (%d): %08x", + id, i, reg); + } + /* RTRU CSRs*/ + for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) { + if (mlx5_devx_regex_register_read(ctx, id, + (MLX5_RXP_CSR_WIDTH * i) + + MLX5_RXP_RTRU_CSR_BASE_ADDRESS, + ®)) { + DRV_LOG(ERR, "Failed to read RTRU CSRs Engine %d!", id); + return; + } + DRV_LOG(DEBUG, "RXP RTRU CSRs (Eng%d) register (%d): %08x", + id, i, reg); + } + /* STAT CSRs */ + for (i = 0; i < MLX5_RXP_CSR_NUM_ENTRIES; i++) { + if (mlx5_devx_regex_register_read(ctx, id, + (MLX5_RXP_CSR_WIDTH * i) + + MLX5_RXP_STATS_CSR_BASE_ADDRESS, + ®)) { + DRV_LOG(ERR, "Failed to read STAT CSRs Engine %d!", id); + return; + } + DRV_LOG(DEBUG, "RXP STAT CSRs (Eng%d) register (%d): %08x", + id, i, reg); + } +} /** * DPDK callback for reading device info. @@ -44,24 +143,168 @@ info->max_payload_size = MLX5_REGEX_MAX_PAYLOAD_SIZE; info->max_rules_per_group = MLX5_REGEX_MAX_RULES_PER_GROUP; info->max_groups = MLX5_REGEX_MAX_GROUPS; + info->max_queue_pairs = 1; info->regexdev_capa = RTE_REGEXDEV_SUPP_PCRE_GREEDY_F; info->rule_flags = 0; return 0; } +/** + * Actual writing of RXP instructions to RXP via CSRs. + * + * @param ctx + * The IBV context. + * @param rules + * RXP instructions to be written. + * @param count + * The number of instructions to be written. + * @param id + * The selected engine. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +rxp_write_rules_via_cp(struct ibv_context *ctx, + struct mlx5_rxp_rof_entry *rules, + int count, uint8_t id) +{ + int i, ret = 0; + uint32_t tmp; + + for (i = 0; i < count; i++) { + tmp = (uint32_t)rules[i].value; + ret |= mlx5_devx_regex_register_write(ctx, id, + MLX5_RXP_RTRU_CSR_DATA_0, + tmp); + tmp = (uint32_t)(rules[i].value >> 32); + ret |= mlx5_devx_regex_register_write(ctx, id, + MLX5_RXP_RTRU_CSR_DATA_0 + + MLX5_RXP_CSR_WIDTH, tmp); + tmp = rules[i].addr; + ret |= mlx5_devx_regex_register_write(ctx, id, + MLX5_RXP_RTRU_CSR_ADDR, + tmp); + if (ret) { + DRV_LOG(ERR, "Failed to copy instructions to RXP."); + return -1; + } + } + DRV_LOG(DEBUG, "Written %d instructions", count); + return 0; +} + +/** + * Flushing the programmed instructions to RXP. + * + * @param ctx + * The IBV context. + * @param rules + * RXP instructions to be written. + * @param count + * The number of instructions to be written. + * @param id + * The selected engine. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +rxp_flush_rules(struct ibv_context *ctx, struct mlx5_rxp_rof_entry *rules, + int count, uint8_t id) +{ + uint32_t val, fifo_depth; + int ret; + + ret = rxp_write_rules_via_cp(ctx, rules, count, id); + if (ret < 0) { + DRV_LOG(ERR, "Failed to write rules via CSRs."); + return -1; + } + ret = mlx5_devx_regex_register_read(ctx, id, + MLX5_RXP_RTRU_CSR_CAPABILITY, + &fifo_depth); + if (ret) { + DRV_LOG(ERR, "CSR read failed!"); + return -1; + } + ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_FIFO_STAT, + count, ~0, + MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id); + if (ret < 0) { + DRV_LOG(ERR, "Rules not rx by RXP: credit: %d, depth: %d", val, + fifo_depth); + return ret; + } + DRV_LOG(DEBUG, "RTRU FIFO depth: 0x%x", fifo_depth); + DRV_LOG(DEBUG, "Rules flush took %d cycles.", ret); + ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, + &val); + if (ret) { + DRV_LOG(ERR, "CSR read failed!"); + return -1; + } + val |= MLX5_RXP_RTRU_CSR_CTRL_GO; + ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, + val); + ret = rxp_poll_csr_for_value(ctx, &val, MLX5_RXP_RTRU_CSR_STATUS, + MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE, + MLX5_RXP_RTRU_CSR_STATUS_UPDATE_DONE, + MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT, id); + if (ret < 0) { + DRV_LOG(ERR, "Rules update timeout: 0x%08X", val); + return ret; + } + DRV_LOG(DEBUG, "Rules update took %d cycles", ret); + if (mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, + &val)) { + DRV_LOG(ERR, "CSR read failed!"); + return -1; + } + val &= ~(MLX5_RXP_RTRU_CSR_CTRL_GO); + if (mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, + val)) { + DRV_LOG(ERR, "CSR write write failed!"); + return -1; + } + + DRV_LOG(DEBUG, "RXP Flush rules finished."); + return 0; +} + +/** + * Poll RXP CSRs for expected value. + * + * @param ctx + * The IBV context. + * @param value + * The returned value from a CSR read. + * @param address + * The RXP CSR register to read from. + * @param expected_value + * The expected value to compare read value against. + * @param expected_mask + * A mask used to only compare particular bits of read value. + * @param timeout_ms + * The number of poll interations. + * @param id + * The selected engine to poll. + * + * @return + * 0 on success, a negative errno value otherwise. + */ static int rxp_poll_csr_for_value(struct ibv_context *ctx, uint32_t *value, uint32_t address, uint32_t expected_value, uint32_t expected_mask, uint32_t timeout_ms, uint8_t id) { unsigned int i; - int ret; + int ret = 0; ret = -EBUSY; for (i = 0; i < timeout_ms; i++) { if (mlx5_devx_regex_register_read(ctx, id, address, value)) return -1; - if ((*value & expected_mask) == expected_value) { ret = 0; break; @@ -92,6 +335,7 @@ if (ret) return ret; ctrl |= MLX5_RXP_CSR_CTRL_GO; + ctrl |= MLX5_RXP_CSR_CTRL_DISABLE_L2C; ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); return ret; } @@ -121,6 +365,18 @@ return ret; } +/** + * Initialise the selected RXP engine to specified init mode. + * + * @param ctx + * The IBV context. + * @param id + * The selected engine. + * @param init_bits + * The RXP initialisation modes. + * @return + * 0 on success, a negative errno value otherwise. + */ static int rxp_init_rtru(struct ibv_context *ctx, uint8_t id, uint32_t init_bits) { @@ -130,23 +386,23 @@ uint32_t expected_mask; int ret = 0; - /* Read the rtru ctrl CSR */ + /* Read the rtru ctrl CSR. */ ret = mlx5_devx_regex_register_read(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, &ctrl_value); if (ret) return -1; - /* Clear any previous init modes */ + /* Clear any previous init modes. */ ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_MASK); if (ctrl_value & MLX5_RXP_RTRU_CSR_CTRL_INIT) { ctrl_value &= ~(MLX5_RXP_RTRU_CSR_CTRL_INIT); mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, ctrl_value); } - /* Set the init_mode bits in the rtru ctrl CSR */ + /* Set the init_mode bits in the rtru ctrl CSR. */ ctrl_value |= init_bits; mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_RTRU_CSR_CTRL, ctrl_value); - /* Need to sleep for a short period after pulsing the rtru init bit. */ + /* Need to sleep for a short period after pulsing the rtru init bit. */ rte_delay_us(20000); /* Poll the rtru status CSR until all the init done bits are set. */ DRV_LOG(DEBUG, "waiting for RXP rule memory to complete init"); @@ -162,11 +418,11 @@ if (init_bits == MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2) { /* Must be incremental mode */ expected_value = MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE | - MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE; + MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE; } else { expected_value = MLX5_RXP_RTRU_CSR_STATUS_IM_INIT_DONE | - MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE | - MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE; + MLX5_RXP_RTRU_CSR_STATUS_L1C_INIT_DONE | + MLX5_RXP_RTRU_CSR_STATUS_L2C_INIT_DONE; } expected_mask = expected_value; ret = rxp_poll_csr_for_value(ctx, &poll_value, @@ -184,6 +440,257 @@ } /** + * Extract instructions from buffer into rules format ready for RXP programming. + * + * @param buf + * The buffer holding RXP instructions. + * @param len + * The actual length of the buffer. + * @param init_bits + * The RXP initialisation modes. + * @return + * 0 on success, a negative errno value otherwise. + */ +static int +rxp_parse_rof(const char *buf, uint32_t len, + struct mlx5_rxp_ctl_rules_pgm **rules) +{ + static const char del[] = "\n\r"; + char *line; + char *tmp; + char *cur_pos; + uint32_t lines = 0; + uint32_t entries; + struct mlx5_rxp_rof_entry *curentry; + + tmp = rte_malloc("", len, 0); + if (!tmp) + return -ENOMEM; + memcpy(tmp, buf, len); + line = strtok(tmp, del); + while (line) { + if (line[0] != '#' && line[0] != '\0') + lines++; + line = strtok(NULL, del); + } + *rules = rte_malloc("", lines * sizeof(*curentry) + sizeof(**rules), 0); + if (!(*rules)) { + rte_free(tmp); + return -ENOMEM; + } + memset(*rules, 0, lines * sizeof(curentry) + sizeof(**rules)); + curentry = (*rules)->rules; + (*rules)->hdr.cmd = MLX5_RXP_CTL_RULES_PGM; + entries = 0; + memcpy(tmp, buf, len); + line = strtok(tmp, del); + while (line) { + if (line[0] == '#' || line[0] == '\0') { + line = strtok(NULL, del); + continue; + } + curentry->type = strtoul(line, &cur_pos, 10); + if (cur_pos == line || cur_pos[0] != ',') + goto parse_error; + cur_pos++; + curentry->addr = strtoul(cur_pos, &cur_pos, 16); + if (cur_pos[0] != ',') + goto parse_error; + cur_pos++; + curentry->value = strtoull(cur_pos, &cur_pos, 16); + if (cur_pos[0] != '\0' && cur_pos[0] != '\n') + goto parse_error; + curentry++; + entries++; + if (entries > lines) + goto parse_error; + line = strtok(NULL, del); + } + (*rules)->count = entries; + (*rules)->hdr.len = entries * sizeof(*curentry) + sizeof(**rules); + rte_free(tmp); + return 0; +parse_error: + rte_free(tmp); + if (*rules) + rte_free(*rules); + return -EINVAL; +} + +/** + * Provide db pointer to EM and ensure idle. + * + * @param priv + * Pointer to the private device data structure. + * @param id + * The RXP engine to setup. + * @param db_to_use + * Index to which db pointer allocated for RXP engine. + * @return + * 0 on success, a negative errno value otherwise. + */ +static int +mlnx_set_database(struct mlx5_regex_priv *priv, uint8_t id, uint8_t db_to_use) +{ + int ret; + uint32_t umem_id; + + ret = mlx5_devx_regex_database_stop(priv->ctx, id); + if (ret < 0) { + DRV_LOG(ERR, "stop engine failed!"); + return ret; + } + umem_id = mlx5_os_get_umem_id(priv->db[db_to_use].umem.umem); + ret = mlx5_devx_regex_database_program(priv->ctx, id, umem_id, 0); + if (ret < 0) { + DRV_LOG(ERR, "program db failed!"); + return ret; + } + return 0; +} + +/** + * Resume engine. + * + * @param priv + * Pointer to the private device data structure. + * @param id + * The RXP engine to resume. + * @return + * 0 on success, a negative errno value otherwise. + */ +static int +mlnx_resume_database(struct mlx5_regex_priv *priv, uint8_t id) +{ + mlx5_devx_regex_database_resume(priv->ctx, id); + return 0; +} + +/** + * Assign db memory for RXP programming. + * + * @param priv + * Pointer to the private device data structure. + * @param id + * The RXP engine to assign db. + * @return + * Index to allocated db buffer on success, a negative errno value otherwise. + */ +static int +mlnx_update_database(struct mlx5_regex_priv *priv, uint8_t id) +{ + unsigned int i; + uint8_t db_free = MLX5_RXP_DB_NOT_ASSIGNED; + uint8_t eng_assigned = MLX5_RXP_DB_NOT_ASSIGNED; + + /* Check which database rxp_eng is currently located if any? */ + for (i = 0; i < (MLX5_RXP_MAX_ENGINES + MLX5_RXP_EM_COUNT); + i++) { + if (priv->db[i].db_assigned_to_eng_num == id) { + eng_assigned = i; + break; + } + } + /* + * If private mode then, we can keep the same db ptr as RXP will be + * programming EM itself if necessary, however need to see if + * programmed yet. + */ + if ((priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) && + (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED)) + return eng_assigned; + /* Check for inactive db memory to use. */ + for (i = 0; i < (MLX5_RXP_MAX_ENGINES + MLX5_RXP_EM_COUNT); + i++) { + if (priv->db[i].active == true) + continue; /* Already in use, so skip db. */ + /* Set this db to active now as free to use. */ + priv->db[i].active = true; + /* Now unassign last db index in use by RXP Eng. */ + if (eng_assigned != MLX5_RXP_DB_NOT_ASSIGNED) { + priv->db[eng_assigned].active = false; + priv->db[eng_assigned].db_assigned_to_eng_num = + MLX5_RXP_DB_NOT_ASSIGNED; + + /* Set all DB memory to 0's before setting up DB. */ + memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE); + } + /* Now reassign new db index with RXP Engine. */ + priv->db[i].db_assigned_to_eng_num = id; + db_free = i; + break; + } + if (db_free == MLX5_RXP_DB_NOT_ASSIGNED) + return -1; + return db_free; +} + +/** + * Program RXP instruction db to RXP engine/s. + * + * @param priv + * Pointer to the private device data structure. + * @param rule_buf + * Pointer to buffer that will be holding RXP instructions. + * @param len + * The length of the rule buffer. + * @param id + * The selected engine. + * @return + * 0 on success, a negative errno value otherwise. + */ +static int +program_rxp_rules(struct mlx5_regex_priv *priv, + struct mlx5_rxp_ctl_rules_pgm *rules, uint8_t id) +{ + int ret, db_free; + uint32_t rule_cnt; + + rule_cnt = rules->count; + db_free = mlnx_update_database(priv, id); + if (db_free < 0) { + DRV_LOG(ERR, "Failed to setup db memory!"); + return db_free; + } + if (priv->prog_mode == MLX5_RXP_PRIVATE_PROG_MODE) { + /* Register early to ensure RXP writes to EM use valid addr. */ + ret = mlnx_set_database(priv, id, db_free); + if (ret < 0) { + DRV_LOG(ERR, "Failed to register db memory!"); + return ret; + } + } + ret = write_private_rules(priv, rules, id); + if (ret < 0) { + DRV_LOG(ERR, "Failed to write rules!"); + return ret; + } + if (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) { + /* Write external rules directly to EM. */ + rules->count = rule_cnt; + /* Now write external instructions to EM. */ + ret = write_shared_rules(priv, rules, rules->hdr.len, db_free); + if (ret < 0) { + DRV_LOG(ERR, "Failed to write EM rules!"); + return ret; + } + ret = mlnx_set_database(priv, id, db_free); + if (ret < 0) { + DRV_LOG(ERR, "Failed to register db memory!"); + return ret; + } + } + ret = mlnx_resume_database(priv, id); + if (ret < 0) { + DRV_LOG(ERR, "Failed to resume engine!"); + return ret; + } + DRV_LOG(DEBUG, "Programmed RXP Engine %d\n", id); + rules->count = rule_cnt; + return 0; +} + +/** * Init the engine. * * @param ctx @@ -192,10 +699,10 @@ * The selected engine. * * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. + * 0 on success, a negative errno value otherwise. */ static int -rxp_init(struct mlx5_regex_priv *priv, uint8_t id) +rxp_init_eng(struct mlx5_regex_priv *priv, uint8_t id) { uint32_t ctrl; uint32_t reg; @@ -219,7 +726,6 @@ ctrl &= ~MLX5_RXP_CSR_CTRL_INIT; ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_CTRL, ctrl); rte_delay_us(20000); - ret = rxp_poll_csr_for_value(ctx, &ctrl, MLX5_RXP_CSR_STATUS, MLX5_RXP_CSR_STATUS_INIT_DONE, MLX5_RXP_CSR_STATUS_INIT_DONE, @@ -234,7 +740,6 @@ ctrl); if (ret) return ret; - rxp_init_rtru(ctx, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2); ret = rxp_init_rtru(ctx, id, MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_IM_L1_L2); if (ret) return ret; @@ -244,8 +749,16 @@ return ret; DRV_LOG(DEBUG, "max matches: %d, DDOS threshold: %d", reg >> 16, reg & 0xffff); - ret = mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_MATCH, - priv->nb_max_matches); + if ((reg >> 16) >= priv->nb_max_matches) + ret = mlx5_devx_regex_register_write(ctx, id, + MLX5_RXP_CSR_MAX_MATCH, + priv->nb_max_matches); + else + ret = mlx5_devx_regex_register_write(ctx, id, + MLX5_RXP_CSR_MAX_MATCH, + (reg >> 16)); + ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_PREFIX, + (reg & 0xFFFF)); ret |= mlx5_devx_regex_register_write(ctx, id, MLX5_RXP_CSR_MAX_LATENCY, 0); ret |= mlx5_devx_regex_register_write(ctx, id, @@ -254,6 +767,451 @@ } /** + * Private programming of RXP, here all private/internal instructions will + * be written to RXP private memories, and all external instructions will be + * written to EM via RXP and not host. + * + * @param priv + * Pointer to private regex structure. + * @param rules + * Pointer to actual rules buffer containing instructions. + * @param count + * The number of instructions to program. + * @param id + * The selected engine. + * + * @return + * 0 on success, a negative errno value otherwise. + */ +static int +write_private_rules(struct mlx5_regex_priv *priv, + struct mlx5_rxp_ctl_rules_pgm *rules, + uint8_t id) +{ + unsigned int pending; + uint32_t block, reg, val, rule_cnt, rule_offset, rtru_max_num_entries; + int ret = 1; + + if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) + return -EINVAL; + if (rules->hdr.len == 0 || rules->hdr.cmd < MLX5_RXP_CTL_RULES_PGM || + rules->hdr.cmd > MLX5_RXP_CTL_RULES_PGM_INCR) + return -EINVAL; + /* For a non-incremental rules program, re-init the RXP. */ + if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM) { + ret = rxp_init_eng(priv, id); + if (ret < 0) + return ret; + } else if (rules->hdr.cmd == MLX5_RXP_CTL_RULES_PGM_INCR) { + /* Flush RXP L1 and L2 cache by using MODE_L1_L2. */ + ret = rxp_init_rtru(priv->ctx, id, + MLX5_RXP_RTRU_CSR_CTRL_INIT_MODE_L1_L2); + if (ret < 0) + return ret; + } + if (rules->count == 0) + return -EINVAL; + /* Confirm the RXP is initialised. */ + if (mlx5_devx_regex_register_read(priv->ctx, id, + MLX5_RXP_CSR_STATUS, &val)) { + DRV_LOG(ERR, "Failed to read from RXP!"); + return -ENODEV; + } + if (!(val & MLX5_RXP_CSR_STATUS_INIT_DONE)) { + DRV_LOG(ERR, "RXP not initialised..."); + return -EBUSY; + } + /* Get the RTRU maximum number of entries allowed. */ + if (mlx5_devx_regex_register_read(priv->ctx, id, + MLX5_RXP_RTRU_CSR_CAPABILITY, &rtru_max_num_entries)) { + DRV_LOG(ERR, "Failed to read RTRU capability!"); + return -ENODEV; + } + rtru_max_num_entries = (rtru_max_num_entries & 0x00FF); + rule_cnt = 0; + pending = 0; + while (rules->count > 0) { + if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_INST) || + (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_IM) || + (rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM)) { + if ((rules->rules[rule_cnt].type == + MLX5_RXP_ROF_ENTRY_EM) && + (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) { + /* Skip EM rules programming. */ + if (pending > 0) { + /* Flush any rules that are pending. */ + rule_offset = (rule_cnt - pending); + ret = rxp_flush_rules(priv->ctx, + &rules->rules[rule_offset], + pending, id); + if (ret < 0) { + DRV_LOG(ERR, "Flushing rules."); + return -ENODEV; + } + pending = 0; + } + rule_cnt++; + } else { + pending++; + rule_cnt++; + /* + * If parsing the last rule, or if reached the + * maximum number of rules for this batch, then + * flush the rules batch to the RXP. + */ + if ((rules->count == 1) || + (pending == rtru_max_num_entries)) { + rule_offset = (rule_cnt - pending); + ret = rxp_flush_rules(priv->ctx, + &rules->rules[rule_offset], + pending, id); + if (ret < 0) { + DRV_LOG(ERR, "Flushing rules."); + return -ENODEV; + } + pending = 0; + } + } + } else if ((rules->rules[rule_cnt].type == + MLX5_RXP_ROF_ENTRY_EQ) || + (rules->rules[rule_cnt].type == + MLX5_RXP_ROF_ENTRY_GTE) || + (rules->rules[rule_cnt].type == + MLX5_RXP_ROF_ENTRY_LTE) || + (rules->rules[rule_cnt].type == + MLX5_RXP_ROF_ENTRY_CHECKSUM) || + (rules->rules[rule_cnt].type == + MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM)) { + if (pending) { + /* Flush rules before checking reg values. */ + rule_offset = (rule_cnt - pending); + ret = rxp_flush_rules(priv->ctx, + &rules->rules[rule_offset], + pending, id); + if (ret < 0) { + DRV_LOG(ERR, "Failed to flush rules."); + return -ENODEV; + } + } + block = (rules->rules[rule_cnt].addr >> 16) & 0xFFFF; + if (block == 0) + reg = MLX5_RXP_CSR_BASE_ADDRESS; + else if (block == 1) + reg = MLX5_RXP_RTRU_CSR_BASE_ADDRESS; + else { + DRV_LOG(ERR, "Invalid ROF register 0x%08X!", + rules->rules[rule_cnt].addr); + return -EINVAL; + } + reg += (rules->rules[rule_cnt].addr & 0xFFFF) * + MLX5_RXP_CSR_WIDTH; + ret = mlx5_devx_regex_register_read(priv->ctx, id, + reg, &val); + if (ret) { + DRV_LOG(ERR, "RXP CSR read failed!"); + return ret; + } + if ((priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE) && + ((rules->rules[rule_cnt].type == + MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM) && + (val != rules->rules[rule_cnt].value))) { + DRV_LOG(ERR, "Unexpected value for register:"); + DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32, + rules->rules[rule_cnt].addr, val); + DRV_LOG(ERR, "expected %" PRIx64 ".", + rules->rules[rule_cnt].value); + return -EINVAL; + } else if ((priv->prog_mode == + MLX5_RXP_PRIVATE_PROG_MODE) && + (rules->rules[rule_cnt].type == + MLX5_RXP_ROF_ENTRY_CHECKSUM) && + (val != rules->rules[rule_cnt].value)) { + DRV_LOG(ERR, "Unexpected value for register:"); + DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32, + rules->rules[rule_cnt].addr, val); + DRV_LOG(ERR, "expected %" PRIx64 ".", + rules->rules[rule_cnt].value); + return -EINVAL; + } else if ((rules->rules[rule_cnt].type == + MLX5_RXP_ROF_ENTRY_EQ) && + (val != rules->rules[rule_cnt].value)) { + DRV_LOG(ERR, "Unexpected value for register:"); + DRV_LOG(ERR, "reg %x" PRIu32 " got %x" PRIu32, + rules->rules[rule_cnt].addr, val); + DRV_LOG(ERR, "expected %" PRIx64 ".", + rules->rules[rule_cnt].value); + return -EINVAL; + } else if ((rules->rules[rule_cnt].type == + MLX5_RXP_ROF_ENTRY_GTE) && + (val < rules->rules[rule_cnt].value)) { + DRV_LOG(ERR, "Unexpected value reg 0x%08X,", + rules->rules[rule_cnt].addr); + DRV_LOG(ERR, "got %X, expected >= %" PRIx64 ".", + val, rules->rules[rule_cnt].value); + return -EINVAL; + } else if ((rules->rules[rule_cnt].type == + MLX5_RXP_ROF_ENTRY_LTE) && + (val > rules->rules[rule_cnt].value)) { + DRV_LOG(ERR, "Unexpected value reg 0x%08X,", + rules->rules[rule_cnt].addr); + DRV_LOG(ERR, "got %08X expected <= %" PRIx64, + val, rules->rules[rule_cnt].value); + return -EINVAL; + } + rule_cnt++; + pending = 0; + } else { + DRV_LOG(ERR, "Error: Invalid rule type %d!", + rules->rules[rule_cnt].type); + return -EINVAL; + } + rules->count--; + } + return ret; +} + +/** + * Shared memory programming mode, here all external db instructions are written + * to EM via the host. + * + * @param priv + * Pointer to private regex structure. + * @param rules + * Pointer to actual rules buffer containing instructions. + * @param count + * The number of instructions to program. + * @param db_to_program + * The selected db memory to write too. + * + * @return + * 0 on success, a negative errno value otherwise. + */ +static int +write_shared_rules(struct mlx5_regex_priv *priv, + struct mlx5_rxp_ctl_rules_pgm *rules, uint32_t count, + uint8_t db_to_program) +{ + uint32_t rule_cnt, rof_rule_addr; + uint64_t tmp_write_swap[4]; +#ifndef VIPER_BF2 + uint64_t rof_value = 0; +#endif + if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) + return -EINVAL; + if ((rules->count == 0) || (count == 0)) + return -EINVAL; + rule_cnt = 0; + while (rule_cnt < rules->count) { + if ((rules->rules[rule_cnt].type == MLX5_RXP_ROF_ENTRY_EM) && + (priv->prog_mode == MLX5_RXP_SHARED_PROG_MODE)) { +#ifdef VIPER_BF2 + /* + * Note there are always blocks of 8 instructions for + * 7's written sequentially. However there is no + * guarantee that all blocks are sequential! + */ + if (count >= (rule_cnt + MLX5_RXP_INST_BLOCK_SIZE)) { + /* + * Ensure memory write not exceeding boundary + * Check essential to ensure 0x10000 offset + * accounted for! + */ + if ((uint8_t *)((uint8_t *) + priv->db[db_to_program].ptr + + ((rules->rules[rule_cnt + 7].addr << + MLX5_RXP_INST_OFFSET))) >= + ((uint8_t *)((uint8_t *) + priv->db[db_to_program].ptr + + MLX5_MAX_DB_SIZE))) { + DRV_LOG(ERR, "DB exceeded memory!"); + return -ENODEV; + } + /* + * Rule address Offset to align with RXP + * external instruction offset. + */ + rof_rule_addr = (rules->rules[rule_cnt].addr << + MLX5_RXP_INST_OFFSET); + /* 32 byte instruction swap (sw work around)! */ + tmp_write_swap[0] = le64toh( + rules->rules[(rule_cnt + 4)].value); + tmp_write_swap[1] = le64toh( + rules->rules[(rule_cnt + 5)].value); + tmp_write_swap[2] = le64toh( + rules->rules[(rule_cnt + 6)].value); + tmp_write_swap[3] = le64toh( + rules->rules[(rule_cnt + 7)].value); + /* Write only 4 of the 8 instructions. */ + memcpy((uint8_t *)((uint8_t *) + priv->db[db_to_program].ptr + + rof_rule_addr), &tmp_write_swap, + (sizeof(uint64_t) * 4)); + /* Write 1st 4 rules of block after last 4. */ + rof_rule_addr = (rules->rules[ + (rule_cnt + 4)].addr << + MLX5_RXP_INST_OFFSET); + tmp_write_swap[0] = le64toh( + rules->rules[(rule_cnt + 0)].value); + tmp_write_swap[1] = le64toh( + rules->rules[(rule_cnt + 1)].value); + tmp_write_swap[2] = le64toh( + rules->rules[(rule_cnt + 2)].value); + tmp_write_swap[3] = le64toh( + rules->rules[(rule_cnt + 3)].value); + memcpy((uint8_t *)((uint8_t *) + priv->db[db_to_program].ptr + + rof_rule_addr), &tmp_write_swap, + (sizeof(uint64_t) * 4)); + } else + return -1; + /* Fast forward as already handled block of 8. */ + rule_cnt += MLX5_RXP_INST_BLOCK_SIZE; +#else + /* + * Ensure memory write not exceeding boundary check + * essential to ensure 0x10000 offset accounted for! + */ + if ((uint8_t *)((uint8_t *)priv->db[db_to_program].ptr + + (rules->rules[rule_cnt].addr << + MLX5_RXP_INST_OFFSET)) >= + (uint8_t *)((uint8_t *)priv->db[db_to_program].ptr + + MLX5_MAX_DB_SIZE)) { + /* Exceeded database memory boundary. */ + DRV_LOG(ERR, "DB exceeded memory boundary!"); + return -ENODEV; + } + /* + * Rule address Offset to align with RXP external + * instruction offset. + */ + rof_rule_addr = (rules->rules[rule_cnt].addr << + MLX5_RXP_INST_OFFSET); + rof_value = le64toh(rules->rules[rule_cnt].value); + memcpy((uint8_t *)( + (uint8_t *)priv->db[db_to_program].ptr + + rof_rule_addr), &rof_value, sizeof(uint64_t)); + rule_cnt++; +#endif /* VIPER_BF2 */ + } else + rule_cnt++; /* Must be something other than EM rule. */ + } + return 0; +} + +/** + * RXP initial db setup function for EM instructions + * + * @param priv + * Pointer to private RegEx structure. + * + * @return + * 0 on success, a negative errno value otherwise. + */ +static int +rxp_db_setup(struct mlx5_regex_priv *priv) +{ + int ret; + uint8_t i; + + /* Setup database memories for both RXP engines + reprogram memory. */ + for (i = 0; i < (MLX5_RXP_MAX_ENGINES + MLX5_RXP_EM_COUNT); i++) { + priv->db[i].ptr = rte_malloc("", MLX5_MAX_DB_SIZE, 0); + if (!priv->db[i].ptr) { + DRV_LOG(ERR, "Failed to alloc db memory!"); + ret = ENODEV; + goto tidyup_error; + } + /* Register the memory. */ + priv->db[i].umem.umem = mlx5_glue->devx_umem_reg(priv->ctx, + priv->db[i].ptr, + MLX5_MAX_DB_SIZE, 7); + if (!priv->db[i].umem.umem) { + DRV_LOG(ERR, "Failed to register memory!"); + ret = ENODEV; + goto tidyup_error; + } + /* Ensure set all DB memory to 0's before setting up DB. */ + memset(priv->db[i].ptr, 0x00, MLX5_MAX_DB_SIZE); + /* No data currently in database. */ + priv->db[i].len = 0; + priv->db[i].active = false; + priv->db[i].db_assigned_to_eng_num = MLX5_RXP_DB_NOT_ASSIGNED; + } + return 0; +tidyup_error: + for (i = 0; i < (MLX5_RXP_MAX_ENGINES + MLX5_RXP_EM_COUNT); i++) { + if (priv->db[i].ptr) + munmap(priv->db[i].ptr, MLX5_MAX_DB_SIZE); + if (priv->db[i].umem.umem) + mlx5_glue->devx_umem_dereg(priv->db[i].umem.umem); + } + return -ret; +} + +/** + * DPDK callback for loading RXP rules. + * + * @param dev + * Pointer to RegEx device structure. + * @param[in] cfg + * Pointer to the regexdev device configuration structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_regex_rules_db_import(struct rte_regexdev *dev, + const char *rule_db, uint32_t rule_db_len) +{ + struct mlx5_regex_priv *priv = dev->data->dev_private; + struct mlx5_rxp_ctl_rules_pgm *rules = NULL; + uint8_t id; + int ret; + + if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) { + DRV_LOG(ERR, "RXP programming mode not set!"); + return -1; + } + if (rule_db == NULL) { + DRV_LOG(ERR, "Database empty!"); + return -ENODEV; + } + if (rule_db_len == 0) + return -EINVAL; + ret = rxp_parse_rof(rule_db, rule_db_len, &rules); + if (ret) { + DRV_LOG(ERR, "Can't parse ROF file."); + return ret; + } + /* Need to ensure RXP not busy before stop! */ + for (id = 0; id < MLX5_RXP_MAX_ENGINES; id++) { + ret = rxp_stop_engine(priv->ctx, id); + if (ret) { + DRV_LOG(ERR, "Can't stop engine."); + ret = -ENODEV; + goto tidyup_error; + } + ret = program_rxp_rules(priv, rules, id); + if (ret < 0) { + DRV_LOG(ERR, "Failed to program rxp rules."); + ret = -ENODEV; + goto tidyup_error; + } + ret = rxp_start_engine(priv->ctx, id); + if (ret) { + DRV_LOG(ERR, "Can't start engine."); + ret = -ENODEV; + goto tidyup_error; + } + } + rte_free(rules); + return 0; +tidyup_error: + rte_free(rules); + return ret; +} + +/** * DPDK callback for reading device info. * * @param dev @@ -270,9 +1228,11 @@ { struct mlx5_regex_priv *priv = dev->data->dev_private; int ret; - uint8_t id; + if (priv->prog_mode == MLX5_RXP_MODE_NOT_DEFINED) + return -1; priv->nb_queues = cfg->nb_queue_pairs; + dev->data->dev_conf.nb_queue_pairs = priv->nb_queues; priv->qps = rte_zmalloc(NULL, sizeof(struct mlx5_regex_qp) * priv->nb_queues, 0); if (!priv->nb_queues) { @@ -281,35 +1241,22 @@ return -rte_errno; } priv->nb_max_matches = cfg->nb_max_matches; - for (id = 0; id < 2; id++) { - ret = rxp_stop_engine(priv->ctx, id); - if (ret) { - DRV_LOG(ERR, "can't stop engine."); - rte_errno = ENODEV; - return -rte_errno; - } - ret = rxp_init(priv, id); - if (ret) { - DRV_LOG(ERR, "can't init engine."); - rte_errno = ENODEV; - return -rte_errno; - } - ret = mlx5_devx_regex_register_write(priv->ctx, id, - MLX5_RXP_CSR_MAX_MATCH, - priv->nb_max_matches); - if (ret) { - DRV_LOG(ERR, "can't update number of matches."); - rte_errno = ENODEV; - goto configure_error; - } - ret = rxp_start_engine(priv->ctx, id); - if (ret) { - DRV_LOG(ERR, "can't start engine."); + /* Setup rxp db memories. */ + if (rxp_db_setup(priv)) { + DRV_LOG(ERR, "Failed to setup RXP db memory"); + rte_errno = ENOMEM; + return -rte_errno; + } + if (cfg->rule_db != NULL) { + ret = mlx5_regex_rules_db_import(dev, cfg->rule_db, + cfg->rule_db_len); + if (ret < 0) { + DRV_LOG(ERR, "Failed to program rxp rules."); rte_errno = ENODEV; goto configure_error; } - - } + } else + DRV_LOG(DEBUG, "Regex config without rules programming!"); return 0; configure_error: if (priv->qps) diff --git a/drivers/regex/mlx5/mlx5_rxp.h b/drivers/regex/mlx5/mlx5_rxp.h new file mode 100644 index 0000000..fd19d40 --- /dev/null +++ b/drivers/regex/mlx5/mlx5_rxp.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2019 Mellanox Technologies, Ltd + */ + +#ifndef RTE_PMD_MLX5_REGEX_RXP_H_ +#define RTE_PMD_MLX5_REGEX_RXP_H_ + +#define MLX5_RXP_MAX_JOB_LENGTH 16384 +#define MLX5_RXP_MAX_SUBSETS 4095 +#define MLX5_RXP_CSR_NUM_ENTRIES 31 + +#define MLX5_RXP_CTRL_TYPE_MASK 7 +#define MLX5_RXP_CTRL_TYPE_JOB_DESCRIPTOR 0 +#define MLX5_RXP_CTRL_TYPE_RESPONSE_DESCRIPTOR 1 +#define MLX5_RXP_CTRL_TYPE_MEMORY_WRITE 4 +#define MLX5_RXP_CSR_CTRL_DISABLE_L2C (1 << 7) + +#define MLX5_RXP_CTRL_JOB_DESC_SOF 0x0010 +#define MLX5_RXP_CTRL_JOB_DESC_EOF 0x0020 +#define MLX5_RXP_CTRL_JOB_DESC_HPM_ENABLE 0x0100 +#define MLX5_RXP_CTRL_JOB_DESC_ANYMATCH_ENABLE 0x0200 +#define MLX5_RXP_CTRL_JOB_DESC_FLAGS (MLX5_RXP_CTRL_JOB_DESC_SOF | \ + MLX5_RXP_CTRL_JOB_DESC_EOF | \ + MLX5_RXP_CTRL_JOB_DESC_HPM_ENABLE | \ + MLX5_RXP_CTRL_JOB_DESC_ANYMATCH_ENABLE) + +#define MLX5_RXP_CTRL_VALID 0x8000 + +#define MLX5_RXP_RESP_STATUS_MAX_PRI_THREADS (1 << 3) +#define MLX5_RXP_RESP_STATUS_MAX_SEC_THREADS (1 << 4) +#define MLX5_RXP_RESP_STATUS_MAX_LATENCY (1 << 5) +#define MLX5_RXP_RESP_STATUS_MAX_MATCH (1 << 6) +#define MLX5_RXP_RESP_STATUS_MAX_PREFIX (1 << 7) +#define MLX5_RXP_RESP_STATUS_HPM (1 << 8) +#define MLX5_RXP_RESP_STATUS_ANYMATCH (1 << 9) +#define MLX5_RXP_RESP_STATUS_PMI_SOJ (1 << 13) +#define MLX5_RXP_RESP_STATUS_PMI_EOJ (1 << 14) + +/* This describes the header the RXP expects for any search data. */ +struct mlx5_rxp_job_desc { + uint32_t job_id; + uint16_t ctrl; + uint16_t len; + uint16_t subset[4]; +} __rte_packed; + +struct mlx5_rxp_response_desc { + uint32_t job_id; + uint16_t status; + uint8_t detected_match_count; + uint8_t match_count; + uint16_t primary_thread_count; + uint16_t instruction_count; + uint16_t latency_count; + uint16_t pmi_min_byte_ptr; +} __rte_packed; + +struct mlx5_rxp_match_tuple { + uint32_t rule_id; + uint16_t start_ptr; + uint16_t length; +} __rte_packed; + +struct mlx5_rxp_response { + struct mlx5_rxp_response_desc header; + struct mlx5_rxp_match_tuple matches[0]; +}; + +#define MLX5_RXP_MAX_MATCHES 254 + +#define MLX5_RXP_CTL_RULES_PGM 1 +#define MLX5_RXP_CTL_RULES_PGM_INCR 2 + +#define MLX5_RXP_ROF_ENTRY_INST 0 +#define MLX5_RXP_ROF_ENTRY_EQ 1 +#define MLX5_RXP_ROF_ENTRY_GTE 2 +#define MLX5_RXP_ROF_ENTRY_LTE 3 +#define MLX5_RXP_ROF_ENTRY_CHECKSUM 4 +#define MLX5_RXP_ROF_ENTRY_CHECKSUM_EX_EM 5 +#define MLX5_RXP_ROF_ENTRY_IM 6 +#define MLX5_RXP_ROF_ENTRY_EM 7 +#define MLX5_RXP_ROF_ENTRY_TYPE_MAX 7 + +#define MLX5_RXP_INST_OFFSET 3 +#define MLX5_RXP_INST_BLOCK_SIZE 8 +#define MLX5_MAX_SIZE_RES_DES (sizeof(struct mlx5_rxp_response_desc)) +#define MLX5_MAX_DB_SIZE (1u << 27u) +#define MLX5_MAX_SIZE_MATCH_RESP (254 * sizeof(struct mlx5_rxp_match_tuple)) +#define MLX5_RXP_SQ_NOT_BUSY false +#define MLX5_RXP_SQ_BUSY true + + +struct mlx5_rxp_ctl_hdr { + uint16_t cmd; + uint32_t len; +}; + +struct mlx5_rxp_rof_entry { + uint8_t type; + uint32_t addr; + uint64_t value; +}; + +struct mlx5_rxp_rof { + uint32_t rof_version; + char *timestamp; + char *rxp_compiler_version; + uint32_t rof_revision; + uint32_t number_of_entries; + struct mlx5_rxp_rof_entry *rof_entries; +}; + +struct mlx5_rxp_ctl_rules_pgm { + struct mlx5_rxp_ctl_hdr hdr; + uint32_t count; + struct mlx5_rxp_rof_entry rules[0]; +} __rte_packed; + +/* RXP programming mode setting. */ +enum mlx5_rxp_program_mode { + MLX5_RXP_MODE_NOT_DEFINED = 0, + MLX5_RXP_SHARED_PROG_MODE, + MLX5_RXP_PRIVATE_PROG_MODE, +}; + +#define MLX5_RXP_POLL_CSR_FOR_VALUE_TIMEOUT 3000 /* Poll timeout in ms. */ +#define MLX5_RXP_INITIALIZATION_TIMEOUT 60000 /* Initialize timeout in ms. */ +#define MLX5_RXP_MAX_ENGINES 2u /* Number of RXP engines. */ +#define MLX5_RXP_EM_COUNT 1u /* Extra External Memories to use. */ +#define MLX5_RXP_DB_NOT_ASSIGNED 0xFF + +struct mlx5_regex_umem { + struct mlx5dv_devx_umem *umem; + uint32_t id; + uint64_t offset; +}; + +#endif /* RTE_PMD_MLX5_REGEX_RXP_H_ */