From patchwork Sun Jul 12 20:58:59 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ori Kam X-Patchwork-Id: 73855 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 6741AA052A; Sun, 12 Jul 2020 23:02:17 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 4EAB31D57A; Sun, 12 Jul 2020 23:01:24 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id F06211D535 for ; Sun, 12 Jul 2020 23:01:19 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from orika@mellanox.com) with SMTP; 13 Jul 2020 00:00:35 +0300 Received: from pegasus04.mtr.labs.mlnx. (pegasus04.mtr.labs.mlnx [10.210.16.126]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 06CKxFGD025997; Mon, 13 Jul 2020 00:00:34 +0300 From: Ori Kam To: jerinj@marvell.com, xiang.w.wang@intel.com, matan@mellanox.com, viacheslavo@mellanox.com Cc: guyk@marvell.com, dev@dpdk.org, pbhagavatula@marvell.com, shahafs@mellanox.com, hemant.agrawal@nxp.com, opher@mellanox.com, alexr@mellanox.com, dovrat@marvell.com, pkapoor@marvell.com, nipun.gupta@nxp.com, bruce.richardson@intel.com, yang.a.hong@intel.com, harry.chang@intel.com, gu.jian1@zte.com.cn, shanjiangh@chinatelecom.cn, zhangy.yun@chinatelecom.cn, lixingfu@huachentel.com, wushuai@inspur.com, yuyingxia@yxlink.com, fanchenggang@sunyainfo.com, davidfgao@tencent.com, liuzhong1@chinaunicom.cn, zhaoyong11@huawei.com, oc@yunify.com, jim@netgate.com, hongjun.ni@intel.com, deri@ntop.org, fc@napatech.com, arthur.su@lionic.com, thomas@monjalon.net, orika@mellanox.com, rasland@mellanox.com, Yuval Avnery Date: Sun, 12 Jul 2020 20:58:59 +0000 Message-Id: <1594587541-110442-19-git-send-email-orika@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1594587541-110442-1-git-send-email-orika@mellanox.com> References: <1593941027-86651-1-git-send-email-orika@mellanox.com> <1594587541-110442-1-git-send-email-orika@mellanox.com> Subject: [dpdk-dev] [PATCH v2 18/20] regex/mlx5: add enqueue implementation X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Yuval Avnery Will look for a free SQ to send the job on. doorbell will be given when sq is full, or no more jobs on the burst. Signed-off-by: Yuval Avnery Acked-by: Ori Kam --- drivers/regex/mlx5/mlx5_regex.c | 1 + drivers/regex/mlx5/mlx5_regex.h | 6 ++ drivers/regex/mlx5/mlx5_regex_control.c | 2 + drivers/regex/mlx5/mlx5_regex_fastpath.c | 146 +++++++++++++++++++++++++++++-- 4 files changed, 148 insertions(+), 7 deletions(-) diff --git a/drivers/regex/mlx5/mlx5_regex.c b/drivers/regex/mlx5/mlx5_regex.c index f06f817..d823e17 100644 --- a/drivers/regex/mlx5/mlx5_regex.c +++ b/drivers/regex/mlx5/mlx5_regex.c @@ -180,6 +180,7 @@ goto error; } priv->regexdev->dev_ops = &mlx5_regexdev_ops; + priv->regexdev->enqueue = mlx5_regexdev_enqueue; priv->regexdev->device = (struct rte_device *)pci_dev; priv->regexdev->data->dev_private = priv; priv->regexdev->state = RTE_REGEXDEV_READY; diff --git a/drivers/regex/mlx5/mlx5_regex.h b/drivers/regex/mlx5/mlx5_regex.h index cf8863f..468772c 100644 --- a/drivers/regex/mlx5/mlx5_regex.h +++ b/drivers/regex/mlx5/mlx5_regex.h @@ -25,6 +25,9 @@ struct mlx5_regex_sq { uint32_t dbr_umem; /* Door bell record umem id. */ uint8_t *wqe; /* The SQ ring buffer. */ struct mlx5dv_devx_umem *wqe_umem; /* SQ buffer umem. */ + size_t pi, db_pi; + size_t ci; + uint32_t sqn; uint32_t *dbr; }; @@ -49,6 +52,7 @@ struct mlx5_regex_qp { struct ibv_mr *metadata; struct ibv_mr *inputs; struct ibv_mr *outputs; + size_t ci, pi; }; struct mlx5_regex_db { @@ -91,4 +95,6 @@ int mlx5_regex_rules_db_import(struct rte_regexdev *dev, /* mlx5_regex_fastpath.c */ int mlx5_regexdev_setup_fastpath(struct mlx5_regex_priv *priv, uint32_t qp_id); +uint16_t mlx5_regexdev_enqueue(struct rte_regexdev *dev, uint16_t qp_id, + struct rte_regex_ops **ops, uint16_t nb_ops); #endif /* MLX5_REGEX_H */ diff --git a/drivers/regex/mlx5/mlx5_regex_control.c b/drivers/regex/mlx5/mlx5_regex_control.c index 9b3f39e..c2d080f 100644 --- a/drivers/regex/mlx5/mlx5_regex_control.c +++ b/drivers/regex/mlx5/mlx5_regex_control.c @@ -216,6 +216,8 @@ sq->wqe = buf; sq->wqe_umem = mlx5_glue->devx_umem_reg(priv->ctx, buf, 64 * sq_size, 7); + sq->ci = 0; + sq->pi = 0; if (!sq->wqe_umem) { DRV_LOG(ERR, "Can't register wqe mem."); rte_errno = ENOMEM; diff --git a/drivers/regex/mlx5/mlx5_regex_fastpath.c b/drivers/regex/mlx5/mlx5_regex_fastpath.c index b5147ce..1823353 100644 --- a/drivers/regex/mlx5/mlx5_regex_fastpath.c +++ b/drivers/regex/mlx5/mlx5_regex_fastpath.c @@ -33,10 +33,14 @@ #pragma GCC diagnostic error "-Wpedantic" #endif -#define MAX_WQE_INDEX 0xffff +#define MLX5_REGEX_MAX_WQE_INDEX 0xffff #define MLX5_REGEX_METADATA_SIZE 64 -#define MLX5_REGEX_MAX_INPUT (1<<14) -#define MLX5_REGEX_MAX_OUTPUT (1<<11) +#define MLX5_REGEX_MAX_INPUT (1 << 14) +#define MLX5_REGEX_MAX_OUTPUT (1 << 11) +#define MLX5_REGEX_WQE_CTRL_OFFSET 12 +#define MLX5_REGEX_WQE_METADATA_OFFSET 16 +#define MLX5_REGEX_WQE_GATHER_OFFSET 32 +#define MLX5_REGEX_WQE_SCATTER_OFFSET 48 static inline uint32_t @@ -77,6 +81,134 @@ struct mlx5_regex_job { seg->addr = rte_cpu_to_be_64(address); } +static inline void +set_regex_ctrl_seg(void *seg, uint8_t le, uint16_t subset_id0, + uint16_t subset_id1, uint16_t subset_id2, + uint16_t subset_id3, uint8_t ctrl) +{ + DEVX_SET(regexp_mmo_control, seg, le, le); + DEVX_SET(regexp_mmo_control, seg, ctrl, ctrl); + DEVX_SET(regexp_mmo_control, seg, subset_id_0, subset_id0); + DEVX_SET(regexp_mmo_control, seg, subset_id_1, subset_id1); + DEVX_SET(regexp_mmo_control, seg, subset_id_2, subset_id2); + DEVX_SET(regexp_mmo_control, seg, subset_id_3, subset_id3); +} + +static inline void +set_wqe_ctrl_seg(struct mlx5_wqe_ctrl_seg *seg, uint16_t pi, uint8_t opcode, + uint8_t opmod, uint32_t qp_num, uint8_t fm_ce_se, uint8_t ds, + uint8_t signature, uint32_t imm) +{ + seg->opmod_idx_opcode = rte_cpu_to_be_32(((uint32_t)opmod << 24) | + ((uint32_t)pi << 8) | + opcode); + seg->qpn_ds = rte_cpu_to_be_32((qp_num << 8) | ds); + seg->fm_ce_se = fm_ce_se; + seg->signature = signature; + seg->imm = imm; +} + +static inline void +prep_one(struct mlx5_regex_sq *sq, struct rte_regex_ops *op, + struct mlx5_regex_job *job) +{ + size_t wqe_offset = (sq->pi % sq_size_get(sq)) * MLX5_SEND_WQE_BB; + uint8_t *wqe = (uint8_t *)sq->wqe + wqe_offset; + int ds = 4; /* ctrl + meta + input + output */ + + memcpy(job->input, + rte_pktmbuf_mtod(op->mbuf, void *), + rte_pktmbuf_data_len(op->mbuf)); + set_wqe_ctrl_seg((struct mlx5_wqe_ctrl_seg *)wqe, sq->pi, + MLX5_OPCODE_MMO, MLX5_OPC_MOD_MMO_REGEX, sq->obj->id, + 0, ds, 0, 0); + set_regex_ctrl_seg(wqe + 12, 0, op->group_id0, op->group_id1, + op->group_id2, + op->group_id3, 0); + struct mlx5_wqe_data_seg *input_seg = + (struct mlx5_wqe_data_seg *)(wqe + + MLX5_REGEX_WQE_GATHER_OFFSET); + input_seg->byte_count = + rte_cpu_to_be_32(rte_pktmbuf_data_len(op->mbuf)); + job->user_id = op->user_id; + sq->db_pi = sq->pi; + sq->pi = (sq->pi + 1) % MLX5_REGEX_MAX_WQE_INDEX; +} + +static inline void +send_doorbell(struct mlx5dv_devx_uar *uar, struct mlx5_regex_sq *sq) +{ + size_t wqe_offset = (sq->db_pi % sq_size_get(sq)) * MLX5_SEND_WQE_BB; + uint8_t *wqe = (uint8_t *)sq->wqe + wqe_offset; + ((struct mlx5_wqe_ctrl_seg *)wqe)->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; + uint64_t *doorbell_addr = + (uint64_t *)((uint8_t *)uar->base_addr + 0x800); + rte_cio_wmb(); + sq->dbr[MLX5_SND_DBR] = rte_cpu_to_be_32((sq->db_pi + 1) % + MLX5_REGEX_MAX_WQE_INDEX); + rte_wmb(); + *doorbell_addr = *(volatile uint64_t *)wqe; + rte_wmb(); +} + +static inline int +can_send(struct mlx5_regex_sq *sq) { + return unlikely(sq->ci > sq->pi) ? + MLX5_REGEX_MAX_WQE_INDEX + sq->pi - sq->ci < + sq_size_get(sq) : + sq->pi - sq->ci < sq_size_get(sq); +} + +static inline uint32_t +job_id_get(uint32_t qid, size_t sq_size, size_t index) { + return qid*sq_size + index % sq_size; +} + +/** + * DPDK callback for enqueue. + * + * @param dev + * Pointer to the regex dev structure. + * @param qp_id + * The queue to enqueue the traffic to. + * @param ops + * List of regex ops to enqueue. + * @param nb_ops + * Number of ops in ops parameter. + * + * @return + * Number of packets successfully enqueued (<= pkts_n). + */ +uint16_t +mlx5_regexdev_enqueue(struct rte_regexdev *dev, uint16_t qp_id, + struct rte_regex_ops **ops, uint16_t nb_ops) +{ + struct mlx5_regex_priv *priv = dev->data->dev_private; + struct mlx5_regex_qp *queue = &priv->qps[qp_id]; + struct mlx5_regex_sq *sq; + size_t sqid, job_id, i = 0; + + while ((sqid = ffs(queue->free_sqs))) { + sqid--; /* ffs returns 1 for bit 0 */ + sq = &queue->sqs[sqid]; + while (can_send(sq)) { + job_id = job_id_get(sqid, sq_size_get(sq), sq->pi); + prep_one(sq, ops[i], &queue->jobs[job_id]); + i++; + if (unlikely(i == nb_ops)) { + send_doorbell(priv->uar, sq); + goto out; + } + } + queue->free_sqs &= ~(1 << sqid); + send_doorbell(priv->uar, sq); + } + +out: + queue->pi += i; + return i; +} + static void setup_sqs(struct mlx5_regex_qp *queue) { @@ -147,7 +279,7 @@ struct mlx5_regex_job { goto err_output; } qp->outputs = mlx5_glue->reg_mr(pd, ptr, - MLX5_REGEX_MAX_OUTPUT*qp->nb_desc, + MLX5_REGEX_MAX_OUTPUT * qp->nb_desc, IBV_ACCESS_LOCAL_WRITE); if (!qp->outputs) { rte_free(ptr); @@ -159,13 +291,13 @@ struct mlx5_regex_job { for (i = 0; i < qp->nb_desc; i++) { qp->jobs[i].input = (uint8_t *)qp->inputs->addr + - (i%qp->nb_desc)*MLX5_REGEX_MAX_INPUT; + (i % qp->nb_desc) * MLX5_REGEX_MAX_INPUT; qp->jobs[i].output = (uint8_t *)qp->outputs->addr + - (i%qp->nb_desc)*MLX5_REGEX_MAX_OUTPUT; + (i % qp->nb_desc) * MLX5_REGEX_MAX_OUTPUT; qp->jobs[i].metadata = (uint8_t *)qp->metadata->addr + - (i%qp->nb_desc)*MLX5_REGEX_METADATA_SIZE; + (i % qp->nb_desc) * MLX5_REGEX_METADATA_SIZE; } return 0;