From patchwork Sat Mar 2 09:38:12 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Nagadheeraj Rottela X-Patchwork-Id: 137799 X-Patchwork-Delegate: gakhil@marvell.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id E723F43C30; Sat, 2 Mar 2024 10:39:25 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 9320D42DDA; Sat, 2 Mar 2024 10:38:59 +0100 (CET) Received: from mx0b-0016f401.pphosted.com (mx0a-0016f401.pphosted.com [67.231.148.174]) by mails.dpdk.org (Postfix) with ESMTP id 990C042DD6 for ; Sat, 2 Mar 2024 10:38:57 +0100 (CET) Received: from pps.filterd (m0045849.ppops.net [127.0.0.1]) by mx0a-0016f401.pphosted.com (8.17.1.24/8.17.1.24) with ESMTP id 42280ANc013446; Sat, 2 Mar 2024 01:38:56 -0800 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com; h= from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding:content-type; s= pfpt0220; bh=wJXZEJWRLWIBUtoMEQvFFWA+oDQA05ii+qV+YfHkTPM=; b=cz2 ltLlk8TkM+GnnOjgBAT0EFe+WQebTEH/rHAxN/Mb3A5ZDnJDuagpPI5uTcG2t0+J l0Vj5GMNaoIHDmIR8Hd0A6l2RN+6n6n0QFHi3BQD5WN9oA/pPwr2VUMIYVhNxlaN 5l1B5KUiKHncrecL30Kk+XilB91GbwfPawCc1XrY0hkohxWSuvphsZmFfshgY7I9 UzWx/2l8jy06JrVlig4F7oXEI9Wdx0AZRzPTww3oxYqfWoseRaEyt87dmBZuMRim psylk0AL4qqbAbBn4H/ylihcmAsJkj+v/cj5IQnpXEDXwjFjPKChufmxeevA3Lzk In9iMTi3Ze0oEXP+E4Q== Received: from dc5-exch05.marvell.com ([199.233.59.128]) by mx0a-0016f401.pphosted.com (PPS) with ESMTPS id 3wkcq5bh9x-1 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT); Sat, 02 Mar 2024 01:38:56 -0800 (PST) Received: from DC5-EXCH02.marvell.com (10.69.176.39) by DC5-EXCH05.marvell.com (10.69.176.209) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384) id 15.2.1258.12; Sat, 2 Mar 2024 01:38:55 -0800 Received: from DC5-EXCH05.marvell.com (10.69.176.209) by DC5-EXCH02.marvell.com (10.69.176.39) with Microsoft SMTP Server (TLS) id 15.0.1497.48; Sat, 2 Mar 2024 01:38:53 -0800 Received: from hyd1399.caveonetworks.com.com (10.69.176.80) by DC5-EXCH05.marvell.com (10.69.176.209) with Microsoft SMTP Server id 15.2.1258.12 via Frontend Transport; Sat, 2 Mar 2024 01:38:51 -0800 From: Nagadheeraj Rottela To: , , CC: , Nagadheeraj Rottela Subject: [PATCH v5 6/7] compress/nitrox: support stateless request Date: Sat, 2 Mar 2024 15:08:12 +0530 Message-ID: <20240302093813.14922-7-rnagadheeraj@marvell.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20240302093813.14922-1-rnagadheeraj@marvell.com> References: <20240302093813.14922-1-rnagadheeraj@marvell.com> MIME-Version: 1.0 X-Proofpoint-GUID: cfHreudFgL60GTLdiK1tpdTDqfrN4jsf X-Proofpoint-ORIG-GUID: cfHreudFgL60GTLdiK1tpdTDqfrN4jsf X-Proofpoint-Virus-Version: vendor=baseguard engine=ICAP:2.0.272,Aquarius:18.0.1011,Hydra:6.0.619,FMLib:17.11.176.26 definitions=2024-03-02_04,2024-03-01_03,2023-05-22_02 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Implement enqueue and dequeue burst operations for stateless request support. Signed-off-by: Nagadheeraj Rottela --- drivers/compress/nitrox/meson.build | 1 + drivers/compress/nitrox/nitrox_comp.c | 91 ++- drivers/compress/nitrox/nitrox_comp_reqmgr.c | 792 +++++++++++++++++++ drivers/compress/nitrox/nitrox_comp_reqmgr.h | 10 + 4 files changed, 885 insertions(+), 9 deletions(-) create mode 100644 drivers/compress/nitrox/nitrox_comp_reqmgr.c diff --git a/drivers/compress/nitrox/meson.build b/drivers/compress/nitrox/meson.build index f137303689..2c35aba60b 100644 --- a/drivers/compress/nitrox/meson.build +++ b/drivers/compress/nitrox/meson.build @@ -10,6 +10,7 @@ deps += ['common_nitrox', 'bus_pci', 'compressdev'] sources += files( 'nitrox_comp.c', + 'nitrox_comp_reqmgr.c', ) includes += include_directories('../../common/nitrox') diff --git a/drivers/compress/nitrox/nitrox_comp.c b/drivers/compress/nitrox/nitrox_comp.c index 299cb8e783..0ea5ed43ed 100644 --- a/drivers/compress/nitrox/nitrox_comp.c +++ b/drivers/compress/nitrox/nitrox_comp.c @@ -187,10 +187,17 @@ static int nitrox_comp_queue_pair_setup(struct rte_compressdev *dev, if (unlikely(err)) goto qp_setup_err; + qp->sr_mp = nitrox_comp_req_pool_create(dev, qp->count, qp_id, + socket_id); + if (unlikely(!qp->sr_mp)) + goto req_pool_err; + dev->data->queue_pairs[qp_id] = qp; NITROX_LOG(DEBUG, "queue %d setup done\n", qp_id); return 0; +req_pool_err: + nitrox_qp_release(qp, ndev->bar_addr); qp_setup_err: rte_free(qp); return err; @@ -224,6 +231,7 @@ static int nitrox_comp_queue_pair_release(struct rte_compressdev *dev, dev->data->queue_pairs[qp_id] = NULL; err = nitrox_qp_release(qp, ndev->bar_addr); + nitrox_comp_req_pool_free(qp->sr_mp); rte_free(qp); NITROX_LOG(DEBUG, "queue %d release done\n", qp_id); return err; @@ -349,24 +357,89 @@ static int nitrox_comp_private_xform_free(struct rte_compressdev *dev, return 0; } -static uint16_t nitrox_comp_dev_enq_burst(void *qp, +static int nitrox_enq_single_op(struct nitrox_qp *qp, struct rte_comp_op *op) +{ + struct nitrox_softreq *sr; + int err; + + if (unlikely(rte_mempool_get(qp->sr_mp, (void **)&sr))) + return -ENOMEM; + + err = nitrox_process_comp_req(op, sr); + if (unlikely(err)) { + rte_mempool_put(qp->sr_mp, sr); + return err; + } + + nitrox_qp_enqueue(qp, nitrox_comp_instr_addr(sr), sr); + return 0; +} + +static uint16_t nitrox_comp_dev_enq_burst(void *queue_pair, struct rte_comp_op **ops, uint16_t nb_ops) { - RTE_SET_USED(qp); - RTE_SET_USED(ops); - RTE_SET_USED(nb_ops); + struct nitrox_qp *qp = queue_pair; + uint16_t free_slots = 0; + uint16_t cnt = 0; + bool err = false; + + free_slots = nitrox_qp_free_count(qp); + if (nb_ops > free_slots) + nb_ops = free_slots; + + for (cnt = 0; cnt < nb_ops; cnt++) { + if (unlikely(nitrox_enq_single_op(qp, ops[cnt]))) { + err = true; + break; + } + } + + nitrox_ring_dbell(qp, cnt); + qp->stats.enqueued_count += cnt; + if (unlikely(err)) + qp->stats.enqueue_err_count++; + + return cnt; +} + +static int nitrox_deq_single_op(struct nitrox_qp *qp, + struct rte_comp_op **op_ptr) +{ + struct nitrox_softreq *sr; + int err; + + sr = nitrox_qp_get_softreq(qp); + err = nitrox_check_comp_req(sr, op_ptr); + if (err == -EAGAIN) + return err; + + nitrox_qp_dequeue(qp); + rte_mempool_put(qp->sr_mp, sr); + if (err == 0) + qp->stats.dequeued_count++; + else + qp->stats.dequeue_err_count++; + return 0; } -static uint16_t nitrox_comp_dev_deq_burst(void *qp, +static uint16_t nitrox_comp_dev_deq_burst(void *queue_pair, struct rte_comp_op **ops, uint16_t nb_ops) { - RTE_SET_USED(qp); - RTE_SET_USED(ops); - RTE_SET_USED(nb_ops); - return 0; + struct nitrox_qp *qp = queue_pair; + uint16_t filled_slots = nitrox_qp_used_count(qp); + int cnt = 0; + + if (nb_ops > filled_slots) + nb_ops = filled_slots; + + for (cnt = 0; cnt < nb_ops; cnt++) + if (nitrox_deq_single_op(qp, &ops[cnt])) + break; + + return cnt; } static struct rte_compressdev_ops nitrox_compressdev_ops = { diff --git a/drivers/compress/nitrox/nitrox_comp_reqmgr.c b/drivers/compress/nitrox/nitrox_comp_reqmgr.c new file mode 100644 index 0000000000..5ad1a4439a --- /dev/null +++ b/drivers/compress/nitrox/nitrox_comp_reqmgr.c @@ -0,0 +1,792 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2024 Marvell. + */ + +#include +#include +#include + +#include "nitrox_comp_reqmgr.h" +#include "nitrox_logs.h" +#include "rte_comp.h" + +#define NITROX_ZIP_SGL_COUNT 16 +#define NITROX_ZIP_MAX_ZPTRS 2048 +#define NITROX_ZIP_MAX_DATASIZE ((1 << 24) - 1) +#define NITROX_ZIP_MAX_ONFSIZE 1024 +#define CMD_TIMEOUT 2 + +union nitrox_zip_instr_word0 { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t raz0 : 8; + uint64_t tol : 24; + uint64_t raz1 : 5; + uint64_t exn : 3; + uint64_t raz2 : 1; + uint64_t exbits : 7; + uint64_t raz3 : 3; + uint64_t ca : 1; + uint64_t sf : 1; + uint64_t ss : 2; + uint64_t cc : 2; + uint64_t ef : 1; + uint64_t bf : 1; + uint64_t co : 1; + uint64_t raz4 : 1; + uint64_t ds : 1; + uint64_t dg : 1; + uint64_t hg : 1; +#else + uint64_t hg : 1; + uint64_t dg : 1; + uint64_t ds : 1; + uint64_t raz4 : 1; + uint64_t co : 1; + uint64_t bf : 1; + uint64_t ef : 1; + uint64_t cc : 2; + uint64_t ss : 2; + uint64_t sf : 1; + uint64_t ca : 1; + uint64_t raz3 : 3; + uint64_t exbits : 7; + uint64_t raz2 : 1; + uint64_t exn : 3; + uint64_t raz1 : 5; + uint64_t tol : 24; + uint64_t raz0 : 8; +#endif + + }; +}; + +union nitrox_zip_instr_word1 { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t hl : 16; + uint64_t raz0 : 16; + uint64_t adlercrc32 : 32; +#else + uint64_t adlercrc32 : 32; + uint64_t raz0 : 16; + uint64_t hl : 16; +#endif + }; +}; + +union nitrox_zip_instr_word2 { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t raz0 : 20; + uint64_t cptr : 44; +#else + uint64_t cptr : 44; + uint64_t raz0 : 20; +#endif + }; +}; + +union nitrox_zip_instr_word3 { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t raz0 : 4; + uint64_t hlen : 16; + uint64_t hptr : 44; +#else + uint64_t hptr : 44; + uint64_t hlen : 16; + uint64_t raz0 : 4; +#endif + }; +}; + +union nitrox_zip_instr_word4 { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t raz0 : 4; + uint64_t ilen : 16; + uint64_t iptr : 44; +#else + uint64_t iptr : 44; + uint64_t ilen : 16; + uint64_t raz0 : 4; +#endif + }; +}; + +union nitrox_zip_instr_word5 { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t raz0 : 4; + uint64_t olen : 16; + uint64_t optr : 44; +#else + uint64_t optr : 44; + uint64_t olen : 16; + uint64_t raz0 : 4; +#endif + }; +}; + +union nitrox_zip_instr_word6 { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t raz0 : 20; + uint64_t rptr : 44; +#else + uint64_t rptr : 44; + uint64_t raz0 : 20; +#endif + }; +}; + +union nitrox_zip_instr_word7 { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t grp : 3; + uint64_t raz0 : 41; + uint64_t addr_msb: 20; +#else + uint64_t addr_msb: 20; + uint64_t raz0 : 41; + uint64_t grp : 3; +#endif + }; +}; + +struct nitrox_zip_instr { + union nitrox_zip_instr_word0 w0; + union nitrox_zip_instr_word1 w1; + union nitrox_zip_instr_word2 w2; + union nitrox_zip_instr_word3 w3; + union nitrox_zip_instr_word4 w4; + union nitrox_zip_instr_word5 w5; + union nitrox_zip_instr_word6 w6; + union nitrox_zip_instr_word7 w7; +}; + +union nitrox_zip_result_word0 { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t crc32 : 32; + uint64_t adler32: 32; +#else + uint64_t adler32: 32; + uint64_t crc32 : 32; +#endif + }; +}; + +union nitrox_zip_result_word1 { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t tbyteswritten : 32; + uint64_t tbytesread : 32; +#else + uint64_t tbytesread : 32; + uint64_t tbyteswritten : 32; +#endif + }; +}; + +union nitrox_zip_result_word2 { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t tbits : 32; + uint64_t raz0 : 5; + uint64_t exn : 3; + uint64_t raz1 : 1; + uint64_t exbits : 7; + uint64_t raz2 : 7; + uint64_t ef : 1; + uint64_t compcode: 8; +#else + uint64_t compcode: 8; + uint64_t ef : 1; + uint64_t raz2 : 7; + uint64_t exbits : 7; + uint64_t raz1 : 1; + uint64_t exn : 3; + uint64_t raz0 : 5; + uint64_t tbits : 32; +#endif + }; +}; + +struct nitrox_zip_result { + union nitrox_zip_result_word0 w0; + union nitrox_zip_result_word1 w1; + union nitrox_zip_result_word2 w2; +}; + +union nitrox_zip_zptr { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t raz0 : 3; + uint64_t le : 1; + uint64_t length : 16; + uint64_t addr : 44; +#else + uint64_t addr : 44; + uint64_t length : 16; + uint64_t le : 1; + uint64_t raz0 : 3; +#endif + } s; +}; + +struct nitrox_zip_iova_addr { + union { + uint64_t u64; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t addr_msb: 20; + uint64_t addr : 44; +#else + uint64_t addr : 44; + uint64_t addr_msb: 20; +#endif + } zda; + + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t addr_msb: 20; + uint64_t addr : 41; + uint64_t align_8bytes: 3; +#else + uint64_t align_8bytes: 3; + uint64_t addr : 41; + uint64_t addr_msb: 20; +#endif + } z8a; + }; +}; + +enum nitrox_zip_comp_code { + NITROX_CC_NOTDONE = 0, + NITROX_CC_SUCCESS = 1, + NITROX_CC_DTRUNC = 2, + NITROX_CC_STOP = 3, + NITROX_CC_ITRUNK = 4, + NITROX_CC_RBLOCK = 5, + NITROX_CC_NLEN = 6, + NITROX_CC_BADCODE = 7, + NITROX_CC_BADCODE2 = 8, + NITROX_CC_ZERO_LEN = 9, + NITROX_CC_PARITY = 10, + NITROX_CC_FATAL = 11, + NITROX_CC_TIMEOUT = 12, + NITROX_CC_NPCI_ERR = 13, +}; + +struct nitrox_sgtable { + union nitrox_zip_zptr *sgl; + uint64_t addr_msb; + uint32_t total_bytes; + uint16_t nb_sgls; + uint16_t filled_sgls; +}; + +struct nitrox_softreq { + struct nitrox_zip_instr instr; + struct nitrox_zip_result zip_res __rte_aligned(8); + uint8_t decomp_threshold[NITROX_ZIP_MAX_ONFSIZE]; + struct rte_comp_op *op; + struct nitrox_sgtable src; + struct nitrox_sgtable dst; + struct nitrox_comp_xform xform; + uint64_t timeout; +}; + +static int create_sglist_from_mbuf(struct nitrox_sgtable *sgtbl, + struct rte_mbuf *mbuf, uint32_t off, + uint32_t datalen, uint8_t extra_segs, + int socket_id) +{ + struct rte_mbuf *m; + union nitrox_zip_zptr *sgl; + struct nitrox_zip_iova_addr zip_addr; + uint16_t nb_segs; + uint16_t i; + uint32_t mlen; + + if (unlikely(datalen > NITROX_ZIP_MAX_DATASIZE)) { + NITROX_LOG(ERR, "Unsupported datalen %d, max supported %d\n", + datalen, NITROX_ZIP_MAX_DATASIZE); + return -ENOTSUP; + } + + nb_segs = mbuf->nb_segs + extra_segs; + for (m = mbuf; m && off > rte_pktmbuf_data_len(m); m = m->next) { + off -= rte_pktmbuf_data_len(m); + nb_segs--; + } + + if (unlikely(nb_segs > NITROX_ZIP_MAX_ZPTRS)) { + NITROX_LOG(ERR, "Mbuf has more segments %d than supported\n", + nb_segs); + return -ENOTSUP; + } + + if (unlikely(nb_segs > sgtbl->nb_sgls)) { + union nitrox_zip_zptr *sgl; + + NITROX_LOG(INFO, "Mbuf has more segs %d than allocated %d\n", + nb_segs, sgtbl->nb_sgls); + sgl = rte_realloc_socket(sgtbl->sgl, + sizeof(*sgtbl->sgl) * nb_segs, + 8, socket_id); + if (unlikely(!sgl)) { + NITROX_LOG(ERR, "Failed to expand sglist memory\n"); + return -ENOMEM; + } + + sgtbl->sgl = sgl; + sgtbl->nb_sgls = nb_segs; + } + + sgtbl->filled_sgls = 0; + sgtbl->total_bytes = 0; + sgl = sgtbl->sgl; + if (!m) + return 0; + + mlen = rte_pktmbuf_data_len(m) - off; + if (datalen <= mlen) + mlen = datalen; + + i = 0; + zip_addr.u64 = rte_pktmbuf_iova_offset(m, off); + sgl[i].s.addr = zip_addr.zda.addr; + sgl[i].s.length = mlen; + sgl[i].s.le = 0; + sgtbl->total_bytes += mlen; + sgtbl->addr_msb = zip_addr.zda.addr_msb; + datalen -= mlen; + i++; + for (m = m->next; m && datalen; m = m->next) { + mlen = rte_pktmbuf_data_len(m) < datalen ? + rte_pktmbuf_data_len(m) : datalen; + zip_addr.u64 = rte_pktmbuf_iova(m); + if (unlikely(zip_addr.zda.addr_msb != sgtbl->addr_msb)) { + NITROX_LOG(ERR, "zip_ptrs have different msb addr\n"); + return -ENOTSUP; + } + + sgl[i].s.addr = zip_addr.zda.addr; + sgl[i].s.length = mlen; + sgl[i].s.le = 0; + sgtbl->total_bytes += mlen; + datalen -= mlen; + i++; + } + + sgtbl->filled_sgls = i; + return 0; +} + +static int softreq_init(struct nitrox_softreq *sr) +{ + struct rte_mempool *mp; + int err; + + mp = rte_mempool_from_obj(sr); + if (unlikely(mp == NULL)) + return -EINVAL; + + err = create_sglist_from_mbuf(&sr->src, sr->op->m_src, + sr->op->src.offset, + sr->op->src.length, 0, mp->socket_id); + if (unlikely(err)) + return err; + + err = create_sglist_from_mbuf(&sr->dst, sr->op->m_dst, + sr->op->dst.offset, + rte_pktmbuf_pkt_len(sr->op->m_dst) - sr->op->dst.offset, + (sr->xform.op == NITROX_COMP_OP_DECOMPRESS) ? 1 : 0, + mp->socket_id); + if (unlikely(err)) + return err; + + if (sr->xform.op == NITROX_COMP_OP_DECOMPRESS) { + struct nitrox_zip_iova_addr zip_addr; + int i; + + zip_addr.u64 = rte_mempool_virt2iova(sr) + + offsetof(struct nitrox_softreq, decomp_threshold); + i = sr->dst.filled_sgls; + sr->dst.sgl[i].s.addr = zip_addr.zda.addr; + sr->dst.sgl[i].s.length = NITROX_ZIP_MAX_ONFSIZE; + sr->dst.sgl[i].s.le = 0; + sr->dst.total_bytes += NITROX_ZIP_MAX_ONFSIZE; + sr->dst.filled_sgls++; + } + + return 0; +} + +static void nitrox_zip_instr_to_b64(struct nitrox_softreq *sr) +{ + struct nitrox_zip_instr *instr = &sr->instr; + int i; + + for (i = 0; instr->w0.dg && (i < instr->w4.ilen); i++) + sr->src.sgl[i].u64 = rte_cpu_to_be_64(sr->src.sgl[i].u64); + + for (i = 0; instr->w0.ds && (i < instr->w5.olen); i++) + sr->dst.sgl[i].u64 = rte_cpu_to_be_64(sr->dst.sgl[i].u64); + + instr->w0.u64 = rte_cpu_to_be_64(instr->w0.u64); + instr->w1.u64 = rte_cpu_to_be_64(instr->w1.u64); + instr->w2.u64 = rte_cpu_to_be_64(instr->w2.u64); + instr->w3.u64 = rte_cpu_to_be_64(instr->w3.u64); + instr->w4.u64 = rte_cpu_to_be_64(instr->w4.u64); + instr->w5.u64 = rte_cpu_to_be_64(instr->w5.u64); + instr->w6.u64 = rte_cpu_to_be_64(instr->w6.u64); + instr->w7.u64 = rte_cpu_to_be_64(instr->w7.u64); +} + +static int process_zip_stateless(struct nitrox_softreq *sr) +{ + struct nitrox_zip_instr *instr; + struct nitrox_comp_xform *xform; + struct nitrox_zip_iova_addr zip_addr; + uint64_t iptr_msb, optr_msb, rptr_msb; + int err; + + xform = sr->op->private_xform; + if (unlikely(xform == NULL)) { + NITROX_LOG(ERR, "Invalid stateless comp op\n"); + return -EINVAL; + } + + if (unlikely(xform->op == NITROX_COMP_OP_COMPRESS && + sr->op->flush_flag != RTE_COMP_FLUSH_FULL && + sr->op->flush_flag != RTE_COMP_FLUSH_FINAL)) { + NITROX_LOG(ERR, "Invalid flush flag %d in stateless op\n", + sr->op->flush_flag); + return -EINVAL; + } + + sr->xform = *xform; + err = softreq_init(sr); + if (unlikely(err)) + return err; + + instr = &sr->instr; + memset(instr, 0, sizeof(*instr)); + /* word 0 */ + instr->w0.tol = sr->dst.total_bytes; + instr->w0.exn = 0; + instr->w0.exbits = 0; + instr->w0.ca = 0; + if (xform->op == NITROX_COMP_OP_DECOMPRESS || + sr->op->flush_flag == RTE_COMP_FLUSH_FULL) + instr->w0.sf = 1; + else + instr->w0.sf = 0; + + instr->w0.ss = xform->level; + instr->w0.cc = xform->algo; + if (xform->op == NITROX_COMP_OP_COMPRESS && + sr->op->flush_flag == RTE_COMP_FLUSH_FINAL) + instr->w0.ef = 1; + else + instr->w0.ef = 0; + + instr->w0.bf = 1; + instr->w0.co = xform->op; + if (sr->dst.filled_sgls > 1) + instr->w0.ds = 1; + else + instr->w0.ds = 0; + + if (sr->src.filled_sgls > 1) + instr->w0.dg = 1; + else + instr->w0.dg = 0; + + instr->w0.hg = 0; + + /* word 1 */ + instr->w1.hl = 0; + if (sr->op->input_chksum != 0) + instr->w1.adlercrc32 = sr->op->input_chksum; + else if (xform->chksum_type == NITROX_CHKSUM_TYPE_ADLER32) + instr->w1.adlercrc32 = 1; + else if (xform->chksum_type == NITROX_CHKSUM_TYPE_CRC32) + instr->w1.adlercrc32 = 0; + + /* word 2 */ + instr->w2.cptr = 0; + + /* word 3 */ + instr->w3.hlen = 0; + instr->w3.hptr = 0; + + /* word 4 */ + if (sr->src.filled_sgls == 1) { + instr->w4.ilen = sr->src.sgl[0].s.length; + instr->w4.iptr = sr->src.sgl[0].s.addr; + iptr_msb = sr->src.addr_msb; + } else { + zip_addr.u64 = rte_malloc_virt2iova(sr->src.sgl); + instr->w4.ilen = sr->src.filled_sgls; + instr->w4.iptr = zip_addr.zda.addr; + iptr_msb = zip_addr.zda.addr_msb; + } + + /* word 5 */ + if (sr->dst.filled_sgls == 1) { + instr->w5.olen = sr->dst.sgl[0].s.length; + instr->w5.optr = sr->dst.sgl[0].s.addr; + optr_msb = sr->dst.addr_msb; + } else { + zip_addr.u64 = rte_malloc_virt2iova(sr->dst.sgl); + instr->w5.olen = sr->dst.filled_sgls; + instr->w5.optr = zip_addr.zda.addr; + optr_msb = zip_addr.zda.addr_msb; + } + + /* word 6 */ + memset(&sr->zip_res, 0, sizeof(sr->zip_res)); + zip_addr.u64 = rte_mempool_virt2iova(sr) + + offsetof(struct nitrox_softreq, zip_res); + instr->w6.rptr = zip_addr.zda.addr; + rptr_msb = zip_addr.zda.addr_msb; + + if (iptr_msb != optr_msb || iptr_msb != rptr_msb) { + NITROX_LOG(ERR, "addr_msb is not same for all addresses\n"); + return -ENOTSUP; + } + + /* word 7 */ + instr->w7.addr_msb = iptr_msb; + instr->w7.grp = 0; + + nitrox_zip_instr_to_b64(sr); + return 0; +} + +static int process_zip_request(struct nitrox_softreq *sr) +{ + int err; + + switch (sr->op->op_type) { + case RTE_COMP_OP_STATELESS: + err = process_zip_stateless(sr); + break; + default: + err = -EINVAL; + break; + } + + return err; +} + +int +nitrox_process_comp_req(struct rte_comp_op *op, struct nitrox_softreq *sr) +{ + int err; + + sr->op = op; + err = process_zip_request(sr); + if (unlikely(err)) + goto err_exit; + + sr->timeout = rte_get_timer_cycles() + CMD_TIMEOUT * rte_get_timer_hz(); + return 0; +err_exit: + if (err == -ENOMEM) + sr->op->status = RTE_COMP_OP_STATUS_ERROR; + else + sr->op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + + return err; +} + +static struct nitrox_zip_result zip_result_to_cpu64(struct nitrox_zip_result *r) +{ + struct nitrox_zip_result out_res; + + out_res.w2.u64 = rte_be_to_cpu_64(r->w2.u64); + out_res.w1.u64 = rte_be_to_cpu_64(r->w1.u64); + out_res.w0.u64 = rte_be_to_cpu_64(r->w0.u64); + return out_res; +} + +int +nitrox_check_comp_req(struct nitrox_softreq *sr, struct rte_comp_op **op) +{ + struct nitrox_zip_result zip_res; + int output_unused_bytes; + int err = 0; + + zip_res = zip_result_to_cpu64(&sr->zip_res); + if (zip_res.w2.compcode == NITROX_CC_NOTDONE) { + if (rte_get_timer_cycles() >= sr->timeout) { + NITROX_LOG(ERR, "Op timedout\n"); + sr->op->status = RTE_COMP_OP_STATUS_ERROR; + err = -ETIMEDOUT; + goto exit; + } else { + return -EAGAIN; + } + } + + if (unlikely(zip_res.w2.compcode != NITROX_CC_SUCCESS)) { + struct rte_comp_op *op = sr->op; + + NITROX_LOG(ERR, "Op dequeue error 0x%x\n", + zip_res.w2.compcode); + if (zip_res.w2.compcode == NITROX_CC_STOP || + zip_res.w2.compcode == NITROX_CC_DTRUNC) + op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; + else + op->status = RTE_COMP_OP_STATUS_ERROR; + + op->consumed = 0; + op->produced = 0; + err = -EFAULT; + goto exit; + } + + output_unused_bytes = sr->dst.total_bytes - zip_res.w1.tbyteswritten; + if (unlikely(sr->xform.op == NITROX_COMP_OP_DECOMPRESS && + output_unused_bytes < NITROX_ZIP_MAX_ONFSIZE)) { + NITROX_LOG(ERR, "TOL %d, Total bytes written %d\n", + sr->dst.total_bytes, zip_res.w1.tbyteswritten); + sr->op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; + sr->op->consumed = 0; + sr->op->produced = sr->dst.total_bytes - NITROX_ZIP_MAX_ONFSIZE; + err = -EIO; + goto exit; + } + + if (sr->xform.op == NITROX_COMP_OP_COMPRESS && + sr->op->flush_flag == RTE_COMP_FLUSH_FINAL && + zip_res.w2.exn) { + uint32_t datalen = zip_res.w1.tbyteswritten; + uint32_t off = sr->op->dst.offset; + struct rte_mbuf *m = sr->op->m_dst; + uint32_t mlen; + uint8_t *last_byte; + + for (; m && off > rte_pktmbuf_data_len(m); m = m->next) + off -= rte_pktmbuf_data_len(m); + + mlen = rte_pktmbuf_data_len(m) - off; + for (; m && (datalen > mlen); m = m->next) + datalen -= mlen; + + last_byte = rte_pktmbuf_mtod_offset(m, uint8_t *, datalen - 1); + *last_byte = zip_res.w2.exbits & 0xFF; + } + + sr->op->consumed = zip_res.w1.tbytesread; + sr->op->produced = zip_res.w1.tbyteswritten; + if (sr->xform.chksum_type == NITROX_CHKSUM_TYPE_CRC32) + sr->op->output_chksum = zip_res.w0.crc32; + else if (sr->xform.chksum_type == NITROX_CHKSUM_TYPE_ADLER32) + sr->op->output_chksum = zip_res.w0.adler32; + + sr->op->status = RTE_COMP_OP_STATUS_SUCCESS; + err = 0; +exit: + *op = sr->op; + return err; +} + +void * +nitrox_comp_instr_addr(struct nitrox_softreq *sr) +{ + return &sr->instr; +} + +static void req_pool_obj_free(struct rte_mempool *mp, void *opaque, void *obj, + unsigned int obj_idx) +{ + struct nitrox_softreq *sr; + + RTE_SET_USED(mp); + RTE_SET_USED(opaque); + RTE_SET_USED(obj_idx); + sr = obj; + rte_free(sr->src.sgl); + sr->src.sgl = NULL; + rte_free(sr->dst.sgl); + sr->dst.sgl = NULL; +} + +void +nitrox_comp_req_pool_free(struct rte_mempool *mp) +{ + rte_mempool_obj_iter(mp, req_pool_obj_free, NULL); + rte_mempool_free(mp); +} + +static void req_pool_obj_init(struct rte_mempool *mp, void *arg, void *obj, + unsigned int obj_idx) +{ + struct nitrox_softreq *sr; + int *err = arg; + + RTE_SET_USED(mp); + RTE_SET_USED(obj_idx); + sr = obj; + sr->src.sgl = rte_zmalloc_socket(NULL, + sizeof(*sr->src.sgl) * NITROX_ZIP_SGL_COUNT, + 8, mp->socket_id); + sr->dst.sgl = rte_zmalloc_socket(NULL, + sizeof(*sr->dst.sgl) * NITROX_ZIP_SGL_COUNT, + 8, mp->socket_id); + if (sr->src.sgl == NULL || sr->dst.sgl == NULL) { + NITROX_LOG(ERR, "Failed to allocate zip_sgl memory\n"); + *err = -ENOMEM; + } + + sr->src.nb_sgls = NITROX_ZIP_SGL_COUNT; + sr->src.filled_sgls = 0; + sr->dst.nb_sgls = NITROX_ZIP_SGL_COUNT; + sr->dst.filled_sgls = 0; +} + +struct rte_mempool * +nitrox_comp_req_pool_create(struct rte_compressdev *dev, uint32_t nobjs, + uint16_t qp_id, int socket_id) +{ + char softreq_pool_name[RTE_RING_NAMESIZE]; + struct rte_mempool *mp; + int err = 0; + + snprintf(softreq_pool_name, RTE_RING_NAMESIZE, "%s_sr_%d", + dev->data->name, qp_id); + mp = rte_mempool_create(softreq_pool_name, + RTE_ALIGN_MUL_CEIL(nobjs, 64), + sizeof(struct nitrox_softreq), + 64, 0, NULL, NULL, req_pool_obj_init, &err, + socket_id, 0); + if (unlikely(!mp)) + NITROX_LOG(ERR, "Failed to create req pool, qid %d, err %d\n", + qp_id, rte_errno); + + if (unlikely(err)) { + nitrox_comp_req_pool_free(mp); + return NULL; + } + + return mp; +} diff --git a/drivers/compress/nitrox/nitrox_comp_reqmgr.h b/drivers/compress/nitrox/nitrox_comp_reqmgr.h index 14f35a1e5b..07c65f0d5e 100644 --- a/drivers/compress/nitrox/nitrox_comp_reqmgr.h +++ b/drivers/compress/nitrox/nitrox_comp_reqmgr.h @@ -5,6 +5,8 @@ #ifndef _NITROX_COMP_REQMGR_H_ #define _NITROX_COMP_REQMGR_H_ +struct nitrox_softreq; + enum nitrox_comp_op { NITROX_COMP_OP_DECOMPRESS, NITROX_COMP_OP_COMPRESS, @@ -37,4 +39,12 @@ struct nitrox_comp_xform { enum nitrox_chksum_type chksum_type; }; +int nitrox_process_comp_req(struct rte_comp_op *op, struct nitrox_softreq *sr); +int nitrox_check_comp_req(struct nitrox_softreq *sr, struct rte_comp_op **op); +void *nitrox_comp_instr_addr(struct nitrox_softreq *sr); +struct rte_mempool *nitrox_comp_req_pool_create(struct rte_compressdev *cdev, + uint32_t nobjs, uint16_t qp_id, + int socket_id); +void nitrox_comp_req_pool_free(struct rte_mempool *mp); + #endif /* _NITROX_COMP_REQMGR_H_ */