From patchwork Sat Mar 2 09:38:13 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Nagadheeraj Rottela X-Patchwork-Id: 137800 X-Patchwork-Delegate: gakhil@marvell.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 106B443C30; Sat, 2 Mar 2024 10:39:34 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 6AD0042DDF; Sat, 2 Mar 2024 10:39:02 +0100 (CET) Received: from mx0b-0016f401.pphosted.com (mx0b-0016f401.pphosted.com [67.231.156.173]) by mails.dpdk.org (Postfix) with ESMTP id 6A60542DDC for ; Sat, 2 Mar 2024 10:39:00 +0100 (CET) Received: from pps.filterd (m0045851.ppops.net [127.0.0.1]) by mx0b-0016f401.pphosted.com (8.17.1.24/8.17.1.24) with ESMTP id 42289wp8032374; Sat, 2 Mar 2024 01:38:59 -0800 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com; h= from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding:content-type; s= pfpt0220; bh=JmNmm50tt3uOflXvDzdfSnMbIOGWuZ2r7qs+D/pkv+0=; b=Cg/ 1T/PpONk4olWmLL6kB0+4CCNK+yvY9kEhEgh2l19kfIFkF9ta24QQXRfUAAPcDu9 za43zGWnZ3dZlGhQyIzKl8G5G+0fmUvJUkUYXHVJxMvzAWZ/9YmS4brTQPTa7naJ j86MEory+mHPtrS9M18w0LNKLAlbGFFXh6EvsxBNc9BuhhHHcXlMygDkHBP2gh52 y+yh4IaQjFCHiw3aO7fg29xQAc8e2NqD/c1OG58WjrXlrkyxnLHS1OClEIEMKPcY M2E5yRg2Br7ToQ1oUHxRygkGIyXFbBue4uk1rlhxiJbtZnoOFCIFcq7jqgp5+3XA /59upqTsBBbblPzC8qA== Received: from dc5-exch05.marvell.com ([199.233.59.128]) by mx0b-0016f401.pphosted.com (PPS) with ESMTPS id 3wjfayafpu-1 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=NOT); Sat, 02 Mar 2024 01:38:59 -0800 (PST) Received: from DC5-EXCH05.marvell.com (10.69.176.209) by DC5-EXCH05.marvell.com (10.69.176.209) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.1258.12; Sat, 2 Mar 2024 01:38:57 -0800 Received: from hyd1399.caveonetworks.com.com (10.69.176.80) by DC5-EXCH05.marvell.com (10.69.176.209) with Microsoft SMTP Server id 15.2.1258.12 via Frontend Transport; Sat, 2 Mar 2024 01:38:55 -0800 From: Nagadheeraj Rottela To: , , CC: , Nagadheeraj Rottela Subject: [PATCH v5 7/7] compress/nitrox: support stateful request Date: Sat, 2 Mar 2024 15:08:13 +0530 Message-ID: <20240302093813.14922-8-rnagadheeraj@marvell.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20240302093813.14922-1-rnagadheeraj@marvell.com> References: <20240302093813.14922-1-rnagadheeraj@marvell.com> MIME-Version: 1.0 X-Proofpoint-ORIG-GUID: iaIGvmyUCHUdV4bud1x5gQ4E26z9eaeb X-Proofpoint-GUID: iaIGvmyUCHUdV4bud1x5gQ4E26z9eaeb X-Proofpoint-Virus-Version: vendor=baseguard engine=ICAP:2.0.272,Aquarius:18.0.1011,Hydra:6.0.619,FMLib:17.11.176.26 definitions=2024-03-02_04,2024-03-01_03,2023-05-22_02 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Implement enqueue and dequeue burst operations for stateful request support. Signed-off-by: Nagadheeraj Rottela --- drivers/compress/nitrox/nitrox_comp.c | 97 +++- drivers/compress/nitrox/nitrox_comp.h | 1 + drivers/compress/nitrox/nitrox_comp_reqmgr.c | 550 ++++++++++++++++--- drivers/compress/nitrox/nitrox_comp_reqmgr.h | 8 + 4 files changed, 576 insertions(+), 80 deletions(-) diff --git a/drivers/compress/nitrox/nitrox_comp.c b/drivers/compress/nitrox/nitrox_comp.c index 0ea5ed43ed..97d2c4a0e8 100644 --- a/drivers/compress/nitrox/nitrox_comp.c +++ b/drivers/compress/nitrox/nitrox_comp.c @@ -32,7 +32,9 @@ static const struct rte_compressdev_capabilities RTE_COMP_FF_SHAREABLE_PRIV_XFORM | RTE_COMP_FF_OOP_SGL_IN_SGL_OUT | RTE_COMP_FF_OOP_SGL_IN_LB_OUT | - RTE_COMP_FF_OOP_LB_IN_SGL_OUT, + RTE_COMP_FF_OOP_LB_IN_SGL_OUT | + RTE_COMP_FF_STATEFUL_COMPRESSION | + RTE_COMP_FF_STATEFUL_DECOMPRESSION, .window_size = { .min = NITROX_COMP_WINDOW_SIZE_MIN, .max = NITROX_COMP_WINDOW_SIZE_MAX, @@ -334,6 +336,13 @@ static int nitrox_comp_private_xform_create(struct rte_compressdev *dev, goto err_exit; } + nxform->context = NULL; + nxform->history_window = NULL; + nxform->window_size = 0; + nxform->hlen = 0; + nxform->exn = 0; + nxform->exbits = 0; + nxform->bf = true; return 0; err_exit: memset(nxform, 0, sizeof(*nxform)); @@ -357,6 +366,74 @@ static int nitrox_comp_private_xform_free(struct rte_compressdev *dev, return 0; } +static int nitrox_comp_stream_free(struct rte_compressdev *dev, void *stream) +{ + struct nitrox_comp_xform *nxform = stream; + + if (unlikely(nxform == NULL)) + return -EINVAL; + + rte_free(nxform->history_window); + nxform->history_window = NULL; + rte_free(nxform->context); + nxform->context = NULL; + return nitrox_comp_private_xform_free(dev, stream); +} + +static int nitrox_comp_stream_create(struct rte_compressdev *dev, + const struct rte_comp_xform *xform, void **stream) +{ + int err; + struct nitrox_comp_xform *nxform; + struct nitrox_comp_device *comp_dev = dev->data->dev_private; + + err = nitrox_comp_private_xform_create(dev, xform, stream); + if (unlikely(err)) + return err; + + nxform = *stream; + if (xform->type == RTE_COMP_COMPRESS) { + uint8_t window_size = xform->compress.window_size; + + if (unlikely(window_size < NITROX_COMP_WINDOW_SIZE_MIN || + window_size > NITROX_COMP_WINDOW_SIZE_MAX)) { + NITROX_LOG(ERR, "Invalid window size %d\n", + window_size); + return -EINVAL; + } + + if (window_size == NITROX_COMP_WINDOW_SIZE_MAX) + nxform->window_size = NITROX_CONSTANTS_MAX_SEARCH_DEPTH; + else + nxform->window_size = RTE_BIT32(window_size); + } else { + nxform->window_size = NITROX_DEFAULT_DEFLATE_SEARCH_DEPTH; + } + + nxform->history_window = rte_zmalloc_socket(NULL, nxform->window_size, + 8, comp_dev->xform_pool->socket_id); + if (unlikely(nxform->history_window == NULL)) { + err = -ENOMEM; + goto err_exit; + } + + if (xform->type == RTE_COMP_COMPRESS) + return 0; + + nxform->context = rte_zmalloc_socket(NULL, + NITROX_DECOMP_CTX_SIZE, 8, + comp_dev->xform_pool->socket_id); + if (unlikely(nxform->context == NULL)) { + err = -ENOMEM; + goto err_exit; + } + + return 0; +err_exit: + nitrox_comp_stream_free(dev, *stream); + return err; +} + static int nitrox_enq_single_op(struct nitrox_qp *qp, struct rte_comp_op *op) { struct nitrox_softreq *sr; @@ -371,8 +448,12 @@ static int nitrox_enq_single_op(struct nitrox_qp *qp, struct rte_comp_op *op) return err; } - nitrox_qp_enqueue(qp, nitrox_comp_instr_addr(sr), sr); - return 0; + if (op->status == RTE_COMP_OP_STATUS_SUCCESS) + err = nitrox_qp_enqueue_sr(qp, sr); + else + nitrox_qp_enqueue(qp, nitrox_comp_instr_addr(sr), sr); + + return err; } static uint16_t nitrox_comp_dev_enq_burst(void *queue_pair, @@ -382,6 +463,7 @@ static uint16_t nitrox_comp_dev_enq_burst(void *queue_pair, struct nitrox_qp *qp = queue_pair; uint16_t free_slots = 0; uint16_t cnt = 0; + uint16_t dbcnt = 0; bool err = false; free_slots = nitrox_qp_free_count(qp); @@ -393,9 +475,12 @@ static uint16_t nitrox_comp_dev_enq_burst(void *queue_pair, err = true; break; } + + if (ops[cnt]->status != RTE_COMP_OP_STATUS_SUCCESS) + dbcnt++; } - nitrox_ring_dbell(qp, cnt); + nitrox_ring_dbell(qp, dbcnt); qp->stats.enqueued_count += cnt; if (unlikely(err)) qp->stats.enqueue_err_count++; @@ -458,8 +543,8 @@ static struct rte_compressdev_ops nitrox_compressdev_ops = { .private_xform_create = nitrox_comp_private_xform_create, .private_xform_free = nitrox_comp_private_xform_free, - .stream_create = NULL, - .stream_free = NULL + .stream_create = nitrox_comp_stream_create, + .stream_free = nitrox_comp_stream_free, }; int diff --git a/drivers/compress/nitrox/nitrox_comp.h b/drivers/compress/nitrox/nitrox_comp.h index e49debaf6b..83e5902076 100644 --- a/drivers/compress/nitrox/nitrox_comp.h +++ b/drivers/compress/nitrox/nitrox_comp.h @@ -8,6 +8,7 @@ #define COMPRESSDEV_NAME_NITROX_PMD compress_nitrox #define NITROX_DECOMP_CTX_SIZE 2048 #define NITROX_CONSTANTS_MAX_SEARCH_DEPTH 31744 +#define NITROX_DEFAULT_DEFLATE_SEARCH_DEPTH 32768 #define NITROX_COMP_WINDOW_SIZE_MIN 1 #define NITROX_COMP_WINDOW_SIZE_MAX 15 #define NITROX_COMP_LEVEL_LOWEST_START 1 diff --git a/drivers/compress/nitrox/nitrox_comp_reqmgr.c b/drivers/compress/nitrox/nitrox_comp_reqmgr.c index 5ad1a4439a..0a25672d6e 100644 --- a/drivers/compress/nitrox/nitrox_comp_reqmgr.c +++ b/drivers/compress/nitrox/nitrox_comp_reqmgr.c @@ -5,11 +5,13 @@ #include #include #include +#include #include "nitrox_comp_reqmgr.h" #include "nitrox_logs.h" #include "rte_comp.h" +#define NITROX_INSTR_BUFFER_DEBUG 0 #define NITROX_ZIP_SGL_COUNT 16 #define NITROX_ZIP_MAX_ZPTRS 2048 #define NITROX_ZIP_MAX_DATASIZE ((1 << 24) - 1) @@ -307,10 +309,217 @@ struct nitrox_softreq { struct rte_comp_op *op; struct nitrox_sgtable src; struct nitrox_sgtable dst; - struct nitrox_comp_xform xform; uint64_t timeout; }; +#if NITROX_INSTR_BUFFER_DEBUG +static void nitrox_dump_databuf(const char *name, struct rte_mbuf *m, + uint32_t off, uint32_t datalen) +{ + uint32_t mlen; + + if (!rte_log_can_log(nitrox_logtype, RTE_LOG_DEBUG)) + return; + + for (; m && off > rte_pktmbuf_data_len(m); m = m->next) + off -= rte_pktmbuf_data_len(m); + + mlen = rte_pktmbuf_data_len(m) - off; + if (datalen <= mlen) + mlen = datalen; + + rte_hexdump(rte_log_get_stream(), name, + rte_pktmbuf_mtod_offset(m, char *, off), mlen); + for (m = m->next; m && datalen; m = m->next) { + mlen = rte_pktmbuf_data_len(m) < datalen ? + rte_pktmbuf_data_len(m) : datalen; + rte_hexdump(rte_log_get_stream(), name, + rte_pktmbuf_mtod(m, char *), mlen); + } + + NITROX_LOG(DEBUG, "\n"); +} + +static void nitrox_dump_zip_instr(struct nitrox_zip_instr *instr, + union nitrox_zip_zptr *hptr_arr, + union nitrox_zip_zptr *iptr_arr, + union nitrox_zip_zptr *optr_arr) +{ + uint64_t value; + int i = 0; + + NITROX_LOG(DEBUG, "\nZIP instruction..(%p)\n", instr); + NITROX_LOG(DEBUG, "\tWORD0 = 0x%016"PRIx64"\n", instr->w0.u64); + NITROX_LOG(DEBUG, "\t\tTOL = %d\n", instr->w0.tol); + NITROX_LOG(DEBUG, "\t\tEXNUM = %d\n", instr->w0.exn); + NITROX_LOG(DEBUG, "\t\tEXBITS = %x\n", instr->w0.exbits); + NITROX_LOG(DEBUG, "\t\tCA = %d\n", instr->w0.ca); + NITROX_LOG(DEBUG, "\t\tSF = %d\n", instr->w0.sf); + NITROX_LOG(DEBUG, "\t\tSS = %d\n", instr->w0.ss); + NITROX_LOG(DEBUG, "\t\tCC = %d\n", instr->w0.cc); + NITROX_LOG(DEBUG, "\t\tEF = %d\n", instr->w0.ef); + NITROX_LOG(DEBUG, "\t\tBF = %d\n", instr->w0.bf); + NITROX_LOG(DEBUG, "\t\tCO = %d\n", instr->w0.co); + NITROX_LOG(DEBUG, "\t\tDS = %d\n", instr->w0.ds); + NITROX_LOG(DEBUG, "\t\tDG = %d\n", instr->w0.dg); + NITROX_LOG(DEBUG, "\t\tHG = %d\n", instr->w0.hg); + NITROX_LOG(DEBUG, "\n"); + + NITROX_LOG(DEBUG, "\tWORD1 = 0x%016"PRIx64"\n", instr->w1.u64); + NITROX_LOG(DEBUG, "\t\tHL = %d\n", instr->w1.hl); + NITROX_LOG(DEBUG, "\t\tADLERCRC32 = 0x%08x\n", instr->w1.adlercrc32); + NITROX_LOG(DEBUG, "\n"); + + value = instr->w2.cptr; + NITROX_LOG(DEBUG, "\tWORD2 = 0x%016"PRIx64"\n", instr->w2.u64); + NITROX_LOG(DEBUG, "\t\tCPTR = 0x%11"PRIx64"\n", value); + NITROX_LOG(DEBUG, "\n"); + + value = instr->w3.hptr; + NITROX_LOG(DEBUG, "\tWORD3 = 0x%016"PRIx64"\n", instr->w3.u64); + NITROX_LOG(DEBUG, "\t\tHLEN = %d\n", instr->w3.hlen); + NITROX_LOG(DEBUG, "\t\tHPTR = 0x%11"PRIx64"\n", value); + + if (instr->w0.hg && hptr_arr) { + for (i = 0; i < instr->w3.hlen; i++) { + value = hptr_arr[i].s.addr; + NITROX_LOG(DEBUG, "\t\t\tZPTR[%d] : Length = %d Addr = 0x%11"PRIx64"\n", + i, hptr_arr[i].s.length, value); + } + } + + NITROX_LOG(DEBUG, "\n"); + + value = instr->w4.iptr; + NITROX_LOG(DEBUG, "\tWORD4 = 0x%016"PRIx64"\n", instr->w4.u64); + NITROX_LOG(DEBUG, "\t\tILEN = %d\n", instr->w4.ilen); + NITROX_LOG(DEBUG, "\t\tIPTR = 0x%11"PRIx64"\n", value); + if (instr->w0.dg && iptr_arr) { + for (i = 0; i < instr->w4.ilen; i++) { + value = iptr_arr[i].s.addr; + NITROX_LOG(DEBUG, "\t\t\tZPTR[%d] : Length = %d Addr = 0x%11"PRIx64"\n", + i, iptr_arr[i].s.length, value); + } + } + + NITROX_LOG(DEBUG, "\n"); + + value = instr->w5.optr; + NITROX_LOG(DEBUG, "\tWORD5 = 0x%016"PRIx64"\n", instr->w5.u64); + NITROX_LOG(DEBUG, "\t\t OLEN = %d\n", instr->w5.olen); + NITROX_LOG(DEBUG, "\t\t OPTR = 0x%11"PRIx64"\n", value); + if (instr->w0.ds && optr_arr) { + for (i = 0; i < instr->w5.olen; i++) { + value = optr_arr[i].s.addr; + NITROX_LOG(DEBUG, "\t\t\tZPTR[%d] : Length = %d Addr = 0x%11"PRIx64"\n", + i, optr_arr[i].s.length, value); + } + } + + NITROX_LOG(DEBUG, "\n"); + + value = instr->w6.rptr; + NITROX_LOG(DEBUG, "\tWORD6 = 0x%016"PRIx64"\n", instr->w6.u64); + NITROX_LOG(DEBUG, "\t\tRPTR = 0x%11"PRIx64"\n", value); + NITROX_LOG(DEBUG, "\n"); + + NITROX_LOG(DEBUG, "\tWORD7 = 0x%016"PRIx64"\n", instr->w7.u64); + NITROX_LOG(DEBUG, "\t\tGRP = %x\n", instr->w7.grp); + NITROX_LOG(DEBUG, "\t\tADDR_MSB = 0x%5x\n", instr->w7.addr_msb); + NITROX_LOG(DEBUG, "\n"); +} + +static void nitrox_dump_zip_result(struct nitrox_zip_instr *instr, + struct nitrox_zip_result *result) +{ + NITROX_LOG(DEBUG, "ZIP result..(instr %p)\n", instr); + NITROX_LOG(DEBUG, "\tWORD0 = 0x%016"PRIx64"\n", result->w0.u64); + NITROX_LOG(DEBUG, "\t\tCRC32 = 0x%8x\n", result->w0.crc32); + NITROX_LOG(DEBUG, "\t\tADLER32 = 0x%8x\n", result->w0.adler32); + NITROX_LOG(DEBUG, "\n"); + + NITROX_LOG(DEBUG, "\tWORD1 = 0x%016"PRIx64"\n", result->w1.u64); + NITROX_LOG(DEBUG, "\t\tTBYTESWRITTEN = %u\n", result->w1.tbyteswritten); + NITROX_LOG(DEBUG, "\t\tTBYTESREAD = %u\n", result->w1.tbytesread); + NITROX_LOG(DEBUG, "\n"); + + NITROX_LOG(DEBUG, "\tWORD2 = 0x%016"PRIx64"\n", result->w2.u64); + NITROX_LOG(DEBUG, "\t\tTBITS = %u\n", result->w2.tbits); + NITROX_LOG(DEBUG, "\t\tEXN = %d\n", result->w2.exn); + NITROX_LOG(DEBUG, "\t\tEBITS = %x\n", result->w2.exbits); + NITROX_LOG(DEBUG, "\t\tEF = %d\n", result->w2.ef); + NITROX_LOG(DEBUG, "\t\tCOMPCODE = 0x%2x\n", result->w2.compcode); + NITROX_LOG(DEBUG, "\n"); +} +#else +#define nitrox_dump_databuf(name, m, off, datalen) +#define nitrox_dump_zip_instr(instr, hptr_arr, iptr_arr, optr_arr) +#define nitrox_dump_zip_result(instr, result) +#endif + +static int handle_zero_length_compression(struct nitrox_softreq *sr, + struct nitrox_comp_xform *xform) +{ + union { + uint32_t num; + uint8_t bytes[4]; + } fblk; + uint32_t dstlen, rlen; + struct rte_mbuf *m; + uint32_t off; + uint32_t mlen; + uint32_t i = 0; + uint8_t *ptr; + + fblk.num = xform->exn ? (xform->exbits & 0x7F) : 0; + fblk.num |= (0x3 << xform->exn); + memset(&sr->zip_res, 0, sizeof(sr->zip_res)); + sr->zip_res.w1.tbytesread = xform->hlen; + sr->zip_res.w1.tbyteswritten = 2; + sr->zip_res.w2.ef = 1; + if (xform->exn == 7) + sr->zip_res.w1.tbyteswritten++; + + rlen = sr->zip_res.w1.tbyteswritten; + dstlen = rte_pktmbuf_pkt_len(sr->op->m_dst) - sr->op->dst.offset; + if (unlikely(dstlen < rlen)) + return -EIO; + + off = sr->op->dst.offset; + for (m = sr->op->m_dst; m && off > rte_pktmbuf_data_len(m); m = m->next) + off -= rte_pktmbuf_data_len(m); + + if (unlikely(!m)) + return -EIO; + + mlen = rte_pktmbuf_data_len(m) - off; + if (rlen <= mlen) + mlen = rlen; + + ptr = rte_pktmbuf_mtod_offset(m, uint8_t *, off); + memcpy(ptr, fblk.bytes, mlen); + i += mlen; + rlen -= mlen; + for (m = m->next; m && rlen; m = m->next) { + mlen = rte_pktmbuf_data_len(m) < rlen ? + rte_pktmbuf_data_len(m) : rlen; + ptr = rte_pktmbuf_mtod(m, uint8_t *); + memcpy(ptr, &fblk.bytes[i], mlen); + i += mlen; + rlen -= mlen; + } + + if (unlikely(rlen != 0)) + return -EIO; + + sr->zip_res.w2.compcode = NITROX_CC_SUCCESS; + sr->op->status = RTE_COMP_OP_STATUS_SUCCESS; + sr->zip_res.w0.u64 = rte_cpu_to_be_64(sr->zip_res.w0.u64); + sr->zip_res.w1.u64 = rte_cpu_to_be_64(sr->zip_res.w1.u64); + sr->zip_res.w2.u64 = rte_cpu_to_be_64(sr->zip_res.w2.u64); + return 0; +} + static int create_sglist_from_mbuf(struct nitrox_sgtable *sgtbl, struct rte_mbuf *mbuf, uint32_t off, uint32_t datalen, uint8_t extra_segs, @@ -398,10 +607,12 @@ static int create_sglist_from_mbuf(struct nitrox_sgtable *sgtbl, return 0; } -static int softreq_init(struct nitrox_softreq *sr) +static int softreq_init(struct nitrox_softreq *sr, + struct nitrox_comp_xform *xform) { struct rte_mempool *mp; int err; + bool need_decomp_threshold; mp = rte_mempool_from_obj(sr); if (unlikely(mp == NULL)) @@ -413,15 +624,17 @@ static int softreq_init(struct nitrox_softreq *sr) if (unlikely(err)) return err; + need_decomp_threshold = (sr->op->op_type == RTE_COMP_OP_STATELESS && + xform->op == NITROX_COMP_OP_DECOMPRESS); err = create_sglist_from_mbuf(&sr->dst, sr->op->m_dst, sr->op->dst.offset, rte_pktmbuf_pkt_len(sr->op->m_dst) - sr->op->dst.offset, - (sr->xform.op == NITROX_COMP_OP_DECOMPRESS) ? 1 : 0, + need_decomp_threshold ? 1 : 0, mp->socket_id); if (unlikely(err)) return err; - if (sr->xform.op == NITROX_COMP_OP_DECOMPRESS) { + if (need_decomp_threshold) { struct nitrox_zip_iova_addr zip_addr; int i; @@ -459,12 +672,12 @@ static void nitrox_zip_instr_to_b64(struct nitrox_softreq *sr) instr->w7.u64 = rte_cpu_to_be_64(instr->w7.u64); } -static int process_zip_stateless(struct nitrox_softreq *sr) +static int process_zip_request(struct nitrox_softreq *sr) { struct nitrox_zip_instr *instr; struct nitrox_comp_xform *xform; struct nitrox_zip_iova_addr zip_addr; - uint64_t iptr_msb, optr_msb, rptr_msb; + uint64_t iptr_msb, optr_msb, rptr_msb, cptr_msb, hptr_msb; int err; xform = sr->op->private_xform; @@ -473,7 +686,14 @@ static int process_zip_stateless(struct nitrox_softreq *sr) return -EINVAL; } - if (unlikely(xform->op == NITROX_COMP_OP_COMPRESS && + if (unlikely(sr->op->op_type == RTE_COMP_OP_STATEFUL && + xform->op == NITROX_COMP_OP_COMPRESS && + sr->op->flush_flag == RTE_COMP_FLUSH_FINAL && + sr->op->src.length == 0)) + return handle_zero_length_compression(sr, xform); + + if (unlikely(sr->op->op_type == RTE_COMP_OP_STATELESS && + xform->op == NITROX_COMP_OP_COMPRESS && sr->op->flush_flag != RTE_COMP_FLUSH_FULL && sr->op->flush_flag != RTE_COMP_FLUSH_FINAL)) { NITROX_LOG(ERR, "Invalid flush flag %d in stateless op\n", @@ -481,8 +701,7 @@ static int process_zip_stateless(struct nitrox_softreq *sr) return -EINVAL; } - sr->xform = *xform; - err = softreq_init(sr); + err = softreq_init(sr, xform); if (unlikely(err)) return err; @@ -490,10 +709,11 @@ static int process_zip_stateless(struct nitrox_softreq *sr) memset(instr, 0, sizeof(*instr)); /* word 0 */ instr->w0.tol = sr->dst.total_bytes; - instr->w0.exn = 0; - instr->w0.exbits = 0; + instr->w0.exn = xform->exn; + instr->w0.exbits = xform->exbits; instr->w0.ca = 0; if (xform->op == NITROX_COMP_OP_DECOMPRESS || + sr->op->flush_flag == RTE_COMP_FLUSH_SYNC || sr->op->flush_flag == RTE_COMP_FLUSH_FULL) instr->w0.sf = 1; else @@ -501,13 +721,12 @@ static int process_zip_stateless(struct nitrox_softreq *sr) instr->w0.ss = xform->level; instr->w0.cc = xform->algo; - if (xform->op == NITROX_COMP_OP_COMPRESS && - sr->op->flush_flag == RTE_COMP_FLUSH_FINAL) + if (sr->op->flush_flag == RTE_COMP_FLUSH_FINAL) instr->w0.ef = 1; else instr->w0.ef = 0; - instr->w0.bf = 1; + instr->w0.bf = xform->bf; instr->w0.co = xform->op; if (sr->dst.filled_sgls > 1) instr->w0.ds = 1; @@ -522,8 +741,11 @@ static int process_zip_stateless(struct nitrox_softreq *sr) instr->w0.hg = 0; /* word 1 */ - instr->w1.hl = 0; - if (sr->op->input_chksum != 0) + instr->w1.hl = xform->hlen; + if (sr->op->op_type == RTE_COMP_OP_STATEFUL && !xform->bf) + instr->w1.adlercrc32 = xform->chksum; + else if (sr->op->op_type == RTE_COMP_OP_STATELESS && + sr->op->input_chksum != 0) instr->w1.adlercrc32 = sr->op->input_chksum; else if (xform->chksum_type == NITROX_CHKSUM_TYPE_ADLER32) instr->w1.adlercrc32 = 1; @@ -531,11 +753,23 @@ static int process_zip_stateless(struct nitrox_softreq *sr) instr->w1.adlercrc32 = 0; /* word 2 */ - instr->w2.cptr = 0; + if (xform->context) + zip_addr.u64 = rte_malloc_virt2iova(xform->context); + else + zip_addr.u64 = 0; + + instr->w2.cptr = zip_addr.zda.addr; + cptr_msb = zip_addr.zda.addr_msb; /* word 3 */ - instr->w3.hlen = 0; - instr->w3.hptr = 0; + instr->w3.hlen = xform->hlen; + if (xform->history_window) + zip_addr.u64 = rte_malloc_virt2iova(xform->history_window); + else + zip_addr.u64 = 0; + + instr->w3.hptr = zip_addr.zda.addr; + hptr_msb = zip_addr.zda.addr_msb; /* word 4 */ if (sr->src.filled_sgls == 1) { @@ -568,7 +802,9 @@ static int process_zip_stateless(struct nitrox_softreq *sr) instr->w6.rptr = zip_addr.zda.addr; rptr_msb = zip_addr.zda.addr_msb; - if (iptr_msb != optr_msb || iptr_msb != rptr_msb) { + if (unlikely(iptr_msb != optr_msb || iptr_msb != rptr_msb || + (xform->history_window && (iptr_msb != hptr_msb)) || + (xform->context && (iptr_msb != cptr_msb)))) { NITROX_LOG(ERR, "addr_msb is not same for all addresses\n"); return -ENOTSUP; } @@ -577,32 +813,20 @@ static int process_zip_stateless(struct nitrox_softreq *sr) instr->w7.addr_msb = iptr_msb; instr->w7.grp = 0; + nitrox_dump_zip_instr(instr, NULL, sr->src.sgl, sr->dst.sgl); + nitrox_dump_databuf("IN", sr->op->m_src, sr->op->src.offset, + sr->op->src.length); nitrox_zip_instr_to_b64(sr); return 0; } -static int process_zip_request(struct nitrox_softreq *sr) -{ - int err; - - switch (sr->op->op_type) { - case RTE_COMP_OP_STATELESS: - err = process_zip_stateless(sr); - break; - default: - err = -EINVAL; - break; - } - - return err; -} - int nitrox_process_comp_req(struct rte_comp_op *op, struct nitrox_softreq *sr) { int err; sr->op = op; + sr->op->status = RTE_COMP_OP_STATUS_NOT_PROCESSED; err = process_zip_request(sr); if (unlikely(err)) goto err_exit; @@ -628,55 +852,239 @@ static struct nitrox_zip_result zip_result_to_cpu64(struct nitrox_zip_result *r) return out_res; } -int -nitrox_check_comp_req(struct nitrox_softreq *sr, struct rte_comp_op **op) +static int post_process_zip_stateless(struct nitrox_softreq *sr, + struct nitrox_comp_xform *xform, + struct nitrox_zip_result *zip_res) { - struct nitrox_zip_result zip_res; int output_unused_bytes; - int err = 0; - - zip_res = zip_result_to_cpu64(&sr->zip_res); - if (zip_res.w2.compcode == NITROX_CC_NOTDONE) { - if (rte_get_timer_cycles() >= sr->timeout) { - NITROX_LOG(ERR, "Op timedout\n"); - sr->op->status = RTE_COMP_OP_STATUS_ERROR; - err = -ETIMEDOUT; - goto exit; - } else { - return -EAGAIN; - } - } - if (unlikely(zip_res.w2.compcode != NITROX_CC_SUCCESS)) { + if (unlikely(zip_res->w2.compcode != NITROX_CC_SUCCESS)) { struct rte_comp_op *op = sr->op; - NITROX_LOG(ERR, "Op dequeue error 0x%x\n", - zip_res.w2.compcode); - if (zip_res.w2.compcode == NITROX_CC_STOP || - zip_res.w2.compcode == NITROX_CC_DTRUNC) + NITROX_LOG(ERR, "Dequeue error 0x%x\n", + zip_res->w2.compcode); + if (zip_res->w2.compcode == NITROX_CC_STOP || + zip_res->w2.compcode == NITROX_CC_DTRUNC) op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; else op->status = RTE_COMP_OP_STATUS_ERROR; op->consumed = 0; op->produced = 0; - err = -EFAULT; - goto exit; + return -EFAULT; } - output_unused_bytes = sr->dst.total_bytes - zip_res.w1.tbyteswritten; - if (unlikely(sr->xform.op == NITROX_COMP_OP_DECOMPRESS && + output_unused_bytes = sr->dst.total_bytes - zip_res->w1.tbyteswritten; + if (unlikely(xform->op == NITROX_COMP_OP_DECOMPRESS && output_unused_bytes < NITROX_ZIP_MAX_ONFSIZE)) { NITROX_LOG(ERR, "TOL %d, Total bytes written %d\n", - sr->dst.total_bytes, zip_res.w1.tbyteswritten); + sr->dst.total_bytes, zip_res->w1.tbyteswritten); sr->op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; sr->op->consumed = 0; sr->op->produced = sr->dst.total_bytes - NITROX_ZIP_MAX_ONFSIZE; - err = -EIO; - goto exit; + return -EIO; + } + + if (xform->chksum_type == NITROX_CHKSUM_TYPE_CRC32) + sr->op->output_chksum = zip_res->w0.crc32; + else if (xform->chksum_type == NITROX_CHKSUM_TYPE_ADLER32) + sr->op->output_chksum = zip_res->w0.adler32; + + sr->op->consumed = RTE_MIN(sr->op->src.length, + (uint32_t)zip_res->w1.tbytesread); + sr->op->produced = zip_res->w1.tbyteswritten; + sr->op->status = RTE_COMP_OP_STATUS_SUCCESS; + return 0; +} + +static int update_history(struct rte_mbuf *mbuf, uint32_t off, uint16_t datalen, + uint8_t *dst) +{ + struct rte_mbuf *m; + uint32_t mlen; + uint16_t copied = 0; + + for (m = mbuf; m && off > rte_pktmbuf_data_len(m); m = m->next) + off -= rte_pktmbuf_data_len(m); + + if (unlikely(!m)) { + NITROX_LOG(ERR, "Failed to update history. Invalid mbuf\n"); + return -EINVAL; + } + + mlen = rte_pktmbuf_data_len(m) - off; + if (datalen <= mlen) + mlen = datalen; + + memcpy(&dst[copied], rte_pktmbuf_mtod_offset(m, char *, off), mlen); + copied += mlen; + datalen -= mlen; + for (m = m->next; m && datalen; m = m->next) { + mlen = rte_pktmbuf_data_len(m) < datalen ? + rte_pktmbuf_data_len(m) : datalen; + memcpy(&dst[copied], rte_pktmbuf_mtod(m, char *), mlen); + copied += mlen; + datalen -= mlen; + } + + if (unlikely(datalen != 0)) { + NITROX_LOG(ERR, "Failed to update history. Invalid datalen\n"); + return -EINVAL; + } + + return 0; +} + +static void reset_nitrox_xform(struct nitrox_comp_xform *xform) +{ + xform->hlen = 0; + xform->exn = 0; + xform->exbits = 0; + xform->bf = true; +} + +static int post_process_zip_stateful(struct nitrox_softreq *sr, + struct nitrox_comp_xform *xform, + struct nitrox_zip_result *zip_res) +{ + uint32_t bytesread = 0; + uint32_t chksum = 0; + + if (unlikely(zip_res->w2.compcode == NITROX_CC_DTRUNC)) { + sr->op->consumed = 0; + sr->op->produced = 0; + xform->hlen = 0; + sr->op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE; + NITROX_LOG(ERR, "Dequeue compress DTRUNC error\n"); + return 0; + } else if (unlikely(zip_res->w2.compcode == NITROX_CC_STOP)) { + sr->op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE; + NITROX_LOG(NOTICE, "Dequeue decompress dynamic STOP\n"); + } else if (zip_res->w2.compcode == NITROX_CC_SUCCESS) { + sr->op->status = RTE_COMP_OP_STATUS_SUCCESS; + } else { + xform->hlen = 0; + xform->exn = 0; + xform->exbits = 0; + xform->bf = true; + sr->op->status = RTE_COMP_OP_STATUS_ERROR; + NITROX_LOG(ERR, "Dequeue error 0x%x\n", + zip_res->w2.compcode); + return -EFAULT; + } + + if (xform->op == NITROX_COMP_OP_COMPRESS) { + if (zip_res->w1.tbytesread < xform->hlen) { + NITROX_LOG(ERR, "Invalid bytesread\n"); + reset_nitrox_xform(xform); + sr->op->status = RTE_COMP_OP_STATUS_ERROR; + return -EFAULT; + } + + bytesread = zip_res->w1.tbytesread - xform->hlen; + } else { + bytesread = RTE_MIN(sr->op->src.length, + (uint32_t)zip_res->w1.tbytesread); + } + + if ((xform->op == NITROX_COMP_OP_COMPRESS && + (sr->op->flush_flag == RTE_COMP_FLUSH_NONE || + sr->op->flush_flag == RTE_COMP_FLUSH_SYNC)) || + (xform->op == NITROX_COMP_OP_DECOMPRESS && !zip_res->w2.ef)) { + struct rte_mbuf *mbuf; + uint32_t pktlen, m_off; + int err; + + if (xform->op == NITROX_COMP_OP_COMPRESS) { + mbuf = sr->op->m_src; + pktlen = bytesread; + m_off = sr->op->src.offset; + } else { + mbuf = sr->op->m_dst; + pktlen = zip_res->w1.tbyteswritten; + m_off = sr->op->dst.offset; + } + + if (pktlen >= xform->window_size) { + m_off += pktlen - xform->window_size; + err = update_history(mbuf, m_off, xform->window_size, + xform->history_window); + xform->hlen = xform->window_size; + } else if ((xform->hlen + pktlen) <= xform->window_size) { + err = update_history(mbuf, m_off, pktlen, + &xform->history_window[xform->hlen]); + xform->hlen += pktlen; + } else { + uint16_t shift_off, shift_len; + + shift_off = pktlen + xform->hlen - xform->window_size; + shift_len = xform->hlen - shift_off; + memmove(xform->history_window, + &xform->history_window[shift_off], + shift_len); + err = update_history(mbuf, m_off, pktlen, + &xform->history_window[shift_len]); + xform->hlen = xform->window_size; + + } + + if (unlikely(err)) { + sr->op->status = RTE_COMP_OP_STATUS_ERROR; + return err; + } + + if (xform->op == NITROX_COMP_OP_COMPRESS) { + xform->exn = zip_res->w2.exn; + xform->exbits = zip_res->w2.exbits; + } + + xform->bf = false; + } else { + reset_nitrox_xform(xform); } - if (sr->xform.op == NITROX_COMP_OP_COMPRESS && + if (xform->chksum_type == NITROX_CHKSUM_TYPE_CRC32) + chksum = zip_res->w0.crc32; + else if (xform->chksum_type == NITROX_CHKSUM_TYPE_ADLER32) + chksum = zip_res->w0.adler32; + + if (xform->bf) + sr->op->output_chksum = chksum; + else + xform->chksum = chksum; + + sr->op->consumed = bytesread; + sr->op->produced = zip_res->w1.tbyteswritten; + return 0; +} + +int +nitrox_check_comp_req(struct nitrox_softreq *sr, struct rte_comp_op **op) +{ + struct nitrox_zip_result zip_res; + struct nitrox_comp_xform *xform; + int err = 0; + + zip_res = zip_result_to_cpu64(&sr->zip_res); + if (zip_res.w2.compcode == NITROX_CC_NOTDONE) { + if (rte_get_timer_cycles() >= sr->timeout) { + NITROX_LOG(ERR, "Op timedout\n"); + sr->op->status = RTE_COMP_OP_STATUS_ERROR; + err = -ETIMEDOUT; + goto exit; + } else { + return -EAGAIN; + } + } + + xform = sr->op->private_xform; + if (sr->op->op_type == RTE_COMP_OP_STATELESS) + err = post_process_zip_stateless(sr, xform, &zip_res); + else + err = post_process_zip_stateful(sr, xform, &zip_res); + + if (sr->op->status == RTE_COMP_OP_STATUS_SUCCESS && + xform->op == NITROX_COMP_OP_COMPRESS && sr->op->flush_flag == RTE_COMP_FLUSH_FINAL && zip_res.w2.exn) { uint32_t datalen = zip_res.w1.tbyteswritten; @@ -696,17 +1104,11 @@ nitrox_check_comp_req(struct nitrox_softreq *sr, struct rte_comp_op **op) *last_byte = zip_res.w2.exbits & 0xFF; } - sr->op->consumed = zip_res.w1.tbytesread; - sr->op->produced = zip_res.w1.tbyteswritten; - if (sr->xform.chksum_type == NITROX_CHKSUM_TYPE_CRC32) - sr->op->output_chksum = zip_res.w0.crc32; - else if (sr->xform.chksum_type == NITROX_CHKSUM_TYPE_ADLER32) - sr->op->output_chksum = zip_res.w0.adler32; - - sr->op->status = RTE_COMP_OP_STATUS_SUCCESS; - err = 0; exit: *op = sr->op; + nitrox_dump_zip_result(&sr->instr, &zip_res); + nitrox_dump_databuf("OUT after", sr->op->m_dst, sr->op->dst.offset, + sr->op->produced); return err; } diff --git a/drivers/compress/nitrox/nitrox_comp_reqmgr.h b/drivers/compress/nitrox/nitrox_comp_reqmgr.h index 07c65f0d5e..f0cd1eb8fd 100644 --- a/drivers/compress/nitrox/nitrox_comp_reqmgr.h +++ b/drivers/compress/nitrox/nitrox_comp_reqmgr.h @@ -37,6 +37,14 @@ struct nitrox_comp_xform { enum nitrox_comp_algo algo; enum nitrox_comp_level level; enum nitrox_chksum_type chksum_type; + uint8_t *context; + uint8_t *history_window; + uint32_t chksum; + uint16_t window_size; + uint16_t hlen; + uint8_t exn; + uint8_t exbits; + bool bf; }; int nitrox_process_comp_req(struct rte_comp_op *op, struct nitrox_softreq *sr);