Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/75650/?format=api
{ "id": 75650, "url": "http://patches.dpdk.org/api/patches/75650/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1597791894-37041-6-git-send-email-nicolas.chautru@intel.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1597791894-37041-6-git-send-email-nicolas.chautru@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1597791894-37041-6-git-send-email-nicolas.chautru@intel.com", "date": "2020-08-18T23:04:48", "name": "[v2,05/11] baseband/acc100: add LDPC processing functions", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "553b359e6089cc6b914598afd0580880d03174ef", "submitter": { "id": 1314, "url": "http://patches.dpdk.org/api/people/1314/?format=api", "name": "Chautru, Nicolas", "email": "nicolas.chautru@intel.com" }, "delegate": { "id": 6690, "url": "http://patches.dpdk.org/api/users/6690/?format=api", "username": "akhil", "first_name": "akhil", "last_name": "goyal", "email": "gakhil@marvell.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1597791894-37041-6-git-send-email-nicolas.chautru@intel.com/mbox/", "series": [ { "id": 11695, "url": "http://patches.dpdk.org/api/series/11695/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=11695", "date": "2020-08-18T23:04:43", "name": "bbdev PMD ACC100", "version": 2, "mbox": "http://patches.dpdk.org/series/11695/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/75650/comments/", "check": "success", "checks": "http://patches.dpdk.org/api/patches/75650/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 1BD0BA04AF;\n\tWed, 19 Aug 2020 01:07:55 +0200 (CEST)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 83C291C114;\n\tWed, 19 Aug 2020 01:06:56 +0200 (CEST)", "from mga14.intel.com (mga14.intel.com [192.55.52.115])\n by dpdk.org (Postfix) with ESMTP id 42BF0AAB7\n for <dev@dpdk.org>; Wed, 19 Aug 2020 01:06:47 +0200 (CEST)", "from orsmga004.jf.intel.com ([10.7.209.38])\n by fmsmga103.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 18 Aug 2020 16:06:44 -0700", "from skx-5gnr-sc12-4.sc.intel.com ([172.25.69.210])\n by orsmga004.jf.intel.com with ESMTP; 18 Aug 2020 16:06:43 -0700" ], "IronPort-SDR": [ "\n cAlEnczz/sie5kYCJaqKWCOfIb0ipmqWThJ8+VkuHgZ/291PQw09WkWscEe5MybZes7P8nuoQm\n lRxTmaizgWGA==", "\n tfws/IwEz6mADAuaTaDmL9VcYDpAR7AWa/VZSJqex5k33Qk7NO+STKG2/nQ54GbU/ABAA090f6\n gsKzcxQ2VAQg==" ], "X-IronPort-AV": [ "E=McAfee;i=\"6000,8403,9717\"; a=\"154281360\"", "E=Sophos;i=\"5.76,329,1592895600\"; d=\"scan'208\";a=\"154281360\"", "E=Sophos;i=\"5.76,329,1592895600\"; d=\"scan'208\";a=\"441400701\"" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "From": "Nicolas Chautru <nicolas.chautru@intel.com>", "To": "dev@dpdk.org,\n\takhil.goyal@nxp.com", "Cc": "bruce.richardson@intel.com,\n\tNicolas Chautru <nicolas.chautru@intel.com>", "Date": "Tue, 18 Aug 2020 16:04:48 -0700", "Message-Id": "<1597791894-37041-6-git-send-email-nicolas.chautru@intel.com>", "X-Mailer": "git-send-email 1.8.3.1", "In-Reply-To": "<1597791894-37041-1-git-send-email-nicolas.chautru@intel.com>", "References": "<1597791894-37041-1-git-send-email-nicolas.chautru@intel.com>", "Subject": "[dpdk-dev] [PATCH v2 05/11] baseband/acc100: add LDPC processing\n\tfunctions", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Adding LDPC decode and encode processing operations\n\nSigned-off-by: Nicolas Chautru <nicolas.chautru@intel.com>\n---\n drivers/baseband/acc100/rte_acc100_pmd.c | 1625 +++++++++++++++++++++++++++++-\n drivers/baseband/acc100/rte_acc100_pmd.h | 3 +\n 2 files changed, 1626 insertions(+), 2 deletions(-)", "diff": "diff --git a/drivers/baseband/acc100/rte_acc100_pmd.c b/drivers/baseband/acc100/rte_acc100_pmd.c\nindex 7a21c57..5f32813 100644\n--- a/drivers/baseband/acc100/rte_acc100_pmd.c\n+++ b/drivers/baseband/acc100/rte_acc100_pmd.c\n@@ -15,6 +15,9 @@\n #include <rte_hexdump.h>\n #include <rte_pci.h>\n #include <rte_bus_pci.h>\n+#ifdef RTE_BBDEV_OFFLOAD_COST\n+#include <rte_cycles.h>\n+#endif\n \n #include <rte_bbdev.h>\n #include <rte_bbdev_pmd.h>\n@@ -449,7 +452,6 @@\n \treturn 0;\n }\n \n-\n /**\n * Report a ACC100 queue index which is free\n * Return 0 to 16k for a valid queue_idx or -1 when no queue is available\n@@ -634,6 +636,46 @@\n \tstruct acc100_device *d = dev->data->dev_private;\n \n \tstatic const struct rte_bbdev_op_cap bbdev_capabilities[] = {\n+\t\t{\n+\t\t\t.type = RTE_BBDEV_OP_LDPC_ENC,\n+\t\t\t.cap.ldpc_enc = {\n+\t\t\t\t.capability_flags =\n+\t\t\t\t\tRTE_BBDEV_LDPC_RATE_MATCH |\n+\t\t\t\t\tRTE_BBDEV_LDPC_CRC_24B_ATTACH |\n+\t\t\t\t\tRTE_BBDEV_LDPC_INTERLEAVER_BYPASS,\n+\t\t\t\t.num_buffers_src =\n+\t\t\t\t\t\tRTE_BBDEV_LDPC_MAX_CODE_BLOCKS,\n+\t\t\t\t.num_buffers_dst =\n+\t\t\t\t\t\tRTE_BBDEV_LDPC_MAX_CODE_BLOCKS,\n+\t\t\t}\n+\t\t},\n+\t\t{\n+\t\t\t.type = RTE_BBDEV_OP_LDPC_DEC,\n+\t\t\t.cap.ldpc_dec = {\n+\t\t\t.capability_flags =\n+\t\t\t\tRTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK |\n+\t\t\t\tRTE_BBDEV_LDPC_CRC_TYPE_24B_DROP |\n+\t\t\t\tRTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |\n+\t\t\t\tRTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |\n+#ifdef ACC100_EXT_MEM\n+\t\t\t\tRTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE |\n+\t\t\t\tRTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE |\n+#endif\n+\t\t\t\tRTE_BBDEV_LDPC_ITERATION_STOP_ENABLE |\n+\t\t\t\tRTE_BBDEV_LDPC_DEINTERLEAVER_BYPASS |\n+\t\t\t\tRTE_BBDEV_LDPC_DECODE_BYPASS |\n+\t\t\t\tRTE_BBDEV_LDPC_DEC_SCATTER_GATHER |\n+\t\t\t\tRTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION |\n+\t\t\t\tRTE_BBDEV_LDPC_LLR_COMPRESSION,\n+\t\t\t.llr_size = 8,\n+\t\t\t.llr_decimals = 1,\n+\t\t\t.num_buffers_src =\n+\t\t\t\t\tRTE_BBDEV_LDPC_MAX_CODE_BLOCKS,\n+\t\t\t.num_buffers_hard_out =\n+\t\t\t\t\tRTE_BBDEV_LDPC_MAX_CODE_BLOCKS,\n+\t\t\t.num_buffers_soft_out = 0,\n+\t\t\t}\n+\t\t},\n \t\tRTE_BBDEV_END_OF_CAPABILITIES_LIST()\n \t};\n \n@@ -669,9 +711,14 @@\n \tdev_info->cpu_flag_reqs = NULL;\n \tdev_info->min_alignment = 64;\n \tdev_info->capabilities = bbdev_capabilities;\n+#ifdef ACC100_EXT_MEM\n \tdev_info->harq_buffer_size = d->ddr_size;\n+#else\n+\tdev_info->harq_buffer_size = 0;\n+#endif\n }\n \n+\n static const struct rte_bbdev_ops acc100_bbdev_ops = {\n \t.setup_queues = acc100_setup_queues,\n \t.close = acc100_dev_close,\n@@ -696,6 +743,1577 @@\n \t{.device_id = 0},\n };\n \n+/* Read flag value 0/1 from bitmap */\n+static inline bool\n+check_bit(uint32_t bitmap, uint32_t bitmask)\n+{\n+\treturn bitmap & bitmask;\n+}\n+\n+static inline char *\n+mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len)\n+{\n+\tif (unlikely(len > rte_pktmbuf_tailroom(m)))\n+\t\treturn NULL;\n+\n+\tchar *tail = (char *)m->buf_addr + m->data_off + m->data_len;\n+\tm->data_len = (uint16_t)(m->data_len + len);\n+\tm_head->pkt_len = (m_head->pkt_len + len);\n+\treturn tail;\n+}\n+\n+/* Compute value of k0.\n+ * Based on 3GPP 38.212 Table 5.4.2.1-2\n+ * Starting position of different redundancy versions, k0\n+ */\n+static inline uint16_t\n+get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)\n+{\n+\tif (rv_index == 0)\n+\t\treturn 0;\n+\tuint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;\n+\tif (n_cb == n) {\n+\t\tif (rv_index == 1)\n+\t\t\treturn (bg == 1 ? K0_1_1 : K0_1_2) * z_c;\n+\t\telse if (rv_index == 2)\n+\t\t\treturn (bg == 1 ? K0_2_1 : K0_2_2) * z_c;\n+\t\telse\n+\t\t\treturn (bg == 1 ? K0_3_1 : K0_3_2) * z_c;\n+\t}\n+\t/* LBRM case - includes a division by N */\n+\tif (rv_index == 1)\n+\t\treturn (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)\n+\t\t\t\t/ n) * z_c;\n+\telse if (rv_index == 2)\n+\t\treturn (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)\n+\t\t\t\t/ n) * z_c;\n+\telse\n+\t\treturn (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)\n+\t\t\t\t/ n) * z_c;\n+}\n+\n+/* Fill in a frame control word for LDPC encoding. */\n+static inline void\n+acc100_fcw_le_fill(const struct rte_bbdev_enc_op *op,\n+\t\tstruct acc100_fcw_le *fcw, int num_cb)\n+{\n+\tfcw->qm = op->ldpc_enc.q_m;\n+\tfcw->nfiller = op->ldpc_enc.n_filler;\n+\tfcw->BG = (op->ldpc_enc.basegraph - 1);\n+\tfcw->Zc = op->ldpc_enc.z_c;\n+\tfcw->ncb = op->ldpc_enc.n_cb;\n+\tfcw->k0 = get_k0(fcw->ncb, fcw->Zc, op->ldpc_enc.basegraph,\n+\t\t\top->ldpc_enc.rv_index);\n+\tfcw->rm_e = op->ldpc_enc.cb_params.e;\n+\tfcw->crc_select = check_bit(op->ldpc_enc.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_CRC_24B_ATTACH);\n+\tfcw->bypass_intlv = check_bit(op->ldpc_enc.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_INTERLEAVER_BYPASS);\n+\tfcw->mcb_count = num_cb;\n+}\n+\n+/* Fill in a frame control word for LDPC decoding. */\n+static inline void\n+acc100_fcw_ld_fill(const struct rte_bbdev_dec_op *op, struct acc100_fcw_ld *fcw,\n+\t\tunion acc100_harq_layout_data *harq_layout)\n+{\n+\tuint16_t harq_out_length, harq_in_length, ncb_p, k0_p, parity_offset;\n+\tuint16_t harq_index;\n+\tuint32_t l;\n+\tbool harq_prun = false;\n+\n+\tfcw->qm = op->ldpc_dec.q_m;\n+\tfcw->nfiller = op->ldpc_dec.n_filler;\n+\tfcw->BG = (op->ldpc_dec.basegraph - 1);\n+\tfcw->Zc = op->ldpc_dec.z_c;\n+\tfcw->ncb = op->ldpc_dec.n_cb;\n+\tfcw->k0 = get_k0(fcw->ncb, fcw->Zc, op->ldpc_dec.basegraph,\n+\t\t\top->ldpc_dec.rv_index);\n+\tif (op->ldpc_dec.code_block_mode == 1)\n+\t\tfcw->rm_e = op->ldpc_dec.cb_params.e;\n+\telse\n+\t\tfcw->rm_e = (op->ldpc_dec.tb_params.r <\n+\t\t\t\top->ldpc_dec.tb_params.cab) ?\n+\t\t\t\t\t\top->ldpc_dec.tb_params.ea :\n+\t\t\t\t\t\top->ldpc_dec.tb_params.eb;\n+\n+\tfcw->hcin_en = check_bit(op->ldpc_dec.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE);\n+\tfcw->hcout_en = check_bit(op->ldpc_dec.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE);\n+\tfcw->crc_select = check_bit(op->ldpc_dec.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK);\n+\tfcw->bypass_dec = check_bit(op->ldpc_dec.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_DECODE_BYPASS);\n+\tfcw->bypass_intlv = check_bit(op->ldpc_dec.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_DEINTERLEAVER_BYPASS);\n+\tif (op->ldpc_dec.q_m == 1) {\n+\t\tfcw->bypass_intlv = 1;\n+\t\tfcw->qm = 2;\n+\t}\n+\tfcw->hcin_decomp_mode = check_bit(op->ldpc_dec.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION);\n+\tfcw->hcout_comp_mode = check_bit(op->ldpc_dec.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION);\n+\tfcw->llr_pack_mode = check_bit(op->ldpc_dec.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_LLR_COMPRESSION);\n+\tharq_index = op->ldpc_dec.harq_combined_output.offset /\n+\t\t\tACC100_HARQ_OFFSET;\n+#ifdef ACC100_EXT_MEM\n+\t/* Limit cases when HARQ pruning is valid */\n+\tharq_prun = ((op->ldpc_dec.harq_combined_output.offset %\n+\t\t\tACC100_HARQ_OFFSET) == 0) &&\n+\t\t\t(op->ldpc_dec.harq_combined_output.offset <= UINT16_MAX\n+\t\t\t* ACC100_HARQ_OFFSET);\n+#endif\n+\tif (fcw->hcin_en > 0) {\n+\t\tharq_in_length = op->ldpc_dec.harq_combined_input.length;\n+\t\tif (fcw->hcin_decomp_mode > 0)\n+\t\t\tharq_in_length = harq_in_length * 8 / 6;\n+\t\tharq_in_length = RTE_ALIGN(harq_in_length, 64);\n+\t\tif ((harq_layout[harq_index].offset > 0) & harq_prun) {\n+\t\t\trte_bbdev_log_debug(\"HARQ IN offset unexpected for now\\n\");\n+\t\t\tfcw->hcin_size0 = harq_layout[harq_index].size0;\n+\t\t\tfcw->hcin_offset = harq_layout[harq_index].offset;\n+\t\t\tfcw->hcin_size1 = harq_in_length -\n+\t\t\t\t\tharq_layout[harq_index].offset;\n+\t\t} else {\n+\t\t\tfcw->hcin_size0 = harq_in_length;\n+\t\t\tfcw->hcin_offset = 0;\n+\t\t\tfcw->hcin_size1 = 0;\n+\t\t}\n+\t} else {\n+\t\tfcw->hcin_size0 = 0;\n+\t\tfcw->hcin_offset = 0;\n+\t\tfcw->hcin_size1 = 0;\n+\t}\n+\n+\tfcw->itmax = op->ldpc_dec.iter_max;\n+\tfcw->itstop = check_bit(op->ldpc_dec.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_ITERATION_STOP_ENABLE);\n+\tfcw->synd_precoder = fcw->itstop;\n+\t/*\n+\t * These are all implicitly set\n+\t * fcw->synd_post = 0;\n+\t * fcw->so_en = 0;\n+\t * fcw->so_bypass_rm = 0;\n+\t * fcw->so_bypass_intlv = 0;\n+\t * fcw->dec_convllr = 0;\n+\t * fcw->hcout_convllr = 0;\n+\t * fcw->hcout_size1 = 0;\n+\t * fcw->so_it = 0;\n+\t * fcw->hcout_offset = 0;\n+\t * fcw->negstop_th = 0;\n+\t * fcw->negstop_it = 0;\n+\t * fcw->negstop_en = 0;\n+\t * fcw->gain_i = 1;\n+\t * fcw->gain_h = 1;\n+\t */\n+\tif (fcw->hcout_en > 0) {\n+\t\tparity_offset = (op->ldpc_dec.basegraph == 1 ? 20 : 8)\n+\t\t\t* op->ldpc_dec.z_c - op->ldpc_dec.n_filler;\n+\t\tk0_p = (fcw->k0 > parity_offset) ?\n+\t\t\t\tfcw->k0 - op->ldpc_dec.n_filler : fcw->k0;\n+\t\tncb_p = fcw->ncb - op->ldpc_dec.n_filler;\n+\t\tl = k0_p + fcw->rm_e;\n+\t\tharq_out_length = (uint16_t) fcw->hcin_size0;\n+\t\tharq_out_length = RTE_MIN(RTE_MAX(harq_out_length, l), ncb_p);\n+\t\tharq_out_length = (harq_out_length + 0x3F) & 0xFFC0;\n+\t\tif ((k0_p > fcw->hcin_size0 + ACC100_HARQ_OFFSET_THRESHOLD) &&\n+\t\t\t\tharq_prun) {\n+\t\t\tfcw->hcout_size0 = (uint16_t) fcw->hcin_size0;\n+\t\t\tfcw->hcout_offset = k0_p & 0xFFC0;\n+\t\t\tfcw->hcout_size1 = harq_out_length - fcw->hcout_offset;\n+\t\t} else {\n+\t\t\tfcw->hcout_size0 = harq_out_length;\n+\t\t\tfcw->hcout_size1 = 0;\n+\t\t\tfcw->hcout_offset = 0;\n+\t\t}\n+\t\tharq_layout[harq_index].offset = fcw->hcout_offset;\n+\t\tharq_layout[harq_index].size0 = fcw->hcout_size0;\n+\t} else {\n+\t\tfcw->hcout_size0 = 0;\n+\t\tfcw->hcout_size1 = 0;\n+\t\tfcw->hcout_offset = 0;\n+\t}\n+}\n+\n+/**\n+ * Fills descriptor with data pointers of one block type.\n+ *\n+ * @param desc\n+ * Pointer to DMA descriptor.\n+ * @param input\n+ * Pointer to pointer to input data which will be encoded. It can be changed\n+ * and points to next segment in scatter-gather case.\n+ * @param offset\n+ * Input offset in rte_mbuf structure. It is used for calculating the point\n+ * where data is starting.\n+ * @param cb_len\n+ * Length of currently processed Code Block\n+ * @param seg_total_left\n+ * It indicates how many bytes still left in segment (mbuf) for further\n+ * processing.\n+ * @param op_flags\n+ * Store information about device capabilities\n+ * @param next_triplet\n+ * Index for ACC100 DMA Descriptor triplet\n+ *\n+ * @return\n+ * Returns index of next triplet on success, other value if lengths of\n+ * pkt and processed cb do not match.\n+ *\n+ */\n+static inline int\n+acc100_dma_fill_blk_type_in(struct acc100_dma_req_desc *desc,\n+\t\tstruct rte_mbuf **input, uint32_t *offset, uint32_t cb_len,\n+\t\tuint32_t *seg_total_left, int next_triplet)\n+{\n+\tuint32_t part_len;\n+\tstruct rte_mbuf *m = *input;\n+\n+\tpart_len = (*seg_total_left < cb_len) ? *seg_total_left : cb_len;\n+\tcb_len -= part_len;\n+\t*seg_total_left -= part_len;\n+\n+\tdesc->data_ptrs[next_triplet].address =\n+\t\t\trte_pktmbuf_iova_offset(m, *offset);\n+\tdesc->data_ptrs[next_triplet].blen = part_len;\n+\tdesc->data_ptrs[next_triplet].blkid = ACC100_DMA_BLKID_IN;\n+\tdesc->data_ptrs[next_triplet].last = 0;\n+\tdesc->data_ptrs[next_triplet].dma_ext = 0;\n+\t*offset += part_len;\n+\tnext_triplet++;\n+\n+\twhile (cb_len > 0) {\n+\t\tif (next_triplet < ACC100_DMA_MAX_NUM_POINTERS &&\n+\t\t\t\tm->next != NULL) {\n+\n+\t\t\tm = m->next;\n+\t\t\t*seg_total_left = rte_pktmbuf_data_len(m);\n+\t\t\tpart_len = (*seg_total_left < cb_len) ?\n+\t\t\t\t\t*seg_total_left :\n+\t\t\t\t\tcb_len;\n+\t\t\tdesc->data_ptrs[next_triplet].address =\n+\t\t\t\t\trte_pktmbuf_mtophys(m);\n+\t\t\tdesc->data_ptrs[next_triplet].blen = part_len;\n+\t\t\tdesc->data_ptrs[next_triplet].blkid =\n+\t\t\t\t\tACC100_DMA_BLKID_IN;\n+\t\t\tdesc->data_ptrs[next_triplet].last = 0;\n+\t\t\tdesc->data_ptrs[next_triplet].dma_ext = 0;\n+\t\t\tcb_len -= part_len;\n+\t\t\t*seg_total_left -= part_len;\n+\t\t\t/* Initializing offset for next segment (mbuf) */\n+\t\t\t*offset = part_len;\n+\t\t\tnext_triplet++;\n+\t\t} else {\n+\t\t\trte_bbdev_log(ERR,\n+\t\t\t\t\"Some data still left for processing: \"\n+\t\t\t\t\"data_left: %u, next_triplet: %u, next_mbuf: %p\",\n+\t\t\t\tcb_len, next_triplet, m->next);\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t}\n+\t/* Storing new mbuf as it could be changed in scatter-gather case*/\n+\t*input = m;\n+\n+\treturn next_triplet;\n+}\n+\n+/* Fills descriptor with data pointers of one block type.\n+ * Returns index of next triplet on success, other value if lengths of\n+ * output data and processed mbuf do not match.\n+ */\n+static inline int\n+acc100_dma_fill_blk_type_out(struct acc100_dma_req_desc *desc,\n+\t\tstruct rte_mbuf *output, uint32_t out_offset,\n+\t\tuint32_t output_len, int next_triplet, int blk_id)\n+{\n+\tdesc->data_ptrs[next_triplet].address =\n+\t\t\trte_pktmbuf_iova_offset(output, out_offset);\n+\tdesc->data_ptrs[next_triplet].blen = output_len;\n+\tdesc->data_ptrs[next_triplet].blkid = blk_id;\n+\tdesc->data_ptrs[next_triplet].last = 0;\n+\tdesc->data_ptrs[next_triplet].dma_ext = 0;\n+\tnext_triplet++;\n+\n+\treturn next_triplet;\n+}\n+\n+static inline int\n+acc100_dma_desc_le_fill(struct rte_bbdev_enc_op *op,\n+\t\tstruct acc100_dma_req_desc *desc, struct rte_mbuf **input,\n+\t\tstruct rte_mbuf *output, uint32_t *in_offset,\n+\t\tuint32_t *out_offset, uint32_t *out_length,\n+\t\tuint32_t *mbuf_total_left, uint32_t *seg_total_left)\n+{\n+\tint next_triplet = 1; /* FCW already done */\n+\tuint16_t K, in_length_in_bits, in_length_in_bytes;\n+\tstruct rte_bbdev_op_ldpc_enc *enc = &op->ldpc_enc;\n+\n+\tdesc->word0 = ACC100_DMA_DESC_TYPE;\n+\tdesc->word1 = 0; /**< Timestamp could be disabled */\n+\tdesc->word2 = 0;\n+\tdesc->word3 = 0;\n+\tdesc->numCBs = 1;\n+\n+\tK = (enc->basegraph == 1 ? 22 : 10) * enc->z_c;\n+\tin_length_in_bits = K - enc->n_filler;\n+\tif ((enc->op_flags & RTE_BBDEV_LDPC_CRC_24A_ATTACH) ||\n+\t\t\t(enc->op_flags & RTE_BBDEV_LDPC_CRC_24B_ATTACH))\n+\t\tin_length_in_bits -= 24;\n+\tin_length_in_bytes = in_length_in_bits >> 3;\n+\n+\tif (unlikely((*mbuf_total_left == 0) ||\n+\t\t\t(*mbuf_total_left < in_length_in_bytes))) {\n+\t\trte_bbdev_log(ERR,\n+\t\t\t\t\"Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u\",\n+\t\t\t\t*mbuf_total_left, in_length_in_bytes);\n+\t\treturn -1;\n+\t}\n+\n+\tnext_triplet = acc100_dma_fill_blk_type_in(desc, input, in_offset,\n+\t\t\tin_length_in_bytes,\n+\t\t\tseg_total_left, next_triplet);\n+\tif (unlikely(next_triplet < 0)) {\n+\t\trte_bbdev_log(ERR,\n+\t\t\t\t\"Mismatch between data to process and mbuf data length in bbdev_op: %p\",\n+\t\t\t\top);\n+\t\treturn -1;\n+\t}\n+\tdesc->data_ptrs[next_triplet - 1].last = 1;\n+\tdesc->m2dlen = next_triplet;\n+\t*mbuf_total_left -= in_length_in_bytes;\n+\n+\t/* Set output length */\n+\t/* Integer round up division by 8 */\n+\t*out_length = (enc->cb_params.e + 7) >> 3;\n+\n+\tnext_triplet = acc100_dma_fill_blk_type_out(desc, output, *out_offset,\n+\t\t\t*out_length, next_triplet, ACC100_DMA_BLKID_OUT_ENC);\n+\tif (unlikely(next_triplet < 0)) {\n+\t\trte_bbdev_log(ERR,\n+\t\t\t\t\"Mismatch between data to process and mbuf data length in bbdev_op: %p\",\n+\t\t\t\top);\n+\t\treturn -1;\n+\t}\n+\top->ldpc_enc.output.length += *out_length;\n+\t*out_offset += *out_length;\n+\tdesc->data_ptrs[next_triplet - 1].last = 1;\n+\tdesc->data_ptrs[next_triplet - 1].dma_ext = 0;\n+\tdesc->d2mlen = next_triplet - desc->m2dlen;\n+\n+\tdesc->op_addr = op;\n+\n+\treturn 0;\n+}\n+\n+static inline int\n+acc100_dma_desc_ld_fill(struct rte_bbdev_dec_op *op,\n+\t\tstruct acc100_dma_req_desc *desc,\n+\t\tstruct rte_mbuf **input, struct rte_mbuf *h_output,\n+\t\tuint32_t *in_offset, uint32_t *h_out_offset,\n+\t\tuint32_t *h_out_length, uint32_t *mbuf_total_left,\n+\t\tuint32_t *seg_total_left,\n+\t\tstruct acc100_fcw_ld *fcw)\n+{\n+\tstruct rte_bbdev_op_ldpc_dec *dec = &op->ldpc_dec;\n+\tint next_triplet = 1; /* FCW already done */\n+\tuint32_t input_length;\n+\tuint16_t output_length, crc24_overlap = 0;\n+\tuint16_t sys_cols, K, h_p_size, h_np_size;\n+\tbool h_comp = check_bit(dec->op_flags,\n+\t\t\tRTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION);\n+\n+\tdesc->word0 = ACC100_DMA_DESC_TYPE;\n+\tdesc->word1 = 0; /**< Timestamp could be disabled */\n+\tdesc->word2 = 0;\n+\tdesc->word3 = 0;\n+\tdesc->numCBs = 1;\n+\n+\tif (check_bit(op->ldpc_dec.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_CRC_TYPE_24B_DROP))\n+\t\tcrc24_overlap = 24;\n+\n+\t/* Compute some LDPC BG lengths */\n+\tinput_length = dec->cb_params.e;\n+\tif (check_bit(op->ldpc_dec.op_flags,\n+\t\t\tRTE_BBDEV_LDPC_LLR_COMPRESSION))\n+\t\tinput_length = (input_length * 3 + 3) / 4;\n+\tsys_cols = (dec->basegraph == 1) ? 22 : 10;\n+\tK = sys_cols * dec->z_c;\n+\toutput_length = K - dec->n_filler - crc24_overlap;\n+\n+\tif (unlikely((*mbuf_total_left == 0) ||\n+\t\t\t(*mbuf_total_left < input_length))) {\n+\t\trte_bbdev_log(ERR,\n+\t\t\t\t\"Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u\",\n+\t\t\t\t*mbuf_total_left, input_length);\n+\t\treturn -1;\n+\t}\n+\n+\tnext_triplet = acc100_dma_fill_blk_type_in(desc, input,\n+\t\t\tin_offset, input_length,\n+\t\t\tseg_total_left, next_triplet);\n+\n+\tif (unlikely(next_triplet < 0)) {\n+\t\trte_bbdev_log(ERR,\n+\t\t\t\t\"Mismatch between data to process and mbuf data length in bbdev_op: %p\",\n+\t\t\t\top);\n+\t\treturn -1;\n+\t}\n+\n+\tif (check_bit(op->ldpc_dec.op_flags,\n+\t\t\t\tRTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {\n+\t\th_p_size = fcw->hcin_size0 + fcw->hcin_size1;\n+\t\tif (h_comp)\n+\t\t\th_p_size = (h_p_size * 3 + 3) / 4;\n+\t\tdesc->data_ptrs[next_triplet].address =\n+\t\t\t\tdec->harq_combined_input.offset;\n+\t\tdesc->data_ptrs[next_triplet].blen = h_p_size;\n+\t\tdesc->data_ptrs[next_triplet].blkid = ACC100_DMA_BLKID_IN_HARQ;\n+\t\tdesc->data_ptrs[next_triplet].dma_ext = 1;\n+#ifndef ACC100_EXT_MEM\n+\t\tacc100_dma_fill_blk_type_out(\n+\t\t\t\tdesc,\n+\t\t\t\top->ldpc_dec.harq_combined_input.data,\n+\t\t\t\top->ldpc_dec.harq_combined_input.offset,\n+\t\t\t\th_p_size,\n+\t\t\t\tnext_triplet,\n+\t\t\t\tACC100_DMA_BLKID_IN_HARQ);\n+#endif\n+\t\tnext_triplet++;\n+\t}\n+\n+\tdesc->data_ptrs[next_triplet - 1].last = 1;\n+\tdesc->m2dlen = next_triplet;\n+\t*mbuf_total_left -= input_length;\n+\n+\tnext_triplet = acc100_dma_fill_blk_type_out(desc, h_output,\n+\t\t\t*h_out_offset, output_length >> 3, next_triplet,\n+\t\t\tACC100_DMA_BLKID_OUT_HARD);\n+\tif (unlikely(next_triplet < 0)) {\n+\t\trte_bbdev_log(ERR,\n+\t\t\t\t\"Mismatch between data to process and mbuf data length in bbdev_op: %p\",\n+\t\t\t\top);\n+\t\treturn -1;\n+\t}\n+\n+\tif (check_bit(op->ldpc_dec.op_flags,\n+\t\t\t\tRTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) {\n+\t\t/* Pruned size of the HARQ */\n+\t\th_p_size = fcw->hcout_size0 + fcw->hcout_size1;\n+\t\t/* Non-Pruned size of the HARQ */\n+\t\th_np_size = fcw->hcout_offset > 0 ?\n+\t\t\t\tfcw->hcout_offset + fcw->hcout_size1 :\n+\t\t\t\th_p_size;\n+\t\tif (h_comp) {\n+\t\t\th_np_size = (h_np_size * 3 + 3) / 4;\n+\t\t\th_p_size = (h_p_size * 3 + 3) / 4;\n+\t\t}\n+\t\tdec->harq_combined_output.length = h_np_size;\n+\t\tdesc->data_ptrs[next_triplet].address =\n+\t\t\t\tdec->harq_combined_output.offset;\n+\t\tdesc->data_ptrs[next_triplet].blen = h_p_size;\n+\t\tdesc->data_ptrs[next_triplet].blkid = ACC100_DMA_BLKID_OUT_HARQ;\n+\t\tdesc->data_ptrs[next_triplet].dma_ext = 1;\n+#ifndef ACC100_EXT_MEM\n+\t\tacc100_dma_fill_blk_type_out(\n+\t\t\t\tdesc,\n+\t\t\t\tdec->harq_combined_output.data,\n+\t\t\t\tdec->harq_combined_output.offset,\n+\t\t\t\th_p_size,\n+\t\t\t\tnext_triplet,\n+\t\t\t\tACC100_DMA_BLKID_OUT_HARQ);\n+#endif\n+\t\tnext_triplet++;\n+\t}\n+\n+\t*h_out_length = output_length >> 3;\n+\tdec->hard_output.length += *h_out_length;\n+\t*h_out_offset += *h_out_length;\n+\tdesc->data_ptrs[next_triplet - 1].last = 1;\n+\tdesc->d2mlen = next_triplet - desc->m2dlen;\n+\n+\tdesc->op_addr = op;\n+\n+\treturn 0;\n+}\n+\n+static inline void\n+acc100_dma_desc_ld_update(struct rte_bbdev_dec_op *op,\n+\t\tstruct acc100_dma_req_desc *desc,\n+\t\tstruct rte_mbuf *input, struct rte_mbuf *h_output,\n+\t\tuint32_t *in_offset, uint32_t *h_out_offset,\n+\t\tuint32_t *h_out_length,\n+\t\tunion acc100_harq_layout_data *harq_layout)\n+{\n+\tint next_triplet = 1; /* FCW already done */\n+\tdesc->data_ptrs[next_triplet].address =\n+\t\t\trte_pktmbuf_iova_offset(input, *in_offset);\n+\tnext_triplet++;\n+\n+\tif (check_bit(op->ldpc_dec.op_flags,\n+\t\t\t\tRTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {\n+\t\tstruct rte_bbdev_op_data hi = op->ldpc_dec.harq_combined_input;\n+\t\tdesc->data_ptrs[next_triplet].address = hi.offset;\n+#ifndef ACC100_EXT_MEM\n+\t\tdesc->data_ptrs[next_triplet].address =\n+\t\t\t\trte_pktmbuf_iova_offset(hi.data, hi.offset);\n+#endif\n+\t\tnext_triplet++;\n+\t}\n+\n+\tdesc->data_ptrs[next_triplet].address =\n+\t\t\trte_pktmbuf_iova_offset(h_output, *h_out_offset);\n+\t*h_out_length = desc->data_ptrs[next_triplet].blen;\n+\tnext_triplet++;\n+\n+\tif (check_bit(op->ldpc_dec.op_flags,\n+\t\t\t\tRTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) {\n+\t\tdesc->data_ptrs[next_triplet].address =\n+\t\t\t\top->ldpc_dec.harq_combined_output.offset;\n+\t\t/* Adjust based on previous operation */\n+\t\tstruct rte_bbdev_dec_op *prev_op = desc->op_addr;\n+\t\top->ldpc_dec.harq_combined_output.length =\n+\t\t\t\tprev_op->ldpc_dec.harq_combined_output.length;\n+\t\tint16_t hq_idx = op->ldpc_dec.harq_combined_output.offset /\n+\t\t\t\tACC100_HARQ_OFFSET;\n+\t\tint16_t prev_hq_idx =\n+\t\t\t\tprev_op->ldpc_dec.harq_combined_output.offset\n+\t\t\t\t/ ACC100_HARQ_OFFSET;\n+\t\tharq_layout[hq_idx].val = harq_layout[prev_hq_idx].val;\n+#ifndef ACC100_EXT_MEM\n+\t\tstruct rte_bbdev_op_data ho =\n+\t\t\t\top->ldpc_dec.harq_combined_output;\n+\t\tdesc->data_ptrs[next_triplet].address =\n+\t\t\t\trte_pktmbuf_iova_offset(ho.data, ho.offset);\n+#endif\n+\t\tnext_triplet++;\n+\t}\n+\n+\top->ldpc_dec.hard_output.length += *h_out_length;\n+\tdesc->op_addr = op;\n+}\n+\n+\n+/* Enqueue a number of operations to HW and update software rings */\n+static inline void\n+acc100_dma_enqueue(struct acc100_queue *q, uint16_t n,\n+\t\tstruct rte_bbdev_stats *queue_stats)\n+{\n+\tunion acc100_enqueue_reg_fmt enq_req;\n+#ifdef RTE_BBDEV_OFFLOAD_COST\n+\tuint64_t start_time = 0;\n+\tqueue_stats->acc_offload_cycles = 0;\n+\tRTE_SET_USED(queue_stats);\n+#else\n+\tRTE_SET_USED(queue_stats);\n+#endif\n+\n+\tenq_req.val = 0;\n+\t/* Setting offset, 100b for 256 DMA Desc */\n+\tenq_req.addr_offset = ACC100_DESC_OFFSET;\n+\n+\t/* Split ops into batches */\n+\tdo {\n+\t\tunion acc100_dma_desc *desc;\n+\t\tuint16_t enq_batch_size;\n+\t\tuint64_t offset;\n+\t\trte_iova_t req_elem_addr;\n+\n+\t\tenq_batch_size = RTE_MIN(n, MAX_ENQ_BATCH_SIZE);\n+\n+\t\t/* Set flag on last descriptor in a batch */\n+\t\tdesc = q->ring_addr + ((q->sw_ring_head + enq_batch_size - 1) &\n+\t\t\t\tq->sw_ring_wrap_mask);\n+\t\tdesc->req.last_desc_in_batch = 1;\n+\n+\t\t/* Calculate the 1st descriptor's address */\n+\t\toffset = ((q->sw_ring_head & q->sw_ring_wrap_mask) *\n+\t\t\t\tsizeof(union acc100_dma_desc));\n+\t\treq_elem_addr = q->ring_addr_phys + offset;\n+\n+\t\t/* Fill enqueue struct */\n+\t\tenq_req.num_elem = enq_batch_size;\n+\t\t/* low 6 bits are not needed */\n+\t\tenq_req.req_elem_addr = (uint32_t)(req_elem_addr >> 6);\n+\n+#ifdef RTE_LIBRTE_BBDEV_DEBUG\n+\t\trte_memdump(stderr, \"Req sdone\", desc, sizeof(*desc));\n+#endif\n+\t\trte_bbdev_log_debug(\n+\t\t\t\t\"Enqueue %u reqs (phys %#\"PRIx64\") to reg %p\",\n+\t\t\t\tenq_batch_size,\n+\t\t\t\treq_elem_addr,\n+\t\t\t\t(void *)q->mmio_reg_enqueue);\n+\n+\t\trte_wmb();\n+\n+#ifdef RTE_BBDEV_OFFLOAD_COST\n+\t\t/* Start time measurement for enqueue function offload. */\n+\t\tstart_time = rte_rdtsc_precise();\n+#endif\n+\t\trte_bbdev_log(DEBUG, \"Debug : MMIO Enqueue\");\n+\t\tmmio_write(q->mmio_reg_enqueue, enq_req.val);\n+\n+#ifdef RTE_BBDEV_OFFLOAD_COST\n+\t\tqueue_stats->acc_offload_cycles +=\n+\t\t\t\trte_rdtsc_precise() - start_time;\n+#endif\n+\n+\t\tq->aq_enqueued++;\n+\t\tq->sw_ring_head += enq_batch_size;\n+\t\tn -= enq_batch_size;\n+\n+\t} while (n);\n+\n+\n+}\n+\n+/* Enqueue one encode operations for ACC100 device in CB mode */\n+static inline int\n+enqueue_ldpc_enc_n_op_cb(struct acc100_queue *q, struct rte_bbdev_enc_op **ops,\n+\t\tuint16_t total_enqueued_cbs, int16_t num)\n+{\n+\tunion acc100_dma_desc *desc = NULL;\n+\tuint32_t out_length;\n+\tstruct rte_mbuf *output_head, *output;\n+\tint i, next_triplet;\n+\tuint16_t in_length_in_bytes;\n+\tstruct rte_bbdev_op_ldpc_enc *enc = &ops[0]->ldpc_enc;\n+\n+\tuint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\tdesc = q->ring_addr + desc_idx;\n+\tacc100_fcw_le_fill(ops[0], &desc->req.fcw_le, num);\n+\n+\t/** This could be done at polling */\n+\tdesc->req.word0 = ACC100_DMA_DESC_TYPE;\n+\tdesc->req.word1 = 0; /**< Timestamp could be disabled */\n+\tdesc->req.word2 = 0;\n+\tdesc->req.word3 = 0;\n+\tdesc->req.numCBs = num;\n+\n+\tin_length_in_bytes = ops[0]->ldpc_enc.input.data->data_len;\n+\tout_length = (enc->cb_params.e + 7) >> 3;\n+\tdesc->req.m2dlen = 1 + num;\n+\tdesc->req.d2mlen = num;\n+\tnext_triplet = 1;\n+\n+\tfor (i = 0; i < num; i++) {\n+\t\tdesc->req.data_ptrs[next_triplet].address =\n+\t\t\trte_pktmbuf_iova_offset(ops[i]->ldpc_enc.input.data, 0);\n+\t\tdesc->req.data_ptrs[next_triplet].blen = in_length_in_bytes;\n+\t\tnext_triplet++;\n+\t\tdesc->req.data_ptrs[next_triplet].address =\n+\t\t\t\trte_pktmbuf_iova_offset(\n+\t\t\t\tops[i]->ldpc_enc.output.data, 0);\n+\t\tdesc->req.data_ptrs[next_triplet].blen = out_length;\n+\t\tnext_triplet++;\n+\t\tops[i]->ldpc_enc.output.length = out_length;\n+\t\toutput_head = output = ops[i]->ldpc_enc.output.data;\n+\t\tmbuf_append(output_head, output, out_length);\n+\t\toutput->data_len = out_length;\n+\t}\n+\n+\tdesc->req.op_addr = ops[0];\n+\n+#ifdef RTE_LIBRTE_BBDEV_DEBUG\n+\trte_memdump(stderr, \"FCW\", &desc->req.fcw_le,\n+\t\t\tsizeof(desc->req.fcw_le) - 8);\n+\trte_memdump(stderr, \"Req Desc.\", desc, sizeof(*desc));\n+#endif\n+\n+\t/* One CB (one op) was successfully prepared to enqueue */\n+\treturn num;\n+}\n+\n+/* Enqueue one encode operations for ACC100 device in CB mode */\n+static inline int\n+enqueue_ldpc_enc_one_op_cb(struct acc100_queue *q, struct rte_bbdev_enc_op *op,\n+\t\tuint16_t total_enqueued_cbs)\n+{\n+\tunion acc100_dma_desc *desc = NULL;\n+\tint ret;\n+\tuint32_t in_offset, out_offset, out_length, mbuf_total_left,\n+\t\tseg_total_left;\n+\tstruct rte_mbuf *input, *output_head, *output;\n+\n+\tuint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\tdesc = q->ring_addr + desc_idx;\n+\tacc100_fcw_le_fill(op, &desc->req.fcw_le, 1);\n+\n+\tinput = op->ldpc_enc.input.data;\n+\toutput_head = output = op->ldpc_enc.output.data;\n+\tin_offset = op->ldpc_enc.input.offset;\n+\tout_offset = op->ldpc_enc.output.offset;\n+\tout_length = 0;\n+\tmbuf_total_left = op->ldpc_enc.input.length;\n+\tseg_total_left = rte_pktmbuf_data_len(op->ldpc_enc.input.data)\n+\t\t\t- in_offset;\n+\n+\tret = acc100_dma_desc_le_fill(op, &desc->req, &input, output,\n+\t\t\t&in_offset, &out_offset, &out_length, &mbuf_total_left,\n+\t\t\t&seg_total_left);\n+\n+\tif (unlikely(ret < 0))\n+\t\treturn ret;\n+\n+\tmbuf_append(output_head, output, out_length);\n+\n+#ifdef RTE_LIBRTE_BBDEV_DEBUG\n+\trte_memdump(stderr, \"FCW\", &desc->req.fcw_le,\n+\t\t\tsizeof(desc->req.fcw_le) - 8);\n+\trte_memdump(stderr, \"Req Desc.\", desc, sizeof(*desc));\n+\n+\t/* Check if any data left after processing one CB */\n+\tif (mbuf_total_left != 0) {\n+\t\trte_bbdev_log(ERR,\n+\t\t\t\t\"Some date still left after processing one CB: mbuf_total_left = %u\",\n+\t\t\t\tmbuf_total_left);\n+\t\treturn -EINVAL;\n+\t}\n+#endif\n+\t/* One CB (one op) was successfully prepared to enqueue */\n+\treturn 1;\n+}\n+\n+/** Enqueue one decode operations for ACC100 device in CB mode */\n+static inline int\n+enqueue_ldpc_dec_one_op_cb(struct acc100_queue *q, struct rte_bbdev_dec_op *op,\n+\t\tuint16_t total_enqueued_cbs, bool same_op)\n+{\n+\tint ret;\n+\n+\tunion acc100_dma_desc *desc;\n+\tuint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\tdesc = q->ring_addr + desc_idx;\n+\tstruct rte_mbuf *input, *h_output_head, *h_output;\n+\tuint32_t in_offset, h_out_offset, h_out_length, mbuf_total_left;\n+\tinput = op->ldpc_dec.input.data;\n+\th_output_head = h_output = op->ldpc_dec.hard_output.data;\n+\tin_offset = op->ldpc_dec.input.offset;\n+\th_out_offset = op->ldpc_dec.hard_output.offset;\n+\tmbuf_total_left = op->ldpc_dec.input.length;\n+#ifdef RTE_LIBRTE_BBDEV_DEBUG\n+\tif (unlikely(input == NULL)) {\n+\t\trte_bbdev_log(ERR, \"Invalid mbuf pointer\");\n+\t\treturn -EFAULT;\n+\t}\n+#endif\n+\tunion acc100_harq_layout_data *harq_layout = q->d->harq_layout;\n+\n+\tif (same_op) {\n+\t\tunion acc100_dma_desc *prev_desc;\n+\t\tdesc_idx = ((q->sw_ring_head + total_enqueued_cbs - 1)\n+\t\t\t\t& q->sw_ring_wrap_mask);\n+\t\tprev_desc = q->ring_addr + desc_idx;\n+\t\tuint8_t *prev_ptr = (uint8_t *) prev_desc;\n+\t\tuint8_t *new_ptr = (uint8_t *) desc;\n+\t\t/* Copy first 4 words and BDESCs */\n+\t\trte_memcpy(new_ptr, prev_ptr, 16);\n+\t\trte_memcpy(new_ptr + 36, prev_ptr + 36, 40);\n+\t\tdesc->req.op_addr = prev_desc->req.op_addr;\n+\t\t/* Copy FCW */\n+\t\trte_memcpy(new_ptr + ACC100_DESC_FCW_OFFSET,\n+\t\t\t\tprev_ptr + ACC100_DESC_FCW_OFFSET,\n+\t\t\t\tACC100_FCW_LD_BLEN);\n+\t\tacc100_dma_desc_ld_update(op, &desc->req, input, h_output,\n+\t\t\t\t&in_offset, &h_out_offset,\n+\t\t\t\t&h_out_length, harq_layout);\n+\t} else {\n+\t\tstruct acc100_fcw_ld *fcw;\n+\t\tuint32_t seg_total_left;\n+\t\tfcw = &desc->req.fcw_ld;\n+\t\tacc100_fcw_ld_fill(op, fcw, harq_layout);\n+\n+\t\t/* Special handling when overusing mbuf */\n+\t\tif (fcw->rm_e < MAX_E_MBUF)\n+\t\t\tseg_total_left = rte_pktmbuf_data_len(input)\n+\t\t\t\t\t- in_offset;\n+\t\telse\n+\t\t\tseg_total_left = fcw->rm_e;\n+\n+\t\tret = acc100_dma_desc_ld_fill(op, &desc->req, &input, h_output,\n+\t\t\t\t&in_offset, &h_out_offset,\n+\t\t\t\t&h_out_length, &mbuf_total_left,\n+\t\t\t\t&seg_total_left, fcw);\n+\t\tif (unlikely(ret < 0))\n+\t\t\treturn ret;\n+\t}\n+\n+\t/* Hard output */\n+\tmbuf_append(h_output_head, h_output, h_out_length);\n+#ifndef ACC100_EXT_MEM\n+\tif (op->ldpc_dec.harq_combined_output.length > 0) {\n+\t\t/* Push the HARQ output into host memory */\n+\t\tstruct rte_mbuf *hq_output_head, *hq_output;\n+\t\thq_output_head = op->ldpc_dec.harq_combined_output.data;\n+\t\thq_output = op->ldpc_dec.harq_combined_output.data;\n+\t\tmbuf_append(hq_output_head, hq_output,\n+\t\t\t\top->ldpc_dec.harq_combined_output.length);\n+\t}\n+#endif\n+\n+#ifdef RTE_LIBRTE_BBDEV_DEBUG\n+\trte_memdump(stderr, \"FCW\", &desc->req.fcw_ld,\n+\t\t\tsizeof(desc->req.fcw_ld) - 8);\n+\trte_memdump(stderr, \"Req Desc.\", desc, sizeof(*desc));\n+#endif\n+\n+\t/* One CB (one op) was successfully prepared to enqueue */\n+\treturn 1;\n+}\n+\n+\n+/* Enqueue one decode operations for ACC100 device in TB mode */\n+static inline int\n+enqueue_ldpc_dec_one_op_tb(struct acc100_queue *q, struct rte_bbdev_dec_op *op,\n+\t\tuint16_t total_enqueued_cbs, uint8_t cbs_in_tb)\n+{\n+\tunion acc100_dma_desc *desc = NULL;\n+\tint ret;\n+\tuint8_t r, c;\n+\tuint32_t in_offset, h_out_offset,\n+\t\th_out_length, mbuf_total_left, seg_total_left;\n+\tstruct rte_mbuf *input, *h_output_head, *h_output;\n+\tuint16_t current_enqueued_cbs = 0;\n+\n+\tuint16_t desc_idx = ((q->sw_ring_head + total_enqueued_cbs)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\tdesc = q->ring_addr + desc_idx;\n+\tuint64_t fcw_offset = (desc_idx << 8) + ACC100_DESC_FCW_OFFSET;\n+\tunion acc100_harq_layout_data *harq_layout = q->d->harq_layout;\n+\tacc100_fcw_ld_fill(op, &desc->req.fcw_ld, harq_layout);\n+\n+\tinput = op->ldpc_dec.input.data;\n+\th_output_head = h_output = op->ldpc_dec.hard_output.data;\n+\tin_offset = op->ldpc_dec.input.offset;\n+\th_out_offset = op->ldpc_dec.hard_output.offset;\n+\th_out_length = 0;\n+\tmbuf_total_left = op->ldpc_dec.input.length;\n+\tc = op->ldpc_dec.tb_params.c;\n+\tr = op->ldpc_dec.tb_params.r;\n+\n+\twhile (mbuf_total_left > 0 && r < c) {\n+\n+\t\tseg_total_left = rte_pktmbuf_data_len(input) - in_offset;\n+\n+\t\t/* Set up DMA descriptor */\n+\t\tdesc = q->ring_addr + ((q->sw_ring_head + total_enqueued_cbs)\n+\t\t\t\t& q->sw_ring_wrap_mask);\n+\t\tdesc->req.data_ptrs[0].address = q->ring_addr_phys + fcw_offset;\n+\t\tdesc->req.data_ptrs[0].blen = ACC100_FCW_LD_BLEN;\n+\t\tret = acc100_dma_desc_ld_fill(op, &desc->req, &input,\n+\t\t\t\th_output, &in_offset, &h_out_offset,\n+\t\t\t\t&h_out_length,\n+\t\t\t\t&mbuf_total_left, &seg_total_left,\n+\t\t\t\t&desc->req.fcw_ld);\n+\n+\t\tif (unlikely(ret < 0))\n+\t\t\treturn ret;\n+\n+\t\t/* Hard output */\n+\t\tmbuf_append(h_output_head, h_output, h_out_length);\n+\n+\t\t/* Set total number of CBs in TB */\n+\t\tdesc->req.cbs_in_tb = cbs_in_tb;\n+#ifdef RTE_LIBRTE_BBDEV_DEBUG\n+\t\trte_memdump(stderr, \"FCW\", &desc->req.fcw_td,\n+\t\t\t\tsizeof(desc->req.fcw_td) - 8);\n+\t\trte_memdump(stderr, \"Req Desc.\", desc, sizeof(*desc));\n+#endif\n+\n+\t\tif (seg_total_left == 0) {\n+\t\t\t/* Go to the next mbuf */\n+\t\t\tinput = input->next;\n+\t\t\tin_offset = 0;\n+\t\t\th_output = h_output->next;\n+\t\t\th_out_offset = 0;\n+\t\t}\n+\t\ttotal_enqueued_cbs++;\n+\t\tcurrent_enqueued_cbs++;\n+\t\tr++;\n+\t}\n+\n+\tif (unlikely(desc == NULL))\n+\t\treturn current_enqueued_cbs;\n+\n+#ifdef RTE_LIBRTE_BBDEV_DEBUG\n+\t/* Check if any CBs left for processing */\n+\tif (mbuf_total_left != 0) {\n+\t\trte_bbdev_log(ERR,\n+\t\t\t\t\"Some date still left for processing: mbuf_total_left = %u\",\n+\t\t\t\tmbuf_total_left);\n+\t\treturn -EINVAL;\n+\t}\n+#endif\n+\t/* Set SDone on last CB descriptor for TB mode */\n+\tdesc->req.sdone_enable = 1;\n+\tdesc->req.irq_enable = q->irq_enable;\n+\n+\treturn current_enqueued_cbs;\n+}\n+\n+\n+/* Calculates number of CBs in processed encoder TB based on 'r' and input\n+ * length.\n+ */\n+static inline uint8_t\n+get_num_cbs_in_tb_enc(struct rte_bbdev_op_turbo_enc *turbo_enc)\n+{\n+\tuint8_t c, c_neg, r, crc24_bits = 0;\n+\tuint16_t k, k_neg, k_pos;\n+\tuint8_t cbs_in_tb = 0;\n+\tint32_t length;\n+\n+\tlength = turbo_enc->input.length;\n+\tr = turbo_enc->tb_params.r;\n+\tc = turbo_enc->tb_params.c;\n+\tc_neg = turbo_enc->tb_params.c_neg;\n+\tk_neg = turbo_enc->tb_params.k_neg;\n+\tk_pos = turbo_enc->tb_params.k_pos;\n+\tcrc24_bits = 0;\n+\tif (check_bit(turbo_enc->op_flags, RTE_BBDEV_TURBO_CRC_24B_ATTACH))\n+\t\tcrc24_bits = 24;\n+\twhile (length > 0 && r < c) {\n+\t\tk = (r < c_neg) ? k_neg : k_pos;\n+\t\tlength -= (k - crc24_bits) >> 3;\n+\t\tr++;\n+\t\tcbs_in_tb++;\n+\t}\n+\n+\treturn cbs_in_tb;\n+}\n+\n+/* Calculates number of CBs in processed decoder TB based on 'r' and input\n+ * length.\n+ */\n+static inline uint16_t\n+get_num_cbs_in_tb_dec(struct rte_bbdev_op_turbo_dec *turbo_dec)\n+{\n+\tuint8_t c, c_neg, r = 0;\n+\tuint16_t kw, k, k_neg, k_pos, cbs_in_tb = 0;\n+\tint32_t length;\n+\n+\tlength = turbo_dec->input.length;\n+\tr = turbo_dec->tb_params.r;\n+\tc = turbo_dec->tb_params.c;\n+\tc_neg = turbo_dec->tb_params.c_neg;\n+\tk_neg = turbo_dec->tb_params.k_neg;\n+\tk_pos = turbo_dec->tb_params.k_pos;\n+\twhile (length > 0 && r < c) {\n+\t\tk = (r < c_neg) ? k_neg : k_pos;\n+\t\tkw = RTE_ALIGN_CEIL(k + 4, 32) * 3;\n+\t\tlength -= kw;\n+\t\tr++;\n+\t\tcbs_in_tb++;\n+\t}\n+\n+\treturn cbs_in_tb;\n+}\n+\n+/* Calculates number of CBs in processed decoder TB based on 'r' and input\n+ * length.\n+ */\n+static inline uint16_t\n+get_num_cbs_in_tb_ldpc_dec(struct rte_bbdev_op_ldpc_dec *ldpc_dec)\n+{\n+\tuint16_t r, cbs_in_tb = 0;\n+\tint32_t length = ldpc_dec->input.length;\n+\tr = ldpc_dec->tb_params.r;\n+\twhile (length > 0 && r < ldpc_dec->tb_params.c) {\n+\t\tlength -= (r < ldpc_dec->tb_params.cab) ?\n+\t\t\t\tldpc_dec->tb_params.ea :\n+\t\t\t\tldpc_dec->tb_params.eb;\n+\t\tr++;\n+\t\tcbs_in_tb++;\n+\t}\n+\treturn cbs_in_tb;\n+}\n+\n+/* Check we can mux encode operations with common FCW */\n+static inline bool\n+check_mux(struct rte_bbdev_enc_op **ops, uint16_t num) {\n+\tuint16_t i;\n+\tif (num == 1)\n+\t\treturn false;\n+\tfor (i = 1; i < num; ++i) {\n+\t\t/* Only mux compatible code blocks */\n+\t\tif (memcmp((uint8_t *)(&ops[i]->ldpc_enc) + ENC_OFFSET,\n+\t\t\t\t(uint8_t *)(&ops[0]->ldpc_enc) + ENC_OFFSET,\n+\t\t\t\tCMP_ENC_SIZE) != 0)\n+\t\t\treturn false;\n+\t}\n+\treturn true;\n+}\n+\n+/** Enqueue encode operations for ACC100 device in CB mode. */\n+static inline uint16_t\n+acc100_enqueue_ldpc_enc_cb(struct rte_bbdev_queue_data *q_data,\n+\t\tstruct rte_bbdev_enc_op **ops, uint16_t num)\n+{\n+\tstruct acc100_queue *q = q_data->queue_private;\n+\tint32_t avail = q->sw_ring_depth + q->sw_ring_tail - q->sw_ring_head;\n+\tuint16_t i = 0;\n+\tunion acc100_dma_desc *desc;\n+\tint ret, desc_idx = 0;\n+\tint16_t enq, left = num;\n+\n+\twhile (left > 0) {\n+\t\tif (unlikely(avail - 1 < 0))\n+\t\t\tbreak;\n+\t\tavail--;\n+\t\tenq = RTE_MIN(left, MUX_5GDL_DESC);\n+\t\tif (check_mux(&ops[i], enq)) {\n+\t\t\tret = enqueue_ldpc_enc_n_op_cb(q, &ops[i],\n+\t\t\t\t\tdesc_idx, enq);\n+\t\t\tif (ret < 0)\n+\t\t\t\tbreak;\n+\t\t\ti += enq;\n+\t\t} else {\n+\t\t\tret = enqueue_ldpc_enc_one_op_cb(q, ops[i], desc_idx);\n+\t\t\tif (ret < 0)\n+\t\t\t\tbreak;\n+\t\t\ti++;\n+\t\t}\n+\t\tdesc_idx++;\n+\t\tleft = num - i;\n+\t}\n+\n+\tif (unlikely(i == 0))\n+\t\treturn 0; /* Nothing to enqueue */\n+\n+\t/* Set SDone in last CB in enqueued ops for CB mode*/\n+\tdesc = q->ring_addr + ((q->sw_ring_head + desc_idx - 1)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\tdesc->req.sdone_enable = 1;\n+\tdesc->req.irq_enable = q->irq_enable;\n+\n+\tacc100_dma_enqueue(q, desc_idx, &q_data->queue_stats);\n+\n+\t/* Update stats */\n+\tq_data->queue_stats.enqueued_count += i;\n+\tq_data->queue_stats.enqueue_err_count += num - i;\n+\n+\treturn i;\n+}\n+\n+/* Enqueue encode operations for ACC100 device. */\n+static uint16_t\n+acc100_enqueue_ldpc_enc(struct rte_bbdev_queue_data *q_data,\n+\t\tstruct rte_bbdev_enc_op **ops, uint16_t num)\n+{\n+\tif (unlikely(num == 0))\n+\t\treturn 0;\n+\treturn acc100_enqueue_ldpc_enc_cb(q_data, ops, num);\n+}\n+\n+/* Check we can mux encode operations with common FCW */\n+static inline bool\n+cmp_ldpc_dec_op(struct rte_bbdev_dec_op **ops) {\n+\t/* Only mux compatible code blocks */\n+\tif (memcmp((uint8_t *)(&ops[0]->ldpc_dec) + DEC_OFFSET,\n+\t\t\t(uint8_t *)(&ops[1]->ldpc_dec) +\n+\t\t\tDEC_OFFSET, CMP_DEC_SIZE) != 0) {\n+\t\treturn false;\n+\t} else\n+\t\treturn true;\n+}\n+\n+\n+/* Enqueue decode operations for ACC100 device in TB mode */\n+static uint16_t\n+acc100_enqueue_ldpc_dec_tb(struct rte_bbdev_queue_data *q_data,\n+\t\tstruct rte_bbdev_dec_op **ops, uint16_t num)\n+{\n+\tstruct acc100_queue *q = q_data->queue_private;\n+\tint32_t avail = q->sw_ring_depth + q->sw_ring_tail - q->sw_ring_head;\n+\tuint16_t i, enqueued_cbs = 0;\n+\tuint8_t cbs_in_tb;\n+\tint ret;\n+\n+\tfor (i = 0; i < num; ++i) {\n+\t\tcbs_in_tb = get_num_cbs_in_tb_ldpc_dec(&ops[i]->ldpc_dec);\n+\t\t/* Check if there are available space for further processing */\n+\t\tif (unlikely(avail - cbs_in_tb < 0))\n+\t\t\tbreak;\n+\t\tavail -= cbs_in_tb;\n+\n+\t\tret = enqueue_ldpc_dec_one_op_tb(q, ops[i],\n+\t\t\t\tenqueued_cbs, cbs_in_tb);\n+\t\tif (ret < 0)\n+\t\t\tbreak;\n+\t\tenqueued_cbs += ret;\n+\t}\n+\n+\tacc100_dma_enqueue(q, enqueued_cbs, &q_data->queue_stats);\n+\n+\t/* Update stats */\n+\tq_data->queue_stats.enqueued_count += i;\n+\tq_data->queue_stats.enqueue_err_count += num - i;\n+\treturn i;\n+}\n+\n+/* Enqueue decode operations for ACC100 device in CB mode */\n+static uint16_t\n+acc100_enqueue_ldpc_dec_cb(struct rte_bbdev_queue_data *q_data,\n+\t\tstruct rte_bbdev_dec_op **ops, uint16_t num)\n+{\n+\tstruct acc100_queue *q = q_data->queue_private;\n+\tint32_t avail = q->sw_ring_depth + q->sw_ring_tail - q->sw_ring_head;\n+\tuint16_t i;\n+\tunion acc100_dma_desc *desc;\n+\tint ret;\n+\tbool same_op = false;\n+\tfor (i = 0; i < num; ++i) {\n+\t\t/* Check if there are available space for further processing */\n+\t\tif (unlikely(avail - 1 < 0))\n+\t\t\tbreak;\n+\t\tavail -= 1;\n+\n+\t\tif (i > 0)\n+\t\t\tsame_op = cmp_ldpc_dec_op(&ops[i-1]);\n+\t\trte_bbdev_log(INFO, \"Op %d %d %d %d %d %d %d %d %d %d %d %d\\n\",\n+\t\t\ti, ops[i]->ldpc_dec.op_flags, ops[i]->ldpc_dec.rv_index,\n+\t\t\tops[i]->ldpc_dec.iter_max, ops[i]->ldpc_dec.iter_count,\n+\t\t\tops[i]->ldpc_dec.basegraph, ops[i]->ldpc_dec.z_c,\n+\t\t\tops[i]->ldpc_dec.n_cb, ops[i]->ldpc_dec.q_m,\n+\t\t\tops[i]->ldpc_dec.n_filler, ops[i]->ldpc_dec.cb_params.e,\n+\t\t\tsame_op);\n+\t\tret = enqueue_ldpc_dec_one_op_cb(q, ops[i], i, same_op);\n+\t\tif (ret < 0)\n+\t\t\tbreak;\n+\t}\n+\n+\tif (unlikely(i == 0))\n+\t\treturn 0; /* Nothing to enqueue */\n+\n+\t/* Set SDone in last CB in enqueued ops for CB mode*/\n+\tdesc = q->ring_addr + ((q->sw_ring_head + i - 1)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\n+\tdesc->req.sdone_enable = 1;\n+\tdesc->req.irq_enable = q->irq_enable;\n+\n+\tacc100_dma_enqueue(q, i, &q_data->queue_stats);\n+\n+\t/* Update stats */\n+\tq_data->queue_stats.enqueued_count += i;\n+\tq_data->queue_stats.enqueue_err_count += num - i;\n+\treturn i;\n+}\n+\n+/* Enqueue decode operations for ACC100 device. */\n+static uint16_t\n+acc100_enqueue_ldpc_dec(struct rte_bbdev_queue_data *q_data,\n+\t\tstruct rte_bbdev_dec_op **ops, uint16_t num)\n+{\n+\tstruct acc100_queue *q = q_data->queue_private;\n+\tint32_t aq_avail = q->aq_depth +\n+\t\t\t(q->aq_dequeued - q->aq_enqueued) / 128;\n+\n+\tif (unlikely((aq_avail == 0) || (num == 0)))\n+\t\treturn 0;\n+\n+\tif (ops[0]->ldpc_dec.code_block_mode == 0)\n+\t\treturn acc100_enqueue_ldpc_dec_tb(q_data, ops, num);\n+\telse\n+\t\treturn acc100_enqueue_ldpc_dec_cb(q_data, ops, num);\n+}\n+\n+\n+/* Dequeue one encode operations from ACC100 device in CB mode */\n+static inline int\n+dequeue_enc_one_op_cb(struct acc100_queue *q, struct rte_bbdev_enc_op **ref_op,\n+\t\tuint16_t total_dequeued_cbs, uint32_t *aq_dequeued)\n+{\n+\tunion acc100_dma_desc *desc, atom_desc;\n+\tunion acc100_dma_rsp_desc rsp;\n+\tstruct rte_bbdev_enc_op *op;\n+\tint i;\n+\n+\tdesc = q->ring_addr + ((q->sw_ring_tail + total_dequeued_cbs)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\tatom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,\n+\t\t\t__ATOMIC_RELAXED);\n+\n+\t/* Check fdone bit */\n+\tif (!(atom_desc.rsp.val & ACC100_FDONE))\n+\t\treturn -1;\n+\n+\trsp.val = atom_desc.rsp.val;\n+\trte_bbdev_log_debug(\"Resp. desc %p: %x\", desc, rsp.val);\n+\n+\t/* Dequeue */\n+\top = desc->req.op_addr;\n+\n+\t/* Clearing status, it will be set based on response */\n+\top->status = 0;\n+\n+\top->status |= ((rsp.input_err)\n+\t\t\t? (1 << RTE_BBDEV_DATA_ERROR) : 0);\n+\top->status |= ((rsp.dma_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);\n+\top->status |= ((rsp.fcw_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);\n+\n+\tif (desc->req.last_desc_in_batch) {\n+\t\t(*aq_dequeued)++;\n+\t\tdesc->req.last_desc_in_batch = 0;\n+\t}\n+\tdesc->rsp.val = ACC100_DMA_DESC_TYPE;\n+\tdesc->rsp.add_info_0 = 0; /*Reserved bits */\n+\tdesc->rsp.add_info_1 = 0; /*Reserved bits */\n+\n+\t/* Flag that the muxing cause loss of opaque data */\n+\top->opaque_data = (void *)-1;\n+\tfor (i = 0 ; i < desc->req.numCBs; i++)\n+\t\tref_op[i] = op;\n+\n+\t/* One CB (op) was successfully dequeued */\n+\treturn desc->req.numCBs;\n+}\n+\n+/* Dequeue one encode operations from ACC100 device in TB mode */\n+static inline int\n+dequeue_enc_one_op_tb(struct acc100_queue *q, struct rte_bbdev_enc_op **ref_op,\n+\t\tuint16_t total_dequeued_cbs, uint32_t *aq_dequeued)\n+{\n+\tunion acc100_dma_desc *desc, *last_desc, atom_desc;\n+\tunion acc100_dma_rsp_desc rsp;\n+\tstruct rte_bbdev_enc_op *op;\n+\tuint8_t i = 0;\n+\tuint16_t current_dequeued_cbs = 0, cbs_in_tb;\n+\n+\tdesc = q->ring_addr + ((q->sw_ring_tail + total_dequeued_cbs)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\tatom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,\n+\t\t\t__ATOMIC_RELAXED);\n+\n+\t/* Check fdone bit */\n+\tif (!(atom_desc.rsp.val & ACC100_FDONE))\n+\t\treturn -1;\n+\n+\t/* Get number of CBs in dequeued TB */\n+\tcbs_in_tb = desc->req.cbs_in_tb;\n+\t/* Get last CB */\n+\tlast_desc = q->ring_addr + ((q->sw_ring_tail\n+\t\t\t+ total_dequeued_cbs + cbs_in_tb - 1)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\t/* Check if last CB in TB is ready to dequeue (and thus\n+\t * the whole TB) - checking sdone bit. If not return.\n+\t */\n+\tatom_desc.atom_hdr = __atomic_load_n((uint64_t *)last_desc,\n+\t\t\t__ATOMIC_RELAXED);\n+\tif (!(atom_desc.rsp.val & ACC100_SDONE))\n+\t\treturn -1;\n+\n+\t/* Dequeue */\n+\top = desc->req.op_addr;\n+\n+\t/* Clearing status, it will be set based on response */\n+\top->status = 0;\n+\n+\twhile (i < cbs_in_tb) {\n+\t\tdesc = q->ring_addr + ((q->sw_ring_tail\n+\t\t\t\t+ total_dequeued_cbs)\n+\t\t\t\t& q->sw_ring_wrap_mask);\n+\t\tatom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,\n+\t\t\t\t__ATOMIC_RELAXED);\n+\t\trsp.val = atom_desc.rsp.val;\n+\t\trte_bbdev_log_debug(\"Resp. desc %p: %x\", desc,\n+\t\t\t\trsp.val);\n+\n+\t\top->status |= ((rsp.input_err)\n+\t\t\t\t? (1 << RTE_BBDEV_DATA_ERROR) : 0);\n+\t\top->status |= ((rsp.dma_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);\n+\t\top->status |= ((rsp.fcw_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);\n+\n+\t\tif (desc->req.last_desc_in_batch) {\n+\t\t\t(*aq_dequeued)++;\n+\t\t\tdesc->req.last_desc_in_batch = 0;\n+\t\t}\n+\t\tdesc->rsp.val = ACC100_DMA_DESC_TYPE;\n+\t\tdesc->rsp.add_info_0 = 0;\n+\t\tdesc->rsp.add_info_1 = 0;\n+\t\ttotal_dequeued_cbs++;\n+\t\tcurrent_dequeued_cbs++;\n+\t\ti++;\n+\t}\n+\n+\t*ref_op = op;\n+\n+\treturn current_dequeued_cbs;\n+}\n+\n+/* Dequeue one decode operation from ACC100 device in CB mode */\n+static inline int\n+dequeue_dec_one_op_cb(struct rte_bbdev_queue_data *q_data,\n+\t\tstruct acc100_queue *q, struct rte_bbdev_dec_op **ref_op,\n+\t\tuint16_t dequeued_cbs, uint32_t *aq_dequeued)\n+{\n+\tunion acc100_dma_desc *desc, atom_desc;\n+\tunion acc100_dma_rsp_desc rsp;\n+\tstruct rte_bbdev_dec_op *op;\n+\n+\tdesc = q->ring_addr + ((q->sw_ring_tail + dequeued_cbs)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\tatom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,\n+\t\t\t__ATOMIC_RELAXED);\n+\n+\t/* Check fdone bit */\n+\tif (!(atom_desc.rsp.val & ACC100_FDONE))\n+\t\treturn -1;\n+\n+\trsp.val = atom_desc.rsp.val;\n+\trte_bbdev_log_debug(\"Resp. desc %p: %x\", desc, rsp.val);\n+\n+\t/* Dequeue */\n+\top = desc->req.op_addr;\n+\n+\t/* Clearing status, it will be set based on response */\n+\top->status = 0;\n+\top->status |= ((rsp.input_err)\n+\t\t\t? (1 << RTE_BBDEV_DATA_ERROR) : 0);\n+\top->status |= ((rsp.dma_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);\n+\top->status |= ((rsp.fcw_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);\n+\tif (op->status != 0)\n+\t\tq_data->queue_stats.dequeue_err_count++;\n+\n+\t/* CRC invalid if error exists */\n+\tif (!op->status)\n+\t\top->status |= rsp.crc_status << RTE_BBDEV_CRC_ERROR;\n+\top->turbo_dec.iter_count = (uint8_t) rsp.iter_cnt / 2;\n+\t/* Check if this is the last desc in batch (Atomic Queue) */\n+\tif (desc->req.last_desc_in_batch) {\n+\t\t(*aq_dequeued)++;\n+\t\tdesc->req.last_desc_in_batch = 0;\n+\t}\n+\tdesc->rsp.val = ACC100_DMA_DESC_TYPE;\n+\tdesc->rsp.add_info_0 = 0;\n+\tdesc->rsp.add_info_1 = 0;\n+\t*ref_op = op;\n+\n+\t/* One CB (op) was successfully dequeued */\n+\treturn 1;\n+}\n+\n+/* Dequeue one decode operations from ACC100 device in CB mode */\n+static inline int\n+dequeue_ldpc_dec_one_op_cb(struct rte_bbdev_queue_data *q_data,\n+\t\tstruct acc100_queue *q, struct rte_bbdev_dec_op **ref_op,\n+\t\tuint16_t dequeued_cbs, uint32_t *aq_dequeued)\n+{\n+\tunion acc100_dma_desc *desc, atom_desc;\n+\tunion acc100_dma_rsp_desc rsp;\n+\tstruct rte_bbdev_dec_op *op;\n+\n+\tdesc = q->ring_addr + ((q->sw_ring_tail + dequeued_cbs)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\tatom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,\n+\t\t\t__ATOMIC_RELAXED);\n+\n+\t/* Check fdone bit */\n+\tif (!(atom_desc.rsp.val & ACC100_FDONE))\n+\t\treturn -1;\n+\n+\trsp.val = atom_desc.rsp.val;\n+\n+\t/* Dequeue */\n+\top = desc->req.op_addr;\n+\n+\t/* Clearing status, it will be set based on response */\n+\top->status = 0;\n+\top->status |= rsp.input_err << RTE_BBDEV_DATA_ERROR;\n+\top->status |= rsp.dma_err << RTE_BBDEV_DRV_ERROR;\n+\top->status |= rsp.fcw_err << RTE_BBDEV_DRV_ERROR;\n+\tif (op->status != 0)\n+\t\tq_data->queue_stats.dequeue_err_count++;\n+\n+\top->status |= rsp.crc_status << RTE_BBDEV_CRC_ERROR;\n+\tif (op->ldpc_dec.hard_output.length > 0 && !rsp.synd_ok)\n+\t\top->status |= 1 << RTE_BBDEV_SYNDROME_ERROR;\n+\top->ldpc_dec.iter_count = (uint8_t) rsp.iter_cnt;\n+\n+\t/* Check if this is the last desc in batch (Atomic Queue) */\n+\tif (desc->req.last_desc_in_batch) {\n+\t\t(*aq_dequeued)++;\n+\t\tdesc->req.last_desc_in_batch = 0;\n+\t}\n+\n+\tdesc->rsp.val = ACC100_DMA_DESC_TYPE;\n+\tdesc->rsp.add_info_0 = 0;\n+\tdesc->rsp.add_info_1 = 0;\n+\n+\t*ref_op = op;\n+\n+\t/* One CB (op) was successfully dequeued */\n+\treturn 1;\n+}\n+\n+/* Dequeue one decode operations from ACC100 device in TB mode. */\n+static inline int\n+dequeue_dec_one_op_tb(struct acc100_queue *q, struct rte_bbdev_dec_op **ref_op,\n+\t\tuint16_t dequeued_cbs, uint32_t *aq_dequeued)\n+{\n+\tunion acc100_dma_desc *desc, *last_desc, atom_desc;\n+\tunion acc100_dma_rsp_desc rsp;\n+\tstruct rte_bbdev_dec_op *op;\n+\tuint8_t cbs_in_tb = 1, cb_idx = 0;\n+\n+\tdesc = q->ring_addr + ((q->sw_ring_tail + dequeued_cbs)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\tatom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,\n+\t\t\t__ATOMIC_RELAXED);\n+\n+\t/* Check fdone bit */\n+\tif (!(atom_desc.rsp.val & ACC100_FDONE))\n+\t\treturn -1;\n+\n+\t/* Dequeue */\n+\top = desc->req.op_addr;\n+\n+\t/* Get number of CBs in dequeued TB */\n+\tcbs_in_tb = desc->req.cbs_in_tb;\n+\t/* Get last CB */\n+\tlast_desc = q->ring_addr + ((q->sw_ring_tail\n+\t\t\t+ dequeued_cbs + cbs_in_tb - 1)\n+\t\t\t& q->sw_ring_wrap_mask);\n+\t/* Check if last CB in TB is ready to dequeue (and thus\n+\t * the whole TB) - checking sdone bit. If not return.\n+\t */\n+\tatom_desc.atom_hdr = __atomic_load_n((uint64_t *)last_desc,\n+\t\t\t__ATOMIC_RELAXED);\n+\tif (!(atom_desc.rsp.val & ACC100_SDONE))\n+\t\treturn -1;\n+\n+\t/* Clearing status, it will be set based on response */\n+\top->status = 0;\n+\n+\t/* Read remaining CBs if exists */\n+\twhile (cb_idx < cbs_in_tb) {\n+\t\tdesc = q->ring_addr + ((q->sw_ring_tail + dequeued_cbs)\n+\t\t\t\t& q->sw_ring_wrap_mask);\n+\t\tatom_desc.atom_hdr = __atomic_load_n((uint64_t *)desc,\n+\t\t\t\t__ATOMIC_RELAXED);\n+\t\trsp.val = atom_desc.rsp.val;\n+\t\trte_bbdev_log_debug(\"Resp. desc %p: %x\", desc,\n+\t\t\t\trsp.val);\n+\n+\t\top->status |= ((rsp.input_err)\n+\t\t\t\t? (1 << RTE_BBDEV_DATA_ERROR) : 0);\n+\t\top->status |= ((rsp.dma_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);\n+\t\top->status |= ((rsp.fcw_err) ? (1 << RTE_BBDEV_DRV_ERROR) : 0);\n+\n+\t\t/* CRC invalid if error exists */\n+\t\tif (!op->status)\n+\t\t\top->status |= rsp.crc_status << RTE_BBDEV_CRC_ERROR;\n+\t\top->turbo_dec.iter_count = RTE_MAX((uint8_t) rsp.iter_cnt,\n+\t\t\t\top->turbo_dec.iter_count);\n+\n+\t\t/* Check if this is the last desc in batch (Atomic Queue) */\n+\t\tif (desc->req.last_desc_in_batch) {\n+\t\t\t(*aq_dequeued)++;\n+\t\t\tdesc->req.last_desc_in_batch = 0;\n+\t\t}\n+\t\tdesc->rsp.val = ACC100_DMA_DESC_TYPE;\n+\t\tdesc->rsp.add_info_0 = 0;\n+\t\tdesc->rsp.add_info_1 = 0;\n+\t\tdequeued_cbs++;\n+\t\tcb_idx++;\n+\t}\n+\n+\t*ref_op = op;\n+\n+\treturn cb_idx;\n+}\n+\n+/* Dequeue LDPC encode operations from ACC100 device. */\n+static uint16_t\n+acc100_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data,\n+\t\tstruct rte_bbdev_enc_op **ops, uint16_t num)\n+{\n+\tstruct acc100_queue *q = q_data->queue_private;\n+\tuint32_t avail = q->sw_ring_head - q->sw_ring_tail;\n+\tuint32_t aq_dequeued = 0;\n+\tuint16_t dequeue_num, i, dequeued_cbs = 0, dequeued_descs = 0;\n+\tint ret;\n+\n+#ifdef RTE_LIBRTE_BBDEV_DEBUG\n+\tif (unlikely(ops == 0 && q == NULL))\n+\t\treturn 0;\n+#endif\n+\n+\tdequeue_num = (avail < num) ? avail : num;\n+\n+\tfor (i = 0; i < dequeue_num; i++) {\n+\t\tret = dequeue_enc_one_op_cb(q, &ops[dequeued_cbs],\n+\t\t\t\tdequeued_descs, &aq_dequeued);\n+\t\tif (ret < 0)\n+\t\t\tbreak;\n+\t\tdequeued_cbs += ret;\n+\t\tdequeued_descs++;\n+\t\tif (dequeued_cbs >= num)\n+\t\t\tbreak;\n+\t}\n+\n+\tq->aq_dequeued += aq_dequeued;\n+\tq->sw_ring_tail += dequeued_descs;\n+\n+\t/* Update enqueue stats */\n+\tq_data->queue_stats.dequeued_count += dequeued_cbs;\n+\n+\treturn dequeued_cbs;\n+}\n+\n+/* Dequeue decode operations from ACC100 device. */\n+static uint16_t\n+acc100_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data,\n+\t\tstruct rte_bbdev_dec_op **ops, uint16_t num)\n+{\n+\tstruct acc100_queue *q = q_data->queue_private;\n+\tuint16_t dequeue_num;\n+\tuint32_t avail = q->sw_ring_head - q->sw_ring_tail;\n+\tuint32_t aq_dequeued = 0;\n+\tuint16_t i;\n+\tuint16_t dequeued_cbs = 0;\n+\tstruct rte_bbdev_dec_op *op;\n+\tint ret;\n+\n+#ifdef RTE_LIBRTE_BBDEV_DEBUG\n+\tif (unlikely(ops == 0 && q == NULL))\n+\t\treturn 0;\n+#endif\n+\n+\tdequeue_num = (avail < num) ? avail : num;\n+\n+\tfor (i = 0; i < dequeue_num; ++i) {\n+\t\top = (q->ring_addr + ((q->sw_ring_tail + dequeued_cbs)\n+\t\t\t& q->sw_ring_wrap_mask))->req.op_addr;\n+\t\tif (op->ldpc_dec.code_block_mode == 0)\n+\t\t\tret = dequeue_dec_one_op_tb(q, &ops[i], dequeued_cbs,\n+\t\t\t\t\t&aq_dequeued);\n+\t\telse\n+\t\t\tret = dequeue_ldpc_dec_one_op_cb(\n+\t\t\t\t\tq_data, q, &ops[i], dequeued_cbs,\n+\t\t\t\t\t&aq_dequeued);\n+\n+\t\tif (ret < 0)\n+\t\t\tbreak;\n+\t\tdequeued_cbs += ret;\n+\t}\n+\n+\tq->aq_dequeued += aq_dequeued;\n+\tq->sw_ring_tail += dequeued_cbs;\n+\n+\t/* Update enqueue stats */\n+\tq_data->queue_stats.dequeued_count += i;\n+\n+\treturn i;\n+}\n+\n /* Initialization Function */\n static void\n acc100_bbdev_init(struct rte_bbdev *dev, struct rte_pci_driver *drv)\n@@ -703,6 +2321,10 @@\n \tstruct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);\n \n \tdev->dev_ops = &acc100_bbdev_ops;\n+\tdev->enqueue_ldpc_enc_ops = acc100_enqueue_ldpc_enc;\n+\tdev->enqueue_ldpc_dec_ops = acc100_enqueue_ldpc_dec;\n+\tdev->dequeue_ldpc_enc_ops = acc100_dequeue_ldpc_enc;\n+\tdev->dequeue_ldpc_dec_ops = acc100_dequeue_ldpc_dec;\n \n \t((struct acc100_device *) dev->data->dev_private)->pf_device =\n \t\t\t!strcmp(drv->driver.name,\n@@ -815,4 +2437,3 @@ static int acc100_pci_remove(struct rte_pci_device *pci_dev)\n RTE_PMD_REGISTER_PCI_TABLE(ACC100PF_DRIVER_NAME, pci_id_acc100_pf_map);\n RTE_PMD_REGISTER_PCI(ACC100VF_DRIVER_NAME, acc100_pci_vf_driver);\n RTE_PMD_REGISTER_PCI_TABLE(ACC100VF_DRIVER_NAME, pci_id_acc100_vf_map);\n-\ndiff --git a/drivers/baseband/acc100/rte_acc100_pmd.h b/drivers/baseband/acc100/rte_acc100_pmd.h\nindex be699e5..0fb6862 100644\n--- a/drivers/baseband/acc100/rte_acc100_pmd.h\n+++ b/drivers/baseband/acc100/rte_acc100_pmd.h\n@@ -88,6 +88,8 @@\n #define TMPL_PRI_3 0x0f0e0d0c\n #define QUEUE_ENABLE 0x80000000 /* Bit to mark Queue as Enabled */\n #define WORDS_IN_ARAM_SIZE (128 * 1024 / 4)\n+#define ACC100_FDONE 0x80000000\n+#define ACC100_SDONE 0x40000000\n \n #define ACC100_NUM_TMPL 32\n #define VF_OFFSET_QOS 16 /* offset in Memory Space specific to QoS Mon */\n@@ -398,6 +400,7 @@ struct __attribute__((__packed__)) acc100_dma_req_desc {\n union acc100_dma_desc {\n \tstruct acc100_dma_req_desc req;\n \tunion acc100_dma_rsp_desc rsp;\n+\tuint64_t atom_hdr;\n };\n \n \n", "prefixes": [ "v2", "05/11" ] }