@@ -25,6 +25,7 @@ poll mode crypto driver support for the following hardware accelerator devices:
* ``Intel QuickAssist Technology 200xx``
* ``Intel QuickAssist Technology D15xx``
* ``Intel QuickAssist Technology C4xxx``
+* ``Intel QuickAssist Technology 4xxx``
Features
@@ -94,15 +95,16 @@ All the usual chains are supported and also some mixed chains:
+==================+===========+=============+==========+==========+
| NULL CIPHER | Y | 2&3 | 2&3 | Y |
+------------------+-----------+-------------+----------+----------+
- | SNOW3G UEA2 | 2&3 | Y | 2&3 | 2&3 |
+ | SNOW3G UEA2 | 2&3 | 1&2&3 | 2&3 | 2&3 |
+------------------+-----------+-------------+----------+----------+
| ZUC EEA3 | 2&3 | 2&3 | 2&3 | 2&3 |
+------------------+-----------+-------------+----------+----------+
- | AES CTR | Y | 2&3 | 2&3 | Y |
+ | AES CTR | 1&2&3 | 2&3 | 2&3 | Y |
+------------------+-----------+-------------+----------+----------+
* The combinations marked as "Y" are supported on all QAT hardware versions.
-* The combinations marked as "2&3" are supported on GEN2/GEN3 QAT hardware only.
+* The combinations marked as "2&3" are supported on GEN2 and GEN3 QAT hardware only.
+* The combinations marked as "1&2&3" are supported on GEN1, GEN2 and GEN3 QAT hardware only.
Limitations
@@ -373,6 +375,8 @@ to see the full table)
+-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+
| Yes | No | No | 3 | C4xxx | p | qat_c4xxx | c4xxx | 18a0 | 1 | 18a1 | 128 |
+-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+
+ | Yes | No | No | 4 | 4xxx | 5.11.0+ | qat_4xxx | 4xxx | 4940 | 4 | 4941 | 16 |
+ +-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+
* Note: Symmetric mixed crypto algorithms feature on Gen 2 works only with 01.org driver version 4.9.0+
@@ -55,6 +55,12 @@ New Features
Also, make sure to start the actual text at the margin.
=======================================================
+* **Updated Intel QuickAssist PMD.**
+
+ Added fourth generation of QuickAssist Technology devices support.
+ Only symmetric crypto has been currently enabled, compression and asymmetric
+ crypto PMD will fail to create.
+
Removed Items
-------------
new file mode 100644
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
+ * Copyright(c) 2021 Intel Corporation
+ */
+
+#ifndef ADF_TRANSPORT_ACCESS_MACROS_GEN4_H
+#define ADF_TRANSPORT_ACCESS_MACROS_GEN4_H
+
+#include "adf_transport_access_macros.h"
+
+#define ADF_RINGS_PER_INT_SRCSEL_GEN4 2
+#define ADF_BANK_INT_SRC_SEL_MASK_GEN4 0x44UL
+#define ADF_BANK_INT_FLAG_CLEAR_MASK_GEN4 0x3
+#define ADF_RING_BUNDLE_SIZE_GEN4 0x2000
+#define ADF_RING_CSR_ADDR_OFFSET_GEN4 0x100000
+#define ADF_RING_CSR_RING_CONFIG_GEN4 0x1000
+#define ADF_RING_CSR_RING_LBASE_GEN4 0x1040
+#define ADF_RING_CSR_RING_UBASE_GEN4 0x1080
+
+#define BUILD_RING_BASE_ADDR_GEN4(addr, size) \
+ ((((addr) >> 6) & (0xFFFFFFFFFFFFFFFFULL << (size))) << 6)
+
+#define WRITE_CSR_RING_BASE_GEN4(csr_base_addr, bank, ring, value) \
+do { \
+ uint32_t l_base = 0, u_base = 0; \
+ l_base = (uint32_t)(value & 0xFFFFFFFF); \
+ u_base = (uint32_t)((value & 0xFFFFFFFF00000000ULL) >> 32); \
+ ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4, \
+ (ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \
+ ADF_RING_CSR_RING_LBASE_GEN4 + (ring << 2), \
+ l_base); \
+ ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4, \
+ (ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \
+ ADF_RING_CSR_RING_UBASE_GEN4 + (ring << 2), \
+ u_base); \
+} while (0)
+
+#define WRITE_CSR_RING_CONFIG_GEN4(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4, \
+ (ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \
+ ADF_RING_CSR_RING_CONFIG_GEN4 + (ring << 2), value)
+
+#define WRITE_CSR_RING_TAIL_GEN4(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR((u8 *)(csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4, \
+ (ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \
+ ADF_RING_CSR_RING_TAIL + ((ring) << 2), value)
+
+#define WRITE_CSR_RING_HEAD_GEN4(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR((u8 *)(csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4, \
+ (ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \
+ ADF_RING_CSR_RING_HEAD + ((ring) << 2), value)
+
+#endif
new file mode 100644
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
+ * Copyright(c) 2021 Intel Corporation
+ */
+
+#ifndef ADF_TRANSPORT_ACCESS_MACROS_GEN4VF_H
+#define ADF_TRANSPORT_ACCESS_MACROS_GEN4VF_H
+
+#include "adf_transport_access_macros.h"
+#include "adf_transport_access_macros_gen4.h"
+
+#define ADF_RING_CSR_ADDR_OFFSET_GEN4VF 0x0
+
+#define WRITE_CSR_RING_BASE_GEN4VF(csr_base_addr, bank, ring, value) \
+do { \
+ uint32_t l_base = 0, u_base = 0; \
+ l_base = (uint32_t)(value & 0xFFFFFFFF); \
+ u_base = (uint32_t)((value & 0xFFFFFFFF00000000ULL) >> 32); \
+ ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \
+ (ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \
+ ADF_RING_CSR_RING_LBASE_GEN4 + (ring << 2), \
+ l_base); \
+ ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \
+ (ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \
+ ADF_RING_CSR_RING_UBASE_GEN4 + (ring << 2), \
+ u_base); \
+} while (0)
+
+#define WRITE_CSR_RING_CONFIG_GEN4VF(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \
+ (ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \
+ ADF_RING_CSR_RING_CONFIG_GEN4 + (ring << 2), value)
+
+#define WRITE_CSR_RING_TAIL_GEN4VF(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \
+ (ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \
+ ADF_RING_CSR_RING_TAIL + ((ring) << 2), (value))
+
+#define WRITE_CSR_RING_HEAD_GEN4VF(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \
+ (ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \
+ ADF_RING_CSR_RING_HEAD + ((ring) << 2), (value))
+
+#define WRITE_CSR_RING_SRV_ARB_EN_GEN4VF(csr_base_addr, bank, value) \
+ ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \
+ (ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \
+ ADF_RING_CSR_RING_SRV_ARB_EN, (value))
+
+#endif
@@ -18,7 +18,8 @@
enum qat_device_gen {
QAT_GEN1 = 1,
QAT_GEN2,
- QAT_GEN3
+ QAT_GEN3,
+ QAT_GEN4
};
enum qat_service_type {
@@ -30,6 +30,11 @@ struct qat_gen_hw_data qat_gen_config[] = {
.qp_hw_data = qat_gen3_qps,
.comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
},
+ [QAT_GEN4] = {
+ .dev_gen = QAT_GEN4,
+ .qp_hw_data = NULL,
+ .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
+ },
};
/* per-process array of device data */
@@ -59,6 +64,9 @@ static const struct rte_pci_id pci_id_qat_map[] = {
{
RTE_PCI_DEVICE(0x8086, 0x18a1),
},
+ {
+ RTE_PCI_DEVICE(0x8086, 0x4941),
+ },
{.device_id = 0},
};
@@ -232,6 +240,9 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev,
case 0x18a1:
qat_dev->qat_dev_gen = QAT_GEN3;
break;
+ case 0x4941:
+ qat_dev->qat_dev_gen = QAT_GEN4;
+ break;
default:
QAT_LOG(ERR, "Invalid dev_id, can't determine generation");
rte_memzone_free(qat_pci_devs[qat_dev->qat_dev_id].mz);
@@ -241,6 +252,17 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev,
if (devargs && devargs->drv_str)
qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param);
+ if (qat_dev->qat_dev_gen >= QAT_GEN4) {
+ int ret = qat_read_qp_config(qat_dev, qat_dev->qat_dev_gen);
+
+ if (ret) {
+ QAT_LOG(ERR,
+ "Cannot acquire ring configuration for QAT_%d",
+ qat_dev_id);
+ return NULL;
+ }
+ }
+
rte_spinlock_init(&qat_dev->arb_csr_lock);
qat_nb_pci_devices++;
@@ -105,6 +105,11 @@ struct qat_pci_device {
/* Data relating to compression service */
struct qat_comp_dev_private *comp_dev;
/**< link back to compressdev private data */
+ union {
+ struct qat_qp_hw_data qp_gen4_data[QAT_GEN4_BUNDLE_NUM]
+ [QAT_GEN4_QPS_PER_BUNDLE_NUM];
+ };
+ /**< Data of ring configuration on gen4 */
};
struct qat_gen_hw_data {
@@ -19,6 +19,7 @@
#include "qat_asym.h"
#include "qat_comp.h"
#include "adf_transport_access_macros.h"
+#include "adf_transport_access_macros_gen4vf.h"
#define QAT_CQ_MAX_DEQ_RETRIES 10
@@ -138,25 +139,33 @@ static int qat_queue_create(struct qat_pci_device *qat_dev,
struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
uint32_t *queue_size_for_csr);
-static void adf_configure_queues(struct qat_qp *queue);
-static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
- rte_spinlock_t *lock);
-static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
- rte_spinlock_t *lock);
-
+static void adf_configure_queues(struct qat_qp *queue,
+ enum qat_device_gen qat_dev_gen);
+static void adf_queue_arb_enable(enum qat_device_gen qat_dev_gen,
+ struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock);
+static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen,
+ struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock);
int qat_qps_per_service(struct qat_pci_device *qat_dev,
enum qat_service_type service)
{
int i = 0, count = 0, max_ops_per_srv = 0;
- const struct qat_qp_hw_data*
- sym_hw_qps = qat_gen_config[qat_dev->qat_dev_gen]
- .qp_hw_data[service];
- max_ops_per_srv = ADF_MAX_QPS_ON_ANY_SERVICE;
- for (; i < max_ops_per_srv; i++)
- if (sym_hw_qps[i].service_type == service)
- count++;
+ if (qat_dev->qat_dev_gen == QAT_GEN4) {
+ max_ops_per_srv = QAT_GEN4_BUNDLE_NUM;
+ for (i = 0, count = 0; i < max_ops_per_srv; i++)
+ if (qat_dev->qp_gen4_data[i][0].service_type == service)
+ count++;
+ } else {
+ const struct qat_qp_hw_data *sym_hw_qps =
+ qat_gen_config[qat_dev->qat_dev_gen]
+ .qp_hw_data[service];
+
+ max_ops_per_srv = ADF_MAX_QPS_ON_ANY_SERVICE;
+ for (i = 0, count = 0; i < max_ops_per_srv; i++)
+ if (sym_hw_qps[i].service_type == service)
+ count++;
+ }
return count;
}
@@ -195,12 +204,12 @@ int qat_qp_setup(struct qat_pci_device *qat_dev,
struct qat_qp **qp_addr,
uint16_t queue_pair_id,
struct qat_qp_config *qat_qp_conf)
-
{
struct qat_qp *qp;
struct rte_pci_device *pci_dev =
qat_pci_devs[qat_dev->qat_dev_id].pci_dev;
char op_cookie_pool_name[RTE_RING_NAMESIZE];
+ enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
uint32_t i;
QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
@@ -264,8 +273,8 @@ int qat_qp_setup(struct qat_pci_device *qat_dev,
goto create_err;
}
- adf_configure_queues(qp);
- adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr,
+ adf_configure_queues(qp, qat_dev_gen);
+ adf_queue_arb_enable(qat_dev_gen, &qp->tx_q, qp->mmap_bar_addr,
&qat_dev->arb_csr_lock);
snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
@@ -314,7 +323,8 @@ int qat_qp_setup(struct qat_pci_device *qat_dev,
return -EFAULT;
}
-int qat_qp_release(struct qat_qp **qp_addr)
+
+int qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr)
{
struct qat_qp *qp = *qp_addr;
uint32_t i;
@@ -335,8 +345,8 @@ int qat_qp_release(struct qat_qp **qp_addr)
return -EAGAIN;
}
- adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr,
- &qp->qat_dev->arb_csr_lock);
+ adf_queue_arb_disable(qat_dev_gen, &(qp->tx_q), qp->mmap_bar_addr,
+ &qp->qat_dev->arb_csr_lock);
for (i = 0; i < qp->nb_descriptors; i++)
rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
@@ -386,6 +396,7 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
const struct rte_memzone *qp_mz;
struct rte_pci_device *pci_dev =
qat_pci_devs[qat_dev->qat_dev_id].pci_dev;
+ enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
int ret = 0;
uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
@@ -445,14 +456,19 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
* Write an unused pattern to the queue memory.
*/
memset(queue->base_addr, 0x7F, queue_size_bytes);
-
- queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
- queue->queue_size);
-
io_addr = pci_dev->mem_resource[0].addr;
- WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
+ if (qat_dev_gen == QAT_GEN4) {
+ queue_base = BUILD_RING_BASE_ADDR_GEN4(queue->base_phys_addr,
+ queue->queue_size);
+ WRITE_CSR_RING_BASE_GEN4VF(io_addr, queue->hw_bundle_number,
+ queue->hw_queue_number, queue_base);
+ } else {
+ queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
+ queue->queue_size);
+ WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
queue->hw_queue_number, queue_base);
+ }
QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
" nb msgs %u, msg_size %u, modulo mask %u",
@@ -468,6 +484,61 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
return ret;
}
+int
+qat_select_valid_queue(struct qat_pci_device *qat_dev, int qp_id,
+ enum qat_service_type service_type)
+{
+ if (qat_dev->qat_dev_gen == QAT_GEN4) {
+ int i = 0, valid_qps = 0;
+
+ for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
+ if (qat_dev->qp_gen4_data[i][0].service_type ==
+ service_type) {
+ if (valid_qps == qp_id)
+ return i;
+ ++valid_qps;
+ }
+ }
+ }
+ return -1;
+}
+
+int
+qat_read_qp_config(struct qat_pci_device *qat_dev,
+ enum qat_device_gen qat_dev_gen)
+{
+ if (qat_dev_gen == QAT_GEN4) {
+ /* Read default configuration,
+ * until some probe of it can be done
+ */
+ int i = 0;
+
+ for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
+ struct qat_qp_hw_data *hw_data =
+ &qat_dev->qp_gen4_data[i][0];
+ enum qat_service_type service_type =
+ (QAT_GEN4_QP_DEFCON >> (8 * i)) & 0xFF;
+
+ memset(hw_data, 0, sizeof(*hw_data));
+ hw_data->service_type = service_type;
+ if (service_type == QAT_SERVICE_ASYMMETRIC) {
+ hw_data->tx_msg_size = 64;
+ hw_data->rx_msg_size = 32;
+ } else if (service_type == QAT_SERVICE_SYMMETRIC ||
+ service_type ==
+ QAT_SERVICE_COMPRESSION) {
+ hw_data->tx_msg_size = 128;
+ hw_data->rx_msg_size = 32;
+ }
+ hw_data->tx_ring_num = 0;
+ hw_data->rx_ring_num = 1;
+ hw_data->hw_bundle_num = i;
+ }
+ }
+ /* With default config will always return success */
+ return 0;
+}
+
static int qat_qp_check_queue_alignment(uint64_t phys_addr,
uint32_t queue_size_bytes)
{
@@ -491,54 +562,81 @@ static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
return -EINVAL;
}
-static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
- rte_spinlock_t *lock)
+static void
+adf_queue_arb_enable(enum qat_device_gen qat_dev_gen, struct qat_queue *txq,
+ void *base_addr, rte_spinlock_t *lock)
{
- uint32_t arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
- (ADF_ARB_REG_SLOT *
- txq->hw_bundle_number);
- uint32_t value;
+ uint32_t arb_csr_offset = 0, value;
rte_spinlock_lock(lock);
- value = ADF_CSR_RD(base_addr, arb_csr_offset);
+ if (qat_dev_gen == QAT_GEN4) {
+ arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
+ (ADF_RING_BUNDLE_SIZE_GEN4 *
+ txq->hw_bundle_number);
+ value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
+ arb_csr_offset);
+ } else {
+ arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
+ (ADF_ARB_REG_SLOT *
+ txq->hw_bundle_number);
+ value = ADF_CSR_RD(base_addr,
+ arb_csr_offset);
+ }
value |= (0x01 << txq->hw_queue_number);
ADF_CSR_WR(base_addr, arb_csr_offset, value);
rte_spinlock_unlock(lock);
}
-static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
- rte_spinlock_t *lock)
+static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen,
+ struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock)
{
- uint32_t arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
- (ADF_ARB_REG_SLOT *
- txq->hw_bundle_number);
- uint32_t value;
+ uint32_t arb_csr_offset = 0, value;
rte_spinlock_lock(lock);
- value = ADF_CSR_RD(base_addr, arb_csr_offset);
+ if (qat_dev_gen == QAT_GEN4) {
+ arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
+ (ADF_RING_BUNDLE_SIZE_GEN4 *
+ txq->hw_bundle_number);
+ value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
+ arb_csr_offset);
+ } else {
+ arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
+ (ADF_ARB_REG_SLOT *
+ txq->hw_bundle_number);
+ value = ADF_CSR_RD(base_addr,
+ arb_csr_offset);
+ }
value &= ~(0x01 << txq->hw_queue_number);
ADF_CSR_WR(base_addr, arb_csr_offset, value);
rte_spinlock_unlock(lock);
}
-static void adf_configure_queues(struct qat_qp *qp)
+static void adf_configure_queues(struct qat_qp *qp,
+ enum qat_device_gen qat_dev_gen)
{
- uint32_t queue_config;
- struct qat_queue *queue = &qp->tx_q;
-
- queue_config = BUILD_RING_CONFIG(queue->queue_size);
-
- WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
- queue->hw_queue_number, queue_config);
-
- queue = &qp->rx_q;
- queue_config =
- BUILD_RESP_RING_CONFIG(queue->queue_size,
- ADF_RING_NEAR_WATERMARK_512,
- ADF_RING_NEAR_WATERMARK_0);
-
- WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
- queue->hw_queue_number, queue_config);
+ uint32_t q_tx_config, q_resp_config;
+ struct qat_queue *q_tx = &qp->tx_q, *q_rx = &qp->rx_q;
+
+ q_tx_config = BUILD_RING_CONFIG(q_tx->queue_size);
+ q_resp_config = BUILD_RESP_RING_CONFIG(q_rx->queue_size,
+ ADF_RING_NEAR_WATERMARK_512,
+ ADF_RING_NEAR_WATERMARK_0);
+
+ if (qat_dev_gen == QAT_GEN4) {
+ WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
+ q_tx->hw_bundle_number, q_tx->hw_queue_number,
+ q_tx_config);
+ WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
+ q_rx->hw_bundle_number, q_rx->hw_queue_number,
+ q_resp_config);
+ } else {
+ WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr,
+ q_tx->hw_bundle_number, q_tx->hw_queue_number,
+ q_tx_config);
+ WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr,
+ q_rx->hw_bundle_number, q_rx->hw_queue_number,
+ q_resp_config);
+ }
}
static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
@@ -547,14 +645,21 @@ static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
}
static inline void
-txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
- WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
+txq_write_tail(enum qat_device_gen qat_dev_gen,
+ struct qat_qp *qp, struct qat_queue *q) {
+
+ if (qat_dev_gen == QAT_GEN4) {
+ WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr,
+ q->hw_bundle_number, q->hw_queue_number, q->tail);
+ } else {
+ WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
q->hw_queue_number, q->tail);
- q->csr_tail = q->tail;
+ }
}
static inline
-void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
+void rxq_free_desc(enum qat_device_gen qat_dev_gen, struct qat_qp *qp,
+ struct qat_queue *q)
{
uint32_t old_head, new_head;
uint32_t max_head;
@@ -576,8 +681,14 @@ void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
q->csr_head = new_head;
/* write current head to CSR */
- WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
- q->hw_queue_number, new_head);
+ if (qat_dev_gen == QAT_GEN4) {
+ WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr,
+ q->hw_bundle_number, q->hw_queue_number, new_head);
+ } else {
+ WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
+ q->hw_queue_number, new_head);
+ }
+
}
uint16_t
@@ -670,7 +781,7 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
queue->tail = tail;
tmp_qp->enqueued += nb_ops_sent;
tmp_qp->stats.enqueued_count += nb_ops_sent;
- txq_write_tail(tmp_qp, queue);
+ txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue);
return nb_ops_sent;
}
@@ -843,7 +954,7 @@ qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
queue->tail = tail;
tmp_qp->enqueued += total_descriptors_built;
tmp_qp->stats.enqueued_count += nb_ops_sent;
- txq_write_tail(tmp_qp, queue);
+ txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue);
return nb_ops_sent;
}
@@ -909,7 +1020,7 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
rx_queue->head = head;
if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH)
- rxq_free_desc(tmp_qp, rx_queue);
+ rxq_free_desc(tmp_qp->qat_dev_gen, tmp_qp, rx_queue);
QAT_DP_LOG(DEBUG, "Dequeue burst return: %u, QAT responses: %u",
op_resp_counter, fw_resp_counter);
@@ -951,7 +1062,7 @@ qat_cq_dequeue_response(struct qat_qp *qp, void *out_data)
queue->head = adf_modulo(queue->head + queue->msg_size,
queue->modulo_mask);
- rxq_free_desc(qp, queue);
+ rxq_free_desc(qp->qat_dev_gen, qp, queue);
}
return result;
@@ -986,7 +1097,7 @@ qat_cq_get_fw_version(struct qat_qp *qp)
memcpy(base_addr + queue->tail, &null_msg, sizeof(null_msg));
queue->tail = adf_modulo(queue->tail + queue->msg_size,
queue->modulo_mask);
- txq_write_tail(qp, queue);
+ txq_write_tail(qp->qat_dev_gen, qp, queue);
/* receive a response */
if (qat_cq_dequeue_response(qp, &response)) {
@@ -14,6 +14,16 @@ struct qat_pci_device;
#define QAT_QP_MIN_INFL_THRESHOLD 256
+/* Default qp configuration for GEN4 devices */
+#define QAT_GEN4_QP_DEFCON (QAT_SERVICE_SYMMETRIC | \
+ QAT_SERVICE_SYMMETRIC << 8 | \
+ QAT_SERVICE_SYMMETRIC << 16 | \
+ QAT_SERVICE_SYMMETRIC << 24)
+
+/* QAT GEN 4 specific macros */
+#define QAT_GEN4_BUNDLE_NUM 4
+#define QAT_GEN4_QPS_PER_BUNDLE_NUM 1
+
/**
* Structure with data needed for creation of queue pair.
*/
@@ -26,6 +36,15 @@ struct qat_qp_hw_data {
uint16_t rx_msg_size;
};
+/**
+ * Structure with data needed for creation of queue pair on gen4.
+ */
+struct qat_qp_gen4_data {
+ struct qat_qp_hw_data qat_qp_hw_data;
+ uint8_t reserved;
+ uint8_t valid;
+};
+
/**
* Structure with data needed for creation of queue pair.
*/
@@ -90,7 +109,7 @@ uint16_t
qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops);
int
-qat_qp_release(struct qat_qp **qp_addr);
+qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr);
int
qat_qp_setup(struct qat_pci_device *qat_dev,
@@ -110,4 +129,12 @@ qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
void *op_cookie __rte_unused,
uint64_t *dequeue_err_count __rte_unused);
+int
+qat_select_valid_queue(struct qat_pci_device *qat_dev, int qp_id,
+ enum qat_service_type service_type);
+
+int
+qat_read_qp_config(struct qat_pci_device *qat_dev,
+ enum qat_device_gen qat_dev_gen);
+
#endif /* _QAT_QP_H_ */
@@ -74,6 +74,7 @@ qat_comp_qp_release(struct rte_compressdev *dev, uint16_t queue_pair_id)
struct qat_qp **qp_addr =
(struct qat_qp **)&(dev->data->queue_pairs[queue_pair_id]);
struct qat_qp *qp = (struct qat_qp *)*qp_addr;
+ enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen;
uint32_t i;
QAT_LOG(DEBUG, "Release comp qp %u on device %d",
@@ -90,7 +91,7 @@ qat_comp_qp_release(struct rte_compressdev *dev, uint16_t queue_pair_id)
rte_free(cookie->qat_sgl_dst_d);
}
- return qat_qp_release((struct qat_qp **)
+ return qat_qp_release(qat_dev_gen, (struct qat_qp **)
&(dev->data->queue_pairs[queue_pair_id]));
}
@@ -710,6 +711,10 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev,
const struct rte_compressdev_capabilities *capabilities;
uint64_t capa_size;
+ if (qat_pci_dev->qat_dev_gen == QAT_GEN4) {
+ QAT_LOG(ERR, "Compression PMD not supported on QAT 4xxx");
+ return 0;
+ }
snprintf(name, RTE_COMPRESSDEV_NAME_MAX_LEN, "%s_%s",
qat_pci_dev->name, "comp");
QAT_LOG(DEBUG, "Creating QAT COMP device %s", name);
@@ -103,6 +103,7 @@ static int qat_asym_qp_release(struct rte_cryptodev *dev,
uint16_t queue_pair_id)
{
struct qat_asym_dev_private *qat_private = dev->data->dev_private;
+ enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen;
QAT_LOG(DEBUG, "Release asym qp %u on device %d",
queue_pair_id, dev->data->dev_id);
@@ -110,7 +111,7 @@ static int qat_asym_qp_release(struct rte_cryptodev *dev,
qat_private->qat_dev->qps_in_use[QAT_SERVICE_ASYMMETRIC][queue_pair_id]
= NULL;
- return qat_qp_release((struct qat_qp **)
+ return qat_qp_release(qat_dev_gen, (struct qat_qp **)
&(dev->data->queue_pairs[queue_pair_id]));
}
@@ -250,6 +251,10 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
struct rte_cryptodev *cryptodev;
struct qat_asym_dev_private *internals;
+ if (qat_pci_dev->qat_dev_gen == QAT_GEN4) {
+ QAT_LOG(ERR, "Asymmetric crypto PMD not supported on QAT 4xxx");
+ return 0;
+ }
snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
qat_pci_dev->name, "asym");
QAT_LOG(DEBUG, "Creating QAT ASYM device %s\n", name);
@@ -139,6 +139,7 @@ static void qat_sym_stats_reset(struct rte_cryptodev *dev)
static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
{
struct qat_sym_dev_private *qat_private = dev->data->dev_private;
+ enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen;
QAT_LOG(DEBUG, "Release sym qp %u on device %d",
queue_pair_id, dev->data->dev_id);
@@ -146,7 +147,7 @@ static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
qat_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][queue_pair_id]
= NULL;
- return qat_qp_release((struct qat_qp **)
+ return qat_qp_release(qat_dev_gen, (struct qat_qp **)
&(dev->data->queue_pairs[queue_pair_id]));
}
@@ -158,15 +159,33 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
int ret = 0;
uint32_t i;
struct qat_qp_config qat_qp_conf;
+ const struct qat_qp_hw_data *sym_hw_qps = NULL;
+ const struct qat_qp_hw_data *qp_hw_data = NULL;
struct qat_qp **qp_addr =
(struct qat_qp **)&(dev->data->queue_pairs[qp_id]);
struct qat_sym_dev_private *qat_private = dev->data->dev_private;
struct qat_pci_device *qat_dev = qat_private->qat_dev;
- const struct qat_qp_hw_data *sym_hw_qps =
- qat_gen_config[qat_private->qat_dev->qat_dev_gen]
- .qp_hw_data[QAT_SERVICE_SYMMETRIC];
- const struct qat_qp_hw_data *qp_hw_data = sym_hw_qps + qp_id;
+
+ if (qat_dev->qat_dev_gen == QAT_GEN4) {
+ int ring_pair =
+ qat_select_valid_queue(qat_dev, qp_id,
+ QAT_SERVICE_SYMMETRIC);
+ sym_hw_qps =
+ &qat_dev->qp_gen4_data[0][0];
+ qp_hw_data =
+ &qat_dev->qp_gen4_data[ring_pair][0];
+ if (ring_pair < 0) {
+ QAT_LOG(ERR,
+ "qp_id %u invalid for this device, no enough services allocated for GEN4 device",
+ qp_id);
+ return -EINVAL;
+ }
+ } else {
+ sym_hw_qps = qat_gen_config[qat_dev->qat_dev_gen]
+ .qp_hw_data[QAT_SERVICE_SYMMETRIC];
+ qp_hw_data = sym_hw_qps + qp_id;
+ }
/* If qp is already in use free ring memory and qp metadata. */
if (*qp_addr != NULL) {
@@ -430,6 +449,10 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
capabilities = qat_gen3_sym_capabilities;
capa_size = sizeof(qat_gen3_sym_capabilities);
break;
+ case QAT_GEN4:
+ capabilities = NULL;
+ capa_size = 0;
+ break;
default:
QAT_LOG(DEBUG,
"QAT gen %d capabilities unknown",
@@ -550,6 +550,7 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev,
return -EINVAL;
}
+ memset(session, 0, sizeof(*session));
/* Set context descriptor physical address */
session->cd_paddr = session_paddr +
offsetof(struct qat_sym_session, cd);