From patchwork Tue Apr 3 09:43:03 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Zhoujian (jay)" X-Patchwork-Id: 36917 X-Patchwork-Delegate: pablo.de.lara.guarch@intel.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 9C4A81B6B0; Tue, 3 Apr 2018 11:43:31 +0200 (CEST) Received: from huawei.com (unknown [45.249.212.32]) by dpdk.org (Postfix) with ESMTP id 89C931B6AD for ; Tue, 3 Apr 2018 11:43:29 +0200 (CEST) Received: from DGGEMS412-HUB.china.huawei.com (unknown [172.30.72.59]) by Forcepoint Email with ESMTP id C75BD4187B52B; Tue, 3 Apr 2018 17:43:25 +0800 (CST) Received: from localhost (10.177.19.14) by DGGEMS412-HUB.china.huawei.com (10.3.19.212) with Microsoft SMTP Server id 14.3.361.1; Tue, 3 Apr 2018 17:43:19 +0800 From: Jay Zhou To: CC: , , , , , , , , Date: Tue, 3 Apr 2018 17:43:03 +0800 Message-ID: <077b924faee30a928170630db564b179acd7b44a.1522747556.git.jianjay.zhou@huawei.com> X-Mailer: git-send-email 2.6.1.windows.1 In-Reply-To: References: MIME-Version: 1.0 X-Originating-IP: [10.177.19.14] X-CFilter-Loop: Reflected Subject: [dpdk-dev] [PATCH v6 03/10] crypto/virtio: support basic PMD ops X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Signed-off-by: Jay Zhou --- drivers/crypto/virtio/virtio_cryptodev.c | 434 ++++++++++++++++++++++++++++++- drivers/crypto/virtio/virtio_cryptodev.h | 32 ++- drivers/crypto/virtio/virtio_rxtx.c | 68 +++++ 3 files changed, 525 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/virtio/virtio_cryptodev.c b/drivers/crypto/virtio/virtio_cryptodev.c index 4550834..0be99fd 100644 --- a/drivers/crypto/virtio/virtio_cryptodev.c +++ b/drivers/crypto/virtio/virtio_cryptodev.c @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. */ +#include #include #include #include @@ -15,6 +16,22 @@ int virtio_crypto_logtype_tx; int virtio_crypto_logtype_driver; +static int virtio_crypto_dev_configure(struct rte_cryptodev *dev, + struct rte_cryptodev_config *config); +static int virtio_crypto_dev_start(struct rte_cryptodev *dev); +static void virtio_crypto_dev_stop(struct rte_cryptodev *dev); +static int virtio_crypto_dev_close(struct rte_cryptodev *dev); +static void virtio_crypto_dev_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *dev_info); +static int virtio_crypto_qp_setup(struct rte_cryptodev *dev, + uint16_t queue_pair_id, + const struct rte_cryptodev_qp_conf *qp_conf, + int socket_id, + struct rte_mempool *session_pool); +static int virtio_crypto_qp_release(struct rte_cryptodev *dev, + uint16_t queue_pair_id); +static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev); + /* * The set of PCI devices this driver supports */ @@ -26,22 +43,249 @@ uint8_t cryptodev_virtio_driver_id; +void virtio_crypto_queue_release(struct virtqueue *vq) +{ + struct virtio_crypto_hw *hw; + + PMD_INIT_FUNC_TRACE(); + + if (vq) { + hw = vq->hw; + /* Select and deactivate the queue */ + VTPCI_OPS(hw)->del_queue(hw, vq); + + rte_memzone_free(vq->mz); + rte_mempool_free(vq->mpool); + rte_free(vq); + } +} + +#define MPOOL_MAX_NAME_SZ 32 + +int virtio_crypto_queue_setup(struct rte_cryptodev *dev, + int queue_type, + uint16_t vtpci_queue_idx, + uint16_t nb_desc, + int socket_id, + struct virtqueue **pvq) +{ + char vq_name[VIRTQUEUE_MAX_NAME_SZ]; + char mpool_name[MPOOL_MAX_NAME_SZ]; + const struct rte_memzone *mz; + unsigned int vq_size, size; + struct virtio_crypto_hw *hw = dev->data->dev_private; + struct virtqueue *vq = NULL; + uint32_t i = 0; + uint32_t j; + + PMD_INIT_FUNC_TRACE(); + + VIRTIO_CRYPTO_INIT_LOG_DBG("setting up queue: %u", vtpci_queue_idx); + + /* + * Read the virtqueue size from the Queue Size field + * Always power of 2 and if 0 virtqueue does not exist + */ + vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx); + if (vq_size == 0) { + VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue does not exist"); + return -EINVAL; + } + VIRTIO_CRYPTO_INIT_LOG_DBG("vq_size: %u", vq_size); + + if (!rte_is_power_of_2(vq_size)) { + VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue size is not powerof 2"); + return -EINVAL; + } + + if (queue_type == VTCRYPTO_DATAQ) { + snprintf(vq_name, sizeof(vq_name), "dev%d_dataqueue%d", + dev->data->dev_id, vtpci_queue_idx); + snprintf(mpool_name, sizeof(mpool_name), + "dev%d_dataqueue%d_mpool", + dev->data->dev_id, vtpci_queue_idx); + } else if (queue_type == VTCRYPTO_CTRLQ) { + snprintf(vq_name, sizeof(vq_name), "dev%d_controlqueue", + dev->data->dev_id); + snprintf(mpool_name, sizeof(mpool_name), + "dev%d_controlqueue_mpool", + dev->data->dev_id); + } + size = RTE_ALIGN_CEIL(sizeof(*vq) + + vq_size * sizeof(struct vq_desc_extra), + RTE_CACHE_LINE_SIZE); + vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE, + socket_id); + if (vq == NULL) { + VIRTIO_CRYPTO_INIT_LOG_ERR("Can not allocate virtqueue"); + return -ENOMEM; + } + + if (queue_type == VTCRYPTO_DATAQ) { + /* pre-allocate a mempool and use it in the data plane to + * improve performance + */ + vq->mpool = rte_mempool_lookup(mpool_name); + if (vq->mpool == NULL) + vq->mpool = rte_mempool_create(mpool_name, + vq_size, + sizeof(struct virtio_crypto_op_cookie), + RTE_CACHE_LINE_SIZE, 0, + NULL, NULL, NULL, NULL, socket_id, + 0); + if (!vq->mpool) { + VIRTIO_CRYPTO_DRV_LOG_ERR("Virtio Crypto PMD " + "Cannot create mempool"); + goto mpool_create_err; + } + for (i = 0; i < vq_size; i++) { + vq->vq_descx[i].cookie = + rte_zmalloc("crypto PMD op cookie pointer", + sizeof(struct virtio_crypto_op_cookie), + RTE_CACHE_LINE_SIZE); + if (vq->vq_descx[i].cookie == NULL) { + VIRTIO_CRYPTO_DRV_LOG_ERR("Failed to " + "alloc mem for cookie"); + goto cookie_alloc_err; + } + } + } + + vq->hw = hw; + vq->dev_id = dev->data->dev_id; + vq->vq_queue_index = vtpci_queue_idx; + vq->vq_nentries = vq_size; + + /* + * Using part of the vring entries is permitted, but the maximum + * is vq_size + */ + if (nb_desc == 0 || nb_desc > vq_size) + nb_desc = vq_size; + vq->vq_free_cnt = nb_desc; + + /* + * Reserve a memzone for vring elements + */ + size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN); + vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN); + VIRTIO_CRYPTO_INIT_LOG_DBG("%s vring_size: %d, rounded_vring_size: %d", + (queue_type == VTCRYPTO_DATAQ) ? "dataq" : "ctrlq", + size, vq->vq_ring_size); + + mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, + socket_id, 0, VIRTIO_PCI_VRING_ALIGN); + if (mz == NULL) { + if (rte_errno == EEXIST) + mz = rte_memzone_lookup(vq_name); + if (mz == NULL) { + VIRTIO_CRYPTO_INIT_LOG_ERR("not enough memory"); + goto mz_reserve_err; + } + } + + /* + * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit, + * and only accepts 32 bit page frame number. + * Check if the allocated physical memory exceeds 16TB. + */ + if ((mz->phys_addr + vq->vq_ring_size - 1) + >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) { + VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be " + "above 16TB!"); + goto vring_addr_err; + } + + memset(mz->addr, 0, sizeof(mz->len)); + vq->mz = mz; + vq->vq_ring_mem = mz->phys_addr; + vq->vq_ring_virt_mem = mz->addr; + VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_mem(physical): 0x%"PRIx64, + (uint64_t)mz->phys_addr); + VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_virt_mem: 0x%"PRIx64, + (uint64_t)(uintptr_t)mz->addr); + + *pvq = vq; + + return 0; + +vring_addr_err: + rte_memzone_free(mz); +mz_reserve_err: +cookie_alloc_err: + rte_mempool_free(vq->mpool); + if (i != 0) { + for (j = 0; j < i; j++) + rte_free(vq->vq_descx[j].cookie); + } +mpool_create_err: + rte_free(vq); + return -ENOMEM; +} + +static int +virtio_crypto_ctrlq_setup(struct rte_cryptodev *dev, uint16_t queue_idx) +{ + int ret; + struct virtqueue *vq; + struct virtio_crypto_hw *hw = dev->data->dev_private; + + /* if virtio device has started, do not touch the virtqueues */ + if (dev->data->dev_started) + return 0; + + PMD_INIT_FUNC_TRACE(); + + ret = virtio_crypto_queue_setup(dev, VTCRYPTO_CTRLQ, queue_idx, + 0, SOCKET_ID_ANY, &vq); + if (ret < 0) { + VIRTIO_CRYPTO_INIT_LOG_ERR("control vq initialization failed"); + return ret; + } + + hw->cvq = vq; + + return 0; +} + +static void +virtio_crypto_free_queues(struct rte_cryptodev *dev) +{ + unsigned int i; + struct virtio_crypto_hw *hw = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + /* control queue release */ + virtio_crypto_queue_release(hw->cvq); + + /* data queue release */ + for (i = 0; i < hw->max_dataqueues; i++) + virtio_crypto_queue_release(dev->data->queue_pairs[i]); +} + +static int +virtio_crypto_dev_close(struct rte_cryptodev *dev __rte_unused) +{ + return 0; +} + /* * dev_ops for virtio, bare necessities for basic operation */ static struct rte_cryptodev_ops virtio_crypto_dev_ops = { /* Device related operations */ - .dev_configure = NULL, - .dev_start = NULL, - .dev_stop = NULL, - .dev_close = NULL, - .dev_infos_get = NULL, + .dev_configure = virtio_crypto_dev_configure, + .dev_start = virtio_crypto_dev_start, + .dev_stop = virtio_crypto_dev_stop, + .dev_close = virtio_crypto_dev_close, + .dev_infos_get = virtio_crypto_dev_info_get, .stats_get = NULL, .stats_reset = NULL, - .queue_pair_setup = NULL, - .queue_pair_release = NULL, + .queue_pair_setup = virtio_crypto_qp_setup, + .queue_pair_release = virtio_crypto_qp_release, .queue_pair_start = NULL, .queue_pair_stop = NULL, .queue_pair_count = NULL, @@ -55,6 +299,51 @@ }; static int +virtio_crypto_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id, + const struct rte_cryptodev_qp_conf *qp_conf, + int socket_id, + struct rte_mempool *session_pool __rte_unused) +{ + int ret; + struct virtqueue *vq; + + PMD_INIT_FUNC_TRACE(); + + /* if virtio dev is started, do not touch the virtqueues */ + if (dev->data->dev_started) + return 0; + + ret = virtio_crypto_queue_setup(dev, VTCRYPTO_DATAQ, queue_pair_id, + qp_conf->nb_descriptors, socket_id, &vq); + if (ret < 0) { + VIRTIO_CRYPTO_INIT_LOG_ERR( + "virtio crypto data queue initialization failed\n"); + return ret; + } + + dev->data->queue_pairs[queue_pair_id] = vq; + + return 0; +} + +static int +virtio_crypto_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) +{ + struct virtqueue *vq + = (struct virtqueue *)dev->data->queue_pairs[queue_pair_id]; + + PMD_INIT_FUNC_TRACE(); + + if (vq == NULL) { + VIRTIO_CRYPTO_DRV_LOG_DBG("vq already freed"); + return 0; + } + + virtio_crypto_queue_release(vq); + return 0; +} + +static int virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features) { uint64_t host_features; @@ -191,6 +480,135 @@ return 0; } +static int +virtio_crypto_dev_uninit(struct rte_cryptodev *cryptodev) +{ + struct virtio_crypto_hw *hw = cryptodev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + if (rte_eal_process_type() == RTE_PROC_SECONDARY) + return -EPERM; + + if (cryptodev->data->dev_started) { + virtio_crypto_dev_stop(cryptodev); + virtio_crypto_dev_close(cryptodev); + } + + cryptodev->dev_ops = NULL; + cryptodev->enqueue_burst = NULL; + cryptodev->dequeue_burst = NULL; + + /* release control queue */ + virtio_crypto_queue_release(hw->cvq); + + rte_free(cryptodev->data); + cryptodev->data = NULL; + + VIRTIO_CRYPTO_DRV_LOG_INFO("dev_uninit completed"); + + return 0; +} + +static int +virtio_crypto_dev_configure(struct rte_cryptodev *cryptodev, + struct rte_cryptodev_config *config __rte_unused) +{ + struct virtio_crypto_hw *hw = cryptodev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + if (virtio_crypto_init_device(cryptodev, + VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0) + return -1; + + /* setup control queue + * [0, 1, ... ,(config->max_dataqueues - 1)] are data queues + * config->max_dataqueues is the control queue + */ + if (virtio_crypto_ctrlq_setup(cryptodev, hw->max_dataqueues) < 0) { + VIRTIO_CRYPTO_INIT_LOG_ERR("control queue setup error"); + return -1; + } + virtio_crypto_ctrlq_start(cryptodev); + + return 0; +} + +static void +virtio_crypto_dev_stop(struct rte_cryptodev *dev) +{ + struct virtio_crypto_hw *hw = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + VIRTIO_CRYPTO_DRV_LOG_DBG("virtio_dev_stop"); + + vtpci_cryptodev_reset(hw); + + virtio_crypto_dev_free_mbufs(dev); + virtio_crypto_free_queues(dev); + + dev->data->dev_started = 0; +} + +static int +virtio_crypto_dev_start(struct rte_cryptodev *dev) +{ + struct virtio_crypto_hw *hw = dev->data->dev_private; + + if (dev->data->dev_started) + return 0; + + /* Do final configuration before queue engine starts */ + virtio_crypto_dataq_start(dev); + vtpci_cryptodev_reinit_complete(hw); + + dev->data->dev_started = 1; + + return 0; +} + +static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev) +{ + uint32_t i; + struct virtio_crypto_hw *hw = dev->data->dev_private; + + for (i = 0; i < hw->max_dataqueues; i++) { + VIRTIO_CRYPTO_INIT_LOG_DBG("Before freeing dataq[%d] used " + "and unused buf", i); + VIRTQUEUE_DUMP((struct virtqueue *) + dev->data->queue_pairs[i]); + + VIRTIO_CRYPTO_INIT_LOG_DBG("queue_pairs[%d]=%p", + i, dev->data->queue_pairs[i]); + + virtqueue_detatch_unused(dev->data->queue_pairs[i]); + + VIRTIO_CRYPTO_INIT_LOG_DBG("After freeing dataq[%d] used and " + "unused buf", i); + VIRTQUEUE_DUMP( + (struct virtqueue *)dev->data->queue_pairs[i]); + } +} + +static void +virtio_crypto_dev_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *info) +{ + struct virtio_crypto_hw *hw = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + if (info != NULL) { + info->driver_id = cryptodev_virtio_driver_id; + info->pci_dev = RTE_DEV_TO_PCI(dev->device); + info->feature_flags = dev->feature_flags; + info->max_nb_queue_pairs = hw->max_dataqueues; + info->sym.max_nb_sessions = + RTE_VIRTIO_CRYPTO_PMD_MAX_NB_SESSIONS; + } +} + static int crypto_virtio_pci_probe( struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_device *pci_dev) @@ -229,7 +647,7 @@ static int crypto_virtio_pci_remove( if (cryptodev == NULL) return -ENODEV; - return 0; + return virtio_crypto_dev_uninit(cryptodev); } static struct rte_pci_driver rte_virtio_crypto_driver = { diff --git a/drivers/crypto/virtio/virtio_cryptodev.h b/drivers/crypto/virtio/virtio_cryptodev.h index 392db4a..159f58e 100644 --- a/drivers/crypto/virtio/virtio_cryptodev.h +++ b/drivers/crypto/virtio/virtio_cryptodev.h @@ -5,13 +5,43 @@ #ifndef _VIRTIO_CRYPTODEV_H_ #define _VIRTIO_CRYPTODEV_H_ -#include +#include + +#include "virtio_pci.h" +#include "virtio_ring.h" /* Features desired/implemented by this driver. */ #define VIRTIO_CRYPTO_PMD_GUEST_FEATURES (1ULL << VIRTIO_F_VERSION_1) #define CRYPTODEV_NAME_VIRTIO_PMD crypto_virtio +#define NUM_ENTRY_VIRTIO_CRYPTO_OP 7 + +struct virtio_crypto_op_cookie { + struct virtio_crypto_op_data_req data_req; + struct virtio_crypto_inhdr inhdr; + struct vring_desc desc[NUM_ENTRY_VIRTIO_CRYPTO_OP]; +}; + +/* + * Control queue function prototype + */ +void virtio_crypto_ctrlq_start(struct rte_cryptodev *dev); + +/* + * Data queue function prototype + */ +void virtio_crypto_dataq_start(struct rte_cryptodev *dev); + +int virtio_crypto_queue_setup(struct rte_cryptodev *dev, + int queue_type, + uint16_t vtpci_queue_idx, + uint16_t nb_desc, + int socket_id, + struct virtqueue **pvq); + +void virtio_crypto_queue_release(struct virtqueue *vq); + uint16_t virtio_crypto_pkt_tx_burst(void *tx_queue, struct rte_crypto_op **tx_pkts, uint16_t nb_pkts); diff --git a/drivers/crypto/virtio/virtio_rxtx.c b/drivers/crypto/virtio/virtio_rxtx.c index 51f6e09..3b038a7 100644 --- a/drivers/crypto/virtio/virtio_rxtx.c +++ b/drivers/crypto/virtio/virtio_rxtx.c @@ -1,8 +1,76 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. */ +#include "virtqueue.h" #include "virtio_cryptodev.h" +static int +virtio_crypto_vring_start(struct virtqueue *vq) +{ + struct virtio_crypto_hw *hw = vq->hw; + int i, size = vq->vq_nentries; + struct vring *vr = &vq->vq_ring; + uint8_t *ring_mem = vq->vq_ring_virt_mem; + + PMD_INIT_FUNC_TRACE(); + + vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN); + vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1); + vq->vq_free_cnt = vq->vq_nentries; + + /* Chain all the descriptors in the ring with an END */ + for (i = 0; i < size - 1; i++) + vr->desc[i].next = (uint16_t)(i + 1); + vr->desc[i].next = VQ_RING_DESC_CHAIN_END; + + /* + * Disable device(host) interrupting guest + */ + virtqueue_disable_intr(vq); + + /* + * Set guest physical address of the virtqueue + * in VIRTIO_PCI_QUEUE_PFN config register of device + * to share with the backend + */ + if (VTPCI_OPS(hw)->setup_queue(hw, vq) < 0) { + VIRTIO_CRYPTO_INIT_LOG_ERR("setup_queue failed"); + return -EINVAL; + } + + return 0; +} + +void +virtio_crypto_ctrlq_start(struct rte_cryptodev *dev) +{ + struct virtio_crypto_hw *hw = dev->data->dev_private; + + if (hw->cvq) { + virtio_crypto_vring_start(hw->cvq); + VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq); + } +} + +void +virtio_crypto_dataq_start(struct rte_cryptodev *dev) +{ + /* + * Start data vrings + * - Setup vring structure for data queues + */ + uint16_t i; + struct virtio_crypto_hw *hw = dev->data->dev_private; + + PMD_INIT_FUNC_TRACE(); + + /* Start data vring. */ + for (i = 0; i < hw->max_dataqueues; i++) { + virtio_crypto_vring_start(dev->data->queue_pairs[i]); + VIRTQUEUE_DUMP((struct virtqueue *)dev->data->queue_pairs[i]); + } +} + uint16_t virtio_crypto_pkt_rx_burst( void *tx_queue __rte_unused,