From patchwork Wed Jun 19 15:14:46 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Nikos Dragazis X-Patchwork-Id: 54987 X-Patchwork-Delegate: maxime.coquelin@redhat.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 4BC6B1C42E; Wed, 19 Jun 2019 17:16:40 +0200 (CEST) Received: from mx0.arrikto.com (mx0.arrikto.com [212.71.252.59]) by dpdk.org (Postfix) with ESMTP id 9ACD51C3A0 for ; Wed, 19 Jun 2019 17:15:46 +0200 (CEST) Received: from troi.prod.arr (mail.arr [10.99.0.5]) by mx0.arrikto.com (Postfix) with ESMTP id 69350182019; Wed, 19 Jun 2019 18:15:46 +0300 (EEST) Received: from localhost.localdomain (unknown [10.89.50.133]) by troi.prod.arr (Postfix) with ESMTPSA id D58742B2; Wed, 19 Jun 2019 18:15:45 +0300 (EEST) From: Nikos Dragazis To: dev@dpdk.org Cc: Maxime Coquelin , Tiwei Bie , Zhihong Wang , Stefan Hajnoczi , Wei Wang , Stojaczyk Dariusz , Vangelis Koukis Date: Wed, 19 Jun 2019 18:14:46 +0300 Message-Id: <1560957293-17294-22-git-send-email-ndragazis@arrikto.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1560957293-17294-1-git-send-email-ndragazis@arrikto.com> References: <1560957293-17294-1-git-send-email-ndragazis@arrikto.com> Subject: [dpdk-dev] [PATCH 21/28] drivers/virtio_vhost_user: use additional device resources X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Enhance the virtio-vhost-user device driver so that it utilizes the device's additional resource capabilities. In specific, this patch adds support for the doorbells and shared_memory capabilities. The former is used to find the location of the device doorbells. The latter is used to find the location of the vhost memory regions in the device's memory address space. Also, support has been added for the notification capability, though this configuration structure is not currently being used by the virtio-vhost-user driver due to DPDK's poll-mode nature. Signed-off-by: Nikos Dragazis --- .../virtio_vhost_user/trans_virtio_vhost_user.c | 22 ++++++++++++++-------- drivers/virtio_vhost_user/virtio_pci.c | 16 ++++++++++++++++ drivers/virtio_vhost_user/virtio_pci.h | 19 +++++++++++++++++++ 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/drivers/virtio_vhost_user/trans_virtio_vhost_user.c b/drivers/virtio_vhost_user/trans_virtio_vhost_user.c index 72018a4..45863bd 100644 --- a/drivers/virtio_vhost_user/trans_virtio_vhost_user.c +++ b/drivers/virtio_vhost_user/trans_virtio_vhost_user.c @@ -198,11 +198,14 @@ vvu_vring_call(struct virtio_net *dev, struct vhost_virtqueue *vq) struct vvu_connection *conn = container_of(dev, struct vvu_connection, device); struct vvu_socket *vvu_socket = conn->vvu_socket; + struct virtio_hw *hw = &vvu_socket->pdev->hw; uint16_t vq_idx = vq->vring_idx; + uint16_t *notify_addr = (void *)((uint8_t *)vvu_socket->doorbells + + vq_idx * hw->doorbell_off_multiplier); RTE_LOG(DEBUG, VHOST_CONFIG, "%s vq_idx %u\n", __func__, vq_idx); - rte_write16(rte_cpu_to_le_16(vq_idx), &vvu_socket->doorbells[vq_idx]); + rte_write16(rte_cpu_to_le_16(vq_idx), notify_addr); return 0; } @@ -265,14 +268,14 @@ vvu_map_mem_regions(struct virtio_net *dev, struct VhostUserMsg *msg __rte_unuse struct vvu_connection *conn = container_of(dev, struct vvu_connection, device); struct vvu_socket *vvu_socket = conn->vvu_socket; - struct rte_pci_device *pci_dev = vvu_socket->pdev->pci_dev; + struct virtio_hw *hw = &vvu_socket->pdev->hw; uint8_t *mmap_addr; uint32_t i; - /* Memory regions start after the doorbell registers */ - mmap_addr = (uint8_t *)pci_dev->mem_resource[2].addr + - RTE_ALIGN_CEIL((vvu_socket->max_vhost_queues + 1 /* log fd */) * - sizeof(uint16_t), 4096); + /* Get the starting address of vhost memory regions from + * the shared memory virtio PCI capability + */ + mmap_addr = hw->shared_memory_cfg; for (i = 0; i < dev->mem->nregions; i++) { struct rte_vhost_mem_region *reg = &dev->mem->regions[i]; @@ -780,10 +783,13 @@ vvu_virtio_pci_init_intr(struct vvu_socket *vvu_socket) static int vvu_virtio_pci_init_bar(struct vvu_socket *vvu_socket) { - struct rte_pci_device *pci_dev = vvu_socket->pdev->pci_dev; + struct virtio_hw *hw = &vvu_socket->pdev->hw; struct virtio_net *dev = NULL; /* just for sizeof() */ - vvu_socket->doorbells = pci_dev->mem_resource[2].addr; + /* Get the starting address of the doorbells from + * the doorbell virtio PCI capability + */ + vvu_socket->doorbells = hw->doorbell_base; if (!vvu_socket->doorbells) { RTE_LOG(ERR, VHOST_CONFIG, "BAR 2 not available\n"); return -1; diff --git a/drivers/virtio_vhost_user/virtio_pci.c b/drivers/virtio_vhost_user/virtio_pci.c index 9c2c981..7996729 100644 --- a/drivers/virtio_vhost_user/virtio_pci.c +++ b/drivers/virtio_vhost_user/virtio_pci.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation + * Copyright(c) 2019 Arrikto Inc. */ #include @@ -407,6 +408,18 @@ virtio_read_caps(struct rte_pci_device *dev, struct virtio_hw *hw) case VIRTIO_PCI_CAP_ISR_CFG: hw->isr = get_cfg_addr(dev, &cap); break; + case VIRTIO_PCI_CAP_DOORBELL_CFG: + rte_pci_read_config(dev, &hw->doorbell_off_multiplier, + 4, pos + sizeof(cap)); + hw->doorbell_base = get_cfg_addr(dev, &cap); + rte_pci_read_config(dev, &hw->doorbell_length, 4, pos + 10); + break; + case VIRTIO_PCI_CAP_NOTIFICATION_CFG: + hw->notify_cfg = get_cfg_addr(dev, &cap); + break; + case VIRTIO_PCI_CAP_SHARED_MEMORY_CFG: + hw->shared_memory_cfg = get_cfg_addr(dev, &cap); + break; } next: @@ -426,6 +439,9 @@ virtio_read_caps(struct rte_pci_device *dev, struct virtio_hw *hw) RTE_LOG(DEBUG, VIRTIO_PCI_CONFIG, "isr cfg mapped at: %p\n", hw->isr); RTE_LOG(DEBUG, VIRTIO_PCI_CONFIG, "notify base: %p, notify off multiplier: %u\n", hw->notify_base, hw->notify_off_multiplier); + RTE_LOG(DEBUG, VIRTIO_PCI_CONFIG, "doorbell base: %p, doorbell off multiplier: %u\n", hw->doorbell_base, hw->doorbell_off_multiplier); + RTE_LOG(DEBUG, VIRTIO_PCI_CONFIG, "notification cfg mapped at: %p\n", hw->notify_cfg); + RTE_LOG(DEBUG, VIRTIO_PCI_CONFIG, "shared memory region mapped at: %p\n", hw->shared_memory_cfg); return 0; } diff --git a/drivers/virtio_vhost_user/virtio_pci.h b/drivers/virtio_vhost_user/virtio_pci.h index 018e0b7..12373d1 100644 --- a/drivers/virtio_vhost_user/virtio_pci.h +++ b/drivers/virtio_vhost_user/virtio_pci.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation + * Copyright(c) 2019 Arrikto Inc. */ /* XXX This file is based on drivers/net/virtio/virtio_pci.h. It would be @@ -117,6 +118,10 @@ struct virtqueue; #define VIRTIO_PCI_CAP_DEVICE_CFG 4 /* PCI configuration access */ #define VIRTIO_PCI_CAP_PCI_CFG 5 +/* Additional capabilities for the virtio-vhost-user device */ +#define VIRTIO_PCI_CAP_DOORBELL_CFG 6 +#define VIRTIO_PCI_CAP_NOTIFICATION_CFG 7 +#define VIRTIO_PCI_CAP_SHARED_MEMORY_CFG 8 /* This is the PCI capability header: */ struct virtio_pci_cap { @@ -161,6 +166,12 @@ struct virtio_pci_common_cfg { uint32_t queue_used_hi; /* read-write */ }; +/* Fields in VIRTIO_PCI_CAP_NOTIFICATION_CFG */ +struct virtio_pci_notification_cfg { + uint16_t notification_select; /* read-write */ + uint16_t notification_msix_vector; /* read-write */ +}; + struct virtio_hw; struct virtio_pci_ops { @@ -200,6 +211,14 @@ struct virtio_hw { uint16_t *notify_base; struct virtio_pci_common_cfg *common_cfg; void *dev_cfg; + /* virtio-vhost-user additional device resource capabilities + * https://stefanha.github.io/virtio/vhost-user-slave.html#x1-2830007 + */ + uint32_t doorbell_off_multiplier; + uint16_t *doorbell_base; + uint32_t doorbell_length; + struct virtio_pci_notification_cfg *notify_cfg; + uint8_t *shared_memory_cfg; /* * App management thread and virtio interrupt handler thread * both can change device state, this lock is meant to avoid