From patchwork Fri May 17 12:22:16 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Maxime Coquelin X-Patchwork-Id: 53519 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id DADBA6CD8; Fri, 17 May 2019 14:22:44 +0200 (CEST) Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by dpdk.org (Postfix) with ESMTP id 9C949683E for ; Fri, 17 May 2019 14:22:42 +0200 (CEST) Received: from smtp.corp.redhat.com (int-mx07.intmail.prod.int.phx2.redhat.com [10.5.11.22]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id BD28B8CB51; Fri, 17 May 2019 12:22:36 +0000 (UTC) Received: from localhost.localdomain (ovpn-112-59.ams2.redhat.com [10.36.112.59]) by smtp.corp.redhat.com (Postfix) with ESMTP id A539210021B2; Fri, 17 May 2019 12:22:31 +0000 (UTC) From: Maxime Coquelin To: dev@dpdk.org, tiwei.bie@intel.com, jfreimann@redhat.com, zhihong.wang@intel.com, bruce.richardson@intel.com, konstantin.ananyev@intel.com Cc: Maxime Coquelin Date: Fri, 17 May 2019 14:22:16 +0200 Message-Id: <20190517122220.31283-2-maxime.coquelin@redhat.com> In-Reply-To: <20190517122220.31283-1-maxime.coquelin@redhat.com> References: <20190517122220.31283-1-maxime.coquelin@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 2.84 on 10.5.11.22 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.26]); Fri, 17 May 2019 12:22:36 +0000 (UTC) Subject: [dpdk-dev] [PATCH 1/5] vhost: un-inline dirty pages logging functions X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" In order to reduce the I-cache pressure, this patch removes the inlining of the dirty pages logging functions, that we can consider as cold path. Indeed, these functions are only called while doing live migration, so not called most of the time. Signed-off-by: Maxime Coquelin --- lib/librte_vhost/vhost.c | 132 +++++++++++++++++++++++++++++++++++++++ lib/librte_vhost/vhost.h | 129 ++++---------------------------------- 2 files changed, 144 insertions(+), 117 deletions(-) diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c index 163f4595ef..4a54ad6bd1 100644 --- a/lib/librte_vhost/vhost.c +++ b/lib/librte_vhost/vhost.c @@ -69,6 +69,138 @@ __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, return 0; } +#define VHOST_LOG_PAGE 4096 + +/* + * Atomically set a bit in memory. + */ +static __rte_always_inline void +vhost_set_bit(unsigned int nr, volatile uint8_t *addr) +{ +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) + /* + * __sync_ built-ins are deprecated, but __atomic_ ones + * are sub-optimized in older GCC versions. + */ + __sync_fetch_and_or_1(addr, (1U << nr)); +#else + __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED); +#endif +} + +static __rte_always_inline void +vhost_log_page(uint8_t *log_base, uint64_t page) +{ + vhost_set_bit(page % 8, &log_base[page / 8]); +} + +void +__vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) +{ + uint64_t page; + + if (unlikely(!dev->log_base || !len)) + return; + + if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) + return; + + /* To make sure guest memory updates are committed before logging */ + rte_smp_wmb(); + + page = addr / VHOST_LOG_PAGE; + while (page * VHOST_LOG_PAGE < addr + len) { + vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); + page += 1; + } +} + +void +__vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) +{ + unsigned long *log_base; + int i; + + if (unlikely(!dev->log_base)) + return; + + rte_smp_wmb(); + + log_base = (unsigned long *)(uintptr_t)dev->log_base; + + for (i = 0; i < vq->log_cache_nb_elem; i++) { + struct log_cache_entry *elem = vq->log_cache + i; + +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) + /* + * '__sync' builtins are deprecated, but '__atomic' ones + * are sub-optimized in older GCC versions. + */ + __sync_fetch_and_or(log_base + elem->offset, elem->val); +#else + __atomic_fetch_or(log_base + elem->offset, elem->val, + __ATOMIC_RELAXED); +#endif + } + + rte_smp_wmb(); + + vq->log_cache_nb_elem = 0; +} + +static __rte_always_inline void +vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t page) +{ + uint32_t bit_nr = page % (sizeof(unsigned long) << 3); + uint32_t offset = page / (sizeof(unsigned long) << 3); + int i; + + for (i = 0; i < vq->log_cache_nb_elem; i++) { + struct log_cache_entry *elem = vq->log_cache + i; + + if (elem->offset == offset) { + elem->val |= (1UL << bit_nr); + return; + } + } + + if (unlikely(i >= VHOST_LOG_CACHE_NR)) { + /* + * No more room for a new log cache entry, + * so write the dirty log map directly. + */ + rte_smp_wmb(); + vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); + + return; + } + + vq->log_cache[i].offset = offset; + vq->log_cache[i].val = (1UL << bit_nr); + vq->log_cache_nb_elem++; +} + +void +__vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t addr, uint64_t len) +{ + uint64_t page; + + if (unlikely(!dev->log_base || !len)) + return; + + if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) + return; + + page = addr / VHOST_LOG_PAGE; + while (page * VHOST_LOG_PAGE < addr + len) { + vhost_log_cache_page(dev, vq, page); + page += 1; + } +} + + void cleanup_vq(struct vhost_virtqueue *vq, int destroy) { diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index e9138dfab4..3ab7b4950f 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -350,138 +350,33 @@ desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) wrap_counter != !!(flags & VRING_DESC_F_USED); } -#define VHOST_LOG_PAGE 4096 - -/* - * Atomically set a bit in memory. - */ -static __rte_always_inline void -vhost_set_bit(unsigned int nr, volatile uint8_t *addr) -{ -#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) - /* - * __sync_ built-ins are deprecated, but __atomic_ ones - * are sub-optimized in older GCC versions. - */ - __sync_fetch_and_or_1(addr, (1U << nr)); -#else - __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED); -#endif -} - -static __rte_always_inline void -vhost_log_page(uint8_t *log_base, uint64_t page) -{ - vhost_set_bit(page % 8, &log_base[page / 8]); -} +void __vhost_log_cache_write(struct virtio_net *dev, + struct vhost_virtqueue *vq, + uint64_t addr, uint64_t len); +void __vhost_log_cache_sync(struct virtio_net *dev, + struct vhost_virtqueue *vq); +void __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len); static __rte_always_inline void vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) { - uint64_t page; - - if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || - !dev->log_base || !len)) - return; - - if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) - return; - - /* To make sure guest memory updates are committed before logging */ - rte_smp_wmb(); - - page = addr / VHOST_LOG_PAGE; - while (page * VHOST_LOG_PAGE < addr + len) { - vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); - page += 1; - } + if (unlikely(dev->features & (1ULL << VHOST_F_LOG_ALL))) + __vhost_log_write(dev, addr, len); } static __rte_always_inline void vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) { - unsigned long *log_base; - int i; - - if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || - !dev->log_base)) - return; - - rte_smp_wmb(); - - log_base = (unsigned long *)(uintptr_t)dev->log_base; - - for (i = 0; i < vq->log_cache_nb_elem; i++) { - struct log_cache_entry *elem = vq->log_cache + i; - -#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) - /* - * '__sync' builtins are deprecated, but '__atomic' ones - * are sub-optimized in older GCC versions. - */ - __sync_fetch_and_or(log_base + elem->offset, elem->val); -#else - __atomic_fetch_or(log_base + elem->offset, elem->val, - __ATOMIC_RELAXED); -#endif - } - - rte_smp_wmb(); - - vq->log_cache_nb_elem = 0; -} - -static __rte_always_inline void -vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq, - uint64_t page) -{ - uint32_t bit_nr = page % (sizeof(unsigned long) << 3); - uint32_t offset = page / (sizeof(unsigned long) << 3); - int i; - - for (i = 0; i < vq->log_cache_nb_elem; i++) { - struct log_cache_entry *elem = vq->log_cache + i; - - if (elem->offset == offset) { - elem->val |= (1UL << bit_nr); - return; - } - } - - if (unlikely(i >= VHOST_LOG_CACHE_NR)) { - /* - * No more room for a new log cache entry, - * so write the dirty log map directly. - */ - rte_smp_wmb(); - vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); - - return; - } - - vq->log_cache[i].offset = offset; - vq->log_cache[i].val = (1UL << bit_nr); - vq->log_cache_nb_elem++; + if (unlikely(dev->features & (1ULL << VHOST_F_LOG_ALL))) + __vhost_log_cache_sync(dev, vq); } static __rte_always_inline void vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, uint64_t addr, uint64_t len) { - uint64_t page; - - if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || - !dev->log_base || !len)) - return; - - if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) - return; - - page = addr / VHOST_LOG_PAGE; - while (page * VHOST_LOG_PAGE < addr + len) { - vhost_log_cache_page(dev, vq, page); - page += 1; - } + if (unlikely(dev->features & (1ULL << VHOST_F_LOG_ALL))) + __vhost_log_cache_write(dev, vq, addr, len); } static __rte_always_inline void From patchwork Fri May 17 12:22:17 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Maxime Coquelin X-Patchwork-Id: 53520 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id C159B7CE2; Fri, 17 May 2019 14:22:47 +0200 (CEST) Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by dpdk.org (Postfix) with ESMTP id 9AD7E5F34 for ; Fri, 17 May 2019 14:22:44 +0200 (CEST) Received: from smtp.corp.redhat.com (int-mx07.intmail.prod.int.phx2.redhat.com [10.5.11.22]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 1B59330BB37A; Fri, 17 May 2019 12:22:39 +0000 (UTC) Received: from localhost.localdomain (ovpn-112-59.ams2.redhat.com [10.36.112.59]) by smtp.corp.redhat.com (Postfix) with ESMTP id 301E310027B8; Fri, 17 May 2019 12:22:36 +0000 (UTC) From: Maxime Coquelin To: dev@dpdk.org, tiwei.bie@intel.com, jfreimann@redhat.com, zhihong.wang@intel.com, bruce.richardson@intel.com, konstantin.ananyev@intel.com Cc: Maxime Coquelin Date: Fri, 17 May 2019 14:22:17 +0200 Message-Id: <20190517122220.31283-3-maxime.coquelin@redhat.com> In-Reply-To: <20190517122220.31283-1-maxime.coquelin@redhat.com> References: <20190517122220.31283-1-maxime.coquelin@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 2.84 on 10.5.11.22 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.49]); Fri, 17 May 2019 12:22:39 +0000 (UTC) Subject: [dpdk-dev] [PATCH 2/5] vhost: do not inline packed and split functions X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" At runtime either packed Tx/Rx functions will always be called, or split Tx/Rx functions will always be called. This patch removes the forced inlining in order to reduce the I-cache pressure. Signed-off-by: Maxime Coquelin --- lib/librte_vhost/virtio_net.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c index a6a33a1013..35ae4992c2 100644 --- a/lib/librte_vhost/virtio_net.c +++ b/lib/librte_vhost/virtio_net.c @@ -771,7 +771,7 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, return error; } -static __rte_always_inline uint32_t +static uint32_t virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf **pkts, uint32_t count) { @@ -830,7 +830,7 @@ virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, return pkt_idx; } -static __rte_always_inline uint32_t +static uint32_t virtio_dev_rx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf **pkts, uint32_t count) { @@ -1300,7 +1300,7 @@ get_zmbuf(struct vhost_virtqueue *vq) return NULL; } -static __rte_always_inline uint16_t +static uint16_t virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) { @@ -1422,7 +1422,7 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, return i; } -static __rte_always_inline uint16_t +static uint16_t virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) { From patchwork Fri May 17 12:22:18 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Maxime Coquelin X-Patchwork-Id: 53521 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id EEC8AD167; Fri, 17 May 2019 14:22:49 +0200 (CEST) Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by dpdk.org (Postfix) with ESMTP id 9B60F6C9B for ; Fri, 17 May 2019 14:22:44 +0200 (CEST) Received: from smtp.corp.redhat.com (int-mx07.intmail.prod.int.phx2.redhat.com [10.5.11.22]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 3E7192E95B3; Fri, 17 May 2019 12:22:43 +0000 (UTC) Received: from localhost.localdomain (ovpn-112-59.ams2.redhat.com [10.36.112.59]) by smtp.corp.redhat.com (Postfix) with ESMTP id 7B25F10027B8; Fri, 17 May 2019 12:22:39 +0000 (UTC) From: Maxime Coquelin To: dev@dpdk.org, tiwei.bie@intel.com, jfreimann@redhat.com, zhihong.wang@intel.com, bruce.richardson@intel.com, konstantin.ananyev@intel.com Cc: Maxime Coquelin Date: Fri, 17 May 2019 14:22:18 +0200 Message-Id: <20190517122220.31283-4-maxime.coquelin@redhat.com> In-Reply-To: <20190517122220.31283-1-maxime.coquelin@redhat.com> References: <20190517122220.31283-1-maxime.coquelin@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 2.84 on 10.5.11.22 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.29]); Fri, 17 May 2019 12:22:43 +0000 (UTC) Subject: [dpdk-dev] [PATCH 3/5] vhost: do not inline unlikely fragmented buffers code X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Handling of fragmented virtio-net header and indirect descriptors tables was implemented to fix CVE-2018-1059. It should not never happen with healthy guests and so are already considered as unlikely code path. This patch moves these bits into non-inline dedicated functions to reduce the I-cache pressure. Signed-off-by: Maxime Coquelin --- lib/librte_vhost/vhost.c | 33 +++++++++++ lib/librte_vhost/vhost.h | 35 +----------- lib/librte_vhost/virtio_net.c | 102 +++++++++++++++++++--------------- 3 files changed, 91 insertions(+), 79 deletions(-) diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c index 4a54ad6bd1..8a4379bc13 100644 --- a/lib/librte_vhost/vhost.c +++ b/lib/librte_vhost/vhost.c @@ -201,6 +201,39 @@ __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, } +void * +alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t desc_addr, uint64_t desc_len) +{ + void *idesc; + uint64_t src, dst; + uint64_t len, remain = desc_len; + + idesc = rte_malloc(__func__, desc_len, 0); + if (unlikely(!idesc)) + return NULL; + + dst = (uint64_t)(uintptr_t)idesc; + + while (remain) { + len = remain; + src = vhost_iova_to_vva(dev, vq, desc_addr, &len, + VHOST_ACCESS_RO); + if (unlikely(!src || !len)) { + rte_free(idesc); + return NULL; + } + + rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); + + remain -= len; + dst += len; + desc_addr += len; + } + + return idesc; +} + void cleanup_vq(struct vhost_virtqueue *vq, int destroy) { diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index 3ab7b4950f..ab26454e1c 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -488,6 +488,8 @@ void vhost_backend_cleanup(struct virtio_net *dev); uint64_t __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, uint64_t iova, uint64_t *len, uint8_t perm); +void *alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t desc_addr, uint64_t desc_len); int vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq); void vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq); @@ -601,39 +603,6 @@ vhost_vring_call_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) eventfd_write(vq->callfd, (eventfd_t)1); } -static __rte_always_inline void * -alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, - uint64_t desc_addr, uint64_t desc_len) -{ - void *idesc; - uint64_t src, dst; - uint64_t len, remain = desc_len; - - idesc = rte_malloc(__func__, desc_len, 0); - if (unlikely(!idesc)) - return 0; - - dst = (uint64_t)(uintptr_t)idesc; - - while (remain) { - len = remain; - src = vhost_iova_to_vva(dev, vq, desc_addr, &len, - VHOST_ACCESS_RO); - if (unlikely(!src || !len)) { - rte_free(idesc); - return 0; - } - - rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); - - remain -= len; - dst += len; - desc_addr += len; - } - - return idesc; -} - static __rte_always_inline void free_ind_table(void *idesc) { diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c index 35ae4992c2..494dd9957e 100644 --- a/lib/librte_vhost/virtio_net.c +++ b/lib/librte_vhost/virtio_net.c @@ -610,6 +610,35 @@ reserve_avail_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, return 0; } +static void +copy_vnet_hdr_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, + struct buf_vector *buf_vec, + struct virtio_net_hdr_mrg_rxbuf *hdr){ + uint64_t len; + uint64_t remain = dev->vhost_hlen; + uint64_t src = (uint64_t)(uintptr_t)hdr, dst; + uint64_t iova = buf_vec->buf_iova; + + while (remain) { + len = RTE_MIN(remain, + buf_vec->buf_len); + dst = buf_vec->buf_addr; + rte_memcpy((void *)(uintptr_t)dst, + (void *)(uintptr_t)src, + len); + + PRINT_PACKET(dev, (uintptr_t)dst, + (uint32_t)len, 0); + vhost_log_cache_write(dev, vq, + iova, len); + + remain -= len; + iova += len; + src += len; + buf_vec++; + } +} + static __rte_always_inline int copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf *m, struct buf_vector *buf_vec, @@ -703,30 +732,7 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, num_buffers); if (unlikely(hdr == &tmp_hdr)) { - uint64_t len; - uint64_t remain = dev->vhost_hlen; - uint64_t src = (uint64_t)(uintptr_t)hdr, dst; - uint64_t iova = buf_vec[0].buf_iova; - uint16_t hdr_vec_idx = 0; - - while (remain) { - len = RTE_MIN(remain, - buf_vec[hdr_vec_idx].buf_len); - dst = buf_vec[hdr_vec_idx].buf_addr; - rte_memcpy((void *)(uintptr_t)dst, - (void *)(uintptr_t)src, - len); - - PRINT_PACKET(dev, (uintptr_t)dst, - (uint32_t)len, 0); - vhost_log_cache_write(dev, vq, - iova, len); - - remain -= len; - iova += len; - src += len; - hdr_vec_idx++; - } + copy_vnet_hdr_to_desc(dev, vq, buf_vec, hdr); } else { PRINT_PACKET(dev, (uintptr_t)hdr_addr, dev->vhost_hlen, 0); @@ -1063,6 +1069,31 @@ vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m) } } +static void +copy_vnet_hdr_from_desc(struct virtio_net_hdr *hdr, + struct buf_vector *buf_vec) +{ + uint64_t len; + uint64_t remain = sizeof(struct virtio_net_hdr); + uint64_t src; + uint64_t dst = (uint64_t)(uintptr_t)&hdr; + + /* + * No luck, the virtio-net header doesn't fit + * in a contiguous virtual area. + */ + while (remain) { + len = RTE_MIN(remain, buf_vec->buf_len); + src = buf_vec->buf_addr; + rte_memcpy((void *)(uintptr_t)dst, + (void *)(uintptr_t)src, len); + + remain -= len; + dst += len; + buf_vec++; + } +} + static __rte_always_inline int copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, struct buf_vector *buf_vec, uint16_t nr_vec, @@ -1094,28 +1125,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, if (virtio_net_with_host_offload(dev)) { if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) { - uint64_t len; - uint64_t remain = sizeof(struct virtio_net_hdr); - uint64_t src; - uint64_t dst = (uint64_t)(uintptr_t)&tmp_hdr; - uint16_t hdr_vec_idx = 0; - - /* - * No luck, the virtio-net header doesn't fit - * in a contiguous virtual area. - */ - while (remain) { - len = RTE_MIN(remain, - buf_vec[hdr_vec_idx].buf_len); - src = buf_vec[hdr_vec_idx].buf_addr; - rte_memcpy((void *)(uintptr_t)dst, - (void *)(uintptr_t)src, len); - - remain -= len; - dst += len; - hdr_vec_idx++; - } - + copy_vnet_hdr_from_desc(&tmp_hdr, buf_vec); hdr = &tmp_hdr; } else { hdr = (struct virtio_net_hdr *)((uintptr_t)buf_addr); From patchwork Fri May 17 12:22:19 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Maxime Coquelin X-Patchwork-Id: 53522 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 81F07D189; Fri, 17 May 2019 14:22:52 +0200 (CEST) Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by dpdk.org (Postfix) with ESMTP id 3A286D17B for ; Fri, 17 May 2019 14:22:51 +0200 (CEST) Received: from smtp.corp.redhat.com (int-mx07.intmail.prod.int.phx2.redhat.com [10.5.11.22]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 97E30B2DCD; Fri, 17 May 2019 12:22:50 +0000 (UTC) Received: from localhost.localdomain (ovpn-112-59.ams2.redhat.com [10.36.112.59]) by smtp.corp.redhat.com (Postfix) with ESMTP id 9F06010027B9; Fri, 17 May 2019 12:22:43 +0000 (UTC) From: Maxime Coquelin To: dev@dpdk.org, tiwei.bie@intel.com, jfreimann@redhat.com, zhihong.wang@intel.com, bruce.richardson@intel.com, konstantin.ananyev@intel.com Cc: Maxime Coquelin Date: Fri, 17 May 2019 14:22:19 +0200 Message-Id: <20190517122220.31283-5-maxime.coquelin@redhat.com> In-Reply-To: <20190517122220.31283-1-maxime.coquelin@redhat.com> References: <20190517122220.31283-1-maxime.coquelin@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 2.84 on 10.5.11.22 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.26]); Fri, 17 May 2019 12:22:50 +0000 (UTC) Subject: [dpdk-dev] [PATCH 4/5] vhost: simplify descriptor's buffer prefetching X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Now that we have a single function to map the descriptors buffers, let's prefetch them there as it is the earliest place we can do it. Signed-off-by: Maxime Coquelin --- lib/librte_vhost/virtio_net.c | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c index 494dd9957e..d9031fe55c 100644 --- a/lib/librte_vhost/virtio_net.c +++ b/lib/librte_vhost/virtio_net.c @@ -286,6 +286,8 @@ map_one_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, if (unlikely(!desc_addr)) return -1; + rte_prefetch0((void *)(uintptr_t)desc_addr); + buf_vec[vec_id].buf_iova = desc_iova; buf_vec[vec_id].buf_addr = desc_addr; buf_vec[vec_id].buf_len = desc_chunck_len; @@ -664,9 +666,6 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; - if (nr_vec > 1) - rte_prefetch0((void *)(uintptr_t)buf_vec[1].buf_addr); - if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) { error = -1; goto out; @@ -709,10 +708,6 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; - /* Prefetch next buffer address. */ - if (vec_idx + 1 < nr_vec) - rte_prefetch0((void *)(uintptr_t) - buf_vec[vec_idx + 1].buf_addr); buf_offset = 0; buf_avail = buf_len; } @@ -810,8 +805,6 @@ virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, break; } - rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr); - VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n", dev->vid, vq->last_avail_idx, vq->last_avail_idx + num_buffers); @@ -859,8 +852,6 @@ virtio_dev_rx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, break; } - rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr); - VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n", dev->vid, vq->last_avail_idx, vq->last_avail_idx + num_buffers); @@ -1120,16 +1111,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, goto out; } - if (likely(nr_vec > 1)) - rte_prefetch0((void *)(uintptr_t)buf_vec[1].buf_addr); - if (virtio_net_with_host_offload(dev)) { if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) { copy_vnet_hdr_from_desc(&tmp_hdr, buf_vec); hdr = &tmp_hdr; } else { hdr = (struct virtio_net_hdr *)((uintptr_t)buf_addr); - rte_prefetch0(hdr); } } @@ -1159,9 +1146,6 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen; } - rte_prefetch0((void *)(uintptr_t) - (buf_addr + buf_offset)); - PRINT_PACKET(dev, (uintptr_t)(buf_addr + buf_offset), (uint32_t)buf_avail, 0); @@ -1227,14 +1211,6 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; - /* - * Prefecth desc n + 1 buffer while - * desc n buffer is processed. - */ - if (vec_idx + 1 < nr_vec) - rte_prefetch0((void *)(uintptr_t) - buf_vec[vec_idx + 1].buf_addr); - buf_offset = 0; buf_avail = buf_len; @@ -1378,8 +1354,6 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, if (likely(dev->dequeue_zero_copy == 0)) update_shadow_used_ring_split(vq, head_idx, 0); - rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr); - pkts[i] = rte_pktmbuf_alloc(mbuf_pool); if (unlikely(pkts[i] == NULL)) { RTE_LOG(ERR, VHOST_DATA, @@ -1489,8 +1463,6 @@ virtio_dev_tx_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, update_shadow_used_ring_packed(vq, buf_id, 0, desc_count); - rte_prefetch0((void *)(uintptr_t)buf_vec[0].buf_addr); - pkts[i] = rte_pktmbuf_alloc(mbuf_pool); if (unlikely(pkts[i] == NULL)) { RTE_LOG(ERR, VHOST_DATA, From patchwork Fri May 17 12:22:20 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Maxime Coquelin X-Patchwork-Id: 53523 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id B7AB4D1A3; Fri, 17 May 2019 14:22:59 +0200 (CEST) Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by dpdk.org (Postfix) with ESMTP id 81B4CD17B for ; Fri, 17 May 2019 14:22:58 +0200 (CEST) Received: from smtp.corp.redhat.com (int-mx07.intmail.prod.int.phx2.redhat.com [10.5.11.22]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id DBD90B2DCE; Fri, 17 May 2019 12:22:57 +0000 (UTC) Received: from localhost.localdomain (ovpn-112-59.ams2.redhat.com [10.36.112.59]) by smtp.corp.redhat.com (Postfix) with ESMTP id ED3D610021B2; Fri, 17 May 2019 12:22:50 +0000 (UTC) From: Maxime Coquelin To: dev@dpdk.org, tiwei.bie@intel.com, jfreimann@redhat.com, zhihong.wang@intel.com, bruce.richardson@intel.com, konstantin.ananyev@intel.com Cc: root Date: Fri, 17 May 2019 14:22:20 +0200 Message-Id: <20190517122220.31283-6-maxime.coquelin@redhat.com> In-Reply-To: <20190517122220.31283-1-maxime.coquelin@redhat.com> References: <20190517122220.31283-1-maxime.coquelin@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 2.84 on 10.5.11.22 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.26]); Fri, 17 May 2019 12:22:57 +0000 (UTC) Subject: [dpdk-dev] [PATCH 5/5] eal/x86: force inlining of all memcpy and mov helpers X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: root Some helpers in the header file are forced inlined other are only inlined, this patch forces inline for all. It will avoid it to be embedded as functions when called multiple times in the same object file. For example, when we added packed ring support in vhost-user library, rte_memcpy_generic got no more inlined. Signed-off-by: root --- .../common/include/arch/x86/rte_memcpy.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/librte_eal/common/include/arch/x86/rte_memcpy.h b/lib/librte_eal/common/include/arch/x86/rte_memcpy.h index 7b758094df..ba44c4a328 100644 --- a/lib/librte_eal/common/include/arch/x86/rte_memcpy.h +++ b/lib/librte_eal/common/include/arch/x86/rte_memcpy.h @@ -115,7 +115,7 @@ rte_mov256(uint8_t *dst, const uint8_t *src) * Copy 128-byte blocks from one location to another, * locations should not overlap. */ -static inline void +static __rte_always_inline void rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n) { __m512i zmm0, zmm1; @@ -163,7 +163,7 @@ rte_mov512blocks(uint8_t *dst, const uint8_t *src, size_t n) } } -static inline void * +static __rte_always_inline void * rte_memcpy_generic(void *dst, const void *src, size_t n) { uintptr_t dstu = (uintptr_t)dst; @@ -330,7 +330,7 @@ rte_mov64(uint8_t *dst, const uint8_t *src) * Copy 128 bytes from one location to another, * locations should not overlap. */ -static inline void +static __rte_always_inline void rte_mov128(uint8_t *dst, const uint8_t *src) { rte_mov32((uint8_t *)dst + 0 * 32, (const uint8_t *)src + 0 * 32); @@ -343,7 +343,7 @@ rte_mov128(uint8_t *dst, const uint8_t *src) * Copy 128-byte blocks from one location to another, * locations should not overlap. */ -static inline void +static __rte_always_inline void rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n) { __m256i ymm0, ymm1, ymm2, ymm3; @@ -363,7 +363,7 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n) } } -static inline void * +static __rte_always_inline void * rte_memcpy_generic(void *dst, const void *src, size_t n) { uintptr_t dstu = (uintptr_t)dst; @@ -523,7 +523,7 @@ rte_mov64(uint8_t *dst, const uint8_t *src) * Copy 128 bytes from one location to another, * locations should not overlap. */ -static inline void +static __rte_always_inline void rte_mov128(uint8_t *dst, const uint8_t *src) { rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16); @@ -655,7 +655,7 @@ __extension__ ({ \ } \ }) -static inline void * +static __rte_always_inline void * rte_memcpy_generic(void *dst, const void *src, size_t n) { __m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8; @@ -800,7 +800,7 @@ rte_memcpy_generic(void *dst, const void *src, size_t n) #endif /* RTE_MACHINE_CPUFLAG */ -static inline void * +static __rte_always_inline void * rte_memcpy_aligned(void *dst, const void *src, size_t n) { void *ret = dst; @@ -860,7 +860,7 @@ rte_memcpy_aligned(void *dst, const void *src, size_t n) return ret; } -static inline void * +static __rte_always_inline void * rte_memcpy(void *dst, const void *src, size_t n) { if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK))