From patchwork Mon Jan 20 19:16:23 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Slava Ovsiienko X-Patchwork-Id: 64988 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id E9D22A0528; Mon, 20 Jan 2020 20:16:54 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id BA5161C0C3; Mon, 20 Jan 2020 20:16:41 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 5B3731C0C0 for ; Mon, 20 Jan 2020 20:16:40 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from viacheslavo@mellanox.com) with ESMTPS (AES256-SHA encrypted); 20 Jan 2020 21:16:35 +0200 Received: from pegasus11.mtr.labs.mlnx (pegasus11.mtr.labs.mlnx [10.210.16.104]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 00KJGZsq019012; Mon, 20 Jan 2020 21:16:35 +0200 Received: from pegasus11.mtr.labs.mlnx (localhost [127.0.0.1]) by pegasus11.mtr.labs.mlnx (8.14.7/8.14.7) with ESMTP id 00KJGZCv007921; Mon, 20 Jan 2020 19:16:35 GMT Received: (from viacheslavo@localhost) by pegasus11.mtr.labs.mlnx (8.14.7/8.14.7/Submit) id 00KJGZdg007920; Mon, 20 Jan 2020 19:16:35 GMT X-Authentication-Warning: pegasus11.mtr.labs.mlnx: viacheslavo set sender to viacheslavo@mellanox.com using -f From: Viacheslav Ovsiienko To: dev@dpdk.org Cc: matan@mellanox.com, rasland@mellanox.com, orika@mellanox.com, shahafs@mellanox.com, olivier.matz@6wind.com, stephen@networkplumber.org, thomas@mellanox.net Date: Mon, 20 Jan 2020 19:16:23 +0000 Message-Id: <1579547786-7650-3-git-send-email-viacheslavo@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1579547786-7650-1-git-send-email-viacheslavo@mellanox.com> References: <20191118094938.192850-1-shahafs@mellanox.com> <1579547786-7650-1-git-send-email-viacheslavo@mellanox.com> Subject: [dpdk-dev] [PATCH v6 2/5] mbuf: detach mbuf with pinned external buffer X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Update detach routine to check the mbuf pool type. Introduce the special internal version of detach routine to handle the special case of pinned external bufferon mbuf freeing. Signed-off-by: Shahaf Shuler Signed-off-by: Viacheslav Ovsiienko Acked-by: Olivier Matz --- lib/librte_mbuf/rte_mbuf.h | 105 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 96 insertions(+), 9 deletions(-) diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h index 9b0691d..c4f5085 100644 --- a/lib/librte_mbuf/rte_mbuf.h +++ b/lib/librte_mbuf/rte_mbuf.h @@ -323,6 +323,24 @@ struct rte_pktmbuf_pool_private { return mbp_priv->flags; } +/** + * When set, pktmbuf mempool will hold only mbufs with pinned external + * buffer. The external buffer will be attached to the mbuf at the + * memory pool creation and will never be detached by the mbuf free calls. + * mbuf should not contain any room for data after the mbuf structure. + */ +#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0) + +/** + * Returns non zero if given mbuf has a pinned external buffer, or zero + * otherwise. The pinned external buffer is allocated at pool creation + * time and should not be freed on mbuf freeing. + * + * External buffer is a user-provided anonymous buffer. + */ +#define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \ + (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) + #ifdef RTE_LIBRTE_MBUF_DEBUG /** check mbuf type in debug mode */ @@ -588,7 +606,8 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m) { - RTE_ASSERT(RTE_MBUF_DIRECT(m)); + RTE_ASSERT(!RTE_MBUF_CLONED(m) && + (!RTE_MBUF_HAS_EXTBUF(m) || RTE_MBUF_HAS_PINNED_EXTBUF(m))); RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1); RTE_ASSERT(m->next == NULL); RTE_ASSERT(m->nb_segs == 1); @@ -794,7 +813,7 @@ static inline void rte_pktmbuf_reset(struct rte_mbuf *m) m->nb_segs = 1; m->port = MBUF_INVALID_PORT; - m->ol_flags = 0; + m->ol_flags &= EXT_ATTACHED_MBUF; m->packet_type = 0; rte_pktmbuf_reset_headroom(m); @@ -1153,6 +1172,11 @@ static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m) * * All other fields of the given packet mbuf will be left intact. * + * If the packet mbuf was allocated from the pool with pinned + * external buffers the rte_pktmbuf_detach does nothing with the + * mbuf of this kind, because the pinned buffers are not supposed + * to be detached. + * * @param m * The indirect attached packet mbuf. */ @@ -1162,11 +1186,26 @@ static inline void rte_pktmbuf_detach(struct rte_mbuf *m) uint32_t mbuf_size, buf_len; uint16_t priv_size; - if (RTE_MBUF_HAS_EXTBUF(m)) + if (RTE_MBUF_HAS_EXTBUF(m)) { + /* + * The mbuf has the external attached buffer, + * we should check the type of the memory pool where + * the mbuf was allocated from to detect the pinned + * external buffer. + */ + uint32_t flags = rte_pktmbuf_priv_flags(mp); + + if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) { + /* + * The pinned external buffer should not be + * detached from its backing mbuf, just exit. + */ + return; + } __rte_pktmbuf_free_extbuf(m); - else + } else { __rte_pktmbuf_free_direct(m); - + } priv_size = rte_pktmbuf_priv_size(mp); mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size); buf_len = rte_pktmbuf_data_room_size(mp); @@ -1181,6 +1220,44 @@ static inline void rte_pktmbuf_detach(struct rte_mbuf *m) } /** + * @internal Handle the packet mbufs with attached pinned external buffer + * on the mbuf freeing: + * + * - return zero if reference counter in shinfo is one. It means there is + * no more reference to this pinned buffer and mbuf can be returned to + * the pool + * + * - otherwise (if reference counter is not one), decrement reference + * counter and return non-zero value to prevent freeing the backing mbuf. + * + * Returns non zero if mbuf should not be freed. + */ +static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m) +{ + struct rte_mbuf_ext_shared_info *shinfo; + + /* Clear flags, mbuf is being freed. */ + m->ol_flags = EXT_ATTACHED_MBUF; + shinfo = m->shinfo; + + /* Optimize for performance - do not dec/reinit */ + if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) + return 0; + + /* + * Direct usage of add primitive to avoid + * duplication of comparing with one. + */ + if (likely(rte_atomic16_add_return + (&shinfo->refcnt_atomic, -1))) + return 1; + + /* Reinitialize counter before mbuf freeing. */ + rte_mbuf_ext_refcnt_set(shinfo, 1); + return 0; +} + +/** * Decrease reference counter and unlink a mbuf segment * * This function does the same than a free, except that it does not @@ -1201,8 +1278,13 @@ static inline void rte_pktmbuf_detach(struct rte_mbuf *m) if (likely(rte_mbuf_refcnt_read(m) == 1)) { - if (!RTE_MBUF_DIRECT(m)) - rte_pktmbuf_detach(m); + if (!RTE_MBUF_DIRECT(m)) { + if (!RTE_MBUF_HAS_EXTBUF(m) || + !RTE_MBUF_HAS_PINNED_EXTBUF(m)) + rte_pktmbuf_detach(m); + else if (__rte_pktmbuf_pinned_extbuf_decref(m)) + return NULL; + } if (m->next != NULL) { m->next = NULL; @@ -1213,8 +1295,13 @@ static inline void rte_pktmbuf_detach(struct rte_mbuf *m) } else if (__rte_mbuf_refcnt_update(m, -1) == 0) { - if (!RTE_MBUF_DIRECT(m)) - rte_pktmbuf_detach(m); + if (!RTE_MBUF_DIRECT(m)) { + if (!RTE_MBUF_HAS_EXTBUF(m) || + !RTE_MBUF_HAS_PINNED_EXTBUF(m)) + rte_pktmbuf_detach(m); + else if (__rte_pktmbuf_pinned_extbuf_decref(m)) + return NULL; + } if (m->next != NULL) { m->next = NULL;