Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/57675/?format=api
{ "id": 57675, "url": "http://patches.dpdk.org/api/patches/57675/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/patch/1565771263-27353-1-git-send-email-phil.yang@arm.com/", "project": { "id": 1, "url": "http://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1565771263-27353-1-git-send-email-phil.yang@arm.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1565771263-27353-1-git-send-email-phil.yang@arm.com", "date": "2019-08-14T08:27:41", "name": "[v9,1/3] eal/arm64: add 128-bit atomic compare exchange", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "cbe6c99c9d191b431016bad64c767bef2a9f400b", "submitter": { "id": 833, "url": "http://patches.dpdk.org/api/people/833/?format=api", "name": "Phil Yang", "email": "phil.yang@arm.com" }, "delegate": { "id": 24651, "url": "http://patches.dpdk.org/api/users/24651/?format=api", "username": "dmarchand", "first_name": "David", "last_name": "Marchand", "email": "david.marchand@redhat.com" }, "mbox": "http://patches.dpdk.org/project/dpdk/patch/1565771263-27353-1-git-send-email-phil.yang@arm.com/mbox/", "series": [ { "id": 6041, "url": "http://patches.dpdk.org/api/series/6041/?format=api", "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=6041", "date": "2019-08-14T08:27:41", "name": "[v9,1/3] eal/arm64: add 128-bit atomic compare exchange", "version": 9, "mbox": "http://patches.dpdk.org/series/6041/mbox/" } ], "comments": "http://patches.dpdk.org/api/patches/57675/comments/", "check": "warning", "checks": "http://patches.dpdk.org/api/patches/57675/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 404E8378B;\n\tWed, 14 Aug 2019 10:29:26 +0200 (CEST)", "from foss.arm.com (foss.arm.com [217.140.110.172])\n\tby dpdk.org (Postfix) with ESMTP id 7BF8CA69\n\tfor <dev@dpdk.org>; Wed, 14 Aug 2019 10:29:24 +0200 (CEST)", "from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14])\n\tby usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id B4A41337;\n\tWed, 14 Aug 2019 01:29:23 -0700 (PDT)", "from phil-VirtualBox.shanghai.arm.com (unknown [10.169.109.153])\n\tby usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id\n\tC50743F694; Wed, 14 Aug 2019 01:29:21 -0700 (PDT)" ], "From": "Phil Yang <phil.yang@arm.com>", "To": "thomas@monjalon.net, jerinj@marvell.com, gage.eads@intel.com,\n\tdev@dpdk.org", "Cc": "hemant.agrawal@nxp.com, Honnappa.Nagarahalli@arm.com, gavin.hu@arm.com, \n\tnd@arm.com", "Date": "Wed, 14 Aug 2019 16:27:41 +0800", "Message-Id": "<1565771263-27353-1-git-send-email-phil.yang@arm.com>", "X-Mailer": "git-send-email 2.7.4", "In-Reply-To": "<20190723070536.30342-1-jerinj@marvell.com>", "References": "<20190723070536.30342-1-jerinj@marvell.com>", "Subject": "[dpdk-dev] [PATCH v9 1/3] eal/arm64: add 128-bit atomic compare\n\texchange", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Add 128-bit atomic compare exchange on aarch64.\n\nSuggested-by: Jerin Jacob <jerinj@marvell.com>\nSigned-off-by: Phil Yang <phil.yang@arm.com>\nReviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>\nTested-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>\nAcked-by: Jerin Jacob <jerinj@marvell.com>\n---\n\nv9:\nUpdated 19.11 release note.\n\nv8:\nFixed \"WARNING:LONG_LINE: line over 80 characters\" warnings with latest kernel\ncheckpatch.pl\n\nv7:\n1. Adjust code comment.\n\nv6:\n1. Put the RTE_ARM_FEATURE_ATOMICS flag into EAL group. (Jerin Jocob)\n2. Keep rte_stack_lf_stubs.h doing nothing. (Gage Eads)\n3. Fixed 32 bit build issue.\n\nv5:\n1. Enable RTE_ARM_FEATURE_ATOMICS on octeontx2 in default. (Jerin Jocob)\n2. Record the reason of introducing \"rte_stack_lf_stubs.h\" in git\ncommit.\n(Jerin, Jocob)\n3. Fixed a conditional MACRO error in rte_atomic128_cmp_exchange. (Jerin\nJocob)\n\nv4:\n1. Add RTE_ARM_FEATURE_ATOMICS flag to support LSE CASP instructions.\n(Jerin Jocob)\n2. Fix possible arm64 ABI break by making casp_op_name noinline. (Jerin\nJocob)\n3. Add rte_stack_lf_stubs.h to reduce the ifdef clutter. (Gage\nEads/Jerin Jocob)\n\nv3:\n1. Avoid duplication code with macro. (Jerin Jocob)\n2. Make invalid memory order to strongest barrier. (Jerin Jocob)\n3. Update doc/guides/prog_guide/env_abstraction_layer.rst. (Gage Eads)\n4. Fix 32-bit x86 builds issue. (Gage Eads)\n5. Correct documentation issues in UT. (Gage Eads)\n\nv2:\nInitial version.\n\n config/arm/meson.build | 2 +\n config/common_base | 3 +\n config/defconfig_arm64-octeontx2-linuxapp-gcc | 1 +\n config/defconfig_arm64-thunderx2-linuxapp-gcc | 1 +\n .../common/include/arch/arm/rte_atomic_64.h | 163 +++++++++++++++++++++\n .../common/include/arch/x86/rte_atomic_64.h | 12 --\n lib/librte_eal/common/include/generic/rte_atomic.h | 17 ++-\n 7 files changed, 186 insertions(+), 13 deletions(-)", "diff": "diff --git a/config/arm/meson.build b/config/arm/meson.build\nindex 979018e..9f28271 100644\n--- a/config/arm/meson.build\n+++ b/config/arm/meson.build\n@@ -71,11 +71,13 @@ flags_thunderx2_extra = [\n \t['RTE_CACHE_LINE_SIZE', 64],\n \t['RTE_MAX_NUMA_NODES', 2],\n \t['RTE_MAX_LCORE', 256],\n+\t['RTE_ARM_FEATURE_ATOMICS', true],\n \t['RTE_USE_C11_MEM_MODEL', true]]\n flags_octeontx2_extra = [\n \t['RTE_MACHINE', '\"octeontx2\"'],\n \t['RTE_MAX_NUMA_NODES', 1],\n \t['RTE_MAX_LCORE', 24],\n+\t['RTE_ARM_FEATURE_ATOMICS', true],\n \t['RTE_EAL_IGB_UIO', false],\n \t['RTE_USE_C11_MEM_MODEL', true]]\n \ndiff --git a/config/common_base b/config/common_base\nindex 8ef75c2..2054480 100644\n--- a/config/common_base\n+++ b/config/common_base\n@@ -82,6 +82,9 @@ CONFIG_RTE_MAX_LCORE=128\n CONFIG_RTE_MAX_NUMA_NODES=8\n CONFIG_RTE_MAX_HEAPS=32\n CONFIG_RTE_MAX_MEMSEG_LISTS=64\n+\n+# Use ARM LSE ATOMIC instructions\n+CONFIG_RTE_ARM_FEATURE_ATOMICS=n\n # each memseg list will be limited to either RTE_MAX_MEMSEG_PER_LIST pages\n # or RTE_MAX_MEM_MB_PER_LIST megabytes worth of memory, whichever is smaller\n CONFIG_RTE_MAX_MEMSEG_PER_LIST=8192\ndiff --git a/config/defconfig_arm64-octeontx2-linuxapp-gcc b/config/defconfig_arm64-octeontx2-linuxapp-gcc\nindex f20da24..7687dbe 100644\n--- a/config/defconfig_arm64-octeontx2-linuxapp-gcc\n+++ b/config/defconfig_arm64-octeontx2-linuxapp-gcc\n@@ -9,6 +9,7 @@ CONFIG_RTE_MACHINE=\"octeontx2\"\n CONFIG_RTE_CACHE_LINE_SIZE=128\n CONFIG_RTE_MAX_NUMA_NODES=1\n CONFIG_RTE_MAX_LCORE=24\n+CONFIG_RTE_ARM_FEATURE_ATOMICS=y\n \n # Doesn't support NUMA\n CONFIG_RTE_EAL_NUMA_AWARE_HUGEPAGES=n\ndiff --git a/config/defconfig_arm64-thunderx2-linuxapp-gcc b/config/defconfig_arm64-thunderx2-linuxapp-gcc\nindex cc5c64b..af4a89c 100644\n--- a/config/defconfig_arm64-thunderx2-linuxapp-gcc\n+++ b/config/defconfig_arm64-thunderx2-linuxapp-gcc\n@@ -9,3 +9,4 @@ CONFIG_RTE_MACHINE=\"thunderx2\"\n CONFIG_RTE_CACHE_LINE_SIZE=64\n CONFIG_RTE_MAX_NUMA_NODES=2\n CONFIG_RTE_MAX_LCORE=256\n+CONFIG_RTE_ARM_FEATURE_ATOMICS=y\ndiff --git a/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h b/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h\nindex 97060e4..14d869b 100644\n--- a/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h\n+++ b/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h\n@@ -1,5 +1,6 @@\n /* SPDX-License-Identifier: BSD-3-Clause\n * Copyright(c) 2015 Cavium, Inc\n+ * Copyright(c) 2019 Arm Limited\n */\n \n #ifndef _RTE_ATOMIC_ARM64_H_\n@@ -14,6 +15,9 @@ extern \"C\" {\n #endif\n \n #include \"generic/rte_atomic.h\"\n+#include <rte_branch_prediction.h>\n+#include <rte_compat.h>\n+#include <rte_debug.h>\n \n #define dsb(opt) asm volatile(\"dsb \" #opt : : : \"memory\")\n #define dmb(opt) asm volatile(\"dmb \" #opt : : : \"memory\")\n@@ -40,6 +44,165 @@ extern \"C\" {\n \n #define rte_cio_rmb() dmb(oshld)\n \n+/*------------------------ 128 bit atomic operations -------------------------*/\n+\n+#define __HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE)\n+#define __HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \\\n+\t\t\t\t\t (mo) == __ATOMIC_SEQ_CST)\n+\n+#define __MO_LOAD(mo) (__HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED)\n+#define __MO_STORE(mo) (__HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED)\n+\n+#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)\n+#define __ATOMIC128_CAS_OP(cas_op_name, op_string) \\\n+static __rte_noinline rte_int128_t \\\n+cas_op_name(rte_int128_t *dst, rte_int128_t old, \\\n+\t\trte_int128_t updated) \\\n+{ \\\n+\t/* caspX instructions register pair must start from even-numbered\n+\t * register at operand 1.\n+\t * So, specify registers for local variables here.\n+\t */ \\\n+\tregister uint64_t x0 __asm(\"x0\") = (uint64_t)old.val[0]; \\\n+\tregister uint64_t x1 __asm(\"x1\") = (uint64_t)old.val[1]; \\\n+\tregister uint64_t x2 __asm(\"x2\") = (uint64_t)updated.val[0]; \\\n+\tregister uint64_t x3 __asm(\"x3\") = (uint64_t)updated.val[1]; \\\n+\tasm volatile( \\\n+\t\top_string \" %[old0], %[old1], %[upd0], %[upd1], [%[dst]]\" \\\n+\t\t: [old0] \"+r\" (x0), \\\n+\t\t[old1] \"+r\" (x1) \\\n+\t\t: [upd0] \"r\" (x2), \\\n+\t\t[upd1] \"r\" (x3), \\\n+\t\t[dst] \"r\" (dst) \\\n+\t\t: \"memory\"); \\\n+\told.val[0] = x0; \\\n+\told.val[1] = x1; \\\n+\treturn old; \\\n+}\n+\n+__ATOMIC128_CAS_OP(__rte_cas_relaxed, \"casp\")\n+__ATOMIC128_CAS_OP(__rte_cas_acquire, \"caspa\")\n+__ATOMIC128_CAS_OP(__rte_cas_release, \"caspl\")\n+__ATOMIC128_CAS_OP(__rte_cas_acq_rel, \"caspal\")\n+#else\n+#define __ATOMIC128_LDX_OP(ldx_op_name, op_string) \\\n+static inline rte_int128_t \\\n+ldx_op_name(const rte_int128_t *src) \\\n+{ \\\n+\trte_int128_t ret; \\\n+\tasm volatile( \\\n+\t\t\top_string \" %0, %1, %2\" \\\n+\t\t\t: \"=&r\" (ret.val[0]), \\\n+\t\t\t \"=&r\" (ret.val[1]) \\\n+\t\t\t: \"Q\" (src->val[0]) \\\n+\t\t\t: \"memory\"); \\\n+\treturn ret; \\\n+}\n+\n+__ATOMIC128_LDX_OP(__rte_ldx_relaxed, \"ldxp\")\n+__ATOMIC128_LDX_OP(__rte_ldx_acquire, \"ldaxp\")\n+\n+#define __ATOMIC128_STX_OP(stx_op_name, op_string) \\\n+static inline uint32_t \\\n+stx_op_name(rte_int128_t *dst, const rte_int128_t src) \\\n+{ \\\n+\tuint32_t ret; \\\n+\tasm volatile( \\\n+\t\t\top_string \" %w0, %1, %2, %3\" \\\n+\t\t\t: \"=&r\" (ret) \\\n+\t\t\t: \"r\" (src.val[0]), \\\n+\t\t\t \"r\" (src.val[1]), \\\n+\t\t\t \"Q\" (dst->val[0]) \\\n+\t\t\t: \"memory\"); \\\n+\t/* Return 0 on success, 1 on failure */ \\\n+\treturn ret; \\\n+}\n+\n+__ATOMIC128_STX_OP(__rte_stx_relaxed, \"stxp\")\n+__ATOMIC128_STX_OP(__rte_stx_release, \"stlxp\")\n+#endif\n+\n+static inline int __rte_experimental\n+rte_atomic128_cmp_exchange(rte_int128_t *dst,\n+\t\t\t\trte_int128_t *exp,\n+\t\t\t\tconst rte_int128_t *src,\n+\t\t\t\tunsigned int weak,\n+\t\t\t\tint success,\n+\t\t\t\tint failure)\n+{\n+\t/* Always do strong CAS */\n+\tRTE_SET_USED(weak);\n+\t/* Ignore memory ordering for failure, memory order for\n+\t * success must be stronger or equal\n+\t */\n+\tRTE_SET_USED(failure);\n+\t/* Find invalid memory order */\n+\tRTE_ASSERT(success == __ATOMIC_RELAXED\n+\t\t\t|| success == __ATOMIC_ACQUIRE\n+\t\t\t|| success == __ATOMIC_RELEASE\n+\t\t\t|| success == __ATOMIC_ACQ_REL\n+\t\t\t|| success == __ATOMIC_SEQ_CST);\n+\n+#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)\n+\trte_int128_t expected = *exp;\n+\trte_int128_t desired = *src;\n+\trte_int128_t old;\n+\n+\tif (success == __ATOMIC_RELAXED)\n+\t\told = __rte_cas_relaxed(dst, expected, desired);\n+\telse if (success == __ATOMIC_ACQUIRE)\n+\t\told = __rte_cas_acquire(dst, expected, desired);\n+\telse if (success == __ATOMIC_RELEASE)\n+\t\told = __rte_cas_release(dst, expected, desired);\n+\telse\n+\t\told = __rte_cas_acq_rel(dst, expected, desired);\n+#else\n+\tint ldx_mo = __MO_LOAD(success);\n+\tint stx_mo = __MO_STORE(success);\n+\tuint32_t ret = 1;\n+\tregister rte_int128_t expected = *exp;\n+\tregister rte_int128_t desired = *src;\n+\tregister rte_int128_t old;\n+\n+\t/* ldx128 can not guarantee atomic,\n+\t * Must write back src or old to verify atomicity of ldx128;\n+\t */\n+\tdo {\n+\t\tif (ldx_mo == __ATOMIC_RELAXED)\n+\t\t\told = __rte_ldx_relaxed(dst);\n+\t\telse\n+\t\t\told = __rte_ldx_acquire(dst);\n+\n+\t\tif (likely(old.int128 == expected.int128)) {\n+\t\t\tif (stx_mo == __ATOMIC_RELAXED)\n+\t\t\t\tret = __rte_stx_relaxed(dst, desired);\n+\t\t\telse\n+\t\t\t\tret = __rte_stx_release(dst, desired);\n+\t\t} else {\n+\t\t\t/* In the failure case (since 'weak' is ignored and only\n+\t\t\t * weak == 0 is implemented), expected should contain\n+\t\t\t * the atomically read value of dst. This means, 'old'\n+\t\t\t * needs to be stored back to ensure it was read\n+\t\t\t * atomically.\n+\t\t\t */\n+\t\t\tif (stx_mo == __ATOMIC_RELAXED)\n+\t\t\t\tret = __rte_stx_relaxed(dst, old);\n+\t\t\telse\n+\t\t\t\tret = __rte_stx_release(dst, old);\n+\t\t}\n+\t} while (unlikely(ret));\n+#endif\n+\n+\t/* Unconditionally updating expected removes\n+\t * an 'if' statement.\n+\t * expected should already be in register if\n+\t * not in the cache.\n+\t */\n+\t*exp = old;\n+\n+\treturn (old.int128 == expected.int128);\n+}\n+\n #ifdef __cplusplus\n }\n #endif\ndiff --git a/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h\nindex 1335d92..cfe7067 100644\n--- a/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h\n+++ b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h\n@@ -183,18 +183,6 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)\n \n /*------------------------ 128 bit atomic operations -------------------------*/\n \n-/**\n- * 128-bit integer structure.\n- */\n-RTE_STD_C11\n-typedef struct {\n-\tRTE_STD_C11\n-\tunion {\n-\t\tuint64_t val[2];\n-\t\t__extension__ __int128 int128;\n-\t};\n-} __rte_aligned(16) rte_int128_t;\n-\n __rte_experimental\n static inline int\n rte_atomic128_cmp_exchange(rte_int128_t *dst,\ndiff --git a/lib/librte_eal/common/include/generic/rte_atomic.h b/lib/librte_eal/common/include/generic/rte_atomic.h\nindex 24ff7dc..e6ab15a 100644\n--- a/lib/librte_eal/common/include/generic/rte_atomic.h\n+++ b/lib/librte_eal/common/include/generic/rte_atomic.h\n@@ -1081,6 +1081,20 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)\n \n /*------------------------ 128 bit atomic operations -------------------------*/\n \n+/**\n+ * 128-bit integer structure.\n+ */\n+RTE_STD_C11\n+typedef struct {\n+\tRTE_STD_C11\n+\tunion {\n+\t\tuint64_t val[2];\n+#ifdef RTE_ARCH_64\n+\t\t__extension__ __int128 int128;\n+#endif\n+\t};\n+} __rte_aligned(16) rte_int128_t;\n+\n #ifdef __DOXYGEN__\n \n /**\n@@ -1093,7 +1107,8 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)\n * *exp = *dst\n * @endcode\n *\n- * @note This function is currently only available for the x86-64 platform.\n+ * @note This function is currently available for the x86-64 and aarch64\n+ * platforms.\n *\n * @note The success and failure arguments must be one of the __ATOMIC_* values\n * defined in the C++11 standard. For details on their behavior, refer to the\n", "prefixes": [ "v9", "1/3" ] }