[v2,1/1] eal: add 128-bit cmpxchg (x86-64 only)

Message ID 20190222154640.22029-2-gage.eads@intel.com
State Superseded, archived
Delegated to: Thomas Monjalon
Headers show
Series
  • Add 128-bit compare and set
Related show

Checks

Context Check Description
ci/intel-Performance-Testing success Performance Testing PASS
ci/mellanox-Performance-Testing success Performance Testing PASS
ci/Intel-compilation success Compilation OK
ci/checkpatch success coding style OK

Commit Message

Eads, Gage Feb. 22, 2019, 3:46 p.m.
This operation can be used for non-blocking algorithms, such as a
non-blocking stack or ring.

Signed-off-by: Gage Eads <gage.eads@intel.com>
---
 .../common/include/arch/x86/rte_atomic_64.h        | 33 ++++++++++++
 lib/librte_eal/common/include/generic/rte_atomic.h | 59 ++++++++++++++++++++++
 2 files changed, 92 insertions(+)

Comments

Honnappa Nagarahalli March 4, 2019, 8:19 p.m. | #1
> This operation can be used for non-blocking algorithms, such as a non-blocking
> stack or ring.
> 
> Signed-off-by: Gage Eads <gage.eads@intel.com>
> ---
>  .../common/include/arch/x86/rte_atomic_64.h        | 33 ++++++++++++
>  lib/librte_eal/common/include/generic/rte_atomic.h | 59
> ++++++++++++++++++++++
>  2 files changed, 92 insertions(+)
> 
> diff --git a/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
> b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
> index fd2ec9c53..413e5361b 100644
> --- a/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
> +++ b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
> @@ -34,6 +34,7 @@
>  /*
>   * Inspired from FreeBSD src/sys/amd64/include/atomic.h
>   * Copyright (c) 1998 Doug Rabson
> + * Copyright (c) 2019 Intel Corporation
>   * All rights reserved.
>   */
> 
> @@ -46,6 +47,7 @@
> 
>  #include <stdint.h>
>  #include <rte_common.h>
> +#include <rte_compat.h>
>  #include <rte_atomic.h>
> 
>  /*------------------------- 64 bit atomic operations -------------------------*/ @@ -
> 208,4 +210,35 @@ static inline void rte_atomic64_clear(rte_atomic64_t *v)  }
> #endif
> 
> +static inline int __rte_experimental
> +rte_atomic128_cmpxchg(rte_int128_t *dst,
> +		      rte_int128_t *exp,
> +		      const rte_int128_t *src,
> +		      unsigned int weak,
> +		      int success,
> +		      int failure)
> +{
> +	RTE_SET_USED(weak);
> +	RTE_SET_USED(success);
> +	RTE_SET_USED(failure);
> +	uint8_t res;
> +
> +	asm volatile (
> +		      MPLOCKED
> +		      "cmpxchg16b %[dst];"
> +		      " sete %[res]"
> +		      : [dst] "=m" (dst->val[0]),
> +			"=a" (exp->val[0]),
> +			"=d" (exp->val[1]),
> +			[res] "=r" (res)
> +		      : "b" (src->val[0]),
> +			"c" (src->val[1]),
> +			"a" (exp->val[0]),
> +			"d" (exp->val[1]),
> +			"m" (dst->val[0])
> +		      : "memory");
> +
> +	return res;
> +}
> +
>  #endif /* _RTE_ATOMIC_X86_64_H_ */
> diff --git a/lib/librte_eal/common/include/generic/rte_atomic.h
> b/lib/librte_eal/common/include/generic/rte_atomic.h
> index 4afd1acc3..7c43896e9 100644
> --- a/lib/librte_eal/common/include/generic/rte_atomic.h
> +++ b/lib/librte_eal/common/include/generic/rte_atomic.h
> @@ -14,6 +14,7 @@
> 
>  #include <stdint.h>
>  #include <rte_common.h>
> +#include <rte_compat.h>
> 
>  #ifdef __DOXYGEN__
> 
> @@ -1082,4 +1083,62 @@ static inline void
> rte_atomic64_clear(rte_atomic64_t *v)  }  #endif
> 
> +/*------------------------ 128 bit atomic operations
> +-------------------------*/
> +
> +#if defined(RTE_ARCH_X86_64)
> +/**
> + * 128-bit integer structure.
> + */
> +RTE_STD_C11
> +typedef struct {
> +	RTE_STD_C11
> +	union {
> +		uint64_t val[2];
> +		__int128 int128;
> +	};
> +} __rte_aligned(16) rte_int128_t;
> +
> +/**
> + * An atomic compare and set function used by the mutex functions.
> + * (Atomically) Equivalent to:
> + *   if (*dst == exp)
Should be, "if (*dst == *exp)"

> + *     *dst = src
Should be "*dst = *src"

> + *   else
> + *     *exp = *dst
> + *
> + * @note The success and failure arguments must be one of the
> +__ATOMIC_* values
> + * defined in the C++11 standard. For details on their behavior, refer
> +to the
> + * standard.
> + *
> + * @param dst
> + *   The destination into which the value will be written.
> + * @param exp
> + *   Pointer to the expected value. If the operation fails, this memory is
> + *   updated with the actual value.
> + * @param src
> + *   Pointer to the new value.
> + * @param weak
> + *   A value of true allows the comparison to spuriously fail and allows the
> + *   'exp' update to occur non-atomically (i.e. a torn read may occur).
> + *   Implementations may ignore this argument and only implement the strong
> + *   variant.
> + * @param success
> + *   If successful, the operation's memory behavior conforms to this (or a
> + *   stronger) model.
> + * @param failure
> + *   If unsuccessful, the operation's memory behavior conforms to this (or a
> + *   stronger) model. This argument cannot be __ATOMIC_RELEASE,
> + *   __ATOMIC_ACQ_REL, or a stronger model than success.
> + * @return
> + *   Non-zero on success; 0 on failure.
> + */
> +static inline int __rte_experimental
> +rte_atomic128_cmpxchg(rte_int128_t *dst,
Name could be more neutral. May be rte_atomic128_compare/cmp_exchange?

> +		      rte_int128_t *exp,
> +		      const rte_int128_t *src,
> +		      unsigned int weak,
> +		      int success,
> +		      int failure);
> +#endif
> +
>  #endif /* _RTE_ATOMIC_H_ */
> --
> 2.13.6

Few minor comments.
I have not reviewed the x86 implementation.
Otherwise,
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
Eads, Gage March 4, 2019, 8:47 p.m. | #2
[snip]

> > +/**
> > + * An atomic compare and set function used by the mutex functions.
> > + * (Atomically) Equivalent to:
> > + *   if (*dst == exp)
> Should be, "if (*dst == *exp)"
> 
> > + *     *dst = src
> Should be "*dst = *src"
> 

Good catches, will fix both.

> > + *   else
> > + *     *exp = *dst
> > + *
> > + * @note The success and failure arguments must be one of the
> > +__ATOMIC_* values
> > + * defined in the C++11 standard. For details on their behavior,
> > +refer to the
> > + * standard.
> > + *
> > + * @param dst
> > + *   The destination into which the value will be written.
> > + * @param exp
> > + *   Pointer to the expected value. If the operation fails, this memory is
> > + *   updated with the actual value.
> > + * @param src
> > + *   Pointer to the new value.
> > + * @param weak
> > + *   A value of true allows the comparison to spuriously fail and allows the
> > + *   'exp' update to occur non-atomically (i.e. a torn read may occur).
> > + *   Implementations may ignore this argument and only implement the
> strong
> > + *   variant.
> > + * @param success
> > + *   If successful, the operation's memory behavior conforms to this (or a
> > + *   stronger) model.
> > + * @param failure
> > + *   If unsuccessful, the operation's memory behavior conforms to this (or a
> > + *   stronger) model. This argument cannot be __ATOMIC_RELEASE,
> > + *   __ATOMIC_ACQ_REL, or a stronger model than success.
> > + * @return
> > + *   Non-zero on success; 0 on failure.
> > + */
> > +static inline int __rte_experimental
> > +rte_atomic128_cmpxchg(rte_int128_t *dst,
> Name could be more neutral. May be rte_atomic128_compare/cmp_exchange?
> 
> > +		      rte_int128_t *exp,
> > +		      const rte_int128_t *src,
> > +		      unsigned int weak,
> > +		      int success,
> > +		      int failure);
> > +#endif
> > +
> >  #endif /* _RTE_ATOMIC_H_ */
> > --
> > 2.13.6
> 

Sure -- I'll rename it rte_atomic128_cmp_exchange in the next version.

> Few minor comments.
> I have not reviewed the x86 implementation.
> Otherwise,
> Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>

Thanks,
Gage

Patch

diff --git a/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
index fd2ec9c53..413e5361b 100644
--- a/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
+++ b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h
@@ -34,6 +34,7 @@ 
 /*
  * Inspired from FreeBSD src/sys/amd64/include/atomic.h
  * Copyright (c) 1998 Doug Rabson
+ * Copyright (c) 2019 Intel Corporation
  * All rights reserved.
  */
 
@@ -46,6 +47,7 @@ 
 
 #include <stdint.h>
 #include <rte_common.h>
+#include <rte_compat.h>
 #include <rte_atomic.h>
 
 /*------------------------- 64 bit atomic operations -------------------------*/
@@ -208,4 +210,35 @@  static inline void rte_atomic64_clear(rte_atomic64_t *v)
 }
 #endif
 
+static inline int __rte_experimental
+rte_atomic128_cmpxchg(rte_int128_t *dst,
+		      rte_int128_t *exp,
+		      const rte_int128_t *src,
+		      unsigned int weak,
+		      int success,
+		      int failure)
+{
+	RTE_SET_USED(weak);
+	RTE_SET_USED(success);
+	RTE_SET_USED(failure);
+	uint8_t res;
+
+	asm volatile (
+		      MPLOCKED
+		      "cmpxchg16b %[dst];"
+		      " sete %[res]"
+		      : [dst] "=m" (dst->val[0]),
+			"=a" (exp->val[0]),
+			"=d" (exp->val[1]),
+			[res] "=r" (res)
+		      : "b" (src->val[0]),
+			"c" (src->val[1]),
+			"a" (exp->val[0]),
+			"d" (exp->val[1]),
+			"m" (dst->val[0])
+		      : "memory");
+
+	return res;
+}
+
 #endif /* _RTE_ATOMIC_X86_64_H_ */
diff --git a/lib/librte_eal/common/include/generic/rte_atomic.h b/lib/librte_eal/common/include/generic/rte_atomic.h
index 4afd1acc3..7c43896e9 100644
--- a/lib/librte_eal/common/include/generic/rte_atomic.h
+++ b/lib/librte_eal/common/include/generic/rte_atomic.h
@@ -14,6 +14,7 @@ 
 
 #include <stdint.h>
 #include <rte_common.h>
+#include <rte_compat.h>
 
 #ifdef __DOXYGEN__
 
@@ -1082,4 +1083,62 @@  static inline void rte_atomic64_clear(rte_atomic64_t *v)
 }
 #endif
 
+/*------------------------ 128 bit atomic operations -------------------------*/
+
+#if defined(RTE_ARCH_X86_64)
+/**
+ * 128-bit integer structure.
+ */
+RTE_STD_C11
+typedef struct {
+	RTE_STD_C11
+	union {
+		uint64_t val[2];
+		__int128 int128;
+	};
+} __rte_aligned(16) rte_int128_t;
+
+/**
+ * An atomic compare and set function used by the mutex functions.
+ * (Atomically) Equivalent to:
+ *   if (*dst == exp)
+ *     *dst = src
+ *   else
+ *     *exp = *dst
+ *
+ * @note The success and failure arguments must be one of the __ATOMIC_* values
+ * defined in the C++11 standard. For details on their behavior, refer to the
+ * standard.
+ *
+ * @param dst
+ *   The destination into which the value will be written.
+ * @param exp
+ *   Pointer to the expected value. If the operation fails, this memory is
+ *   updated with the actual value.
+ * @param src
+ *   Pointer to the new value.
+ * @param weak
+ *   A value of true allows the comparison to spuriously fail and allows the
+ *   'exp' update to occur non-atomically (i.e. a torn read may occur).
+ *   Implementations may ignore this argument and only implement the strong
+ *   variant.
+ * @param success
+ *   If successful, the operation's memory behavior conforms to this (or a
+ *   stronger) model.
+ * @param failure
+ *   If unsuccessful, the operation's memory behavior conforms to this (or a
+ *   stronger) model. This argument cannot be __ATOMIC_RELEASE,
+ *   __ATOMIC_ACQ_REL, or a stronger model than success.
+ * @return
+ *   Non-zero on success; 0 on failure.
+ */
+static inline int __rte_experimental
+rte_atomic128_cmpxchg(rte_int128_t *dst,
+		      rte_int128_t *exp,
+		      const rte_int128_t *src,
+		      unsigned int weak,
+		      int success,
+		      int failure);
+#endif
+
 #endif /* _RTE_ATOMIC_H_ */