#$@%#$%!

Ngiler SH3LL 360
: / opt / alt / ruby34 / include / ruby /

Information Server
MySQL : OFF Perl : OFF CURL : ON WGET : OFF PKEXEC : OFF

Directive	Local Value
IP Address	162.0.209.101
System	Linux premium169.web-hosting.com 4.18.0-553.50.1.lve.el8.x86_64 #1 SMP Thu Apr 17 19:10:24 UTC 2025 x86_64
User	luxeelil
PHP Version	8.0.30
Software	LiteSpeed
Doc root	Writable

Edit File : atomic.h \| Size : 27.62 KB
#ifndef RUBY_ATOMIC_H /--C++--vi:se ft=cpp:/ #define RUBY_ATOMIC_H /** * @file * @author Ruby developers <ruby-core@ruby-lang.org> * @copyright This file is a part of the programming language Ruby. * Permission is hereby granted, to either redistribute and/or * modify this file, provided that the conditions mentioned in the * file COPYING are met. Consult the file for details. * @warning Symbols prefixed with either `RBIMPL` or `rbimpl` are * implementation details. Don't take them as canon. They could * rapidly appear then vanish. The name (path) of this header file * is also an implementation detail. Do not expect it to persist * at the place it is now. Developers are free to move it anywhere * anytime at will. * @note To ruby-core: remember that this header can be possibly * recursively included from extension libraries written in C++. * Do not expect for instance `__VA_ARGS__` is always available. * We assume C99 for ruby itself but we don't assume languages of * extension libraries. They could be written in C++98. * @brief Atomic operations * * Basically, if we could assume either C11 or C++11, these macros are just * redundant. Sadly we cannot. We have to do them ourselves. / #include "ruby/internal/config.h" #ifdef STDC_HEADERS # include <stddef.h> / size_t / #endif #ifdef HAVE_SYS_TYPES_H # include <sys/types.h> / ssize_t / #endif #if RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0) # pragma intrinsic(_InterlockedOr) #elif defined(__sun) && defined(HAVE_ATOMIC_H) # include <atomic.h> #endif #include "ruby/assert.h" #include "ruby/backward/2/limits.h" #include "ruby/internal/attr/artificial.h" #include "ruby/internal/attr/noalias.h" #include "ruby/internal/attr/nonnull.h" #include "ruby/internal/compiler_since.h" #include "ruby/internal/cast.h" #include "ruby/internal/value.h" #include "ruby/internal/static_assert.h" #include "ruby/internal/stdbool.h" / * Asserts that your environment supports more than one atomic types. These * days systems tend to have such property (C11 was a standard of decades ago, * right?) but we still support older ones. / #if defined(__DOXYGEN__) \|\| defined(HAVE_GCC_ATOMIC_BUILTINS) \|\| defined(HAVE_GCC_SYNC_BUILTINS) # define RUBY_ATOMIC_GENERIC_MACRO 1 #endif /* * Type that is eligible for atomic operations. Depending on your host * platform you might have more than one such type, but we choose one of them * anyways. / #if defined(__DOXYGEN__) using rb_atomic_t = std::atomic<unsigned>; #elif defined(HAVE_GCC_ATOMIC_BUILTINS) typedef unsigned int rb_atomic_t; #elif defined(HAVE_GCC_SYNC_BUILTINS) typedef unsigned int rb_atomic_t; #elif defined(_WIN32) # include <winsock2.h> // to prevent macro redefinitions # include <windows.h> // for `LONG` and `Interlocked` functions typedef LONG rb_atomic_t; #elif defined(__sun) && defined(HAVE_ATOMIC_H) typedef unsigned int rb_atomic_t; #else # error No atomic operation found #endif /* * Atomically replaces the value pointed by `var` with the result of addition * of `val` to the old value of `var`. * * @param var A variable of ::rb_atomic_t. * @param val Value to add. * @return What was stored in `var` before the addition. * @post `var` holds `var + val`. / #define RUBY_ATOMIC_FETCH_ADD(var, val) rbimpl_atomic_fetch_add(&(var), (val)) /* * Atomically replaces the value pointed by `var` with the result of * subtraction of `val` to the old value of `var`. * * @param var A variable of ::rb_atomic_t. * @param val Value to subtract. * @return What was stored in `var` before the subtraction. * @post `var` holds `var - val`. / #define RUBY_ATOMIC_FETCH_SUB(var, val) rbimpl_atomic_fetch_sub(&(var), (val)) /* * Atomically replaces the value pointed by `var` with the result of * bitwise OR between `val` and the old value of `var`. * * @param var A variable of ::rb_atomic_t. * @param val Value to mix. * @return void * @post `var` holds `var \| val`. * @note For portability, this macro can return void. / #define RUBY_ATOMIC_OR(var, val) rbimpl_atomic_or(&(var), (val)) /* * Atomically replaces the value pointed by `var` with `val`. This is just an * assignment, but you can additionally know the previous value. * * @param var A variable of ::rb_atomic_t. * @param val Value to set. * @return What was stored in `var` before the assignment. * @post `var` holds `val`. / #define RUBY_ATOMIC_EXCHANGE(var, val) rbimpl_atomic_exchange(&(var), (val)) /* * Atomic compare-and-swap. This stores `val` to `var` if and only if the * assignment changes the value of `var` from `oldval` to `newval`. You can * detect whether the assignment happened or not using the return value. * * @param var A variable of ::rb_atomic_t. * @param oldval Expected value of `var` before the assignment. * @param newval What you want to store at `var`. * @retval oldval Successful assignment (`var` is now `newval`). * @retval otherwise Something else is at `var`; not updated. / #define RUBY_ATOMIC_CAS(var, oldval, newval) \ rbimpl_atomic_cas(&(var), (oldval), (newval)) /* * Atomic load. This loads `var` with an atomic intrinsic and returns * its value. * * @param var A variable of ::rb_atomic_t * @return What was stored in `var`j / #define RUBY_ATOMIC_LOAD(var) rbimpl_atomic_load(&(var)) /* * Identical to #RUBY_ATOMIC_EXCHANGE, except for the return type. * * @param var A variable of ::rb_atomic_t. * @param val Value to set. * @return void * @post `var` holds `val`. / #define RUBY_ATOMIC_SET(var, val) rbimpl_atomic_set(&(var), (val)) /* * Identical to #RUBY_ATOMIC_FETCH_ADD, except for the return type. * * @param var A variable of ::rb_atomic_t. * @param val Value to add. * @return void * @post `var` holds `var + val`. / #define RUBY_ATOMIC_ADD(var, val) rbimpl_atomic_add(&(var), (val)) /* * Identical to #RUBY_ATOMIC_FETCH_SUB, except for the return type. * * @param var A variable of ::rb_atomic_t. * @param val Value to subtract. * @return void * @post `var` holds `var - val`. / #define RUBY_ATOMIC_SUB(var, val) rbimpl_atomic_sub(&(var), (val)) /* * Atomically increments the value pointed by `var`. * * @param var A variable of ::rb_atomic_t. * @return void * @post `var` holds `var + 1`. / #define RUBY_ATOMIC_INC(var) rbimpl_atomic_inc(&(var)) /* * Atomically decrements the value pointed by `var`. * * @param var A variable of ::rb_atomic_t. * @return void * @post `var` holds `var - 1`. / #define RUBY_ATOMIC_DEC(var) rbimpl_atomic_dec(&(var)) /* * Identical to #RUBY_ATOMIC_INC, except it expects its argument is `size_t`. * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit. This * should be used for size related operations to support such platforms. * * @param var A variable of `size_t`. * @return void * @post `var` holds `var + 1`. / #define RUBY_ATOMIC_SIZE_INC(var) rbimpl_atomic_size_inc(&(var)) /* * Identical to #RUBY_ATOMIC_DEC, except it expects its argument is `size_t`. * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit. This * should be used for size related operations to support such platforms. * * @param var A variable of `size_t`. * @return void * @post `var` holds `var - 1`. / #define RUBY_ATOMIC_SIZE_DEC(var) rbimpl_atomic_size_dec(&(var)) /* * Identical to #RUBY_ATOMIC_EXCHANGE, except it expects its arguments are * `size_t`. There are cases where ::rb_atomic_t is 32bit while `size_t` is * 64bit. This should be used for size related operations to support such * platforms. * * @param var A variable of `size_t`. * @param val Value to set. * @return What was stored in `var` before the assignment. * @post `var` holds `val`. / #define RUBY_ATOMIC_SIZE_EXCHANGE(var, val) \ rbimpl_atomic_size_exchange(&(var), (val)) /* * Identical to #RUBY_ATOMIC_CAS, except it expects its arguments are `size_t`. * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit. This * should be used for size related operations to support such platforms. * * @param var A variable of `size_t`. * @param oldval Expected value of `var` before the assignment. * @param newval What you want to store at `var`. * @retval oldval Successful assignment (`var` is now `newval`). * @retval otherwise Something else is at `var`; not updated. / #define RUBY_ATOMIC_SIZE_CAS(var, oldval, newval) \ rbimpl_atomic_size_cas(&(var), (oldval), (newval)) /* * Identical to #RUBY_ATOMIC_ADD, except it expects its arguments are `size_t`. * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit. This * should be used for size related operations to support such platforms. * * @param var A variable of `size_t`. * @param val Value to add. * @return void * @post `var` holds `var + val`. / #define RUBY_ATOMIC_SIZE_ADD(var, val) rbimpl_atomic_size_add(&(var), (val)) /* * Identical to #RUBY_ATOMIC_SUB, except it expects its arguments are `size_t`. * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit. This * should be used for size related operations to support such platforms. * * @param var A variable of `size_t`. * @param val Value to subtract. * @return void * @post `var` holds `var - val`. / #define RUBY_ATOMIC_SIZE_SUB(var, val) rbimpl_atomic_size_sub(&(var), (val)) /* * Identical to #RUBY_ATOMIC_EXCHANGE, except it expects its arguments are * `void`. There are cases where ::rb_atomic_t is 32bit while `void` is * 64bit. This should be used for pointer related operations to support such * platforms. * * @param var A variable of `void `. @param val Value to set. * @return What was stored in `var` before the assignment. * @post `var` holds `val`. * * @internal * * :FIXME: this `(void)` cast is evil! However `void` is incompatible with * some pointers, most notably function pointers. / #define RUBY_ATOMIC_PTR_EXCHANGE(var, val) \ RBIMPL_CAST(rbimpl_atomic_ptr_exchange((void )&(var), (void )val)) /** * Identical to #RUBY_ATOMIC_LOAD, except it expects its arguments are `void`. There are cases where ::rb_atomic_t is 32bit while `void` is 64bit. This should be used for size related operations to support such platforms. * * @param var A variable of `void` @return The value of `var` (without tearing) / #define RUBY_ATOMIC_PTR_LOAD(var) \ RBIMPL_CAST(rbimpl_atomic_ptr_load((void )&var)) /* * Identical to #RUBY_ATOMIC_CAS, except it expects its arguments are `void`. There are cases where ::rb_atomic_t is 32bit while `void` is 64bit. This should be used for size related operations to support such platforms. * * @param var A variable of `void`. @param oldval Expected value of `var` before the assignment. * @param newval What you want to store at `var`. * @retval oldval Successful assignment (`var` is now `newval`). * @retval otherwise Something else is at `var`; not updated. / #define RUBY_ATOMIC_PTR_CAS(var, oldval, newval) \ RBIMPL_CAST(rbimpl_atomic_ptr_cas((void )&(var), (void )(oldval), (void )(newval))) /* * Identical to #RUBY_ATOMIC_EXCHANGE, except it expects its arguments are * ::VALUE. There are cases where ::rb_atomic_t is 32bit while ::VALUE is * 64bit. This should be used for pointer related operations to support such * platforms. * * @param var A variable of ::VALUE. * @param val Value to set. * @return What was stored in `var` before the assignment. * @post `var` holds `val`. / #define RUBY_ATOMIC_VALUE_EXCHANGE(var, val) \ rbimpl_atomic_value_exchange(&(var), (val)) /* * Identical to #RUBY_ATOMIC_CAS, except it expects its arguments are ::VALUE. * There are cases where ::rb_atomic_t is 32bit while ::VALUE is 64bit. This * should be used for size related operations to support such platforms. * * @param var A variable of `void`. @param oldval Expected value of `var` before the assignment. * @param newval What you want to store at `var`. * @retval oldval Successful assignment (`var` is now `newval`). * @retval otherwise Something else is at `var`; not updated. / #define RUBY_ATOMIC_VALUE_CAS(var, oldval, newval) \ rbimpl_atomic_value_cas(&(var), (oldval), (newval)) /* @cond INTERNAL_MACRO / RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline rb_atomic_t rbimpl_atomic_fetch_add(volatile rb_atomic_t ptr, rb_atomic_t val) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) return __atomic_fetch_add(ptr, val, __ATOMIC_SEQ_CST); #elif defined(HAVE_GCC_SYNC_BUILTINS) return __sync_fetch_and_add(ptr, val); #elif defined(_WIN32) return InterlockedExchangeAdd(ptr, val); #elif defined(__sun) && defined(HAVE_ATOMIC_H) /* * `atomic_add_int_nv` takes its second argument as `int`! Meanwhile our * `rb_atomic_t` is unsigned. We cannot pass `val` as-is. We have to * manually check integer overflow. / RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX); return atomic_add_int_nv(ptr, val) - val; #else # error Unsupported platform. #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void rbimpl_atomic_add(volatile rb_atomic_t ptr, rb_atomic_t val) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) /* * GCC on amd64 is smart enough to detect this `__atomic_add_fetch`'s * return value is not used, then compiles it into single `LOCK ADD` * instruction. / __atomic_add_fetch(ptr, val, __ATOMIC_SEQ_CST); #elif defined(HAVE_GCC_SYNC_BUILTINS) __sync_add_and_fetch(ptr, val); #elif defined(_WIN32) / * `InterlockedExchangeAdd` is `LOCK XADD`. It seems there also is * `_InterlockedAdd` intrinsic in ARM Windows but not for x86? Sticking to * `InterlockedExchangeAdd` for better portability. / InterlockedExchangeAdd(ptr, val); #elif defined(__sun) && defined(HAVE_ATOMIC_H) / Ditto for `atomic_add_int_nv`. / RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX); atomic_add_int(ptr, val); #else # error Unsupported platform. #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void rbimpl_atomic_size_add(volatile size_t ptr, size_t val) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) __atomic_add_fetch(ptr, val, __ATOMIC_SEQ_CST); #elif defined(HAVE_GCC_SYNC_BUILTINS) __sync_add_and_fetch(ptr, val); #elif defined(_WIN64) /* Ditto for `InterlockeExchangedAdd`. / InterlockedExchangeAdd64(ptr, val); #elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) \|\| defined(_I32LPx)) / Ditto for `atomic_add_int_nv`. / RBIMPL_ASSERT_OR_ASSUME(val <= LONG_MAX); atomic_add_long(ptr, val); #else RBIMPL_STATIC_ASSERT(size_of_rb_atomic_t, sizeof ptr == sizeof(rb_atomic_t)); volatile rb_atomic_t const tmp = RBIMPL_CAST((volatile rb_atomic_t )ptr); rbimpl_atomic_add(tmp, val); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void rbimpl_atomic_inc(volatile rb_atomic_t ptr) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) \|\| defined(HAVE_GCC_SYNC_BUILTINS) rbimpl_atomic_add(ptr, 1); #elif defined(_WIN32) InterlockedIncrement(ptr); #elif defined(__sun) && defined(HAVE_ATOMIC_H) atomic_inc_uint(ptr); #else rbimpl_atomic_add(ptr, 1); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void rbimpl_atomic_size_inc(volatile size_t ptr) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) \|\| defined(HAVE_GCC_SYNC_BUILTINS) rbimpl_atomic_size_add(ptr, 1); #elif defined(_WIN64) InterlockedIncrement64(ptr); #elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) \|\| defined(_I32LPx)) atomic_inc_ulong(ptr); #else RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof ptr == sizeof(rb_atomic_t)); rbimpl_atomic_size_add(ptr, 1); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline rb_atomic_t rbimpl_atomic_fetch_sub(volatile rb_atomic_t ptr, rb_atomic_t val) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) return __atomic_fetch_sub(ptr, val, __ATOMIC_SEQ_CST); #elif defined(HAVE_GCC_SYNC_BUILTINS) return __sync_fetch_and_sub(ptr, val); #elif defined(_WIN32) /* rb_atomic_t is signed here! Safe to do `-val`. / return InterlockedExchangeAdd(ptr, -val); #elif defined(__sun) && defined(HAVE_ATOMIC_H) / Ditto for `rbimpl_atomic_fetch_add`. / const signed neg = -1; RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX); return atomic_add_int_nv(ptr, neg val) + val; #else # error Unsupported platform. #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void rbimpl_atomic_sub(volatile rb_atomic_t ptr, rb_atomic_t val) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) __atomic_sub_fetch(ptr, val, __ATOMIC_SEQ_CST); #elif defined(HAVE_GCC_SYNC_BUILTINS) __sync_sub_and_fetch(ptr, val); #elif defined(_WIN32) InterlockedExchangeAdd(ptr, -val); #elif defined(__sun) && defined(HAVE_ATOMIC_H) const signed neg = -1; RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX); atomic_add_int(ptr, neg val); #else # error Unsupported platform. #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void rbimpl_atomic_size_sub(volatile size_t ptr, size_t val) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) __atomic_sub_fetch(ptr, val, __ATOMIC_SEQ_CST); #elif defined(HAVE_GCC_SYNC_BUILTINS) __sync_sub_and_fetch(ptr, val); #elif defined(_WIN64) const ssize_t neg = -1; InterlockedExchangeAdd64(ptr, neg val); #elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) \|\| defined(_I32LPx)) const signed neg = -1; RBIMPL_ASSERT_OR_ASSUME(val <= LONG_MAX); atomic_add_long(ptr, neg * val); #else RBIMPL_STATIC_ASSERT(size_of_rb_atomic_t, sizeof ptr == sizeof(rb_atomic_t)); volatile rb_atomic_t const tmp = RBIMPL_CAST((volatile rb_atomic_t )ptr); rbimpl_atomic_sub(tmp, val); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void rbimpl_atomic_dec(volatile rb_atomic_t ptr) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) \|\| defined(HAVE_GCC_SYNC_BUILTINS) rbimpl_atomic_sub(ptr, 1); #elif defined(_WIN32) InterlockedDecrement(ptr); #elif defined(__sun) && defined(HAVE_ATOMIC_H) atomic_dec_uint(ptr); #else rbimpl_atomic_sub(ptr, 1); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void rbimpl_atomic_size_dec(volatile size_t ptr) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) \|\| defined(HAVE_GCC_SYNC_BUILTINS) rbimpl_atomic_size_sub(ptr, 1); #elif defined(_WIN64) InterlockedDecrement64(ptr); #elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) \|\| defined(_I32LPx)) atomic_dec_ulong(ptr); #else RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof ptr == sizeof(rb_atomic_t)); rbimpl_atomic_size_sub(ptr, 1); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void rbimpl_atomic_or(volatile rb_atomic_t ptr, rb_atomic_t val) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) __atomic_or_fetch(ptr, val, __ATOMIC_SEQ_CST); #elif defined(HAVE_GCC_SYNC_BUILTINS) __sync_or_and_fetch(ptr, val); #elif RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0) _InterlockedOr(ptr, val); #elif defined(_WIN32) && defined(__GNUC__) / This was for old MinGW. Maybe not needed any longer? / __asm__( "lock\n\t" "orl\t%1, %0" : "=m"(ptr) : "Ir"(val)); #elif defined(_WIN32) && defined(_M_IX86) __asm mov eax, ptr; __asm mov ecx, val; __asm lock or [eax], ecx; #elif defined(__sun) && defined(HAVE_ATOMIC_H) atomic_or_uint(ptr, val); #else # error Unsupported platform. #endif } / Nobody uses this but for theoretical backwards compatibility... / #if RBIMPL_COMPILER_BEFORE(MSVC, 13, 0, 0) static inline rb_atomic_t rb_w32_atomic_or(volatile rb_atomic_t var, rb_atomic_t val) { return rbimpl_atomic_or(var, val); } #endif RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline rb_atomic_t rbimpl_atomic_exchange(volatile rb_atomic_t ptr, rb_atomic_t val) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) return __atomic_exchange_n(ptr, val, __ATOMIC_SEQ_CST); #elif defined(HAVE_GCC_SYNC_BUILTINS) return __sync_lock_test_and_set(ptr, val); #elif defined(_WIN32) return InterlockedExchange(ptr, val); #elif defined(__sun) && defined(HAVE_ATOMIC_H) return atomic_swap_uint(ptr, val); #else # error Unsupported platform. #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline size_t rbimpl_atomic_size_exchange(volatile size_t ptr, size_t val) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) return __atomic_exchange_n(ptr, val, __ATOMIC_SEQ_CST); #elif defined(HAVE_GCC_SYNC_BUILTINS) return __sync_lock_test_and_set(ptr, val); #elif defined(_WIN64) return InterlockedExchange64(ptr, val); #elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) \|\| defined(_I32LPx)) return atomic_swap_ulong(ptr, val); #else RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof ptr == sizeof(rb_atomic_t)); volatile rb_atomic_t const tmp = RBIMPL_CAST((volatile rb_atomic_t )ptr); const rb_atomic_t ret = rbimpl_atomic_exchange(tmp, val); return RBIMPL_CAST((size_t)ret); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void rbimpl_atomic_ptr_exchange(void volatile ptr, const void val) { #if 0 #elif defined(InterlockedExchangePointer) / const_cast / PVOID pptr = RBIMPL_CAST((PVOID )ptr); PVOID pval = RBIMPL_CAST((PVOID)val); return InterlockedExchangePointer(pptr, pval); #elif defined(__sun) && defined(HAVE_ATOMIC_H) return atomic_swap_ptr(ptr, RBIMPL_CAST((void )val)); #else RBIMPL_STATIC_ASSERT(sizeof_voidp, sizeof ptr == sizeof(size_t)); const size_t sval = RBIMPL_CAST((size_t)val); volatile size_t const sptr = RBIMPL_CAST((volatile size_t )ptr); const size_t sret = rbimpl_atomic_size_exchange(sptr, sval); return RBIMPL_CAST((void )sret); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline VALUE rbimpl_atomic_value_exchange(volatile VALUE ptr, VALUE val) { RBIMPL_STATIC_ASSERT(sizeof_value, sizeof ptr == sizeof(size_t)); const size_t sval = RBIMPL_CAST((size_t)val); volatile size_t const sptr = RBIMPL_CAST((volatile size_t )ptr); const size_t sret = rbimpl_atomic_size_exchange(sptr, sval); return RBIMPL_CAST((VALUE)sret); } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline rb_atomic_t rbimpl_atomic_load(volatile rb_atomic_t ptr) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); #else return rbimpl_atomic_fetch_add(ptr, 0); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void rbimpl_atomic_set(volatile rb_atomic_t ptr, rb_atomic_t val) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) __atomic_store_n(ptr, val, __ATOMIC_SEQ_CST); #else /* Maybe std::atomic<rb_atomic_t>::store can be faster? / rbimpl_atomic_exchange(ptr, val); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline rb_atomic_t rbimpl_atomic_cas(volatile rb_atomic_t ptr, rb_atomic_t oldval, rb_atomic_t newval) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) __atomic_compare_exchange_n( ptr, &oldval, newval, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); return oldval; #elif defined(HAVE_GCC_SYNC_BUILTINS) return __sync_val_compare_and_swap(ptr, oldval, newval); #elif RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0) return InterlockedCompareExchange(ptr, newval, oldval); #elif defined(_WIN32) PVOID pptr = RBIMPL_CAST((PVOID )ptr); PVOID pold = RBIMPL_CAST((PVOID)oldval); PVOID pnew = RBIMPL_CAST((PVOID)newval); PVOID pret = InterlockedCompareExchange(pptr, pnew, pold); return RBIMPL_CAST((rb_atomic_t)pret); #elif defined(__sun) && defined(HAVE_ATOMIC_H) return atomic_cas_uint(ptr, oldval, newval); #else # error Unsupported platform. #endif } /* Nobody uses this but for theoretical backwards compatibility... / #if RBIMPL_COMPILER_BEFORE(MSVC, 13, 0, 0) static inline rb_atomic_t rb_w32_atomic_cas(volatile rb_atomic_t var, rb_atomic_t oldval, rb_atomic_t newval) { return rbimpl_atomic_cas(var, oldval, newval); } #endif RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline size_t rbimpl_atomic_size_cas(volatile size_t ptr, size_t oldval, size_t newval) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) __atomic_compare_exchange_n( ptr, &oldval, newval, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); return oldval; #elif defined(HAVE_GCC_SYNC_BUILTINS) return __sync_val_compare_and_swap(ptr, oldval, newval); #elif defined(_WIN64) return InterlockedCompareExchange64(ptr, newval, oldval); #elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) \|\| defined(_I32LPx)) return atomic_cas_ulong(ptr, oldval, newval); #else RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof ptr == sizeof(rb_atomic_t)); volatile rb_atomic_t tmp = RBIMPL_CAST((volatile rb_atomic_t )ptr); return rbimpl_atomic_cas(tmp, oldval, newval); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void * rbimpl_atomic_ptr_cas(void *ptr, const void oldval, const void newval) { #if 0 #elif defined(InterlockedExchangePointer) / ... Can we say that InterlockedCompareExchangePtr surly exists when * InterlockedExchangePointer is defined? Seems so but...?/ PVOID pptr = RBIMPL_CAST((PVOID )ptr); PVOID pold = RBIMPL_CAST((PVOID)oldval); PVOID pnew = RBIMPL_CAST((PVOID)newval); return InterlockedCompareExchangePointer(pptr, pnew, pold); #elif defined(__sun) && defined(HAVE_ATOMIC_H) void pold = RBIMPL_CAST((void )oldval); void pnew = RBIMPL_CAST((void )newval); return atomic_cas_ptr(ptr, pold, pnew); #else RBIMPL_STATIC_ASSERT(sizeof_voidp, sizeof ptr == sizeof(size_t)); const size_t snew = RBIMPL_CAST((size_t)newval); const size_t sold = RBIMPL_CAST((size_t)oldval); volatile size_t const sptr = RBIMPL_CAST((volatile size_t )ptr); const size_t sret = rbimpl_atomic_size_cas(sptr, sold, snew); return RBIMPL_CAST((void )sret); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline void rbimpl_atomic_ptr_load(void *ptr) { #if 0 #elif defined(HAVE_GCC_ATOMIC_BUILTINS) return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); #else void val = ptr; return rbimpl_atomic_ptr_cas(ptr, val, val); #endif } RBIMPL_ATTR_ARTIFICIAL() RBIMPL_ATTR_NOALIAS() RBIMPL_ATTR_NONNULL((1)) static inline VALUE rbimpl_atomic_value_cas(volatile VALUE ptr, VALUE oldval, VALUE newval) { RBIMPL_STATIC_ASSERT(sizeof_value, sizeof ptr == sizeof(size_t)); const size_t snew = RBIMPL_CAST((size_t)newval); const size_t sold = RBIMPL_CAST((size_t)oldval); volatile size_t const sptr = RBIMPL_CAST((volatile size_t )ptr); const size_t sret = rbimpl_atomic_size_cas(sptr, sold, snew); return RBIMPL_CAST((VALUE)sret); } /* @endcond / #endif / RUBY_ATOMIC_H */ Back

Edit File : atomic.h | Size : 27.62 KB

#ifndef RUBY_ATOMIC_H                                /*-*-C++-*-vi:se ft=cpp:*/
#define RUBY_ATOMIC_H
/**
 * @file
 * @author     Ruby developers <ruby-core@ruby-lang.org>
 * @copyright  This  file  is   a  part  of  the   programming  language  Ruby.
 *             Permission  is hereby  granted,  to  either redistribute  and/or
 *             modify this file, provided that  the conditions mentioned in the
 *             file COPYING are met.  Consult the file for details.
 * @warning    Symbols   prefixed  with   either  `RBIMPL`   or  `rbimpl`   are
 *             implementation details.   Don't take  them as canon.  They could
 *             rapidly appear then vanish.  The name (path) of this header file
 *             is also an  implementation detail.  Do not expect  it to persist
 *             at the place it is now.  Developers are free to move it anywhere
 *             anytime at will.
 * @note       To  ruby-core:  remember  that   this  header  can  be  possibly
 *             recursively included  from extension  libraries written  in C++.
 *             Do not  expect for  instance `__VA_ARGS__` is  always available.
 *             We assume C99  for ruby itself but we don't  assume languages of
 *             extension libraries.  They could be written in C++98.
 * @brief      Atomic operations
 *
 * Basically, if  we could assume  either C11 or  C++11, these macros  are just
 * redundant.  Sadly we cannot.  We have to do them ourselves.
 */

#include "ruby/internal/config.h"

#ifdef STDC_HEADERS
# include <stddef.h>            /* size_t */
#endif

#ifdef HAVE_SYS_TYPES_H
# include <sys/types.h>         /* ssize_t */
#endif

#if RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0)
# pragma intrinsic(_InterlockedOr)
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
# include <atomic.h>
#endif

#include "ruby/assert.h"
#include "ruby/backward/2/limits.h"
#include "ruby/internal/attr/artificial.h"
#include "ruby/internal/attr/noalias.h"
#include "ruby/internal/attr/nonnull.h"
#include "ruby/internal/compiler_since.h"
#include "ruby/internal/cast.h"
#include "ruby/internal/value.h"
#include "ruby/internal/static_assert.h"
#include "ruby/internal/stdbool.h"

/*
 * Asserts that  your environment supports  more than one atomic  types.  These
 * days systems tend to have such property  (C11 was a standard of decades ago,
 * right?) but we still support older ones.
 */
#if defined(__DOXYGEN__) || defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
# define RUBY_ATOMIC_GENERIC_MACRO 1
#endif

/**
 * Type  that  is eligible  for  atomic  operations.   Depending on  your  host
 * platform you might have  more than one such type, but we  choose one of them
 * anyways.
 */
#if defined(__DOXYGEN__)
using rb_atomic_t = std::atomic<unsigned>;
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
typedef unsigned int rb_atomic_t;
#elif defined(HAVE_GCC_SYNC_BUILTINS)
typedef unsigned int rb_atomic_t;
#elif defined(_WIN32)
# include <winsock2.h>       // to prevent macro redefinitions
# include <windows.h>        // for `LONG` and `Interlocked` functions
typedef LONG rb_atomic_t;
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
typedef unsigned int rb_atomic_t;
#else
# error No atomic operation found
#endif

/**
 * Atomically replaces the  value pointed by `var` with the  result of addition
 * of `val` to the old value of `var`.
 *
 * @param   var  A variable of ::rb_atomic_t.
 * @param   val  Value to add.
 * @return  What was stored in `var` before the addition.
 * @post    `var` holds `var + val`.
 */
#define RUBY_ATOMIC_FETCH_ADD(var, val) rbimpl_atomic_fetch_add(&(var), (val))

/**
 * Atomically  replaces  the  value  pointed   by  `var`  with  the  result  of
 * subtraction of `val` to the old value of `var`.
 *
 * @param   var  A variable of ::rb_atomic_t.
 * @param   val  Value to subtract.
 * @return  What was stored in `var` before the subtraction.
 * @post    `var` holds `var - val`.
 */
#define RUBY_ATOMIC_FETCH_SUB(var, val) rbimpl_atomic_fetch_sub(&(var), (val))

/**
 * Atomically  replaces  the  value  pointed   by  `var`  with  the  result  of
 * bitwise OR between `val` and the old value of `var`.
 *
 * @param   var   A variable of ::rb_atomic_t.
 * @param   val   Value to mix.
 * @return  void
 * @post    `var` holds `var | val`.
 * @note    For portability, this macro can return void.
 */
#define RUBY_ATOMIC_OR(var, val) rbimpl_atomic_or(&(var), (val))

/**
 * Atomically replaces the value pointed by  `var` with `val`.  This is just an
 * assignment, but you can additionally know the previous value.
 *
 * @param   var   A variable of ::rb_atomic_t.
 * @param   val   Value to set.
 * @return  What was stored in `var` before the assignment.
 * @post    `var` holds `val`.
 */
#define RUBY_ATOMIC_EXCHANGE(var, val) rbimpl_atomic_exchange(&(var), (val))

/**
 * Atomic compare-and-swap.   This stores  `val` to  `var` if  and only  if the
 * assignment changes  the value of `var`  from `oldval` to `newval`.   You can
 * detect whether the assignment happened or not using the return value.
 *
 * @param   var        A variable of ::rb_atomic_t.
 * @param   oldval     Expected value of `var` before the assignment.
 * @param   newval     What you want to store at `var`.
 * @retval  oldval     Successful assignment (`var` is now `newval`).
 * @retval  otherwise  Something else is at `var`; not updated.
 */
#define RUBY_ATOMIC_CAS(var, oldval, newval) \
    rbimpl_atomic_cas(&(var), (oldval), (newval))

/**
 * Atomic load. This loads `var` with an atomic intrinsic and returns
 * its value.
 *
 * @param var  A variable of ::rb_atomic_t
 * @return     What was stored in `var`j
 */
#define RUBY_ATOMIC_LOAD(var) rbimpl_atomic_load(&(var))

/**
 * Identical to #RUBY_ATOMIC_EXCHANGE, except for the return type.
 *
 * @param   var   A variable of ::rb_atomic_t.
 * @param   val   Value to set.
 * @return  void
 * @post    `var` holds `val`.
 */
#define RUBY_ATOMIC_SET(var, val) rbimpl_atomic_set(&(var), (val))

/**
 * Identical to #RUBY_ATOMIC_FETCH_ADD, except for the return type.
 *
 * @param   var  A variable of ::rb_atomic_t.
 * @param   val  Value to add.
 * @return  void
 * @post    `var` holds `var + val`.
 */
#define RUBY_ATOMIC_ADD(var, val) rbimpl_atomic_add(&(var), (val))

/**
 * Identical to #RUBY_ATOMIC_FETCH_SUB, except for the return type.
 *
 * @param   var  A variable of ::rb_atomic_t.
 * @param   val  Value to subtract.
 * @return  void
 * @post    `var` holds `var - val`.
 */
#define RUBY_ATOMIC_SUB(var, val) rbimpl_atomic_sub(&(var), (val))

/**
 * Atomically increments the value pointed by `var`.
 *
 * @param   var  A variable of ::rb_atomic_t.
 * @return  void
 * @post    `var` holds `var + 1`.
 */
#define RUBY_ATOMIC_INC(var) rbimpl_atomic_inc(&(var))

/**
 * Atomically decrements the value pointed by `var`.
 *
 * @param   var  A variable of ::rb_atomic_t.
 * @return  void
 * @post    `var` holds `var - 1`.
 */
#define RUBY_ATOMIC_DEC(var) rbimpl_atomic_dec(&(var))

/**
 * Identical to #RUBY_ATOMIC_INC,  except it expects its  argument is `size_t`.
 * There are cases where ::rb_atomic_t is  32bit while `size_t` is 64bit.  This
 * should be used for size related operations to support such platforms.
 *
 * @param   var  A variable of `size_t`.
 * @return  void
 * @post    `var` holds `var + 1`.
 */
#define RUBY_ATOMIC_SIZE_INC(var) rbimpl_atomic_size_inc(&(var))

/**
 * Identical to #RUBY_ATOMIC_DEC,  except it expects its  argument is `size_t`.
 * There are cases where ::rb_atomic_t is  32bit while `size_t` is 64bit.  This
 * should be used for size related operations to support such platforms.
 *
 * @param   var  A variable of `size_t`.
 * @return  void
 * @post    `var` holds `var - 1`.
 */
#define RUBY_ATOMIC_SIZE_DEC(var) rbimpl_atomic_size_dec(&(var))

/**
 * Identical  to #RUBY_ATOMIC_EXCHANGE,  except  it expects  its arguments  are
 * `size_t`.  There  are cases where  ::rb_atomic_t is 32bit while  `size_t` is
 * 64bit.  This  should be  used for  size related  operations to  support such
 * platforms.
 *
 * @param   var  A variable of `size_t`.
 * @param   val   Value to set.
 * @return  What was stored in `var` before the assignment.
 * @post    `var` holds `val`.
 */
#define RUBY_ATOMIC_SIZE_EXCHANGE(var, val) \
    rbimpl_atomic_size_exchange(&(var), (val))

/**
 * Identical to #RUBY_ATOMIC_CAS, except it expects its arguments are `size_t`.
 * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit.  This
 * should be used for size related operations to support such platforms.
 *
 * @param   var        A variable of `size_t`.
 * @param   oldval     Expected value of `var` before the assignment.
 * @param   newval     What you want to store at `var`.
 * @retval  oldval     Successful assignment (`var` is now `newval`).
 * @retval  otherwise  Something else is at `var`; not updated.
 */
#define RUBY_ATOMIC_SIZE_CAS(var, oldval, newval) \
    rbimpl_atomic_size_cas(&(var), (oldval), (newval))

/**
 * Identical to #RUBY_ATOMIC_ADD, except it expects its arguments are `size_t`.
 * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit.  This
 * should be used for size related operations to support such platforms.
 *
 * @param   var  A variable of `size_t`.
 * @param   val  Value to add.
 * @return  void
 * @post    `var` holds `var + val`.
 */
#define RUBY_ATOMIC_SIZE_ADD(var, val) rbimpl_atomic_size_add(&(var), (val))

/**
 * Identical to #RUBY_ATOMIC_SUB, except it expects its arguments are `size_t`.
 * There are cases where ::rb_atomic_t is 32bit while `size_t` is 64bit.  This
 * should be used for size related operations to support such platforms.
 *
 * @param   var  A variable of `size_t`.
 * @param   val  Value to subtract.
 * @return  void
 * @post    `var` holds `var - val`.
 */
#define RUBY_ATOMIC_SIZE_SUB(var, val) rbimpl_atomic_size_sub(&(var), (val))

/**
 * Identical  to #RUBY_ATOMIC_EXCHANGE,  except  it expects  its arguments  are
 * `void*`.   There are  cases where  ::rb_atomic_t is  32bit while  `void*` is
 * 64bit.  This should  be used for pointer related operations  to support such
 * platforms.
 *
 * @param   var  A variable of `void *`.
 * @param   val   Value to set.
 * @return  What was stored in `var` before the assignment.
 * @post    `var` holds `val`.
 *
 * @internal
 *
 * :FIXME: this `(void*)` cast is evil!  However `void*` is incompatible with
 * some pointers, most notably function pointers.
 */
#define RUBY_ATOMIC_PTR_EXCHANGE(var, val) \
    RBIMPL_CAST(rbimpl_atomic_ptr_exchange((void **)&(var), (void *)val))

/**
 * Identical to #RUBY_ATOMIC_LOAD, except it expects its arguments are `void*`.
 * There are cases where ::rb_atomic_t is 32bit while `void*` is 64bit.  This
 * should be used for size related operations to support such platforms.
 *
 * @param   var        A variable of `void*`
 * @return             The value of `var` (without tearing)
 */
#define RUBY_ATOMIC_PTR_LOAD(var) \
    RBIMPL_CAST(rbimpl_atomic_ptr_load((void **)&var))

/**
 * Identical to #RUBY_ATOMIC_CAS, except it expects its arguments are `void*`.
 * There are cases where ::rb_atomic_t is 32bit while `void*` is 64bit.  This
 * should be used for size related operations to support such platforms.
 *
 * @param   var        A variable of `void*`.
 * @param   oldval     Expected value of `var` before the assignment.
 * @param   newval     What you want to store at `var`.
 * @retval  oldval     Successful assignment (`var` is now `newval`).
 * @retval  otherwise  Something else is at `var`; not updated.
 */
#define RUBY_ATOMIC_PTR_CAS(var, oldval, newval) \
    RBIMPL_CAST(rbimpl_atomic_ptr_cas((void **)&(var), (void *)(oldval), (void *)(newval)))

/**
 * Identical  to #RUBY_ATOMIC_EXCHANGE,  except  it expects  its arguments  are
 * ::VALUE.   There are  cases where  ::rb_atomic_t is  32bit while  ::VALUE is
 * 64bit.  This should  be used for pointer related operations  to support such
 * platforms.
 *
 * @param   var  A variable of ::VALUE.
 * @param   val   Value to set.
 * @return  What was stored in `var` before the assignment.
 * @post    `var` holds `val`.
 */
#define RUBY_ATOMIC_VALUE_EXCHANGE(var, val) \
    rbimpl_atomic_value_exchange(&(var), (val))

/**
 * Identical to #RUBY_ATOMIC_CAS, except it  expects its arguments are ::VALUE.
 * There are cases  where ::rb_atomic_t is 32bit while ::VALUE  is 64bit.  This
 * should be used for size related operations to support such platforms.
 *
 * @param   var        A variable of `void*`.
 * @param   oldval     Expected value of `var` before the assignment.
 * @param   newval     What you want to store at `var`.
 * @retval  oldval     Successful assignment (`var` is now `newval`).
 * @retval  otherwise  Something else is at `var`; not updated.
 */
#define RUBY_ATOMIC_VALUE_CAS(var, oldval, newval) \
    rbimpl_atomic_value_cas(&(var), (oldval), (newval))

/** @cond INTERNAL_MACRO */
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline rb_atomic_t
rbimpl_atomic_fetch_add(volatile rb_atomic_t *ptr, rb_atomic_t val)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_fetch_add(ptr, val, __ATOMIC_SEQ_CST);

#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_fetch_and_add(ptr, val);

#elif defined(_WIN32)
    return InterlockedExchangeAdd(ptr, val);

#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    /*
     * `atomic_add_int_nv` takes its second argument as `int`!  Meanwhile our
     * `rb_atomic_t` is unsigned.  We cannot pass `val` as-is.  We have to
     * manually check integer overflow.
     */
    RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX);
    return atomic_add_int_nv(ptr, val) - val;

#else
# error Unsupported platform.
#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_add(volatile rb_atomic_t *ptr, rb_atomic_t val)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    /*
     * GCC on amd64 is smart enough to detect this `__atomic_add_fetch`'s
     * return value is not used, then compiles it into single `LOCK ADD`
     * instruction.
     */
    __atomic_add_fetch(ptr, val, __ATOMIC_SEQ_CST);

#elif defined(HAVE_GCC_SYNC_BUILTINS)
    __sync_add_and_fetch(ptr, val);

#elif defined(_WIN32)
    /*
     * `InterlockedExchangeAdd` is `LOCK XADD`.  It seems there also is
     * `_InterlockedAdd` intrinsic in ARM Windows but not for x86?  Sticking to
     * `InterlockedExchangeAdd` for better portability.
     */
    InterlockedExchangeAdd(ptr, val);

#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    /* Ditto for `atomic_add_int_nv`. */
    RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX);
    atomic_add_int(ptr, val);

#else
# error Unsupported platform.
#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_size_add(volatile size_t *ptr, size_t val)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_add_fetch(ptr, val, __ATOMIC_SEQ_CST);

#elif defined(HAVE_GCC_SYNC_BUILTINS)
    __sync_add_and_fetch(ptr, val);

#elif defined(_WIN64)
    /* Ditto for `InterlockeExchangedAdd`. */
    InterlockedExchangeAdd64(ptr, val);

#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    /* Ditto for `atomic_add_int_nv`. */
    RBIMPL_ASSERT_OR_ASSUME(val <= LONG_MAX);
    atomic_add_long(ptr, val);

#else
    RBIMPL_STATIC_ASSERT(size_of_rb_atomic_t, sizeof *ptr == sizeof(rb_atomic_t));

volatile rb_atomic_t *const tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr);
    rbimpl_atomic_add(tmp, val);

#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_inc(volatile rb_atomic_t *ptr)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
    rbimpl_atomic_add(ptr, 1);

#elif defined(_WIN32)
    InterlockedIncrement(ptr);

#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    atomic_inc_uint(ptr);

#else
    rbimpl_atomic_add(ptr, 1);

#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_size_inc(volatile size_t *ptr)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
    rbimpl_atomic_size_add(ptr, 1);

#elif defined(_WIN64)
    InterlockedIncrement64(ptr);

#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    atomic_inc_ulong(ptr);

#else
    RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof *ptr == sizeof(rb_atomic_t));

rbimpl_atomic_size_add(ptr, 1);

#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline rb_atomic_t
rbimpl_atomic_fetch_sub(volatile rb_atomic_t *ptr, rb_atomic_t val)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_fetch_sub(ptr, val, __ATOMIC_SEQ_CST);

#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_fetch_and_sub(ptr, val);

#elif defined(_WIN32)
    /* rb_atomic_t is signed here! Safe to do `-val`. */
    return InterlockedExchangeAdd(ptr, -val);

#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    /* Ditto for `rbimpl_atomic_fetch_add`. */
    const signed neg = -1;
    RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX);
    return atomic_add_int_nv(ptr, neg * val) + val;

#else
# error Unsupported platform.
#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_sub(volatile rb_atomic_t *ptr, rb_atomic_t val)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_sub_fetch(ptr, val, __ATOMIC_SEQ_CST);

#elif defined(HAVE_GCC_SYNC_BUILTINS)
    __sync_sub_and_fetch(ptr, val);

#elif defined(_WIN32)
    InterlockedExchangeAdd(ptr, -val);

#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    const signed neg = -1;
    RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX);
    atomic_add_int(ptr, neg * val);

#else
# error Unsupported platform.
#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_size_sub(volatile size_t *ptr, size_t val)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_sub_fetch(ptr, val, __ATOMIC_SEQ_CST);

#elif defined(HAVE_GCC_SYNC_BUILTINS)
    __sync_sub_and_fetch(ptr, val);

#elif defined(_WIN64)
    const ssize_t neg = -1;
    InterlockedExchangeAdd64(ptr, neg * val);

#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    const signed neg = -1;
    RBIMPL_ASSERT_OR_ASSUME(val <= LONG_MAX);
    atomic_add_long(ptr, neg * val);

#else
    RBIMPL_STATIC_ASSERT(size_of_rb_atomic_t, sizeof *ptr == sizeof(rb_atomic_t));

volatile rb_atomic_t *const tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr);
    rbimpl_atomic_sub(tmp, val);

#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_dec(volatile rb_atomic_t *ptr)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
    rbimpl_atomic_sub(ptr, 1);

#elif defined(_WIN32)
    InterlockedDecrement(ptr);

#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    atomic_dec_uint(ptr);

#else
    rbimpl_atomic_sub(ptr, 1);

#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_size_dec(volatile size_t *ptr)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
    rbimpl_atomic_size_sub(ptr, 1);

#elif defined(_WIN64)
    InterlockedDecrement64(ptr);

#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    atomic_dec_ulong(ptr);

#else
    RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof *ptr == sizeof(rb_atomic_t));

rbimpl_atomic_size_sub(ptr, 1);

#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_or(volatile rb_atomic_t *ptr, rb_atomic_t val)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_or_fetch(ptr, val, __ATOMIC_SEQ_CST);

#elif defined(HAVE_GCC_SYNC_BUILTINS)
    __sync_or_and_fetch(ptr, val);

#elif RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0)
    _InterlockedOr(ptr, val);

#elif defined(_WIN32) && defined(__GNUC__)
    /* This was for old MinGW.  Maybe not needed any longer? */
    __asm__(
        "lock\n\t"
        "orl\t%1, %0"
        : "=m"(ptr)
        : "Ir"(val));

#elif defined(_WIN32) && defined(_M_IX86)
    __asm mov eax, ptr;
    __asm mov ecx, val;
    __asm lock or [eax], ecx;

#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    atomic_or_uint(ptr, val);

#else
# error Unsupported platform.
#endif
}

/* Nobody uses this but for theoretical backwards compatibility... */
#if RBIMPL_COMPILER_BEFORE(MSVC, 13, 0, 0)
static inline rb_atomic_t
rb_w32_atomic_or(volatile rb_atomic_t *var, rb_atomic_t val)
{
    return rbimpl_atomic_or(var, val);
}
#endif

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline rb_atomic_t
rbimpl_atomic_exchange(volatile rb_atomic_t *ptr, rb_atomic_t val)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_exchange_n(ptr, val, __ATOMIC_SEQ_CST);

#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_lock_test_and_set(ptr, val);

#elif defined(_WIN32)
    return InterlockedExchange(ptr, val);

#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    return atomic_swap_uint(ptr, val);

#else
# error Unsupported platform.
#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline size_t
rbimpl_atomic_size_exchange(volatile size_t *ptr, size_t val)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_exchange_n(ptr, val, __ATOMIC_SEQ_CST);

#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_lock_test_and_set(ptr, val);

#elif defined(_WIN64)
    return InterlockedExchange64(ptr, val);

#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    return atomic_swap_ulong(ptr, val);

#else
    RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof *ptr == sizeof(rb_atomic_t));

volatile rb_atomic_t *const tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr);
    const rb_atomic_t ret = rbimpl_atomic_exchange(tmp, val);
    return RBIMPL_CAST((size_t)ret);

#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void *
rbimpl_atomic_ptr_exchange(void *volatile *ptr, const void *val)
{
#if 0

#elif defined(InterlockedExchangePointer)
    /* const_cast */
    PVOID *pptr = RBIMPL_CAST((PVOID *)ptr);
    PVOID pval = RBIMPL_CAST((PVOID)val);
    return InterlockedExchangePointer(pptr, pval);

#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    return atomic_swap_ptr(ptr, RBIMPL_CAST((void *)val));

#else
    RBIMPL_STATIC_ASSERT(sizeof_voidp, sizeof *ptr == sizeof(size_t));

const size_t sval = RBIMPL_CAST((size_t)val);
    volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr);
    const size_t sret = rbimpl_atomic_size_exchange(sptr, sval);
    return RBIMPL_CAST((void *)sret);

#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline VALUE
rbimpl_atomic_value_exchange(volatile VALUE *ptr, VALUE val)
{
    RBIMPL_STATIC_ASSERT(sizeof_value, sizeof *ptr == sizeof(size_t));

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline rb_atomic_t
rbimpl_atomic_load(volatile rb_atomic_t *ptr)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_load_n(ptr, __ATOMIC_SEQ_CST);
#else
    return rbimpl_atomic_fetch_add(ptr, 0);
#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_set(volatile rb_atomic_t *ptr, rb_atomic_t val)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_store_n(ptr, val, __ATOMIC_SEQ_CST);

#else
    /* Maybe std::atomic<rb_atomic_t>::store can be faster? */
    rbimpl_atomic_exchange(ptr, val);

#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline rb_atomic_t
rbimpl_atomic_cas(volatile rb_atomic_t *ptr, rb_atomic_t oldval, rb_atomic_t newval)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_compare_exchange_n(
        ptr, &oldval, newval, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
    return oldval;

#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_val_compare_and_swap(ptr, oldval, newval);

#elif RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0)
    return InterlockedCompareExchange(ptr, newval, oldval);

#elif defined(_WIN32)
    PVOID *pptr = RBIMPL_CAST((PVOID *)ptr);
    PVOID pold = RBIMPL_CAST((PVOID)oldval);
    PVOID pnew = RBIMPL_CAST((PVOID)newval);
    PVOID pret = InterlockedCompareExchange(pptr, pnew, pold);
    return RBIMPL_CAST((rb_atomic_t)pret);

#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    return atomic_cas_uint(ptr, oldval, newval);

#else
# error Unsupported platform.
#endif
}

/* Nobody uses this but for theoretical backwards compatibility... */
#if RBIMPL_COMPILER_BEFORE(MSVC, 13, 0, 0)
static inline rb_atomic_t
rb_w32_atomic_cas(volatile rb_atomic_t *var, rb_atomic_t oldval, rb_atomic_t newval)
{
    return rbimpl_atomic_cas(var, oldval, newval);
}
#endif

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline size_t
rbimpl_atomic_size_cas(volatile size_t *ptr, size_t oldval, size_t newval)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_compare_exchange_n(
        ptr, &oldval, newval, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
    return oldval;

#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_val_compare_and_swap(ptr, oldval, newval);

#elif defined(_WIN64)
    return InterlockedCompareExchange64(ptr, newval, oldval);

#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    return atomic_cas_ulong(ptr, oldval, newval);

#else
    RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof *ptr == sizeof(rb_atomic_t));

volatile rb_atomic_t *tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr);
    return rbimpl_atomic_cas(tmp, oldval, newval);

#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void *
rbimpl_atomic_ptr_cas(void **ptr, const void *oldval, const void *newval)
{
#if 0

#elif defined(InterlockedExchangePointer)
    /* ... Can we say that InterlockedCompareExchangePtr surly exists when
     * InterlockedExchangePointer is defined?  Seems so but...?*/
    PVOID *pptr = RBIMPL_CAST((PVOID *)ptr);
    PVOID pold = RBIMPL_CAST((PVOID)oldval);
    PVOID pnew = RBIMPL_CAST((PVOID)newval);
    return InterlockedCompareExchangePointer(pptr, pnew, pold);

#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    void *pold = RBIMPL_CAST((void *)oldval);
    void *pnew = RBIMPL_CAST((void *)newval);
    return atomic_cas_ptr(ptr, pold, pnew);

#else
    RBIMPL_STATIC_ASSERT(sizeof_voidp, sizeof *ptr == sizeof(size_t));

const size_t snew = RBIMPL_CAST((size_t)newval);
    const size_t sold = RBIMPL_CAST((size_t)oldval);
    volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr);
    const size_t sret = rbimpl_atomic_size_cas(sptr, sold, snew);
    return RBIMPL_CAST((void *)sret);

#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void *
rbimpl_atomic_ptr_load(void **ptr)
{
#if 0

#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_load_n(ptr, __ATOMIC_SEQ_CST);
#else
    void *val = *ptr;
    return rbimpl_atomic_ptr_cas(ptr, val, val);
#endif
}

RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline VALUE
rbimpl_atomic_value_cas(volatile VALUE *ptr, VALUE oldval, VALUE newval)
{
    RBIMPL_STATIC_ASSERT(sizeof_value, sizeof *ptr == sizeof(size_t));

Back