#$@%#$%!

Ngiler SH3LL 360
: / usr / include / python3.8 / internal /

Information Server
MySQL : OFF Perl : OFF CURL : ON WGET : OFF PKEXEC : OFF

Directive	Local Value
IP Address	162.0.209.101
System	Linux premium169.web-hosting.com 4.18.0-553.50.1.lve.el8.x86_64 #1 SMP Thu Apr 17 19:10:24 UTC 2025 x86_64
User	luxeelil
PHP Version	8.0.30
Software	LiteSpeed
Doc root	Writable

Edit File : pycore_atomic.h \| Size : 16.55 KB
#ifndef Py_ATOMIC_H #define Py_ATOMIC_H #ifdef __cplusplus extern "C" { #endif #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif #include "dynamic_annotations.h" #include "pyconfig.h" #if defined(HAVE_STD_ATOMIC) #include <stdatomic.h> #endif #if defined(_MSC_VER) #include <intrin.h> #if defined(_M_IX86) \|\| defined(_M_X64) # include <immintrin.h> #endif #endif /* This is modeled after the atomics interface from C1x, according to * the draft at * http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1425.pdf. * Operations and types are named the same except with a _Py_ prefix * and have the same semantics. * * Beware, the implementations here are deep magic. / #if defined(HAVE_STD_ATOMIC) typedef enum _Py_memory_order { _Py_memory_order_relaxed = memory_order_relaxed, _Py_memory_order_acquire = memory_order_acquire, _Py_memory_order_release = memory_order_release, _Py_memory_order_acq_rel = memory_order_acq_rel, _Py_memory_order_seq_cst = memory_order_seq_cst } _Py_memory_order; typedef struct _Py_atomic_address { atomic_uintptr_t _value; } _Py_atomic_address; typedef struct _Py_atomic_int { atomic_int _value; } _Py_atomic_int; #define _Py_atomic_signal_fence(/memory_order/ ORDER) \ atomic_signal_fence(ORDER) #define _Py_atomic_thread_fence(/memory_order/ ORDER) \ atomic_thread_fence(ORDER) #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ atomic_store_explicit(&((ATOMIC_VAL)->_value), NEW_VAL, ORDER) #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ atomic_load_explicit(&((ATOMIC_VAL)->_value), ORDER) / Use builtin atomic operations in GCC >= 4.7 / #elif defined(HAVE_BUILTIN_ATOMIC) typedef enum _Py_memory_order { _Py_memory_order_relaxed = __ATOMIC_RELAXED, _Py_memory_order_acquire = __ATOMIC_ACQUIRE, _Py_memory_order_release = __ATOMIC_RELEASE, _Py_memory_order_acq_rel = __ATOMIC_ACQ_REL, _Py_memory_order_seq_cst = __ATOMIC_SEQ_CST } _Py_memory_order; typedef struct _Py_atomic_address { uintptr_t _value; } _Py_atomic_address; typedef struct _Py_atomic_int { int _value; } _Py_atomic_int; #define _Py_atomic_signal_fence(/memory_order/ ORDER) \ __atomic_signal_fence(ORDER) #define _Py_atomic_thread_fence(/memory_order/ ORDER) \ __atomic_thread_fence(ORDER) #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ (assert((ORDER) == __ATOMIC_RELAXED \ \|\| (ORDER) == __ATOMIC_SEQ_CST \ \|\| (ORDER) == __ATOMIC_RELEASE), \ __atomic_store_n(&((ATOMIC_VAL)->_value), NEW_VAL, ORDER)) #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ (assert((ORDER) == __ATOMIC_RELAXED \ \|\| (ORDER) == __ATOMIC_SEQ_CST \ \|\| (ORDER) == __ATOMIC_ACQUIRE \ \|\| (ORDER) == __ATOMIC_CONSUME), \ __atomic_load_n(&((ATOMIC_VAL)->_value), ORDER)) / Only support GCC (for expression statements) and x86 (for simple * atomic semantics) and MSVC x86/x64/ARM / #elif defined(__GNUC__) && (defined(__i386__) \|\| defined(__amd64)) typedef enum _Py_memory_order { _Py_memory_order_relaxed, _Py_memory_order_acquire, _Py_memory_order_release, _Py_memory_order_acq_rel, _Py_memory_order_seq_cst } _Py_memory_order; typedef struct _Py_atomic_address { uintptr_t _value; } _Py_atomic_address; typedef struct _Py_atomic_int { int _value; } _Py_atomic_int; static __inline__ void _Py_atomic_signal_fence(_Py_memory_order order) { if (order != _Py_memory_order_relaxed) __asm__ volatile("":::"memory"); } static __inline__ void _Py_atomic_thread_fence(_Py_memory_order order) { if (order != _Py_memory_order_relaxed) __asm__ volatile("mfence":::"memory"); } / Tell the race checker about this operation's effects. / static __inline__ void _Py_ANNOTATE_MEMORY_ORDER(const volatile void address, _Py_memory_order order) { (void)address; /* shut up -Wunused-parameter / switch(order) { case _Py_memory_order_release: case _Py_memory_order_acq_rel: case _Py_memory_order_seq_cst: _Py_ANNOTATE_HAPPENS_BEFORE(address); break; case _Py_memory_order_relaxed: case _Py_memory_order_acquire: break; } switch(order) { case _Py_memory_order_acquire: case _Py_memory_order_acq_rel: case _Py_memory_order_seq_cst: _Py_ANNOTATE_HAPPENS_AFTER(address); break; case _Py_memory_order_relaxed: case _Py_memory_order_release: break; } } #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ __extension__ ({ \ __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \ __typeof__(atomic_val->_value) new_val = NEW_VAL;\ volatile __typeof__(new_val) volatile_data = &atomic_val->_value; \ _Py_memory_order order = ORDER; \ _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \ \ /* Perform the operation. / \ _Py_ANNOTATE_IGNORE_WRITES_BEGIN(); \ switch(order) { \ case _Py_memory_order_release: \ _Py_atomic_signal_fence(_Py_memory_order_release); \ / fallthrough / \ case _Py_memory_order_relaxed: \ volatile_data = new_val; \ break; \ \ case _Py_memory_order_acquire: \ case _Py_memory_order_acq_rel: \ case _Py_memory_order_seq_cst: \ __asm__ volatile("xchg %0, %1" \ : "+r"(new_val) \ : "m"(atomic_val->_value) \ : "memory"); \ break; \ } \ _Py_ANNOTATE_IGNORE_WRITES_END(); \ }) #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ __extension__ ({ \ __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \ __typeof__(atomic_val->_value) result; \ volatile __typeof__(result) volatile_data = &atomic_val->_value; \ _Py_memory_order order = ORDER; \ _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \ \ / Perform the operation. / \ _Py_ANNOTATE_IGNORE_READS_BEGIN(); \ switch(order) { \ case _Py_memory_order_release: \ case _Py_memory_order_acq_rel: \ case _Py_memory_order_seq_cst: \ / Loads on x86 are not releases by default, so need a / \ / thread fence. / \ _Py_atomic_thread_fence(_Py_memory_order_release); \ break; \ default: \ / No fence / \ break; \ } \ result = volatile_data; \ switch(order) { \ case _Py_memory_order_acquire: \ case _Py_memory_order_acq_rel: \ case _Py_memory_order_seq_cst: \ /* Loads on x86 are automatically acquire operations so / \ / can get by with just a compiler fence. / \ _Py_atomic_signal_fence(_Py_memory_order_acquire); \ break; \ default: \ / No fence / \ break; \ } \ _Py_ANNOTATE_IGNORE_READS_END(); \ result; \ }) #elif defined(_MSC_VER) / _Interlocked* functions provide a full memory barrier and are therefore enough for acq_rel and seq_cst. If the HLE variants aren't available in hardware they will fall back to a full memory barrier as well. This might affect performance but likely only in some very specific and hard to meassure scenario. / #if defined(_M_IX86) \|\| defined(_M_X64) typedef enum _Py_memory_order { _Py_memory_order_relaxed, _Py_memory_order_acquire, _Py_memory_order_release, _Py_memory_order_acq_rel, _Py_memory_order_seq_cst } _Py_memory_order; typedef struct _Py_atomic_address { volatile uintptr_t _value; } _Py_atomic_address; typedef struct _Py_atomic_int { volatile int _value; } _Py_atomic_int; #if defined(_M_X64) #define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) \ switch (ORDER) { \ case _Py_memory_order_acquire: \ _InterlockedExchange64_HLEAcquire((__int64 volatile)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \ break; \ case _Py_memory_order_release: \ _InterlockedExchange64_HLERelease((__int64 volatile)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \ break; \ default: \ _InterlockedExchange64((__int64 volatile)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \ break; \ } #else #define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) ((void)0); #endif #define _Py_atomic_store_32bit(ATOMIC_VAL, NEW_VAL, ORDER) \ switch (ORDER) { \ case _Py_memory_order_acquire: \ _InterlockedExchange_HLEAcquire((volatile long)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \ break; \ case _Py_memory_order_release: \ _InterlockedExchange_HLERelease((volatile long)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \ break; \ default: \ _InterlockedExchange((volatile long)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \ break; \ } #if defined(_M_X64) / This has to be an intptr_t for now. gil_created() uses -1 as a sentinel value, if this returns a uintptr_t it will do an unsigned compare and crash / inline intptr_t _Py_atomic_load_64bit_impl(volatile uintptr_t value, int order) { __int64 old; switch (order) { case _Py_memory_order_acquire: { do { old = value; } while(_InterlockedCompareExchange64_HLEAcquire((volatile __int64)value, old, old) != old); break; } case _Py_memory_order_release: { do { old = value; } while(_InterlockedCompareExchange64_HLERelease((volatile __int64)value, old, old) != old); break; } case _Py_memory_order_relaxed: old = value; break; default: { do { old = value; } while(_InterlockedCompareExchange64((volatile __int64)value, old, old) != old); break; } } return old; } #define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) \ _Py_atomic_load_64bit_impl((volatile uintptr_t)&((ATOMIC_VAL)->_value), (ORDER)) #else #define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) ((ATOMIC_VAL)->_value) #endif inline int _Py_atomic_load_32bit_impl(volatile int* value, int order) { long old; switch (order) { case _Py_memory_order_acquire: { do { old = value; } while(_InterlockedCompareExchange_HLEAcquire((volatile long)value, old, old) != old); break; } case _Py_memory_order_release: { do { old = value; } while(_InterlockedCompareExchange_HLERelease((volatile long)value, old, old) != old); break; } case _Py_memory_order_relaxed: old = value; break; default: { do { old = value; } while(_InterlockedCompareExchange((volatile long)value, old, old) != old); break; } } return old; } #define _Py_atomic_load_32bit(ATOMIC_VAL, ORDER) \ _Py_atomic_load_32bit_impl((volatile int)&((ATOMIC_VAL)->_value), (ORDER)) #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ if (sizeof((ATOMIC_VAL)->_value) == 8) { \ _Py_atomic_store_64bit((ATOMIC_VAL), NEW_VAL, ORDER) } else { \ _Py_atomic_store_32bit((ATOMIC_VAL), NEW_VAL, ORDER) } #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ ( \ sizeof((ATOMIC_VAL)->_value) == 8 ? \ _Py_atomic_load_64bit((ATOMIC_VAL), ORDER) : \ _Py_atomic_load_32bit((ATOMIC_VAL), ORDER) \ ) #elif defined(_M_ARM) \|\| defined(_M_ARM64) typedef enum _Py_memory_order { _Py_memory_order_relaxed, _Py_memory_order_acquire, _Py_memory_order_release, _Py_memory_order_acq_rel, _Py_memory_order_seq_cst } _Py_memory_order; typedef struct _Py_atomic_address { volatile uintptr_t _value; } _Py_atomic_address; typedef struct _Py_atomic_int { volatile int _value; } _Py_atomic_int; #if defined(_M_ARM64) #define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) \ switch (ORDER) { \ case _Py_memory_order_acquire: \ _InterlockedExchange64_acq((__int64 volatile)&((ATOMIC_VAL)->_value), (__int64)NEW_VAL); \ break; \ case _Py_memory_order_release: \ _InterlockedExchange64_rel((__int64 volatile)&((ATOMIC_VAL)->_value), (__int64)NEW_VAL); \ break; \ default: \ _InterlockedExchange64((__int64 volatile)&((ATOMIC_VAL)->_value), (__int64)NEW_VAL); \ break; \ } #else #define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) ((void)0); #endif #define _Py_atomic_store_32bit(ATOMIC_VAL, NEW_VAL, ORDER) \ switch (ORDER) { \ case _Py_memory_order_acquire: \ _InterlockedExchange_acq((volatile long)&((ATOMIC_VAL)->_value), (int)NEW_VAL); \ break; \ case _Py_memory_order_release: \ _InterlockedExchange_rel((volatile long)&((ATOMIC_VAL)->_value), (int)NEW_VAL); \ break; \ default: \ _InterlockedExchange((volatile long)&((ATOMIC_VAL)->_value), (int)NEW_VAL); \ break; \ } #if defined(_M_ARM64) /* This has to be an intptr_t for now. gil_created() uses -1 as a sentinel value, if this returns a uintptr_t it will do an unsigned compare and crash / inline intptr_t _Py_atomic_load_64bit_impl(volatile uintptr_t value, int order) { uintptr_t old; switch (order) { case _Py_memory_order_acquire: { do { old = value; } while(_InterlockedCompareExchange64_acq(value, old, old) != old); break; } case _Py_memory_order_release: { do { old = value; } while(_InterlockedCompareExchange64_rel(value, old, old) != old); break; } case _Py_memory_order_relaxed: old = value; break; default: { do { old = value; } while(_InterlockedCompareExchange64(value, old, old) != old); break; } } return old; } #define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) \ _Py_atomic_load_64bit_impl((volatile uintptr_t)&((ATOMIC_VAL)->_value), (ORDER)) #else #define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) ((ATOMIC_VAL)->_value) #endif inline int _Py_atomic_load_32bit_impl(volatile int value, int order) { int old; switch (order) { case _Py_memory_order_acquire: { do { old = value; } while(_InterlockedCompareExchange_acq(value, old, old) != old); break; } case _Py_memory_order_release: { do { old = value; } while(_InterlockedCompareExchange_rel(value, old, old) != old); break; } case _Py_memory_order_relaxed: old = value; break; default: { do { old = value; } while(_InterlockedCompareExchange(value, old, old) != old); break; } } return old; } #define _Py_atomic_load_32bit(ATOMIC_VAL, ORDER) \ _Py_atomic_load_32bit_impl((volatile int)&((ATOMIC_VAL)->_value), (ORDER)) #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ if (sizeof((ATOMIC_VAL)->_value) == 8) { \ _Py_atomic_store_64bit((ATOMIC_VAL), (NEW_VAL), (ORDER)) } else { \ _Py_atomic_store_32bit((ATOMIC_VAL), (NEW_VAL), (ORDER)) } #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ ( \ sizeof((ATOMIC_VAL)->_value) == 8 ? \ _Py_atomic_load_64bit((ATOMIC_VAL), (ORDER)) : \ _Py_atomic_load_32bit((ATOMIC_VAL), (ORDER)) \ ) #endif #else / !gcc x86 !_msc_ver / typedef enum _Py_memory_order { _Py_memory_order_relaxed, _Py_memory_order_acquire, _Py_memory_order_release, _Py_memory_order_acq_rel, _Py_memory_order_seq_cst } _Py_memory_order; typedef struct _Py_atomic_address { uintptr_t _value; } _Py_atomic_address; typedef struct _Py_atomic_int { int _value; } _Py_atomic_int; / Fall back to other compilers and processors by assuming that simple volatile accesses are atomic. This is false, so people should port this. / #define _Py_atomic_signal_fence(/memory_order/ ORDER) ((void)0) #define _Py_atomic_thread_fence(/memory_order/ ORDER) ((void)0) #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ ((ATOMIC_VAL)->_value = NEW_VAL) #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ ((ATOMIC_VAL)->_value) #endif / Standardized shortcuts. / #define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \ _Py_atomic_store_explicit((ATOMIC_VAL), (NEW_VAL), _Py_memory_order_seq_cst) #define _Py_atomic_load(ATOMIC_VAL) \ _Py_atomic_load_explicit((ATOMIC_VAL), _Py_memory_order_seq_cst) / Python-local extensions / #define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \ _Py_atomic_store_explicit((ATOMIC_VAL), (NEW_VAL), _Py_memory_order_relaxed) #define _Py_atomic_load_relaxed(ATOMIC_VAL) \ _Py_atomic_load_explicit((ATOMIC_VAL), _Py_memory_order_relaxed) #ifdef __cplusplus } #endif #endif / Py_ATOMIC_H */ Back

Edit File : pycore_atomic.h | Size : 16.55 KB

#ifndef Py_ATOMIC_H
#define Py_ATOMIC_H
#ifdef __cplusplus
extern "C" {
#endif

#ifndef Py_BUILD_CORE
#  error "this header requires Py_BUILD_CORE define"
#endif

#include "dynamic_annotations.h"

#include "pyconfig.h"

#if defined(HAVE_STD_ATOMIC)
#include <stdatomic.h>
#endif

#if defined(_MSC_VER)
#include <intrin.h>
#if defined(_M_IX86) || defined(_M_X64)
#  include <immintrin.h>
#endif
#endif

/* This is modeled after the atomics interface from C1x, according to
 * the draft at
 * http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1425.pdf.
 * Operations and types are named the same except with a _Py_ prefix
 * and have the same semantics.
 *
 * Beware, the implementations here are deep magic.
 */

#if defined(HAVE_STD_ATOMIC)

typedef enum _Py_memory_order {
    _Py_memory_order_relaxed = memory_order_relaxed,
    _Py_memory_order_acquire = memory_order_acquire,
    _Py_memory_order_release = memory_order_release,
    _Py_memory_order_acq_rel = memory_order_acq_rel,
    _Py_memory_order_seq_cst = memory_order_seq_cst
} _Py_memory_order;

typedef struct _Py_atomic_address {
    atomic_uintptr_t _value;
} _Py_atomic_address;

typedef struct _Py_atomic_int {
    atomic_int _value;
} _Py_atomic_int;

#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \
    atomic_signal_fence(ORDER)

#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \
    atomic_thread_fence(ORDER)

#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
    atomic_store_explicit(&((ATOMIC_VAL)->_value), NEW_VAL, ORDER)

#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
    atomic_load_explicit(&((ATOMIC_VAL)->_value), ORDER)

/* Use builtin atomic operations in GCC >= 4.7 */
#elif defined(HAVE_BUILTIN_ATOMIC)

typedef enum _Py_memory_order {
    _Py_memory_order_relaxed = __ATOMIC_RELAXED,
    _Py_memory_order_acquire = __ATOMIC_ACQUIRE,
    _Py_memory_order_release = __ATOMIC_RELEASE,
    _Py_memory_order_acq_rel = __ATOMIC_ACQ_REL,
    _Py_memory_order_seq_cst = __ATOMIC_SEQ_CST
} _Py_memory_order;

typedef struct _Py_atomic_address {
    uintptr_t _value;
} _Py_atomic_address;

typedef struct _Py_atomic_int {
    int _value;
} _Py_atomic_int;

#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \
    __atomic_signal_fence(ORDER)

#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \
    __atomic_thread_fence(ORDER)

#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
    (assert((ORDER) == __ATOMIC_RELAXED                       \
            || (ORDER) == __ATOMIC_SEQ_CST                    \
            || (ORDER) == __ATOMIC_RELEASE),                  \
     __atomic_store_n(&((ATOMIC_VAL)->_value), NEW_VAL, ORDER))

#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER)           \
    (assert((ORDER) == __ATOMIC_RELAXED                       \
            || (ORDER) == __ATOMIC_SEQ_CST                    \
            || (ORDER) == __ATOMIC_ACQUIRE                    \
            || (ORDER) == __ATOMIC_CONSUME),                  \
     __atomic_load_n(&((ATOMIC_VAL)->_value), ORDER))

/* Only support GCC (for expression statements) and x86 (for simple
 * atomic semantics) and MSVC x86/x64/ARM */
#elif defined(__GNUC__) && (defined(__i386__) || defined(__amd64))
typedef enum _Py_memory_order {
    _Py_memory_order_relaxed,
    _Py_memory_order_acquire,
    _Py_memory_order_release,
    _Py_memory_order_acq_rel,
    _Py_memory_order_seq_cst
} _Py_memory_order;

typedef struct _Py_atomic_address {
    uintptr_t _value;
} _Py_atomic_address;

typedef struct _Py_atomic_int {
    int _value;
} _Py_atomic_int;

static __inline__ void
_Py_atomic_signal_fence(_Py_memory_order order)
{
    if (order != _Py_memory_order_relaxed)
        __asm__ volatile("":::"memory");
}

static __inline__ void
_Py_atomic_thread_fence(_Py_memory_order order)
{
    if (order != _Py_memory_order_relaxed)
        __asm__ volatile("mfence":::"memory");
}

/* Tell the race checker about this operation's effects. */
static __inline__ void
_Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order)
{
    (void)address;              /* shut up -Wunused-parameter */
    switch(order) {
    case _Py_memory_order_release:
    case _Py_memory_order_acq_rel:
    case _Py_memory_order_seq_cst:
        _Py_ANNOTATE_HAPPENS_BEFORE(address);
        break;
    case _Py_memory_order_relaxed:
    case _Py_memory_order_acquire:
        break;
    }
    switch(order) {
    case _Py_memory_order_acquire:
    case _Py_memory_order_acq_rel:
    case _Py_memory_order_seq_cst:
        _Py_ANNOTATE_HAPPENS_AFTER(address);
        break;
    case _Py_memory_order_relaxed:
    case _Py_memory_order_release:
        break;
    }
}

#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
    __extension__ ({ \
        __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \
        __typeof__(atomic_val->_value) new_val = NEW_VAL;\
        volatile __typeof__(new_val) *volatile_data = &atomic_val->_value; \
        _Py_memory_order order = ORDER; \
        _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \
        \
        /* Perform the operation. */ \
        _Py_ANNOTATE_IGNORE_WRITES_BEGIN(); \
        switch(order) { \
        case _Py_memory_order_release: \
            _Py_atomic_signal_fence(_Py_memory_order_release); \
            /* fallthrough */ \
        case _Py_memory_order_relaxed: \
            *volatile_data = new_val; \
            break; \
        \
        case _Py_memory_order_acquire: \
        case _Py_memory_order_acq_rel: \
        case _Py_memory_order_seq_cst: \
            __asm__ volatile("xchg %0, %1" \
                         : "+r"(new_val) \
                         : "m"(atomic_val->_value) \
                         : "memory"); \
            break; \
        } \
        _Py_ANNOTATE_IGNORE_WRITES_END(); \
    })

#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
    __extension__ ({  \
        __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \
        __typeof__(atomic_val->_value) result; \
        volatile __typeof__(result) *volatile_data = &atomic_val->_value; \
        _Py_memory_order order = ORDER; \
        _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \
        \
        /* Perform the operation. */ \
        _Py_ANNOTATE_IGNORE_READS_BEGIN(); \
        switch(order) { \
        case _Py_memory_order_release: \
        case _Py_memory_order_acq_rel: \
        case _Py_memory_order_seq_cst: \
            /* Loads on x86 are not releases by default, so need a */ \
            /* thread fence. */ \
            _Py_atomic_thread_fence(_Py_memory_order_release); \
            break; \
        default: \
            /* No fence */ \
            break; \
        } \
        result = *volatile_data; \
        switch(order) { \
        case _Py_memory_order_acquire: \
        case _Py_memory_order_acq_rel: \
        case _Py_memory_order_seq_cst: \
            /* Loads on x86 are automatically acquire operations so */ \
            /* can get by with just a compiler fence. */ \
            _Py_atomic_signal_fence(_Py_memory_order_acquire); \
            break; \
        default: \
            /* No fence */ \
            break; \
        } \
        _Py_ANNOTATE_IGNORE_READS_END(); \
        result; \
    })

#elif defined(_MSC_VER)
/*  _Interlocked* functions provide a full memory barrier and are therefore
    enough for acq_rel and seq_cst. If the HLE variants aren't available
    in hardware they will fall back to a full memory barrier as well.

This might affect performance but likely only in some very specific and
    hard to meassure scenario.
*/
#if defined(_M_IX86) || defined(_M_X64)
typedef enum _Py_memory_order {
    _Py_memory_order_relaxed,
    _Py_memory_order_acquire,
    _Py_memory_order_release,
    _Py_memory_order_acq_rel,
    _Py_memory_order_seq_cst
} _Py_memory_order;

typedef struct _Py_atomic_address {
    volatile uintptr_t _value;
} _Py_atomic_address;

typedef struct _Py_atomic_int {
    volatile int _value;
} _Py_atomic_int;

#if defined(_M_X64)
#define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) \
    switch (ORDER) { \
    case _Py_memory_order_acquire: \
      _InterlockedExchange64_HLEAcquire((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \
      break; \
    case _Py_memory_order_release: \
      _InterlockedExchange64_HLERelease((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \
      break; \
    default: \
      _InterlockedExchange64((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)(NEW_VAL)); \
      break; \
  }
#else
#define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) ((void)0);
#endif

#define _Py_atomic_store_32bit(ATOMIC_VAL, NEW_VAL, ORDER) \
  switch (ORDER) { \
  case _Py_memory_order_acquire: \
    _InterlockedExchange_HLEAcquire((volatile long*)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \
    break; \
  case _Py_memory_order_release: \
    _InterlockedExchange_HLERelease((volatile long*)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \
    break; \
  default: \
    _InterlockedExchange((volatile long*)&((ATOMIC_VAL)->_value), (int)(NEW_VAL)); \
    break; \
  }

#if defined(_M_X64)
/*  This has to be an intptr_t for now.
    gil_created() uses -1 as a sentinel value, if this returns
    a uintptr_t it will do an unsigned compare and crash
*/
inline intptr_t _Py_atomic_load_64bit_impl(volatile uintptr_t* value, int order) {
    __int64 old;
    switch (order) {
    case _Py_memory_order_acquire:
    {
      do {
        old = *value;
      } while(_InterlockedCompareExchange64_HLEAcquire((volatile __int64*)value, old, old) != old);
      break;
    }
    case _Py_memory_order_release:
    {
      do {
        old = *value;
      } while(_InterlockedCompareExchange64_HLERelease((volatile __int64*)value, old, old) != old);
      break;
    }
    case _Py_memory_order_relaxed:
      old = *value;
      break;
    default:
    {
      do {
        old = *value;
      } while(_InterlockedCompareExchange64((volatile __int64*)value, old, old) != old);
      break;
    }
    }
    return old;
}

#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) \
    _Py_atomic_load_64bit_impl((volatile uintptr_t*)&((ATOMIC_VAL)->_value), (ORDER))

#else
#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) ((ATOMIC_VAL)->_value)
#endif

inline int _Py_atomic_load_32bit_impl(volatile int* value, int order) {
    long old;
    switch (order) {
    case _Py_memory_order_acquire:
    {
      do {
        old = *value;
      } while(_InterlockedCompareExchange_HLEAcquire((volatile long*)value, old, old) != old);
      break;
    }
    case _Py_memory_order_release:
    {
      do {
        old = *value;
      } while(_InterlockedCompareExchange_HLERelease((volatile long*)value, old, old) != old);
      break;
    }
    case _Py_memory_order_relaxed:
      old = *value;
      break;
    default:
    {
      do {
        old = *value;
      } while(_InterlockedCompareExchange((volatile long*)value, old, old) != old);
      break;
    }
    }
    return old;
}

#define _Py_atomic_load_32bit(ATOMIC_VAL, ORDER) \
    _Py_atomic_load_32bit_impl((volatile int*)&((ATOMIC_VAL)->_value), (ORDER))

#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
  if (sizeof((ATOMIC_VAL)->_value) == 8) { \
    _Py_atomic_store_64bit((ATOMIC_VAL), NEW_VAL, ORDER) } else { \
    _Py_atomic_store_32bit((ATOMIC_VAL), NEW_VAL, ORDER) }

#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
  ( \
    sizeof((ATOMIC_VAL)->_value) == 8 ? \
    _Py_atomic_load_64bit((ATOMIC_VAL), ORDER) : \
    _Py_atomic_load_32bit((ATOMIC_VAL), ORDER) \
  )
#elif defined(_M_ARM) || defined(_M_ARM64)
typedef enum _Py_memory_order {
    _Py_memory_order_relaxed,
    _Py_memory_order_acquire,
    _Py_memory_order_release,
    _Py_memory_order_acq_rel,
    _Py_memory_order_seq_cst
} _Py_memory_order;

typedef struct _Py_atomic_address {
    volatile uintptr_t _value;
} _Py_atomic_address;

typedef struct _Py_atomic_int {
    volatile int _value;
} _Py_atomic_int;

#if defined(_M_ARM64)
#define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) \
    switch (ORDER) { \
    case _Py_memory_order_acquire: \
      _InterlockedExchange64_acq((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)NEW_VAL); \
      break; \
    case _Py_memory_order_release: \
      _InterlockedExchange64_rel((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)NEW_VAL); \
      break; \
    default: \
      _InterlockedExchange64((__int64 volatile*)&((ATOMIC_VAL)->_value), (__int64)NEW_VAL); \
      break; \
  }
#else
#define _Py_atomic_store_64bit(ATOMIC_VAL, NEW_VAL, ORDER) ((void)0);
#endif

#define _Py_atomic_store_32bit(ATOMIC_VAL, NEW_VAL, ORDER) \
  switch (ORDER) { \
  case _Py_memory_order_acquire: \
    _InterlockedExchange_acq((volatile long*)&((ATOMIC_VAL)->_value), (int)NEW_VAL); \
    break; \
  case _Py_memory_order_release: \
    _InterlockedExchange_rel((volatile long*)&((ATOMIC_VAL)->_value), (int)NEW_VAL); \
    break; \
  default: \
    _InterlockedExchange((volatile long*)&((ATOMIC_VAL)->_value), (int)NEW_VAL); \
    break; \
  }

#if defined(_M_ARM64)
/*  This has to be an intptr_t for now.
    gil_created() uses -1 as a sentinel value, if this returns
    a uintptr_t it will do an unsigned compare and crash
*/
inline intptr_t _Py_atomic_load_64bit_impl(volatile uintptr_t* value, int order) {
    uintptr_t old;
    switch (order) {
    case _Py_memory_order_acquire:
    {
      do {
        old = *value;
      } while(_InterlockedCompareExchange64_acq(value, old, old) != old);
      break;
    }
    case _Py_memory_order_release:
    {
      do {
        old = *value;
      } while(_InterlockedCompareExchange64_rel(value, old, old) != old);
      break;
    }
    case _Py_memory_order_relaxed:
      old = *value;
      break;
    default:
    {
      do {
        old = *value;
      } while(_InterlockedCompareExchange64(value, old, old) != old);
      break;
    }
    }
    return old;
}

#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) \
    _Py_atomic_load_64bit_impl((volatile uintptr_t*)&((ATOMIC_VAL)->_value), (ORDER))

#else
#define _Py_atomic_load_64bit(ATOMIC_VAL, ORDER) ((ATOMIC_VAL)->_value)
#endif

inline int _Py_atomic_load_32bit_impl(volatile int* value, int order) {
    int old;
    switch (order) {
    case _Py_memory_order_acquire:
    {
      do {
        old = *value;
      } while(_InterlockedCompareExchange_acq(value, old, old) != old);
      break;
    }
    case _Py_memory_order_release:
    {
      do {
        old = *value;
      } while(_InterlockedCompareExchange_rel(value, old, old) != old);
      break;
    }
    case _Py_memory_order_relaxed:
      old = *value;
      break;
    default:
    {
      do {
        old = *value;
      } while(_InterlockedCompareExchange(value, old, old) != old);
      break;
    }
    }
    return old;
}

#define _Py_atomic_load_32bit(ATOMIC_VAL, ORDER) \
    _Py_atomic_load_32bit_impl((volatile int*)&((ATOMIC_VAL)->_value), (ORDER))

#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
  if (sizeof((ATOMIC_VAL)->_value) == 8) { \
    _Py_atomic_store_64bit((ATOMIC_VAL), (NEW_VAL), (ORDER)) } else { \
    _Py_atomic_store_32bit((ATOMIC_VAL), (NEW_VAL), (ORDER)) }

#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
  ( \
    sizeof((ATOMIC_VAL)->_value) == 8 ? \
    _Py_atomic_load_64bit((ATOMIC_VAL), (ORDER)) : \
    _Py_atomic_load_32bit((ATOMIC_VAL), (ORDER)) \
  )
#endif
#else  /* !gcc x86  !_msc_ver */
typedef enum _Py_memory_order {
    _Py_memory_order_relaxed,
    _Py_memory_order_acquire,
    _Py_memory_order_release,
    _Py_memory_order_acq_rel,
    _Py_memory_order_seq_cst
} _Py_memory_order;

typedef struct _Py_atomic_address {
    uintptr_t _value;
} _Py_atomic_address;

typedef struct _Py_atomic_int {
    int _value;
} _Py_atomic_int;
/* Fall back to other compilers and processors by assuming that simple
   volatile accesses are atomic.  This is false, so people should port
   this. */
#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) ((void)0)
#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) ((void)0)
#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \
    ((ATOMIC_VAL)->_value = NEW_VAL)
#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \
    ((ATOMIC_VAL)->_value)
#endif

/* Standardized shortcuts. */
#define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \
    _Py_atomic_store_explicit((ATOMIC_VAL), (NEW_VAL), _Py_memory_order_seq_cst)
#define _Py_atomic_load(ATOMIC_VAL) \
    _Py_atomic_load_explicit((ATOMIC_VAL), _Py_memory_order_seq_cst)

/* Python-local extensions */

#define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \
    _Py_atomic_store_explicit((ATOMIC_VAL), (NEW_VAL), _Py_memory_order_relaxed)
#define _Py_atomic_load_relaxed(ATOMIC_VAL) \
    _Py_atomic_load_explicit((ATOMIC_VAL), _Py_memory_order_relaxed)

#ifdef __cplusplus
}
#endif
#endif  /* Py_ATOMIC_H */

Back