1/** 2 * Many similar implementations exist. See for example libwsbm 3 * or the linux kernel include/atomic.h 4 * 5 * No copyright claimed on this file. 6 * 7 */ 8 9#ifndef U_ATOMIC_H 10#define U_ATOMIC_H 11 12#include "pipe/p_compiler.h" 13#include "pipe/p_defines.h" 14 15/* Favor OS-provided implementations. 16 * 17 * Where no OS-provided implementation is available, fall back to 18 * locally coded assembly, compiler intrinsic or ultimately a 19 * mutex-based implementation. 20 */ 21#if defined(PIPE_OS_SOLARIS) 22#define PIPE_ATOMIC_OS_SOLARIS 23#elif defined(PIPE_CC_MSVC) 24#define PIPE_ATOMIC_MSVC_INTRINSIC 25#elif (defined(PIPE_CC_MSVC) && defined(PIPE_ARCH_X86)) 26#define PIPE_ATOMIC_ASM_MSVC_X86 27#elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86)) 28#define PIPE_ATOMIC_ASM_GCC_X86 29#elif (defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64)) 30#define PIPE_ATOMIC_ASM_GCC_X86_64 31#elif defined(PIPE_CC_GCC) && (PIPE_CC_GCC_VERSION >= 401) 32#define PIPE_ATOMIC_GCC_INTRINSIC 33#else 34#error "Unsupported platform" 35#endif 36 37 38#if defined(PIPE_ATOMIC_ASM_GCC_X86_64) 39#define PIPE_ATOMIC "GCC x86_64 assembly" 40 41#ifdef __cplusplus 42extern "C" { 43#endif 44 45#define p_atomic_set(_v, _i) (*(_v) = (_i)) 46#define p_atomic_read(_v) (*(_v)) 47 48static INLINE boolean 49p_atomic_dec_zero(int32_t *v) 50{ 51 unsigned char c; 52 53 __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c) 54 ::"memory"); 55 56 return c != 0; 57} 58 59static INLINE void 60p_atomic_inc(int32_t *v) 61{ 62 __asm__ __volatile__("lock; incl %0":"+m"(*v)); 63} 64 65static INLINE void 66p_atomic_dec(int32_t *v) 67{ 68 __asm__ __volatile__("lock; decl %0":"+m"(*v)); 69} 70 71static INLINE int32_t 72p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new) 73{ 74 return __sync_val_compare_and_swap(v, old, _new); 75} 76 77#ifdef __cplusplus 78} 79#endif 80 81#endif /* PIPE_ATOMIC_ASM_GCC_X86_64 */ 82 83 84#if defined(PIPE_ATOMIC_ASM_GCC_X86) 85 86#define PIPE_ATOMIC "GCC x86 assembly" 87 88#ifdef __cplusplus 89extern "C" { 90#endif 91 92#define p_atomic_set(_v, _i) (*(_v) = (_i)) 93#define p_atomic_read(_v) (*(_v)) 94 95static INLINE boolean 96p_atomic_dec_zero(int32_t *v) 97{ 98 unsigned char c; 99 100 __asm__ __volatile__("lock; decl %0; sete %1":"+m"(*v), "=qm"(c) 101 ::"memory"); 102 103 return c != 0; 104} 105 106static INLINE void 107p_atomic_inc(int32_t *v) 108{ 109 __asm__ __volatile__("lock; incl %0":"+m"(*v)); 110} 111 112static INLINE void 113p_atomic_dec(int32_t *v) 114{ 115 __asm__ __volatile__("lock; decl %0":"+m"(*v)); 116} 117 118static INLINE int32_t 119p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new) 120{ 121 return __sync_val_compare_and_swap(v, old, _new); 122} 123 124#ifdef __cplusplus 125} 126#endif 127 128#endif 129 130 131 132/* Implementation using GCC-provided synchronization intrinsics 133 */ 134#if defined(PIPE_ATOMIC_GCC_INTRINSIC) 135 136#define PIPE_ATOMIC "GCC Sync Intrinsics" 137 138#ifdef __cplusplus 139extern "C" { 140#endif 141 142#define p_atomic_set(_v, _i) (*(_v) = (_i)) 143#define p_atomic_read(_v) (*(_v)) 144 145static INLINE boolean 146p_atomic_dec_zero(int32_t *v) 147{ 148 return (__sync_sub_and_fetch(v, 1) == 0); 149} 150 151static INLINE void 152p_atomic_inc(int32_t *v) 153{ 154 (void) __sync_add_and_fetch(v, 1); 155} 156 157static INLINE void 158p_atomic_dec(int32_t *v) 159{ 160 (void) __sync_sub_and_fetch(v, 1); 161} 162 163static INLINE int32_t 164p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new) 165{ 166 return __sync_val_compare_and_swap(v, old, _new); 167} 168 169#ifdef __cplusplus 170} 171#endif 172 173#endif 174 175 176 177/* Unlocked version for single threaded environments, such as some 178 * windows kernel modules. 179 */ 180#if defined(PIPE_ATOMIC_OS_UNLOCKED) 181 182#define PIPE_ATOMIC "Unlocked" 183 184#define p_atomic_set(_v, _i) (*(_v) = (_i)) 185#define p_atomic_read(_v) (*(_v)) 186#define p_atomic_dec_zero(_v) ((boolean) --(*(_v))) 187#define p_atomic_inc(_v) ((void) (*(_v))++) 188#define p_atomic_dec(_v) ((void) (*(_v))--) 189#define p_atomic_cmpxchg(_v, old, _new) (*(_v) == old ? *(_v) = (_new) : *(_v)) 190 191#endif 192 193 194/* Locally coded assembly for MSVC on x86: 195 */ 196#if defined(PIPE_ATOMIC_ASM_MSVC_X86) 197 198#define PIPE_ATOMIC "MSVC x86 assembly" 199 200#ifdef __cplusplus 201extern "C" { 202#endif 203 204#define p_atomic_set(_v, _i) (*(_v) = (_i)) 205#define p_atomic_read(_v) (*(_v)) 206 207static INLINE boolean 208p_atomic_dec_zero(int32_t *v) 209{ 210 unsigned char c; 211 212 __asm { 213 mov eax, [v] 214 lock dec dword ptr [eax] 215 sete byte ptr [c] 216 } 217 218 return c != 0; 219} 220 221static INLINE void 222p_atomic_inc(int32_t *v) 223{ 224 __asm { 225 mov eax, [v] 226 lock inc dword ptr [eax] 227 } 228} 229 230static INLINE void 231p_atomic_dec(int32_t *v) 232{ 233 __asm { 234 mov eax, [v] 235 lock dec dword ptr [eax] 236 } 237} 238 239static INLINE int32_t 240p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new) 241{ 242 int32_t orig; 243 244 __asm { 245 mov ecx, [v] 246 mov eax, [old] 247 mov edx, [_new] 248 lock cmpxchg [ecx], edx 249 mov [orig], eax 250 } 251 252 return orig; 253} 254 255#ifdef __cplusplus 256} 257#endif 258 259#endif 260 261 262#if defined(PIPE_ATOMIC_MSVC_INTRINSIC) 263 264#define PIPE_ATOMIC "MSVC Intrinsics" 265 266#include <intrin.h> 267 268#pragma intrinsic(_InterlockedIncrement) 269#pragma intrinsic(_InterlockedDecrement) 270#pragma intrinsic(_InterlockedCompareExchange) 271 272#ifdef __cplusplus 273extern "C" { 274#endif 275 276#define p_atomic_set(_v, _i) (*(_v) = (_i)) 277#define p_atomic_read(_v) (*(_v)) 278 279static INLINE boolean 280p_atomic_dec_zero(int32_t *v) 281{ 282 return _InterlockedDecrement((long *)v) == 0; 283} 284 285static INLINE void 286p_atomic_inc(int32_t *v) 287{ 288 _InterlockedIncrement((long *)v); 289} 290 291static INLINE void 292p_atomic_dec(int32_t *v) 293{ 294 _InterlockedDecrement((long *)v); 295} 296 297static INLINE int32_t 298p_atomic_cmpxchg(int32_t *v, int32_t old, int32_t _new) 299{ 300 return _InterlockedCompareExchange((long *)v, _new, old); 301} 302 303#ifdef __cplusplus 304} 305#endif 306 307#endif 308 309#if defined(PIPE_ATOMIC_OS_SOLARIS) 310 311#define PIPE_ATOMIC "Solaris OS atomic functions" 312 313#include <atomic.h> 314 315#ifdef __cplusplus 316extern "C" { 317#endif 318 319#define p_atomic_set(_v, _i) (*(_v) = (_i)) 320#define p_atomic_read(_v) (*(_v)) 321 322static INLINE boolean 323p_atomic_dec_zero(int32_t *v) 324{ 325 uint32_t n = atomic_dec_32_nv((uint32_t *) v); 326 327 return n != 0; 328} 329 330#define p_atomic_inc(_v) atomic_inc_32((uint32_t *) _v) 331#define p_atomic_dec(_v) atomic_dec_32((uint32_t *) _v) 332 333#define p_atomic_cmpxchg(_v, _old, _new) \ 334 atomic_cas_32( (uint32_t *) _v, (uint32_t) _old, (uint32_t) _new) 335 336#ifdef __cplusplus 337} 338#endif 339 340#endif 341 342 343#ifndef PIPE_ATOMIC 344#error "No pipe_atomic implementation selected" 345#endif 346 347 348 349#endif /* U_ATOMIC_H */ 350