stdatomic.h revision f0f66c0264eb4b6ee56072af34c91a78a9184f23
1/*- 2 * Copyright (c) 2011 Ed Schouten <ed@FreeBSD.org> 3 * David Chisnall <theraven@FreeBSD.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD$ 28 */ 29 30#ifndef _STDATOMIC_H_ 31#define _STDATOMIC_H_ 32 33#include <sys/cdefs.h> 34 35#if defined(__cplusplus) && defined(_USING_LIBCXX) && \ 36 (__has_feature(cxx_atomic) || _GNUC_VER >= 407) 37 38/* We have a usable C++ <atomic>; use it instead. */ 39 40#include <atomic> 41 42#define _Atomic(t) std::atomic<t> 43 44using std::atomic_is_lock_free; 45using std::atomic_init; 46using std::atomic_store; 47using std::atomic_store_explicit; 48using std::atomic_load; 49using std::atomic_load_explicit; 50using std::atomic_exchange; 51using std::atomic_exchange_explicit; 52using std::atomic_compare_exchange_strong; 53using std::atomic_compare_exchange_strong_explicit; 54using std::atomic_compare_exchange_weak; 55using std::atomic_compare_exchange_weak_explicit; 56using std::atomic_fetch_add; 57using std::atomic_fetch_add_explicit; 58using std::atomic_fetch_sub; 59using std::atomic_fetch_sub_explicit; 60using std::atomic_fetch_or; 61using std::atomic_fetch_or_explicit; 62using std::atomic_fetch_xor; 63using std::atomic_fetch_xor_explicit; 64using std::atomic_fetch_and; 65using std::atomic_fetch_and_explicit; 66using std::atomic_thread_fence; 67using std::atomic_signal_fence; 68 69using std::memory_order; 70using std::memory_order_relaxed; 71using std::memory_order_consume; 72using std::memory_order_release; 73using std::memory_order_acq_rel; 74using std::memory_order_seq_cst; 75 76using std::atomic_bool; 77using std::atomic_char; 78using std::atomic_schar; 79using std::atomic_uchar; 80using std::atomic_short; 81using std::atomic_ushort; 82using std::atomic_int; 83using std::atomic_uint; 84using std::atomic_long; 85using std::atomic_ulong; 86using std::atomic_llong; 87using std::atomic_ullong; 88using std::atomic_char16_t; 89using std::atomic_char32_t; 90using std::atomic_wchar_t; 91using std::atomic_int_least8_t; 92using std::atomic_uint_least8_t; 93using std::atomic_int_least16_t; 94using std::atomic_uint_least16_t; 95using std::atomic_int_least32_t; 96using std::atomic_uint_least32_t; 97using std::atomic_int_least64_t; 98using std::atomic_uint_least64_t; 99using std::atomic_int_fast8_t; 100using std::atomic_uint_fast8_t; 101using std::atomic_int_fast16_t; 102using std::atomic_uint_fast16_t; 103using std::atomic_int_fast32_t; 104using std::atomic_uint_fast32_t; 105using std::atomic_int_fast64_t; 106using std::atomic_uint_fast64_t; 107using std::atomic_intptr_t; 108using std::atomic_uintptr_t; 109using std::atomic_size_t; 110using std::atomic_ptrdiff_t; 111using std::atomic_intmax_t; 112using std::atomic_uintmax_t; 113 114#else /* <atomic> unavailable, possibly because this is C, not C++ */ 115 116#include <sys/types.h> 117#include <stdbool.h> 118 119/* 120 * C: Do it ourselves. 121 * Note that the runtime representation defined here should be compatible 122 * with the C++ one, i.e. an _Atomic(T) needs to contain the same 123 * bits as a T. 124 */ 125 126#if __has_extension(c_atomic) || __has_extension(cxx_atomic) 127#define __CLANG_ATOMICS 128#elif __GNUC_PREREQ__(4, 7) 129#define __GNUC_ATOMICS 130#elif defined(__GNUC__) 131#define __SYNC_ATOMICS 132#else 133#error "stdatomic.h does not support your compiler" 134#endif 135 136/* 137 * 7.17.1 Atomic lock-free macros. 138 */ 139 140#ifdef __GCC_ATOMIC_BOOL_LOCK_FREE 141#define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE 142#endif 143#ifdef __GCC_ATOMIC_CHAR_LOCK_FREE 144#define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE 145#endif 146#ifdef __GCC_ATOMIC_CHAR16_T_LOCK_FREE 147#define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE 148#endif 149#ifdef __GCC_ATOMIC_CHAR32_T_LOCK_FREE 150#define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE 151#endif 152#ifdef __GCC_ATOMIC_WCHAR_T_LOCK_FREE 153#define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE 154#endif 155#ifdef __GCC_ATOMIC_SHORT_LOCK_FREE 156#define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE 157#endif 158#ifdef __GCC_ATOMIC_INT_LOCK_FREE 159#define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE 160#endif 161#ifdef __GCC_ATOMIC_LONG_LOCK_FREE 162#define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE 163#endif 164#ifdef __GCC_ATOMIC_LLONG_LOCK_FREE 165#define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE 166#endif 167#ifdef __GCC_ATOMIC_POINTER_LOCK_FREE 168#define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE 169#endif 170 171/* 172 * 7.17.2 Initialization. 173 */ 174 175#if defined(__CLANG_ATOMICS) 176#define ATOMIC_VAR_INIT(value) (value) 177#define atomic_init(obj, value) __c11_atomic_init(obj, value) 178#else 179#define ATOMIC_VAR_INIT(value) { .__val = (value) } 180#define atomic_init(obj, value) ((void)((obj)->__val = (value))) 181#endif 182 183/* 184 * Clang and recent GCC both provide predefined macros for the memory 185 * orderings. If we are using a compiler that doesn't define them, use the 186 * clang values - these will be ignored in the fallback path. 187 */ 188 189#ifndef __ATOMIC_RELAXED 190#define __ATOMIC_RELAXED 0 191#endif 192#ifndef __ATOMIC_CONSUME 193#define __ATOMIC_CONSUME 1 194#endif 195#ifndef __ATOMIC_ACQUIRE 196#define __ATOMIC_ACQUIRE 2 197#endif 198#ifndef __ATOMIC_RELEASE 199#define __ATOMIC_RELEASE 3 200#endif 201#ifndef __ATOMIC_ACQ_REL 202#define __ATOMIC_ACQ_REL 4 203#endif 204#ifndef __ATOMIC_SEQ_CST 205#define __ATOMIC_SEQ_CST 5 206#endif 207 208/* 209 * 7.17.3 Order and consistency. 210 * 211 * The memory_order_* constants that denote the barrier behaviour of the 212 * atomic operations. 213 * The enum values must be identical to those used by the 214 * C++ <atomic> header. 215 */ 216 217typedef enum { 218 memory_order_relaxed = __ATOMIC_RELAXED, 219 memory_order_consume = __ATOMIC_CONSUME, 220 memory_order_acquire = __ATOMIC_ACQUIRE, 221 memory_order_release = __ATOMIC_RELEASE, 222 memory_order_acq_rel = __ATOMIC_ACQ_REL, 223 memory_order_seq_cst = __ATOMIC_SEQ_CST 224} memory_order; 225 226/* 227 * 7.17.4 Fences. 228 */ 229 230static __inline void 231atomic_thread_fence(memory_order __order __unused) 232{ 233 234#ifdef __CLANG_ATOMICS 235 __c11_atomic_thread_fence(__order); 236#elif defined(__GNUC_ATOMICS) 237 __atomic_thread_fence(__order); 238#else 239 __sync_synchronize(); 240#endif 241} 242 243static __inline void 244atomic_signal_fence(memory_order __order __unused) 245{ 246 247#ifdef __CLANG_ATOMICS 248 __c11_atomic_signal_fence(__order); 249#elif defined(__GNUC_ATOMICS) 250 __atomic_signal_fence(__order); 251#else 252 __asm volatile ("" ::: "memory"); 253#endif 254} 255 256/* 257 * 7.17.5 Lock-free property. 258 */ 259 260#if defined(_KERNEL) 261/* Atomics in kernelspace are always lock-free. */ 262#define atomic_is_lock_free(obj) \ 263 ((void)(obj), (_Bool)1) 264#elif defined(__CLANG_ATOMICS) 265#define atomic_is_lock_free(obj) \ 266 __atomic_is_lock_free(sizeof(*(obj)), obj) 267#elif defined(__GNUC_ATOMICS) 268#define atomic_is_lock_free(obj) \ 269 __atomic_is_lock_free(sizeof((obj)->__val), &(obj)->__val) 270#else 271#define atomic_is_lock_free(obj) \ 272 ((void)(obj), sizeof((obj)->__val) <= sizeof(void *)) 273#endif 274 275/* 276 * 7.17.6 Atomic integer types. 277 */ 278 279#if !__has_extension(c_atomic) && !__has_extension(cxx_atomic) 280/* 281 * No native support for _Atomic(). Place object in structure to prevent 282 * most forms of direct non-atomic access. 283 */ 284#define _Atomic(T) struct { T volatile __val; } 285#endif 286 287typedef _Atomic(bool) atomic_bool; 288typedef _Atomic(char) atomic_char; 289typedef _Atomic(signed char) atomic_schar; 290typedef _Atomic(unsigned char) atomic_uchar; 291typedef _Atomic(short) atomic_short; 292typedef _Atomic(unsigned short) atomic_ushort; 293typedef _Atomic(int) atomic_int; 294typedef _Atomic(unsigned int) atomic_uint; 295typedef _Atomic(long) atomic_long; 296typedef _Atomic(unsigned long) atomic_ulong; 297typedef _Atomic(long long) atomic_llong; 298typedef _Atomic(unsigned long long) atomic_ullong; 299#if __STDC_VERSION__ >= 201112L || __cplusplus >= 201103L 300 typedef _Atomic(char16_t) atomic_char16_t; 301 typedef _Atomic(char32_t) atomic_char32_t; 302#endif 303typedef _Atomic(wchar_t) atomic_wchar_t; 304typedef _Atomic(int_least8_t) atomic_int_least8_t; 305typedef _Atomic(uint_least8_t) atomic_uint_least8_t; 306typedef _Atomic(int_least16_t) atomic_int_least16_t; 307typedef _Atomic(uint_least16_t) atomic_uint_least16_t; 308typedef _Atomic(int_least32_t) atomic_int_least32_t; 309typedef _Atomic(uint_least32_t) atomic_uint_least32_t; 310typedef _Atomic(int_least64_t) atomic_int_least64_t; 311typedef _Atomic(uint_least64_t) atomic_uint_least64_t; 312typedef _Atomic(int_fast8_t) atomic_int_fast8_t; 313typedef _Atomic(uint_fast8_t) atomic_uint_fast8_t; 314typedef _Atomic(int_fast16_t) atomic_int_fast16_t; 315typedef _Atomic(uint_fast16_t) atomic_uint_fast16_t; 316typedef _Atomic(int_fast32_t) atomic_int_fast32_t; 317typedef _Atomic(uint_fast32_t) atomic_uint_fast32_t; 318typedef _Atomic(int_fast64_t) atomic_int_fast64_t; 319typedef _Atomic(uint_fast64_t) atomic_uint_fast64_t; 320typedef _Atomic(intptr_t) atomic_intptr_t; 321typedef _Atomic(uintptr_t) atomic_uintptr_t; 322typedef _Atomic(size_t) atomic_size_t; 323typedef _Atomic(ptrdiff_t) atomic_ptrdiff_t; 324typedef _Atomic(intmax_t) atomic_intmax_t; 325typedef _Atomic(uintmax_t) atomic_uintmax_t; 326 327/* 328 * 7.17.7 Operations on atomic types. 329 */ 330 331/* 332 * Compiler-specific operations. 333 */ 334 335#if defined(__CLANG_ATOMICS) 336#define atomic_compare_exchange_strong_explicit(object, expected, \ 337 desired, success, failure) \ 338 __c11_atomic_compare_exchange_strong(object, expected, desired, \ 339 success, failure) 340#define atomic_compare_exchange_weak_explicit(object, expected, \ 341 desired, success, failure) \ 342 __c11_atomic_compare_exchange_weak(object, expected, desired, \ 343 success, failure) 344#define atomic_exchange_explicit(object, desired, order) \ 345 __c11_atomic_exchange(object, desired, order) 346#define atomic_fetch_add_explicit(object, operand, order) \ 347 __c11_atomic_fetch_add(object, operand, order) 348#define atomic_fetch_and_explicit(object, operand, order) \ 349 __c11_atomic_fetch_and(object, operand, order) 350#define atomic_fetch_or_explicit(object, operand, order) \ 351 __c11_atomic_fetch_or(object, operand, order) 352#define atomic_fetch_sub_explicit(object, operand, order) \ 353 __c11_atomic_fetch_sub(object, operand, order) 354#define atomic_fetch_xor_explicit(object, operand, order) \ 355 __c11_atomic_fetch_xor(object, operand, order) 356#define atomic_load_explicit(object, order) \ 357 __c11_atomic_load(object, order) 358#define atomic_store_explicit(object, desired, order) \ 359 __c11_atomic_store(object, desired, order) 360#elif defined(__GNUC_ATOMICS) 361#define atomic_compare_exchange_strong_explicit(object, expected, \ 362 desired, success, failure) \ 363 __atomic_compare_exchange_n(&(object)->__val, expected, \ 364 desired, 0, success, failure) 365#define atomic_compare_exchange_weak_explicit(object, expected, \ 366 desired, success, failure) \ 367 __atomic_compare_exchange_n(&(object)->__val, expected, \ 368 desired, 1, success, failure) 369#define atomic_exchange_explicit(object, desired, order) \ 370 __atomic_exchange_n(&(object)->__val, desired, order) 371#define atomic_fetch_add_explicit(object, operand, order) \ 372 __atomic_fetch_add(&(object)->__val, operand, order) 373#define atomic_fetch_and_explicit(object, operand, order) \ 374 __atomic_fetch_and(&(object)->__val, operand, order) 375#define atomic_fetch_or_explicit(object, operand, order) \ 376 __atomic_fetch_or(&(object)->__val, operand, order) 377#define atomic_fetch_sub_explicit(object, operand, order) \ 378 __atomic_fetch_sub(&(object)->__val, operand, order) 379#define atomic_fetch_xor_explicit(object, operand, order) \ 380 __atomic_fetch_xor(&(object)->__val, operand, order) 381#define atomic_load_explicit(object, order) \ 382 __atomic_load_n(&(object)->__val, order) 383#define atomic_store_explicit(object, desired, order) \ 384 __atomic_store_n(&(object)->__val, desired, order) 385#else 386#define __atomic_apply_stride(object, operand) \ 387 (((__typeof__((object)->__val))0) + (operand)) 388#define atomic_compare_exchange_strong_explicit(object, expected, \ 389 desired, success, failure) __extension__ ({ \ 390 __typeof__(expected) __ep = (expected); \ 391 __typeof__(*__ep) __e = *__ep; \ 392 (void)(success); (void)(failure); \ 393 (bool)((*__ep = __sync_val_compare_and_swap(&(object)->__val, \ 394 __e, desired)) == __e); \ 395}) 396#define atomic_compare_exchange_weak_explicit(object, expected, \ 397 desired, success, failure) \ 398 atomic_compare_exchange_strong_explicit(object, expected, \ 399 desired, success, failure) 400#if __has_builtin(__sync_swap) 401/* Clang provides a full-barrier atomic exchange - use it if available. */ 402#define atomic_exchange_explicit(object, desired, order) \ 403 ((void)(order), __sync_swap(&(object)->__val, desired)) 404#else 405/* 406 * __sync_lock_test_and_set() is only an acquire barrier in theory (although in 407 * practice it is usually a full barrier) so we need an explicit barrier before 408 * it. 409 */ 410#define atomic_exchange_explicit(object, desired, order) \ 411__extension__ ({ \ 412 __typeof__(object) __o = (object); \ 413 __typeof__(desired) __d = (desired); \ 414 (void)(order); \ 415 __sync_synchronize(); \ 416 __sync_lock_test_and_set(&(__o)->__val, __d); \ 417}) 418#endif 419#define atomic_fetch_add_explicit(object, operand, order) \ 420 ((void)(order), __sync_fetch_and_add(&(object)->__val, \ 421 __atomic_apply_stride(object, operand))) 422#define atomic_fetch_and_explicit(object, operand, order) \ 423 ((void)(order), __sync_fetch_and_and(&(object)->__val, operand)) 424#define atomic_fetch_or_explicit(object, operand, order) \ 425 ((void)(order), __sync_fetch_and_or(&(object)->__val, operand)) 426#define atomic_fetch_sub_explicit(object, operand, order) \ 427 ((void)(order), __sync_fetch_and_sub(&(object)->__val, \ 428 __atomic_apply_stride(object, operand))) 429#define atomic_fetch_xor_explicit(object, operand, order) \ 430 ((void)(order), __sync_fetch_and_xor(&(object)->__val, operand)) 431#define atomic_load_explicit(object, order) \ 432 ((void)(order), __sync_fetch_and_add(&(object)->__val, 0)) 433#define atomic_store_explicit(object, desired, order) \ 434 ((void)atomic_exchange_explicit(object, desired, order)) 435#endif 436 437/* 438 * Convenience functions. 439 * 440 * Don't provide these in kernel space. In kernel space, we should be 441 * disciplined enough to always provide explicit barriers. 442 */ 443 444#ifndef _KERNEL 445#define atomic_compare_exchange_strong(object, expected, desired) \ 446 atomic_compare_exchange_strong_explicit(object, expected, \ 447 desired, memory_order_seq_cst, memory_order_seq_cst) 448#define atomic_compare_exchange_weak(object, expected, desired) \ 449 atomic_compare_exchange_weak_explicit(object, expected, \ 450 desired, memory_order_seq_cst, memory_order_seq_cst) 451#define atomic_exchange(object, desired) \ 452 atomic_exchange_explicit(object, desired, memory_order_seq_cst) 453#define atomic_fetch_add(object, operand) \ 454 atomic_fetch_add_explicit(object, operand, memory_order_seq_cst) 455#define atomic_fetch_and(object, operand) \ 456 atomic_fetch_and_explicit(object, operand, memory_order_seq_cst) 457#define atomic_fetch_or(object, operand) \ 458 atomic_fetch_or_explicit(object, operand, memory_order_seq_cst) 459#define atomic_fetch_sub(object, operand) \ 460 atomic_fetch_sub_explicit(object, operand, memory_order_seq_cst) 461#define atomic_fetch_xor(object, operand) \ 462 atomic_fetch_xor_explicit(object, operand, memory_order_seq_cst) 463#define atomic_load(object) \ 464 atomic_load_explicit(object, memory_order_seq_cst) 465#define atomic_store(object, desired) \ 466 atomic_store_explicit(object, desired, memory_order_seq_cst) 467#endif /* !_KERNEL */ 468 469/* 470 * 7.17.8 Atomic flag type and operations. 471 * 472 * XXX: Assume atomic_bool can be used as an atomic_flag. Is there some 473 * kind of compiler built-in type we could use? 474 */ 475 476typedef struct { 477 atomic_bool __flag; 478} atomic_flag; 479 480#define ATOMIC_FLAG_INIT { ATOMIC_VAR_INIT(0) } 481 482static __inline bool 483atomic_flag_test_and_set_explicit(volatile atomic_flag *__object, 484 memory_order __order) 485{ 486 return (atomic_exchange_explicit(&__object->__flag, 1, __order)); 487} 488 489static __inline void 490atomic_flag_clear_explicit(volatile atomic_flag *__object, memory_order __order) 491{ 492 493 atomic_store_explicit(&__object->__flag, 0, __order); 494} 495 496#ifndef _KERNEL 497static __inline bool 498atomic_flag_test_and_set(volatile atomic_flag *__object) 499{ 500 501 return (atomic_flag_test_and_set_explicit(__object, 502 memory_order_seq_cst)); 503} 504 505static __inline void 506atomic_flag_clear(volatile atomic_flag *__object) 507{ 508 509 atomic_flag_clear_explicit(__object, memory_order_seq_cst); 510} 511#endif /* !_KERNEL */ 512 513#endif /* <atomic> unavailable */ 514 515#endif /* !_STDATOMIC_H_ */ 516