stdatomic.h revision c8cf3513ecb265ba3aadc846aa2113290a504c44
1/*- 2 * Copyright (c) 2011 Ed Schouten <ed@FreeBSD.org> 3 * David Chisnall <theraven@FreeBSD.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD$ 28 */ 29 30#ifndef _STDATOMIC_H_ 31#define _STDATOMIC_H_ 32 33#include <sys/cdefs.h> 34 35#if defined(__cplusplus) && defined(_USING_LIBCXX) && \ 36 (__has_feature(cxx_atomic) || _GNUC_VER >= 407) 37 38/* We have a usable C++ <atomic>; use it instead. */ 39 40#include <atomic> 41 42#define _Atomic(t) std::atomic<t> 43 44using std::atomic_is_lock_free; 45using std::atomic_init; 46using std::atomic_store; 47using std::atomic_store_explicit; 48using std::atomic_load; 49using std::atomic_load_explicit; 50using std::atomic_exchange; 51using std::atomic_exchange_explicit; 52using std::atomic_compare_exchange_strong; 53using std::atomic_compare_exchange_strong_explicit; 54using std::atomic_compare_exchange_weak; 55using std::atomic_compare_exchange_weak_explicit; 56using std::atomic_fetch_add; 57using std::atomic_fetch_add_explicit; 58using std::atomic_fetch_sub; 59using std::atomic_fetch_sub_explicit; 60using std::atomic_fetch_or; 61using std::atomic_fetch_or_explicit; 62using std::atomic_fetch_xor; 63using std::atomic_fetch_xor_explicit; 64using std::atomic_fetch_and; 65using std::atomic_fetch_and_explicit; 66using std::atomic_thread_fence; 67using std::atomic_signal_fence; 68 69using std::memory_order; 70using std::memory_order_relaxed; 71using std::memory_order_consume; 72using std::memory_order_release; 73using std::memory_order_acq_rel; 74using std::memory_order_seq_cst; 75 76using std::atomic_bool; 77using std::atomic_char; 78using std::atomic_schar; 79using std::atomic_uchar; 80using std::atomic_short; 81using std::atomic_ushort; 82using std::atomic_int; 83using std::atomic_uint; 84using std::atomic_long; 85using std::atomic_ulong; 86using std::atomic_llong; 87using std::atomic_ullong; 88using std::atomic_char16_t; 89using std::atomic_char32_t; 90using std::atomic_wchar_t; 91using std::atomic_int_least8_t; 92using std::atomic_uint_least8_t; 93using std::atomic_int_least16_t; 94using std::atomic_uint_least16_t; 95using std::atomic_int_least32_t; 96using std::atomic_uint_least32_t; 97using std::atomic_int_least64_t; 98using std::atomic_uint_least64_t; 99using std::atomic_int_fast8_t; 100using std::atomic_uint_fast8_t; 101using std::atomic_int_fast16_t; 102using std::atomic_uint_fast16_t; 103using std::atomic_int_fast32_t; 104using std::atomic_uint_fast32_t; 105using std::atomic_int_fast64_t; 106using std::atomic_uint_fast64_t; 107using std::atomic_intptr_t; 108using std::atomic_uintptr_t; 109using std::atomic_size_t; 110using std::atomic_ptrdiff_t; 111using std::atomic_intmax_t; 112using std::atomic_uintmax_t; 113 114#else /* <atomic> unavailable, possibly because this is C, not C++ */ 115 116#include <sys/types.h> 117#include <stdbool.h> 118 119/* 120 * C: Do it ourselves. 121 * Note that the runtime representation defined here should be compatible 122 * with the C++ one, i.e. an _Atomic(T) needs to contain the same 123 * bits as a T. 124 */ 125 126#include <stddef.h> /* For ptrdiff_t. */ 127#include <stdint.h> /* TODO: Should pollute namespace less. */ 128#if __STDC_VERSION__ >= 201112L 129# include <uchar.h> /* For char16_t and char32_t. */ 130#endif 131 132#if __has_extension(c_atomic) || __has_extension(cxx_atomic) 133#define __CLANG_ATOMICS 134#elif __GNUC_PREREQ(4, 7) 135#define __GNUC_ATOMICS 136#elif defined(__GNUC__) 137#define __SYNC_ATOMICS 138#else 139#error "stdatomic.h does not support your compiler" 140#endif 141 142/* 143 * 7.17.1 Atomic lock-free macros. 144 */ 145 146#ifdef __GCC_ATOMIC_BOOL_LOCK_FREE 147#define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE 148#endif 149#ifdef __GCC_ATOMIC_CHAR_LOCK_FREE 150#define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE 151#endif 152#ifdef __GCC_ATOMIC_CHAR16_T_LOCK_FREE 153#define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE 154#endif 155#ifdef __GCC_ATOMIC_CHAR32_T_LOCK_FREE 156#define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE 157#endif 158#ifdef __GCC_ATOMIC_WCHAR_T_LOCK_FREE 159#define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE 160#endif 161#ifdef __GCC_ATOMIC_SHORT_LOCK_FREE 162#define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE 163#endif 164#ifdef __GCC_ATOMIC_INT_LOCK_FREE 165#define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE 166#endif 167#ifdef __GCC_ATOMIC_LONG_LOCK_FREE 168#define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE 169#endif 170#ifdef __GCC_ATOMIC_LLONG_LOCK_FREE 171#define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE 172#endif 173#ifdef __GCC_ATOMIC_POINTER_LOCK_FREE 174#define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE 175#endif 176 177/* 178 * 7.17.2 Initialization. 179 */ 180 181#if defined(__CLANG_ATOMICS) 182#define ATOMIC_VAR_INIT(value) (value) 183#define atomic_init(obj, value) __c11_atomic_init(obj, value) 184#else 185#define ATOMIC_VAR_INIT(value) { .__val = (value) } 186#define atomic_init(obj, value) ((void)((obj)->__val = (value))) 187#endif 188 189/* 190 * Clang and recent GCC both provide predefined macros for the memory 191 * orderings. If we are using a compiler that doesn't define them, use the 192 * clang values - these will be ignored in the fallback path. 193 */ 194 195#ifndef __ATOMIC_RELAXED 196#define __ATOMIC_RELAXED 0 197#endif 198#ifndef __ATOMIC_CONSUME 199#define __ATOMIC_CONSUME 1 200#endif 201#ifndef __ATOMIC_ACQUIRE 202#define __ATOMIC_ACQUIRE 2 203#endif 204#ifndef __ATOMIC_RELEASE 205#define __ATOMIC_RELEASE 3 206#endif 207#ifndef __ATOMIC_ACQ_REL 208#define __ATOMIC_ACQ_REL 4 209#endif 210#ifndef __ATOMIC_SEQ_CST 211#define __ATOMIC_SEQ_CST 5 212#endif 213 214/* 215 * 7.17.3 Order and consistency. 216 * 217 * The memory_order_* constants that denote the barrier behaviour of the 218 * atomic operations. 219 * The enum values must be identical to those used by the 220 * C++ <atomic> header. 221 */ 222 223typedef enum { 224 memory_order_relaxed = __ATOMIC_RELAXED, 225 memory_order_consume = __ATOMIC_CONSUME, 226 memory_order_acquire = __ATOMIC_ACQUIRE, 227 memory_order_release = __ATOMIC_RELEASE, 228 memory_order_acq_rel = __ATOMIC_ACQ_REL, 229 memory_order_seq_cst = __ATOMIC_SEQ_CST 230} memory_order; 231 232/* 233 * 7.17.4 Fences. 234 */ 235 236static __inline void 237atomic_thread_fence(memory_order __order __attribute__((unused))) 238{ 239 240#ifdef __CLANG_ATOMICS 241 __c11_atomic_thread_fence(__order); 242#elif defined(__GNUC_ATOMICS) 243 __atomic_thread_fence(__order); 244#else 245 __sync_synchronize(); 246#endif 247} 248 249static __inline void 250atomic_signal_fence(memory_order __order __attribute__((unused))) 251{ 252 253#ifdef __CLANG_ATOMICS 254 __c11_atomic_signal_fence(__order); 255#elif defined(__GNUC_ATOMICS) 256 __atomic_signal_fence(__order); 257#else 258 __asm volatile ("" ::: "memory"); 259#endif 260} 261 262/* 263 * 7.17.5 Lock-free property. 264 */ 265 266#if defined(_KERNEL) 267/* Atomics in kernelspace are always lock-free. */ 268#define atomic_is_lock_free(obj) \ 269 ((void)(obj), (_Bool)1) 270#elif defined(__CLANG_ATOMICS) 271#define atomic_is_lock_free(obj) \ 272 __c11_atomic_is_lock_free(sizeof(*(obj))) 273#elif defined(__GNUC_ATOMICS) 274#define atomic_is_lock_free(obj) \ 275 __atomic_is_lock_free(sizeof((obj)->__val), &(obj)->__val) 276#else 277#define atomic_is_lock_free(obj) \ 278 ((void)(obj), sizeof((obj)->__val) <= sizeof(void *)) 279#endif 280 281/* 282 * 7.17.6 Atomic integer types. 283 */ 284 285#if !__has_extension(c_atomic) && !__has_extension(cxx_atomic) 286/* 287 * No native support for _Atomic(). Place object in structure to prevent 288 * most forms of direct non-atomic access. 289 */ 290#define _Atomic(T) struct { T volatile __val; } 291#endif 292 293typedef _Atomic(bool) atomic_bool; 294typedef _Atomic(char) atomic_char; 295typedef _Atomic(signed char) atomic_schar; 296typedef _Atomic(unsigned char) atomic_uchar; 297typedef _Atomic(short) atomic_short; 298typedef _Atomic(unsigned short) atomic_ushort; 299typedef _Atomic(int) atomic_int; 300typedef _Atomic(unsigned int) atomic_uint; 301typedef _Atomic(long) atomic_long; 302typedef _Atomic(unsigned long) atomic_ulong; 303typedef _Atomic(long long) atomic_llong; 304typedef _Atomic(unsigned long long) atomic_ullong; 305#if __STDC_VERSION__ >= 201112L || __cplusplus >= 201103L 306 typedef _Atomic(char16_t) atomic_char16_t; 307 typedef _Atomic(char32_t) atomic_char32_t; 308#endif 309typedef _Atomic(wchar_t) atomic_wchar_t; 310typedef _Atomic(int_least8_t) atomic_int_least8_t; 311typedef _Atomic(uint_least8_t) atomic_uint_least8_t; 312typedef _Atomic(int_least16_t) atomic_int_least16_t; 313typedef _Atomic(uint_least16_t) atomic_uint_least16_t; 314typedef _Atomic(int_least32_t) atomic_int_least32_t; 315typedef _Atomic(uint_least32_t) atomic_uint_least32_t; 316typedef _Atomic(int_least64_t) atomic_int_least64_t; 317typedef _Atomic(uint_least64_t) atomic_uint_least64_t; 318typedef _Atomic(int_fast8_t) atomic_int_fast8_t; 319typedef _Atomic(uint_fast8_t) atomic_uint_fast8_t; 320typedef _Atomic(int_fast16_t) atomic_int_fast16_t; 321typedef _Atomic(uint_fast16_t) atomic_uint_fast16_t; 322typedef _Atomic(int_fast32_t) atomic_int_fast32_t; 323typedef _Atomic(uint_fast32_t) atomic_uint_fast32_t; 324typedef _Atomic(int_fast64_t) atomic_int_fast64_t; 325typedef _Atomic(uint_fast64_t) atomic_uint_fast64_t; 326typedef _Atomic(intptr_t) atomic_intptr_t; 327typedef _Atomic(uintptr_t) atomic_uintptr_t; 328typedef _Atomic(size_t) atomic_size_t; 329typedef _Atomic(ptrdiff_t) atomic_ptrdiff_t; 330typedef _Atomic(intmax_t) atomic_intmax_t; 331typedef _Atomic(uintmax_t) atomic_uintmax_t; 332 333/* 334 * 7.17.7 Operations on atomic types. 335 */ 336 337/* 338 * Compiler-specific operations. 339 */ 340 341#if defined(__CLANG_ATOMICS) 342#define atomic_compare_exchange_strong_explicit(object, expected, \ 343 desired, success, failure) \ 344 __c11_atomic_compare_exchange_strong(object, expected, desired, \ 345 success, failure) 346#define atomic_compare_exchange_weak_explicit(object, expected, \ 347 desired, success, failure) \ 348 __c11_atomic_compare_exchange_weak(object, expected, desired, \ 349 success, failure) 350#define atomic_exchange_explicit(object, desired, order) \ 351 __c11_atomic_exchange(object, desired, order) 352#define atomic_fetch_add_explicit(object, operand, order) \ 353 __c11_atomic_fetch_add(object, operand, order) 354#define atomic_fetch_and_explicit(object, operand, order) \ 355 __c11_atomic_fetch_and(object, operand, order) 356#define atomic_fetch_or_explicit(object, operand, order) \ 357 __c11_atomic_fetch_or(object, operand, order) 358#define atomic_fetch_sub_explicit(object, operand, order) \ 359 __c11_atomic_fetch_sub(object, operand, order) 360#define atomic_fetch_xor_explicit(object, operand, order) \ 361 __c11_atomic_fetch_xor(object, operand, order) 362#define atomic_load_explicit(object, order) \ 363 __c11_atomic_load(object, order) 364#define atomic_store_explicit(object, desired, order) \ 365 __c11_atomic_store(object, desired, order) 366#elif defined(__GNUC_ATOMICS) 367#define atomic_compare_exchange_strong_explicit(object, expected, \ 368 desired, success, failure) \ 369 __atomic_compare_exchange_n(&(object)->__val, expected, \ 370 desired, 0, success, failure) 371#define atomic_compare_exchange_weak_explicit(object, expected, \ 372 desired, success, failure) \ 373 __atomic_compare_exchange_n(&(object)->__val, expected, \ 374 desired, 1, success, failure) 375#define atomic_exchange_explicit(object, desired, order) \ 376 __atomic_exchange_n(&(object)->__val, desired, order) 377#define atomic_fetch_add_explicit(object, operand, order) \ 378 __atomic_fetch_add(&(object)->__val, operand, order) 379#define atomic_fetch_and_explicit(object, operand, order) \ 380 __atomic_fetch_and(&(object)->__val, operand, order) 381#define atomic_fetch_or_explicit(object, operand, order) \ 382 __atomic_fetch_or(&(object)->__val, operand, order) 383#define atomic_fetch_sub_explicit(object, operand, order) \ 384 __atomic_fetch_sub(&(object)->__val, operand, order) 385#define atomic_fetch_xor_explicit(object, operand, order) \ 386 __atomic_fetch_xor(&(object)->__val, operand, order) 387#define atomic_load_explicit(object, order) \ 388 __atomic_load_n(&(object)->__val, order) 389#define atomic_store_explicit(object, desired, order) \ 390 __atomic_store_n(&(object)->__val, desired, order) 391#else 392#define __atomic_apply_stride(object, operand) \ 393 (((__typeof__((object)->__val))0) + (operand)) 394#define atomic_compare_exchange_strong_explicit(object, expected, \ 395 desired, success, failure) __extension__ ({ \ 396 __typeof__(expected) __ep = (expected); \ 397 __typeof__(*__ep) __e = *__ep; \ 398 (void)(success); (void)(failure); \ 399 (bool)((*__ep = __sync_val_compare_and_swap(&(object)->__val, \ 400 __e, desired)) == __e); \ 401}) 402#define atomic_compare_exchange_weak_explicit(object, expected, \ 403 desired, success, failure) \ 404 atomic_compare_exchange_strong_explicit(object, expected, \ 405 desired, success, failure) 406#if __has_builtin(__sync_swap) 407/* Clang provides a full-barrier atomic exchange - use it if available. */ 408#define atomic_exchange_explicit(object, desired, order) \ 409 ((void)(order), __sync_swap(&(object)->__val, desired)) 410#else 411/* 412 * __sync_lock_test_and_set() is only an acquire barrier in theory (although in 413 * practice it is usually a full barrier) so we need an explicit barrier before 414 * it. 415 */ 416#define atomic_exchange_explicit(object, desired, order) \ 417__extension__ ({ \ 418 __typeof__(object) __o = (object); \ 419 __typeof__(desired) __d = (desired); \ 420 (void)(order); \ 421 __sync_synchronize(); \ 422 __sync_lock_test_and_set(&(__o)->__val, __d); \ 423}) 424#endif 425#define atomic_fetch_add_explicit(object, operand, order) \ 426 ((void)(order), __sync_fetch_and_add(&(object)->__val, \ 427 __atomic_apply_stride(object, operand))) 428#define atomic_fetch_and_explicit(object, operand, order) \ 429 ((void)(order), __sync_fetch_and_and(&(object)->__val, operand)) 430#define atomic_fetch_or_explicit(object, operand, order) \ 431 ((void)(order), __sync_fetch_and_or(&(object)->__val, operand)) 432#define atomic_fetch_sub_explicit(object, operand, order) \ 433 ((void)(order), __sync_fetch_and_sub(&(object)->__val, \ 434 __atomic_apply_stride(object, operand))) 435#define atomic_fetch_xor_explicit(object, operand, order) \ 436 ((void)(order), __sync_fetch_and_xor(&(object)->__val, operand)) 437#define atomic_load_explicit(object, order) \ 438 ((void)(order), __sync_fetch_and_add(&(object)->__val, 0)) 439#define atomic_store_explicit(object, desired, order) \ 440 ((void)atomic_exchange_explicit(object, desired, order)) 441#endif 442 443/* 444 * Convenience functions. 445 * 446 * Don't provide these in kernel space. In kernel space, we should be 447 * disciplined enough to always provide explicit barriers. 448 */ 449 450#ifndef _KERNEL 451#define atomic_compare_exchange_strong(object, expected, desired) \ 452 atomic_compare_exchange_strong_explicit(object, expected, \ 453 desired, memory_order_seq_cst, memory_order_seq_cst) 454#define atomic_compare_exchange_weak(object, expected, desired) \ 455 atomic_compare_exchange_weak_explicit(object, expected, \ 456 desired, memory_order_seq_cst, memory_order_seq_cst) 457#define atomic_exchange(object, desired) \ 458 atomic_exchange_explicit(object, desired, memory_order_seq_cst) 459#define atomic_fetch_add(object, operand) \ 460 atomic_fetch_add_explicit(object, operand, memory_order_seq_cst) 461#define atomic_fetch_and(object, operand) \ 462 atomic_fetch_and_explicit(object, operand, memory_order_seq_cst) 463#define atomic_fetch_or(object, operand) \ 464 atomic_fetch_or_explicit(object, operand, memory_order_seq_cst) 465#define atomic_fetch_sub(object, operand) \ 466 atomic_fetch_sub_explicit(object, operand, memory_order_seq_cst) 467#define atomic_fetch_xor(object, operand) \ 468 atomic_fetch_xor_explicit(object, operand, memory_order_seq_cst) 469#define atomic_load(object) \ 470 atomic_load_explicit(object, memory_order_seq_cst) 471#define atomic_store(object, desired) \ 472 atomic_store_explicit(object, desired, memory_order_seq_cst) 473#endif /* !_KERNEL */ 474 475/* 476 * 7.17.8 Atomic flag type and operations. 477 * 478 * XXX: Assume atomic_bool can be used as an atomic_flag. Is there some 479 * kind of compiler built-in type we could use? 480 */ 481 482typedef struct { 483 atomic_bool __flag; 484} atomic_flag; 485 486#define ATOMIC_FLAG_INIT { ATOMIC_VAR_INIT(false) } 487 488static __inline bool 489atomic_flag_test_and_set_explicit(volatile atomic_flag *__object, 490 memory_order __order) 491{ 492 return (atomic_exchange_explicit(&__object->__flag, 1, __order)); 493} 494 495static __inline void 496atomic_flag_clear_explicit(volatile atomic_flag *__object, memory_order __order) 497{ 498 499 atomic_store_explicit(&__object->__flag, 0, __order); 500} 501 502#ifndef _KERNEL 503static __inline bool 504atomic_flag_test_and_set(volatile atomic_flag *__object) 505{ 506 507 return (atomic_flag_test_and_set_explicit(__object, 508 memory_order_seq_cst)); 509} 510 511static __inline void 512atomic_flag_clear(volatile atomic_flag *__object) 513{ 514 515 atomic_flag_clear_explicit(__object, memory_order_seq_cst); 516} 517#endif /* !_KERNEL */ 518 519#endif /* <atomic> unavailable */ 520 521#endif /* !_STDATOMIC_H_ */ 522