111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/*- 211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * Copyright (c) 2011 Ed Schouten <ed@FreeBSD.org> 311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * David Chisnall <theraven@FreeBSD.org> 411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * All rights reserved. 511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * Redistribution and use in source and binary forms, with or without 711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * modification, are permitted provided that the following conditions 811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * are met: 911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 1. Redistributions of source code must retain the above copyright 1011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * notice, this list of conditions and the following disclaimer. 1111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 2. Redistributions in binary form must reproduce the above copyright 1211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * notice, this list of conditions and the following disclaimer in the 1311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * documentation and/or other materials provided with the distribution. 1411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 1511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * SUCH DAMAGE. 2611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 2711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * $FreeBSD$ 2811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 2911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 3011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifndef _STDATOMIC_H_ 3111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define _STDATOMIC_H_ 3211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 3311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#include <sys/cdefs.h> 3411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 3511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 3611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#if defined(__cplusplus) && __cplusplus >= 201103L && defined(_USING_LIBCXX) 3711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# ifdef __clang__ 3811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# if __has_feature(cxx_atomic) 3911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# define _STDATOMIC_HAVE_ATOMIC 4011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# endif 4111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# else /* gcc */ 4211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# if __GNUC_PREREQ(4, 7) 4311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# define _STDATOMIC_HAVE_ATOMIC 4411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# endif 4511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# endif 4611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 4711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 4811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef _STDATOMIC_HAVE_ATOMIC 4911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 5011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* We have a usable C++ <atomic>; use it instead. */ 5111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 5211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#include <atomic> 5311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 5411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#undef _Atomic 5511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert /* Also defined by <atomic> for gcc. But not used in macros. */ 5611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert /* Also a clang intrinsic. */ 5711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert /* Should not be used by client code before this file is */ 5811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert /* included. The definitions in <atomic> themselves see */ 5911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert /* the old definition, as they should. */ 6011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert /* Client code sees the following definition. */ 6111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 6211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define _Atomic(t) std::atomic<t> 6311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 6411cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_is_lock_free; 6511cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_init; 6611cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_store; 6711cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_store_explicit; 6811cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_load; 6911cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_load_explicit; 7011cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_exchange; 7111cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_exchange_explicit; 7211cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_compare_exchange_strong; 7311cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_compare_exchange_strong_explicit; 7411cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_compare_exchange_weak; 7511cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_compare_exchange_weak_explicit; 7611cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_fetch_add; 7711cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_fetch_add_explicit; 7811cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_fetch_sub; 7911cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_fetch_sub_explicit; 8011cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_fetch_or; 8111cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_fetch_or_explicit; 8211cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_fetch_xor; 8311cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_fetch_xor_explicit; 8411cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_fetch_and; 8511cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_fetch_and_explicit; 8611cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_thread_fence; 8711cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_signal_fence; 8811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 8911cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::memory_order; 9011cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::memory_order_relaxed; 9111cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::memory_order_consume; 9211cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::memory_order_acquire; 9311cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::memory_order_release; 9411cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::memory_order_acq_rel; 9511cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::memory_order_seq_cst; 9611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 9711cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_bool; 9811cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_char; 9911cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_schar; 10011cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_uchar; 10111cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_short; 10211cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_ushort; 10311cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_int; 10411cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_uint; 10511cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_long; 10611cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_ulong; 10711cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_llong; 10811cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_ullong; 10911cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_char16_t; 11011cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_char32_t; 11111cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_wchar_t; 11211cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_int_least8_t; 11311cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_uint_least8_t; 11411cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_int_least16_t; 11511cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_uint_least16_t; 11611cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_int_least32_t; 11711cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_uint_least32_t; 11811cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_int_least64_t; 11911cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_uint_least64_t; 12011cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_int_fast8_t; 12111cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_uint_fast8_t; 12211cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_int_fast16_t; 12311cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_uint_fast16_t; 12411cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_int_fast32_t; 12511cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_uint_fast32_t; 12611cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_int_fast64_t; 12711cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_uint_fast64_t; 12811cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_intptr_t; 12911cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_uintptr_t; 13011cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_size_t; 13111cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_ptrdiff_t; 13211cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_intmax_t; 13311cd02dfb91661c65134cac258cf5924270e9d2Dan Albertusing std::atomic_uintmax_t; 13411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 13511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#else /* <atomic> unavailable, possibly because this is C, not C++ */ 13611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 13711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#include <sys/types.h> 13811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#include <stdbool.h> 13911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 14011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 14111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * C: Do it ourselves. 14211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * Note that the runtime representation defined here should be compatible 14311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * with the C++ one, i.e. an _Atomic(T) needs to contain the same 14411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * bits as a T. 14511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 14611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 14711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#include <stddef.h> /* For ptrdiff_t. */ 14811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#include <stdint.h> /* TODO: Should pollute namespace less. */ 14911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#if __STDC_VERSION__ >= 201112L 15011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# include <uchar.h> /* For char16_t and char32_t. */ 15111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 15211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 15311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __clang__ 15411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# if __has_extension(c_atomic) || __has_extension(cxx_atomic) 15511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# define __CLANG_ATOMICS 15611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# else 15711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# error "stdatomic.h does not support your compiler" 15811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# endif 15911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# if __has_builtin(__sync_swap) 16011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# define __HAS_BUILTIN_SYNC_SWAP 16111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# endif 16211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#else 16311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# if __GNUC_PREREQ(4, 7) 16411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# define __GNUC_ATOMICS 16511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# else 16611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# define __SYNC_ATOMICS 16711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# ifdef __cplusplus 16811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# define __ATOMICS_AVOID_DOT_INIT 16911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# endif 17011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert# endif 17111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 17211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 17311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 17411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 7.17.1 Atomic lock-free macros. 17511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 17611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 17711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __GCC_ATOMIC_BOOL_LOCK_FREE 17811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE 17911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__SYNC_ATOMICS) 18011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_BOOL_LOCK_FREE 2 /* For all modern platforms */ 18111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 18211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __GCC_ATOMIC_CHAR_LOCK_FREE 18311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE 18411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__SYNC_ATOMICS) 18511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_CHAR_LOCK_FREE 2 18611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 18711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __GCC_ATOMIC_CHAR16_T_LOCK_FREE 18811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE 18911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__SYNC_ATOMICS) 19011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_CHAR16_T_LOCK_FREE 2 19111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 19211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __GCC_ATOMIC_CHAR32_T_LOCK_FREE 19311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE 19411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__SYNC_ATOMICS) 19511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_CHAR32_T_LOCK_FREE 2 19611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 19711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __GCC_ATOMIC_WCHAR_T_LOCK_FREE 19811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE 19911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__SYNC_ATOMICS) 20011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_WCHAR_T_LOCK_FREE 2 20111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 20211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __GCC_ATOMIC_SHORT_LOCK_FREE 20311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE 20411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__SYNC_ATOMICS) 20511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_SHORT_LOCK_FREE 2 20611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 20711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __GCC_ATOMIC_INT_LOCK_FREE 20811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE 20911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__SYNC_ATOMICS) 21011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_INT_LOCK_FREE 2 21111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 21211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __GCC_ATOMIC_LONG_LOCK_FREE 21311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE 21411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__SYNC_ATOMICS) 21511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_LONG_LOCK_FREE 2 21611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 21711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __GCC_ATOMIC_LLONG_LOCK_FREE 21811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE 21911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__SYNC_ATOMICS) 22011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_LLONG_LOCK_FREE 1 /* maybe */ 22111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 22211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __GCC_ATOMIC_POINTER_LOCK_FREE 22311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE 22411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__SYNC_ATOMICS) 22511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_POINTER_LOCK_FREE 2 22611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 22711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 22811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 22911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 7.17.2 Initialization. 23011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 23111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 23211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#if defined(__CLANG_ATOMICS) 23311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_VAR_INIT(value) (value) 23411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_init(obj, value) __c11_atomic_init(obj, value) 23511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#else 23611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __ATOMICS_AVOID_DOT_INIT 23711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_VAR_INIT(value) { value } 23811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#else 23911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_VAR_INIT(value) { .__val = (value) } 24011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 24111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_init(obj, value) ((void)((obj)->__val = (value))) 24211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 24311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 24411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 24511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * Clang and recent GCC both provide predefined macros for the memory 24611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * orderings. If we are using a compiler that doesn't define them, use the 24711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * clang values - these will be ignored in the fallback path. 24811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 24911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 25011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifndef __ATOMIC_RELAXED 25111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define __ATOMIC_RELAXED 0 25211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 25311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifndef __ATOMIC_CONSUME 25411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define __ATOMIC_CONSUME 1 25511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 25611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifndef __ATOMIC_ACQUIRE 25711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define __ATOMIC_ACQUIRE 2 25811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 25911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifndef __ATOMIC_RELEASE 26011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define __ATOMIC_RELEASE 3 26111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 26211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifndef __ATOMIC_ACQ_REL 26311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define __ATOMIC_ACQ_REL 4 26411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 26511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifndef __ATOMIC_SEQ_CST 26611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define __ATOMIC_SEQ_CST 5 26711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 26811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 26911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 27011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 7.17.3 Order and consistency. 27111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 27211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * The memory_order_* constants that denote the barrier behaviour of the 27311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * atomic operations. 27411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * The enum values must be identical to those used by the 27511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * C++ <atomic> header. 27611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 27711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 27811cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef enum { 27911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert memory_order_relaxed = __ATOMIC_RELAXED, 28011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert memory_order_consume = __ATOMIC_CONSUME, 28111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert memory_order_acquire = __ATOMIC_ACQUIRE, 28211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert memory_order_release = __ATOMIC_RELEASE, 28311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert memory_order_acq_rel = __ATOMIC_ACQ_REL, 28411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert memory_order_seq_cst = __ATOMIC_SEQ_CST 28511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert} memory_order; 28611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 28711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 28811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 7.17.4 Fences. 28911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 29011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 29111cd02dfb91661c65134cac258cf5924270e9d2Dan Albertstatic __inline void 29211cd02dfb91661c65134cac258cf5924270e9d2Dan Albertatomic_thread_fence(memory_order __order __attribute__((unused))) 29311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert{ 29411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 29511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __CLANG_ATOMICS 29611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_thread_fence(__order); 29711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__GNUC_ATOMICS) 29811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_thread_fence(__order); 29911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#else 30011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __sync_synchronize(); 30111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 30211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert} 30311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 30411cd02dfb91661c65134cac258cf5924270e9d2Dan Albertstatic __inline void 30511cd02dfb91661c65134cac258cf5924270e9d2Dan Albertatomic_signal_fence(memory_order __order __attribute__((unused))) 30611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert{ 30711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 30811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __CLANG_ATOMICS 30911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_signal_fence(__order); 31011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__GNUC_ATOMICS) 31111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_signal_fence(__order); 31211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#else 31311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __asm volatile ("" ::: "memory"); 31411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 31511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert} 31611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 31711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 31811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 7.17.5 Lock-free property. 31911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 32011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 32111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#if defined(_KERNEL) 32211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* Atomics in kernelspace are always lock-free. */ 32311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_is_lock_free(obj) \ 32411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert ((void)(obj), (_Bool)1) 32511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__CLANG_ATOMICS) 32611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_is_lock_free(obj) \ 32711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_is_lock_free(sizeof(*(obj))) 32811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__GNUC_ATOMICS) 32911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_is_lock_free(obj) \ 33011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_is_lock_free(sizeof((obj)->__val), &(obj)->__val) 33111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#else 33211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_is_lock_free(obj) \ 33311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert ((void)(obj), sizeof((obj)->__val) <= sizeof(void *)) 33411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 33511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 33611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 33711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 7.17.6 Atomic integer types. 33811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 33911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 34011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifndef __CLANG_ATOMICS 34111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 34211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * No native support for _Atomic(). Place object in structure to prevent 34311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * most forms of direct non-atomic access. 34411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 34511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define _Atomic(T) struct { T volatile __val; } 34611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 34711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 34811cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(bool) atomic_bool; 34911cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(char) atomic_char; 35011cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(signed char) atomic_schar; 35111cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(unsigned char) atomic_uchar; 35211cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(short) atomic_short; 35311cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(unsigned short) atomic_ushort; 35411cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(int) atomic_int; 35511cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(unsigned int) atomic_uint; 35611cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(long) atomic_long; 35711cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(unsigned long) atomic_ulong; 35811cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(long long) atomic_llong; 35911cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(unsigned long long) atomic_ullong; 36011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#if __STDC_VERSION__ >= 201112L || __cplusplus >= 201103L 36111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert typedef _Atomic(char16_t) atomic_char16_t; 36211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert typedef _Atomic(char32_t) atomic_char32_t; 36311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 36411cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(wchar_t) atomic_wchar_t; 36511cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(int_least8_t) atomic_int_least8_t; 36611cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(uint_least8_t) atomic_uint_least8_t; 36711cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(int_least16_t) atomic_int_least16_t; 36811cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(uint_least16_t) atomic_uint_least16_t; 36911cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(int_least32_t) atomic_int_least32_t; 37011cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(uint_least32_t) atomic_uint_least32_t; 37111cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(int_least64_t) atomic_int_least64_t; 37211cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(uint_least64_t) atomic_uint_least64_t; 37311cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(int_fast8_t) atomic_int_fast8_t; 37411cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(uint_fast8_t) atomic_uint_fast8_t; 37511cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(int_fast16_t) atomic_int_fast16_t; 37611cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(uint_fast16_t) atomic_uint_fast16_t; 37711cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(int_fast32_t) atomic_int_fast32_t; 37811cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(uint_fast32_t) atomic_uint_fast32_t; 37911cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(int_fast64_t) atomic_int_fast64_t; 38011cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(uint_fast64_t) atomic_uint_fast64_t; 38111cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(intptr_t) atomic_intptr_t; 38211cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(uintptr_t) atomic_uintptr_t; 38311cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(size_t) atomic_size_t; 38411cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(ptrdiff_t) atomic_ptrdiff_t; 38511cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(intmax_t) atomic_intmax_t; 38611cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef _Atomic(uintmax_t) atomic_uintmax_t; 38711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 38811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 38911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 7.17.7 Operations on atomic types. 39011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 39111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 39211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 39311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * Compiler-specific operations. 39411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 39511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 39611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#if defined(__CLANG_ATOMICS) 39711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_compare_exchange_strong_explicit(object, expected, \ 39811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert desired, success, failure) \ 39911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_compare_exchange_strong(object, expected, desired, \ 40011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert success, failure) 40111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_compare_exchange_weak_explicit(object, expected, \ 40211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert desired, success, failure) \ 40311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_compare_exchange_weak(object, expected, desired, \ 40411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert success, failure) 40511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_exchange_explicit(object, desired, order) \ 40611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_exchange(object, desired, order) 40711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_add_explicit(object, operand, order) \ 40811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_fetch_add(object, operand, order) 40911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_and_explicit(object, operand, order) \ 41011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_fetch_and(object, operand, order) 41111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_or_explicit(object, operand, order) \ 41211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_fetch_or(object, operand, order) 41311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_sub_explicit(object, operand, order) \ 41411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_fetch_sub(object, operand, order) 41511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_xor_explicit(object, operand, order) \ 41611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_fetch_xor(object, operand, order) 41711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_load_explicit(object, order) \ 41811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_load(object, order) 41911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_store_explicit(object, desired, order) \ 42011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __c11_atomic_store(object, desired, order) 42111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#elif defined(__GNUC_ATOMICS) 42211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_compare_exchange_strong_explicit(object, expected, \ 42311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert desired, success, failure) \ 42411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_compare_exchange_n(&(object)->__val, expected, \ 42511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert desired, 0, success, failure) 42611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_compare_exchange_weak_explicit(object, expected, \ 42711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert desired, success, failure) \ 42811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_compare_exchange_n(&(object)->__val, expected, \ 42911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert desired, 1, success, failure) 43011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_exchange_explicit(object, desired, order) \ 43111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_exchange_n(&(object)->__val, desired, order) 43211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_add_explicit(object, operand, order) \ 43311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_fetch_add(&(object)->__val, operand, order) 43411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_and_explicit(object, operand, order) \ 43511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_fetch_and(&(object)->__val, operand, order) 43611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_or_explicit(object, operand, order) \ 43711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_fetch_or(&(object)->__val, operand, order) 43811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_sub_explicit(object, operand, order) \ 43911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_fetch_sub(&(object)->__val, operand, order) 44011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_xor_explicit(object, operand, order) \ 44111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_fetch_xor(&(object)->__val, operand, order) 44211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_load_explicit(object, order) \ 44311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_load_n(&(object)->__val, order) 44411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_store_explicit(object, desired, order) \ 44511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_store_n(&(object)->__val, desired, order) 44611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#else 44711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define __atomic_apply_stride(object, operand) \ 44811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert (((__typeof__((object)->__val))0) + (operand)) 44911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_compare_exchange_strong_explicit(object, expected, \ 45011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert desired, success, failure) __extension__ ({ \ 45111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __typeof__(expected) __ep = (expected); \ 45211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __typeof__(*__ep) __e = *__ep; \ 45311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert (void)(success); (void)(failure); \ 45411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert (bool)((*__ep = __sync_val_compare_and_swap(&(object)->__val, \ 45511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __e, desired)) == __e); \ 45611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert}) 45711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_compare_exchange_weak_explicit(object, expected, \ 45811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert desired, success, failure) \ 45911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_compare_exchange_strong_explicit(object, expected, \ 46011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert desired, success, failure) 46111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifdef __HAS_BUILTIN_SYNC_SWAP 46211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* Clang provides a full-barrier atomic exchange - use it if available. */ 46311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_exchange_explicit(object, desired, order) \ 46411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert ((void)(order), __sync_swap(&(object)->__val, desired)) 46511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#else 46611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 46711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * __sync_lock_test_and_set() is only an acquire barrier in theory (although in 46811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * practice it is usually a full barrier) so we need an explicit barrier before 46911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * it. 47011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 47111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_exchange_explicit(object, desired, order) \ 47211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert__extension__ ({ \ 47311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __typeof__(object) __o = (object); \ 47411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __typeof__(desired) __d = (desired); \ 47511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert (void)(order); \ 47611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __sync_synchronize(); \ 47711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __sync_lock_test_and_set(&(__o)->__val, __d); \ 47811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert}) 47911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 48011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_add_explicit(object, operand, order) \ 48111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert ((void)(order), __sync_fetch_and_add(&(object)->__val, \ 48211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_apply_stride(object, operand))) 48311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_and_explicit(object, operand, order) \ 48411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert ((void)(order), __sync_fetch_and_and(&(object)->__val, operand)) 48511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_or_explicit(object, operand, order) \ 48611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert ((void)(order), __sync_fetch_and_or(&(object)->__val, operand)) 48711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_sub_explicit(object, operand, order) \ 48811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert ((void)(order), __sync_fetch_and_sub(&(object)->__val, \ 48911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert __atomic_apply_stride(object, operand))) 49011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_xor_explicit(object, operand, order) \ 49111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert ((void)(order), __sync_fetch_and_xor(&(object)->__val, operand)) 49211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_load_explicit(object, order) \ 49311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert ((void)(order), __sync_fetch_and_add(&(object)->__val, 0)) 49411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_store_explicit(object, desired, order) \ 49511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert ((void)atomic_exchange_explicit(object, desired, order)) 49611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif 49711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 49811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 49911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * Convenience functions. 50011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 50111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * Don't provide these in kernel space. In kernel space, we should be 50211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * disciplined enough to always provide explicit barriers. 50311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 50411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 50511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifndef _KERNEL 50611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_compare_exchange_strong(object, expected, desired) \ 50711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_compare_exchange_strong_explicit(object, expected, \ 50811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert desired, memory_order_seq_cst, memory_order_seq_cst) 50911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_compare_exchange_weak(object, expected, desired) \ 51011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_compare_exchange_weak_explicit(object, expected, \ 51111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert desired, memory_order_seq_cst, memory_order_seq_cst) 51211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_exchange(object, desired) \ 51311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_exchange_explicit(object, desired, memory_order_seq_cst) 51411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_add(object, operand) \ 51511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_fetch_add_explicit(object, operand, memory_order_seq_cst) 51611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_and(object, operand) \ 51711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_fetch_and_explicit(object, operand, memory_order_seq_cst) 51811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_or(object, operand) \ 51911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_fetch_or_explicit(object, operand, memory_order_seq_cst) 52011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_sub(object, operand) \ 52111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_fetch_sub_explicit(object, operand, memory_order_seq_cst) 52211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_fetch_xor(object, operand) \ 52311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_fetch_xor_explicit(object, operand, memory_order_seq_cst) 52411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_load(object) \ 52511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_load_explicit(object, memory_order_seq_cst) 52611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define atomic_store(object, desired) \ 52711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_store_explicit(object, desired, memory_order_seq_cst) 52811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif /* !_KERNEL */ 52911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 53011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert/* 53111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 7.17.8 Atomic flag type and operations. 53211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * 53311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * XXX: Assume atomic_bool can be used as an atomic_flag. Is there some 53411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert * kind of compiler built-in type we could use? 53511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert */ 53611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 53711cd02dfb91661c65134cac258cf5924270e9d2Dan Alberttypedef struct { 53811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_bool __flag; 53911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert} atomic_flag; 54011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 54111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#define ATOMIC_FLAG_INIT { ATOMIC_VAR_INIT(false) } 54211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 54311cd02dfb91661c65134cac258cf5924270e9d2Dan Albertstatic __inline bool 54411cd02dfb91661c65134cac258cf5924270e9d2Dan Albertatomic_flag_test_and_set_explicit(volatile atomic_flag *__object, 54511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert memory_order __order) 54611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert{ 54711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert return (atomic_exchange_explicit(&__object->__flag, 1, __order)); 54811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert} 54911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 55011cd02dfb91661c65134cac258cf5924270e9d2Dan Albertstatic __inline void 55111cd02dfb91661c65134cac258cf5924270e9d2Dan Albertatomic_flag_clear_explicit(volatile atomic_flag *__object, memory_order __order) 55211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert{ 55311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 55411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_store_explicit(&__object->__flag, 0, __order); 55511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert} 55611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 55711cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#ifndef _KERNEL 55811cd02dfb91661c65134cac258cf5924270e9d2Dan Albertstatic __inline bool 55911cd02dfb91661c65134cac258cf5924270e9d2Dan Albertatomic_flag_test_and_set(volatile atomic_flag *__object) 56011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert{ 56111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 56211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert return (atomic_flag_test_and_set_explicit(__object, 56311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert memory_order_seq_cst)); 56411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert} 56511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 56611cd02dfb91661c65134cac258cf5924270e9d2Dan Albertstatic __inline void 56711cd02dfb91661c65134cac258cf5924270e9d2Dan Albertatomic_flag_clear(volatile atomic_flag *__object) 56811cd02dfb91661c65134cac258cf5924270e9d2Dan Albert{ 56911cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 57011cd02dfb91661c65134cac258cf5924270e9d2Dan Albert atomic_flag_clear_explicit(__object, memory_order_seq_cst); 57111cd02dfb91661c65134cac258cf5924270e9d2Dan Albert} 57211cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif /* !_KERNEL */ 57311cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 57411cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif /* <atomic> unavailable */ 57511cd02dfb91661c65134cac258cf5924270e9d2Dan Albert 57611cd02dfb91661c65134cac258cf5924270e9d2Dan Albert#endif /* !_STDATOMIC_H_ */ 577