12aff73868669a46a84ea617801beea47dda0b581David Chisnall/*===-- atomic.c - Implement support functions for atomic operations.------=== 22aff73868669a46a84ea617801beea47dda0b581David Chisnall * 32aff73868669a46a84ea617801beea47dda0b581David Chisnall * The LLVM Compiler Infrastructure 42aff73868669a46a84ea617801beea47dda0b581David Chisnall * 52aff73868669a46a84ea617801beea47dda0b581David Chisnall * This file is dual licensed under the MIT and the University of Illinois Open 62aff73868669a46a84ea617801beea47dda0b581David Chisnall * Source Licenses. See LICENSE.TXT for details. 72aff73868669a46a84ea617801beea47dda0b581David Chisnall * 82aff73868669a46a84ea617801beea47dda0b581David Chisnall *===----------------------------------------------------------------------=== 92aff73868669a46a84ea617801beea47dda0b581David Chisnall * 102aff73868669a46a84ea617801beea47dda0b581David Chisnall * atomic.c defines a set of functions for performing atomic accesses on 112aff73868669a46a84ea617801beea47dda0b581David Chisnall * arbitrary-sized memory locations. This design uses locks that should 122aff73868669a46a84ea617801beea47dda0b581David Chisnall * be fast in the uncontended case, for two reasons: 132aff73868669a46a84ea617801beea47dda0b581David Chisnall * 142aff73868669a46a84ea617801beea47dda0b581David Chisnall * 1) This code must work with C programs that do not link to anything 152aff73868669a46a84ea617801beea47dda0b581David Chisnall * (including pthreads) and so it should not depend on any pthread 162aff73868669a46a84ea617801beea47dda0b581David Chisnall * functions. 172aff73868669a46a84ea617801beea47dda0b581David Chisnall * 2) Atomic operations, rather than explicit mutexes, are most commonly used 182aff73868669a46a84ea617801beea47dda0b581David Chisnall * on code where contended operations are rate. 192aff73868669a46a84ea617801beea47dda0b581David Chisnall * 202aff73868669a46a84ea617801beea47dda0b581David Chisnall * To avoid needing a per-object lock, this code allocates an array of 212aff73868669a46a84ea617801beea47dda0b581David Chisnall * locks and hashes the object pointers to find the one that it should use. 222aff73868669a46a84ea617801beea47dda0b581David Chisnall * For operations that must be atomic on two locations, the lower lock is 232aff73868669a46a84ea617801beea47dda0b581David Chisnall * always acquired first, to avoid deadlock. 242aff73868669a46a84ea617801beea47dda0b581David Chisnall * 252aff73868669a46a84ea617801beea47dda0b581David Chisnall *===----------------------------------------------------------------------=== 262aff73868669a46a84ea617801beea47dda0b581David Chisnall */ 272aff73868669a46a84ea617801beea47dda0b581David Chisnall 282aff73868669a46a84ea617801beea47dda0b581David Chisnall#include <stdint.h> 292aff73868669a46a84ea617801beea47dda0b581David Chisnall#include <string.h> 302aff73868669a46a84ea617801beea47dda0b581David Chisnall 312aff73868669a46a84ea617801beea47dda0b581David Chisnall// Clang objects if you redefine a builtin. This little hack allows us to 322aff73868669a46a84ea617801beea47dda0b581David Chisnall// define a function with the same name as an intrinsic. 33e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik#if __APPLE__ 34e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik// mach-o has extra leading underscore 35e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik#pragma redefine_extname __atomic_load_c ___atomic_load 36e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik#pragma redefine_extname __atomic_store_c ___atomic_store 37e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik#pragma redefine_extname __atomic_exchange_c ___atomic_exchange 38e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik#pragma redefine_extname __atomic_compare_exchange_c ___atomic_compare_exchange 39e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik#else 40bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall#pragma redefine_extname __atomic_load_c __atomic_load 41bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall#pragma redefine_extname __atomic_store_c __atomic_store 42bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall#pragma redefine_extname __atomic_exchange_c __atomic_exchange 43bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall#pragma redefine_extname __atomic_compare_exchange_c __atomic_compare_exchange 44e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik#endif 452aff73868669a46a84ea617801beea47dda0b581David Chisnall 462aff73868669a46a84ea617801beea47dda0b581David Chisnall/// Number of locks. This allocates one page on 32-bit platforms, two on 472aff73868669a46a84ea617801beea47dda0b581David Chisnall/// 64-bit. This can be specified externally if a different trade between 482aff73868669a46a84ea617801beea47dda0b581David Chisnall/// memory usage and contention probability is required for a given platform. 492aff73868669a46a84ea617801beea47dda0b581David Chisnall#ifndef SPINLOCK_COUNT 502aff73868669a46a84ea617801beea47dda0b581David Chisnall#define SPINLOCK_COUNT (1<<10) 512aff73868669a46a84ea617801beea47dda0b581David Chisnall#endif 522aff73868669a46a84ea617801beea47dda0b581David Chisnallstatic const long SPINLOCK_MASK = SPINLOCK_COUNT - 1; 532aff73868669a46a84ea617801beea47dda0b581David Chisnall 542aff73868669a46a84ea617801beea47dda0b581David Chisnall//////////////////////////////////////////////////////////////////////////////// 552aff73868669a46a84ea617801beea47dda0b581David Chisnall// Platform-specific lock implementation. Falls back to spinlocks if none is 562aff73868669a46a84ea617801beea47dda0b581David Chisnall// defined. Each platform should define the Lock type, and corresponding 572aff73868669a46a84ea617801beea47dda0b581David Chisnall// lock() and unlock() functions. 582aff73868669a46a84ea617801beea47dda0b581David Chisnall//////////////////////////////////////////////////////////////////////////////// 592aff73868669a46a84ea617801beea47dda0b581David Chisnall#ifdef __FreeBSD__ 602aff73868669a46a84ea617801beea47dda0b581David Chisnall#include <errno.h> 612aff73868669a46a84ea617801beea47dda0b581David Chisnall#include <sys/types.h> 622aff73868669a46a84ea617801beea47dda0b581David Chisnall#include <machine/atomic.h> 632aff73868669a46a84ea617801beea47dda0b581David Chisnall#include <sys/umtx.h> 642aff73868669a46a84ea617801beea47dda0b581David Chisnalltypedef struct _usem Lock; 652aff73868669a46a84ea617801beea47dda0b581David Chisnallinline static void unlock(Lock *l) { 66bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall __c11_atomic_store((_Atomic(uint32_t)*)&l->_count, 1, __ATOMIC_RELEASE); 67bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); 682aff73868669a46a84ea617801beea47dda0b581David Chisnall if (l->_has_waiters) 692aff73868669a46a84ea617801beea47dda0b581David Chisnall _umtx_op(l, UMTX_OP_SEM_WAKE, 1, 0, 0); 702aff73868669a46a84ea617801beea47dda0b581David Chisnall} 712aff73868669a46a84ea617801beea47dda0b581David Chisnallinline static void lock(Lock *l) { 722aff73868669a46a84ea617801beea47dda0b581David Chisnall uint32_t old = 1; 73bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall while (!__c11_atomic_compare_exchange_weak((_Atomic(uint32_t)*)&l->_count, &old, 742aff73868669a46a84ea617801beea47dda0b581David Chisnall 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { 752aff73868669a46a84ea617801beea47dda0b581David Chisnall _umtx_op(l, UMTX_OP_SEM_WAIT, 0, 0, 0); 762aff73868669a46a84ea617801beea47dda0b581David Chisnall old = 1; 772aff73868669a46a84ea617801beea47dda0b581David Chisnall } 782aff73868669a46a84ea617801beea47dda0b581David Chisnall} 792aff73868669a46a84ea617801beea47dda0b581David Chisnall/// locks for atomic operations 802aff73868669a46a84ea617801beea47dda0b581David Chisnallstatic Lock locks[SPINLOCK_COUNT] = { [0 ... SPINLOCK_COUNT-1] = {0,1,0} }; 81e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik 82e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik#elif defined(__APPLE__) 83e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik#include <libkern/OSAtomic.h> 84e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledziktypedef OSSpinLock Lock; 85e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzikinline static void unlock(Lock *l) { 86e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik OSSpinLockUnlock(l); 87e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik} 88e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik/// Locks a lock. In the current implementation, this is potentially 89e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik/// unbounded in the contended case. 90e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzikinline static void lock(Lock *l) { 91e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik OSSpinLockLock(l); 92e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik} 93e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzikstatic Lock locks[SPINLOCK_COUNT]; // initialized to OS_SPINLOCK_INIT which is 0 94e04a1fdc5d173586408421144a0cd79d1767f918Nick Kledzik 952aff73868669a46a84ea617801beea47dda0b581David Chisnall#else 962aff73868669a46a84ea617801beea47dda0b581David Chisnalltypedef _Atomic(uintptr_t) Lock; 972aff73868669a46a84ea617801beea47dda0b581David Chisnall/// Unlock a lock. This is a release operation. 982aff73868669a46a84ea617801beea47dda0b581David Chisnallinline static void unlock(Lock *l) { 99bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall __c11_atomic_store(l, 0, __ATOMIC_RELEASE); 1002aff73868669a46a84ea617801beea47dda0b581David Chisnall} 1012aff73868669a46a84ea617801beea47dda0b581David Chisnall/// Locks a lock. In the current implementation, this is potentially 1022aff73868669a46a84ea617801beea47dda0b581David Chisnall/// unbounded in the contended case. 1032aff73868669a46a84ea617801beea47dda0b581David Chisnallinline static void lock(Lock *l) { 1042aff73868669a46a84ea617801beea47dda0b581David Chisnall uintptr_t old = 0; 105bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall while (!__c11_atomic_compare_exchange_weak(l, &old, 1, __ATOMIC_ACQUIRE, 1062aff73868669a46a84ea617801beea47dda0b581David Chisnall __ATOMIC_RELAXED)) 1072aff73868669a46a84ea617801beea47dda0b581David Chisnall old = 0; 1082aff73868669a46a84ea617801beea47dda0b581David Chisnall} 1092aff73868669a46a84ea617801beea47dda0b581David Chisnall/// locks for atomic operations 1102aff73868669a46a84ea617801beea47dda0b581David Chisnallstatic Lock locks[SPINLOCK_COUNT]; 1112aff73868669a46a84ea617801beea47dda0b581David Chisnall#endif 1122aff73868669a46a84ea617801beea47dda0b581David Chisnall 1132aff73868669a46a84ea617801beea47dda0b581David Chisnall 1142aff73868669a46a84ea617801beea47dda0b581David Chisnall/// Returns a lock to use for a given pointer. 1152aff73868669a46a84ea617801beea47dda0b581David Chisnallstatic inline Lock *lock_for_pointer(void *ptr) { 1162aff73868669a46a84ea617801beea47dda0b581David Chisnall intptr_t hash = (intptr_t)ptr; 1172aff73868669a46a84ea617801beea47dda0b581David Chisnall // Disregard the lowest 4 bits. We want all values that may be part of the 1182aff73868669a46a84ea617801beea47dda0b581David Chisnall // same memory operation to hash to the same value and therefore use the same 1192aff73868669a46a84ea617801beea47dda0b581David Chisnall // lock. 1202aff73868669a46a84ea617801beea47dda0b581David Chisnall hash >>= 4; 1212aff73868669a46a84ea617801beea47dda0b581David Chisnall // Use the next bits as the basis for the hash 1222aff73868669a46a84ea617801beea47dda0b581David Chisnall intptr_t low = hash & SPINLOCK_MASK; 1232aff73868669a46a84ea617801beea47dda0b581David Chisnall // Now use the high(er) set of bits to perturb the hash, so that we don't 1242aff73868669a46a84ea617801beea47dda0b581David Chisnall // get collisions from atomic fields in a single object 1252aff73868669a46a84ea617801beea47dda0b581David Chisnall hash >>= 16; 1262aff73868669a46a84ea617801beea47dda0b581David Chisnall hash ^= low; 1272aff73868669a46a84ea617801beea47dda0b581David Chisnall // Return a pointer to the word to use 1282aff73868669a46a84ea617801beea47dda0b581David Chisnall return locks + (hash & SPINLOCK_MASK); 1292aff73868669a46a84ea617801beea47dda0b581David Chisnall} 1302aff73868669a46a84ea617801beea47dda0b581David Chisnall 1312aff73868669a46a84ea617801beea47dda0b581David Chisnall/// Macros for determining whether a size is lock free. Clang can not yet 1322aff73868669a46a84ea617801beea47dda0b581David Chisnall/// codegen __atomic_is_lock_free(16), so for now we assume 16-byte values are 1332aff73868669a46a84ea617801beea47dda0b581David Chisnall/// not lock free. 134bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall#define IS_LOCK_FREE_1 __c11_atomic_is_lock_free(1) 135bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall#define IS_LOCK_FREE_2 __c11_atomic_is_lock_free(2) 136bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall#define IS_LOCK_FREE_4 __c11_atomic_is_lock_free(4) 137bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall#define IS_LOCK_FREE_8 __c11_atomic_is_lock_free(8) 1382aff73868669a46a84ea617801beea47dda0b581David Chisnall#define IS_LOCK_FREE_16 0 1392aff73868669a46a84ea617801beea47dda0b581David Chisnall 1402aff73868669a46a84ea617801beea47dda0b581David Chisnall/// Macro that calls the compiler-generated lock-free versions of functions 1412aff73868669a46a84ea617801beea47dda0b581David Chisnall/// when they exist. 1422aff73868669a46a84ea617801beea47dda0b581David Chisnall#define LOCK_FREE_CASES() \ 1432aff73868669a46a84ea617801beea47dda0b581David Chisnall do {\ 1442aff73868669a46a84ea617801beea47dda0b581David Chisnall switch (size) {\ 1452aff73868669a46a84ea617801beea47dda0b581David Chisnall case 2:\ 1462aff73868669a46a84ea617801beea47dda0b581David Chisnall if (IS_LOCK_FREE_2) {\ 1472aff73868669a46a84ea617801beea47dda0b581David Chisnall LOCK_FREE_ACTION(uint16_t);\ 1482aff73868669a46a84ea617801beea47dda0b581David Chisnall }\ 1492aff73868669a46a84ea617801beea47dda0b581David Chisnall case 4:\ 1502aff73868669a46a84ea617801beea47dda0b581David Chisnall if (IS_LOCK_FREE_4) {\ 1512aff73868669a46a84ea617801beea47dda0b581David Chisnall LOCK_FREE_ACTION(uint32_t);\ 1522aff73868669a46a84ea617801beea47dda0b581David Chisnall }\ 1532aff73868669a46a84ea617801beea47dda0b581David Chisnall case 8:\ 1542aff73868669a46a84ea617801beea47dda0b581David Chisnall if (IS_LOCK_FREE_8) {\ 1552aff73868669a46a84ea617801beea47dda0b581David Chisnall LOCK_FREE_ACTION(uint64_t);\ 1562aff73868669a46a84ea617801beea47dda0b581David Chisnall }\ 1572aff73868669a46a84ea617801beea47dda0b581David Chisnall case 16:\ 1582aff73868669a46a84ea617801beea47dda0b581David Chisnall if (IS_LOCK_FREE_16) {\ 159fa3eee4101f39661b4a88ac0bc7f184ca8c1ecb1Benjamin Kramer /* FIXME: __uint128_t isn't available on 32 bit platforms. 160fa3eee4101f39661b4a88ac0bc7f184ca8c1ecb1Benjamin Kramer LOCK_FREE_ACTION(__uint128_t);*/\ 1612aff73868669a46a84ea617801beea47dda0b581David Chisnall }\ 1622aff73868669a46a84ea617801beea47dda0b581David Chisnall }\ 1632aff73868669a46a84ea617801beea47dda0b581David Chisnall } while (0) 1642aff73868669a46a84ea617801beea47dda0b581David Chisnall 1652aff73868669a46a84ea617801beea47dda0b581David Chisnall 1662aff73868669a46a84ea617801beea47dda0b581David Chisnall/// An atomic load operation. This is atomic with respect to the source 1672aff73868669a46a84ea617801beea47dda0b581David Chisnall/// pointer only. 168bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnallvoid __atomic_load_c(int size, void *src, void *dest, int model) { 1692aff73868669a46a84ea617801beea47dda0b581David Chisnall#define LOCK_FREE_ACTION(type) \ 170bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall *((type*)dest) = __c11_atomic_load((_Atomic(type)*)src, model);\ 1712aff73868669a46a84ea617801beea47dda0b581David Chisnall return; 1722aff73868669a46a84ea617801beea47dda0b581David Chisnall LOCK_FREE_CASES(); 1732aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef LOCK_FREE_ACTION 1742aff73868669a46a84ea617801beea47dda0b581David Chisnall Lock *l = lock_for_pointer(src); 1752aff73868669a46a84ea617801beea47dda0b581David Chisnall lock(l); 1762aff73868669a46a84ea617801beea47dda0b581David Chisnall memcpy(dest, src, size); 1772aff73868669a46a84ea617801beea47dda0b581David Chisnall unlock(l); 1782aff73868669a46a84ea617801beea47dda0b581David Chisnall} 1792aff73868669a46a84ea617801beea47dda0b581David Chisnall 1802aff73868669a46a84ea617801beea47dda0b581David Chisnall/// An atomic store operation. This is atomic with respect to the destination 1812aff73868669a46a84ea617801beea47dda0b581David Chisnall/// pointer only. 182bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnallvoid __atomic_store_c(int size, void *dest, void *src, int model) { 1832aff73868669a46a84ea617801beea47dda0b581David Chisnall#define LOCK_FREE_ACTION(type) \ 184bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall __c11_atomic_store((_Atomic(type)*)dest, *(type*)dest, model);\ 1852aff73868669a46a84ea617801beea47dda0b581David Chisnall return; 1862aff73868669a46a84ea617801beea47dda0b581David Chisnall LOCK_FREE_CASES(); 1872aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef LOCK_FREE_ACTION 1882aff73868669a46a84ea617801beea47dda0b581David Chisnall Lock *l = lock_for_pointer(dest); 1892aff73868669a46a84ea617801beea47dda0b581David Chisnall lock(l); 1902aff73868669a46a84ea617801beea47dda0b581David Chisnall memcpy(dest, src, size); 1912aff73868669a46a84ea617801beea47dda0b581David Chisnall unlock(l); 1922aff73868669a46a84ea617801beea47dda0b581David Chisnall} 1932aff73868669a46a84ea617801beea47dda0b581David Chisnall 1942aff73868669a46a84ea617801beea47dda0b581David Chisnall/// Atomic compare and exchange operation. If the value at *ptr is identical 1952aff73868669a46a84ea617801beea47dda0b581David Chisnall/// to the value at *expected, then this copies value at *desired to *ptr. If 1962aff73868669a46a84ea617801beea47dda0b581David Chisnall/// they are not, then this stores the current value from *ptr in *expected. 1972aff73868669a46a84ea617801beea47dda0b581David Chisnall/// 1982aff73868669a46a84ea617801beea47dda0b581David Chisnall/// This function returns 1 if the exchange takes place or 0 if it fails. 199bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnallint __atomic_compare_exchange_c(int size, void *ptr, void *expected, 2002aff73868669a46a84ea617801beea47dda0b581David Chisnall void *desired, int success, int failure) { 2012aff73868669a46a84ea617801beea47dda0b581David Chisnall#define LOCK_FREE_ACTION(type) \ 202bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall return __c11_atomic_compare_exchange_strong((_Atomic(type)*)ptr, (type*)expected,\ 2032aff73868669a46a84ea617801beea47dda0b581David Chisnall *(type*)desired, success, failure) 2042aff73868669a46a84ea617801beea47dda0b581David Chisnall LOCK_FREE_CASES(); 2052aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef LOCK_FREE_ACTION 2062aff73868669a46a84ea617801beea47dda0b581David Chisnall Lock *l = lock_for_pointer(ptr); 2072aff73868669a46a84ea617801beea47dda0b581David Chisnall lock(l); 2082aff73868669a46a84ea617801beea47dda0b581David Chisnall if (memcmp(ptr, expected, size) == 0) { 2092aff73868669a46a84ea617801beea47dda0b581David Chisnall memcpy(ptr, desired, size); 2102aff73868669a46a84ea617801beea47dda0b581David Chisnall unlock(l); 2112aff73868669a46a84ea617801beea47dda0b581David Chisnall return 1; 2122aff73868669a46a84ea617801beea47dda0b581David Chisnall } 2132aff73868669a46a84ea617801beea47dda0b581David Chisnall memcpy(expected, ptr, size); 2142aff73868669a46a84ea617801beea47dda0b581David Chisnall unlock(l); 2152aff73868669a46a84ea617801beea47dda0b581David Chisnall return 0; 2162aff73868669a46a84ea617801beea47dda0b581David Chisnall} 2172aff73868669a46a84ea617801beea47dda0b581David Chisnall 2182aff73868669a46a84ea617801beea47dda0b581David Chisnall/// Performs an atomic exchange operation between two pointers. This is atomic 2192aff73868669a46a84ea617801beea47dda0b581David Chisnall/// with respect to the target address. 220bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnallvoid __atomic_exchange_c(int size, void *ptr, void *val, void *old, int model) { 2212aff73868669a46a84ea617801beea47dda0b581David Chisnall#define LOCK_FREE_ACTION(type) \ 222bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall *(type*)old = __c11_atomic_exchange((_Atomic(type)*)ptr, *(type*)val,\ 2232aff73868669a46a84ea617801beea47dda0b581David Chisnall model);\ 2242aff73868669a46a84ea617801beea47dda0b581David Chisnall return; 2252aff73868669a46a84ea617801beea47dda0b581David Chisnall LOCK_FREE_CASES(); 2262aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef LOCK_FREE_ACTION 2272aff73868669a46a84ea617801beea47dda0b581David Chisnall Lock *l = lock_for_pointer(ptr); 2282aff73868669a46a84ea617801beea47dda0b581David Chisnall lock(l); 2292aff73868669a46a84ea617801beea47dda0b581David Chisnall memcpy(old, ptr, size); 2302aff73868669a46a84ea617801beea47dda0b581David Chisnall memcpy(ptr, val, size); 2312aff73868669a46a84ea617801beea47dda0b581David Chisnall unlock(l); 2322aff73868669a46a84ea617801beea47dda0b581David Chisnall} 2332aff73868669a46a84ea617801beea47dda0b581David Chisnall 2342aff73868669a46a84ea617801beea47dda0b581David Chisnall//////////////////////////////////////////////////////////////////////////////// 2352aff73868669a46a84ea617801beea47dda0b581David Chisnall// Where the size is known at compile time, the compiler may emit calls to 2362aff73868669a46a84ea617801beea47dda0b581David Chisnall// specialised versions of the above functions. 2372aff73868669a46a84ea617801beea47dda0b581David Chisnall//////////////////////////////////////////////////////////////////////////////// 2382aff73868669a46a84ea617801beea47dda0b581David Chisnall#define OPTIMISED_CASES\ 2392aff73868669a46a84ea617801beea47dda0b581David Chisnall OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t)\ 2402aff73868669a46a84ea617801beea47dda0b581David Chisnall OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t)\ 2412aff73868669a46a84ea617801beea47dda0b581David Chisnall OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t)\ 2422aff73868669a46a84ea617801beea47dda0b581David Chisnall OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t)\ 243fa3eee4101f39661b4a88ac0bc7f184ca8c1ecb1Benjamin Kramer /* FIXME: __uint128_t isn't available on 32 bit platforms. 244fa3eee4101f39661b4a88ac0bc7f184ca8c1ecb1Benjamin Kramer OPTIMISED_CASE(16, IS_LOCK_FREE_16, __uint128_t)*/\ 2452aff73868669a46a84ea617801beea47dda0b581David Chisnall 2462aff73868669a46a84ea617801beea47dda0b581David Chisnall#define OPTIMISED_CASE(n, lockfree, type)\ 2472aff73868669a46a84ea617801beea47dda0b581David Chisnalltype __atomic_load_##n(type *src, int model) {\ 2482aff73868669a46a84ea617801beea47dda0b581David Chisnall if (lockfree)\ 249bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall return __c11_atomic_load((_Atomic(type)*)src, model);\ 2502aff73868669a46a84ea617801beea47dda0b581David Chisnall Lock *l = lock_for_pointer(src);\ 2512aff73868669a46a84ea617801beea47dda0b581David Chisnall lock(l);\ 2522aff73868669a46a84ea617801beea47dda0b581David Chisnall type val = *src;\ 2532aff73868669a46a84ea617801beea47dda0b581David Chisnall unlock(l);\ 2542aff73868669a46a84ea617801beea47dda0b581David Chisnall return val;\ 2552aff73868669a46a84ea617801beea47dda0b581David Chisnall} 2562aff73868669a46a84ea617801beea47dda0b581David ChisnallOPTIMISED_CASES 2572aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef OPTIMISED_CASE 2582aff73868669a46a84ea617801beea47dda0b581David Chisnall 2592aff73868669a46a84ea617801beea47dda0b581David Chisnall#define OPTIMISED_CASE(n, lockfree, type)\ 2602aff73868669a46a84ea617801beea47dda0b581David Chisnallvoid __atomic_store_##n(type *dest, type val, int model) {\ 2612aff73868669a46a84ea617801beea47dda0b581David Chisnall if (lockfree) {\ 262bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall __c11_atomic_store((_Atomic(type)*)dest, val, model);\ 2632aff73868669a46a84ea617801beea47dda0b581David Chisnall return;\ 2642aff73868669a46a84ea617801beea47dda0b581David Chisnall }\ 2652aff73868669a46a84ea617801beea47dda0b581David Chisnall Lock *l = lock_for_pointer(dest);\ 2662aff73868669a46a84ea617801beea47dda0b581David Chisnall lock(l);\ 2672aff73868669a46a84ea617801beea47dda0b581David Chisnall *dest = val;\ 2682aff73868669a46a84ea617801beea47dda0b581David Chisnall unlock(l);\ 2692aff73868669a46a84ea617801beea47dda0b581David Chisnall return;\ 2702aff73868669a46a84ea617801beea47dda0b581David Chisnall} 2712aff73868669a46a84ea617801beea47dda0b581David ChisnallOPTIMISED_CASES 2722aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef OPTIMISED_CASE 2732aff73868669a46a84ea617801beea47dda0b581David Chisnall 2742aff73868669a46a84ea617801beea47dda0b581David Chisnall#define OPTIMISED_CASE(n, lockfree, type)\ 2752aff73868669a46a84ea617801beea47dda0b581David Chisnalltype __atomic_exchange_##n(type *dest, type val, int model) {\ 2762aff73868669a46a84ea617801beea47dda0b581David Chisnall if (lockfree)\ 277bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall return __c11_atomic_exchange((_Atomic(type)*)dest, val, model);\ 2782aff73868669a46a84ea617801beea47dda0b581David Chisnall Lock *l = lock_for_pointer(dest);\ 2792aff73868669a46a84ea617801beea47dda0b581David Chisnall lock(l);\ 2802aff73868669a46a84ea617801beea47dda0b581David Chisnall type tmp = *dest;\ 2812aff73868669a46a84ea617801beea47dda0b581David Chisnall *dest = val;\ 2822aff73868669a46a84ea617801beea47dda0b581David Chisnall unlock(l);\ 2832aff73868669a46a84ea617801beea47dda0b581David Chisnall return tmp;\ 2842aff73868669a46a84ea617801beea47dda0b581David Chisnall} 2852aff73868669a46a84ea617801beea47dda0b581David ChisnallOPTIMISED_CASES 2862aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef OPTIMISED_CASE 2872aff73868669a46a84ea617801beea47dda0b581David Chisnall 2882aff73868669a46a84ea617801beea47dda0b581David Chisnall#define OPTIMISED_CASE(n, lockfree, type)\ 2892aff73868669a46a84ea617801beea47dda0b581David Chisnallint __atomic_compare_exchange_##n(type *ptr, type *expected, type desired,\ 2902aff73868669a46a84ea617801beea47dda0b581David Chisnall int success, int failure) {\ 2912aff73868669a46a84ea617801beea47dda0b581David Chisnall if (lockfree)\ 292bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall return __c11_atomic_compare_exchange_strong((_Atomic(type)*)ptr, expected, desired,\ 2932aff73868669a46a84ea617801beea47dda0b581David Chisnall success, failure);\ 2942aff73868669a46a84ea617801beea47dda0b581David Chisnall Lock *l = lock_for_pointer(ptr);\ 2952aff73868669a46a84ea617801beea47dda0b581David Chisnall lock(l);\ 2962aff73868669a46a84ea617801beea47dda0b581David Chisnall if (*ptr == *expected) {\ 2972aff73868669a46a84ea617801beea47dda0b581David Chisnall *ptr = desired;\ 2982aff73868669a46a84ea617801beea47dda0b581David Chisnall unlock(l);\ 2992aff73868669a46a84ea617801beea47dda0b581David Chisnall return 1;\ 3002aff73868669a46a84ea617801beea47dda0b581David Chisnall }\ 3012aff73868669a46a84ea617801beea47dda0b581David Chisnall *expected = *ptr;\ 3022aff73868669a46a84ea617801beea47dda0b581David Chisnall unlock(l);\ 3032aff73868669a46a84ea617801beea47dda0b581David Chisnall return 0;\ 3042aff73868669a46a84ea617801beea47dda0b581David Chisnall} 3052aff73868669a46a84ea617801beea47dda0b581David ChisnallOPTIMISED_CASES 3062aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef OPTIMISED_CASE 3072aff73868669a46a84ea617801beea47dda0b581David Chisnall 3082aff73868669a46a84ea617801beea47dda0b581David Chisnall//////////////////////////////////////////////////////////////////////////////// 3092aff73868669a46a84ea617801beea47dda0b581David Chisnall// Atomic read-modify-write operations for integers of various sizes. 3102aff73868669a46a84ea617801beea47dda0b581David Chisnall//////////////////////////////////////////////////////////////////////////////// 3112aff73868669a46a84ea617801beea47dda0b581David Chisnall#define ATOMIC_RMW(n, lockfree, type, opname, op) \ 3122aff73868669a46a84ea617801beea47dda0b581David Chisnalltype __atomic_fetch_##opname##_##n(type *ptr, type val, int model) {\ 3132aff73868669a46a84ea617801beea47dda0b581David Chisnall if (lockfree) \ 314bc7ab9ce95a703ef732beb4ca4d0ea8778a60fddDavid Chisnall return __c11_atomic_fetch_##opname((_Atomic(type)*)ptr, val, model);\ 3152aff73868669a46a84ea617801beea47dda0b581David Chisnall Lock *l = lock_for_pointer(ptr);\ 3162aff73868669a46a84ea617801beea47dda0b581David Chisnall lock(l);\ 3172aff73868669a46a84ea617801beea47dda0b581David Chisnall type tmp = *ptr;\ 3182aff73868669a46a84ea617801beea47dda0b581David Chisnall *ptr = tmp op val;\ 3192aff73868669a46a84ea617801beea47dda0b581David Chisnall unlock(l);\ 3202aff73868669a46a84ea617801beea47dda0b581David Chisnall return tmp;\ 3212aff73868669a46a84ea617801beea47dda0b581David Chisnall} 3222aff73868669a46a84ea617801beea47dda0b581David Chisnall 3232aff73868669a46a84ea617801beea47dda0b581David Chisnall#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, add, +) 3242aff73868669a46a84ea617801beea47dda0b581David ChisnallOPTIMISED_CASES 3252aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef OPTIMISED_CASE 3262aff73868669a46a84ea617801beea47dda0b581David Chisnall#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, sub, -) 3272aff73868669a46a84ea617801beea47dda0b581David ChisnallOPTIMISED_CASES 3282aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef OPTIMISED_CASE 3292aff73868669a46a84ea617801beea47dda0b581David Chisnall#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, and, &) 3302aff73868669a46a84ea617801beea47dda0b581David ChisnallOPTIMISED_CASES 3312aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef OPTIMISED_CASE 3322aff73868669a46a84ea617801beea47dda0b581David Chisnall#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, or, |) 3332aff73868669a46a84ea617801beea47dda0b581David ChisnallOPTIMISED_CASES 3342aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef OPTIMISED_CASE 3352aff73868669a46a84ea617801beea47dda0b581David Chisnall#define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, xor, ^) 3362aff73868669a46a84ea617801beea47dda0b581David ChisnallOPTIMISED_CASES 3372aff73868669a46a84ea617801beea47dda0b581David Chisnall#undef OPTIMISED_CASE 338