atomic.h revision b57d3ec571c6551231be62b7bf92c084a8c8291c
1/******************************************************************************/ 2#ifdef JEMALLOC_H_TYPES 3 4#endif /* JEMALLOC_H_TYPES */ 5/******************************************************************************/ 6#ifdef JEMALLOC_H_STRUCTS 7 8#endif /* JEMALLOC_H_STRUCTS */ 9/******************************************************************************/ 10#ifdef JEMALLOC_H_EXTERNS 11 12#define atomic_read_uint64(p) atomic_add_uint64(p, 0) 13#define atomic_read_uint32(p) atomic_add_uint32(p, 0) 14#define atomic_read_z(p) atomic_add_z(p, 0) 15#define atomic_read_u(p) atomic_add_u(p, 0) 16 17#endif /* JEMALLOC_H_EXTERNS */ 18/******************************************************************************/ 19#ifdef JEMALLOC_H_INLINES 20 21#ifndef JEMALLOC_ENABLE_INLINE 22uint64_t atomic_add_uint64(uint64_t *p, uint64_t x); 23uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); 24uint32_t atomic_add_uint32(uint32_t *p, uint32_t x); 25uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); 26size_t atomic_add_z(size_t *p, size_t x); 27size_t atomic_sub_z(size_t *p, size_t x); 28unsigned atomic_add_u(unsigned *p, unsigned x); 29unsigned atomic_sub_u(unsigned *p, unsigned x); 30#endif 31 32#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_)) 33/******************************************************************************/ 34/* 64-bit operations. */ 35#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) 36# ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 37JEMALLOC_INLINE uint64_t 38atomic_add_uint64(uint64_t *p, uint64_t x) 39{ 40 41 return (__sync_add_and_fetch(p, x)); 42} 43 44JEMALLOC_INLINE uint64_t 45atomic_sub_uint64(uint64_t *p, uint64_t x) 46{ 47 48 return (__sync_sub_and_fetch(p, x)); 49} 50#elif (defined(JEMALLOC_OSATOMIC)) 51JEMALLOC_INLINE uint64_t 52atomic_add_uint64(uint64_t *p, uint64_t x) 53{ 54 55 return (OSAtomicAdd64((int64_t)x, (int64_t *)p)); 56} 57 58JEMALLOC_INLINE uint64_t 59atomic_sub_uint64(uint64_t *p, uint64_t x) 60{ 61 62 return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); 63} 64# elif (defined(__amd64__) || defined(__x86_64__)) 65JEMALLOC_INLINE uint64_t 66atomic_add_uint64(uint64_t *p, uint64_t x) 67{ 68 69 asm volatile ( 70 "lock; xaddq %0, %1;" 71 : "+r" (x), "=m" (*p) /* Outputs. */ 72 : "m" (*p) /* Inputs. */ 73 ); 74 75 return (x); 76} 77 78JEMALLOC_INLINE uint64_t 79atomic_sub_uint64(uint64_t *p, uint64_t x) 80{ 81 82 x = (uint64_t)(-(int64_t)x); 83 asm volatile ( 84 "lock; xaddq %0, %1;" 85 : "+r" (x), "=m" (*p) /* Outputs. */ 86 : "m" (*p) /* Inputs. */ 87 ); 88 89 return (x); 90} 91# elif (defined(JEMALLOC_ATOMIC9)) 92JEMALLOC_INLINE uint64_t 93atomic_add_uint64(uint64_t *p, uint64_t x) 94{ 95 96 /* 97 * atomic_fetchadd_64() doesn't exist, but we only ever use this 98 * function on LP64 systems, so atomic_fetchadd_long() will do. 99 */ 100 assert(sizeof(uint64_t) == sizeof(unsigned long)); 101 102 return (atomic_fetchadd_long(p, (unsigned long)x) + x); 103} 104 105JEMALLOC_INLINE uint64_t 106atomic_sub_uint64(uint64_t *p, uint64_t x) 107{ 108 109 assert(sizeof(uint64_t) == sizeof(unsigned long)); 110 111 return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x); 112} 113# elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8)) 114JEMALLOC_INLINE uint64_t 115atomic_add_uint64(uint64_t *p, uint64_t x) 116{ 117 118 return (__sync_add_and_fetch(p, x)); 119} 120 121JEMALLOC_INLINE uint64_t 122atomic_sub_uint64(uint64_t *p, uint64_t x) 123{ 124 125 return (__sync_sub_and_fetch(p, x)); 126} 127# else 128# error "Missing implementation for 64-bit atomic operations" 129# endif 130#endif 131 132/******************************************************************************/ 133/* 32-bit operations. */ 134#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 135JEMALLOC_INLINE uint32_t 136atomic_add_uint32(uint32_t *p, uint32_t x) 137{ 138 139 return (__sync_add_and_fetch(p, x)); 140} 141 142JEMALLOC_INLINE uint32_t 143atomic_sub_uint32(uint32_t *p, uint32_t x) 144{ 145 146 return (__sync_sub_and_fetch(p, x)); 147} 148#elif (defined(JEMALLOC_OSATOMIC)) 149JEMALLOC_INLINE uint32_t 150atomic_add_uint32(uint32_t *p, uint32_t x) 151{ 152 153 return (OSAtomicAdd32((int32_t)x, (int32_t *)p)); 154} 155 156JEMALLOC_INLINE uint32_t 157atomic_sub_uint32(uint32_t *p, uint32_t x) 158{ 159 160 return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); 161} 162#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) 163JEMALLOC_INLINE uint32_t 164atomic_add_uint32(uint32_t *p, uint32_t x) 165{ 166 167 asm volatile ( 168 "lock; xaddl %0, %1;" 169 : "+r" (x), "=m" (*p) /* Outputs. */ 170 : "m" (*p) /* Inputs. */ 171 ); 172 173 return (x); 174} 175 176JEMALLOC_INLINE uint32_t 177atomic_sub_uint32(uint32_t *p, uint32_t x) 178{ 179 180 x = (uint32_t)(-(int32_t)x); 181 asm volatile ( 182 "lock; xaddl %0, %1;" 183 : "+r" (x), "=m" (*p) /* Outputs. */ 184 : "m" (*p) /* Inputs. */ 185 ); 186 187 return (x); 188} 189#elif (defined(JEMALLOC_ATOMIC9)) 190JEMALLOC_INLINE uint32_t 191atomic_add_uint32(uint32_t *p, uint32_t x) 192{ 193 194 return (atomic_fetchadd_32(p, x) + x); 195} 196 197JEMALLOC_INLINE uint32_t 198atomic_sub_uint32(uint32_t *p, uint32_t x) 199{ 200 201 return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x); 202} 203#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4)) 204JEMALLOC_INLINE uint32_t 205atomic_add_uint32(uint32_t *p, uint32_t x) 206{ 207 208 return (__sync_add_and_fetch(p, x)); 209} 210 211JEMALLOC_INLINE uint32_t 212atomic_sub_uint32(uint32_t *p, uint32_t x) 213{ 214 215 return (__sync_sub_and_fetch(p, x)); 216} 217#else 218# error "Missing implementation for 32-bit atomic operations" 219#endif 220 221/******************************************************************************/ 222/* size_t operations. */ 223JEMALLOC_INLINE size_t 224atomic_add_z(size_t *p, size_t x) 225{ 226 227#if (LG_SIZEOF_PTR == 3) 228 return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); 229#elif (LG_SIZEOF_PTR == 2) 230 return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); 231#endif 232} 233 234JEMALLOC_INLINE size_t 235atomic_sub_z(size_t *p, size_t x) 236{ 237 238#if (LG_SIZEOF_PTR == 3) 239 return ((size_t)atomic_add_uint64((uint64_t *)p, 240 (uint64_t)-((int64_t)x))); 241#elif (LG_SIZEOF_PTR == 2) 242 return ((size_t)atomic_add_uint32((uint32_t *)p, 243 (uint32_t)-((int32_t)x))); 244#endif 245} 246 247/******************************************************************************/ 248/* unsigned operations. */ 249JEMALLOC_INLINE unsigned 250atomic_add_u(unsigned *p, unsigned x) 251{ 252 253#if (LG_SIZEOF_INT == 3) 254 return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); 255#elif (LG_SIZEOF_INT == 2) 256 return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); 257#endif 258} 259 260JEMALLOC_INLINE unsigned 261atomic_sub_u(unsigned *p, unsigned x) 262{ 263 264#if (LG_SIZEOF_INT == 3) 265 return ((unsigned)atomic_add_uint64((uint64_t *)p, 266 (uint64_t)-((int64_t)x))); 267#elif (LG_SIZEOF_INT == 2) 268 return ((unsigned)atomic_add_uint32((uint32_t *)p, 269 (uint32_t)-((int32_t)x))); 270#endif 271} 272/******************************************************************************/ 273#endif 274 275#endif /* JEMALLOC_H_INLINES */ 276/******************************************************************************/ 277