1/* Copyright (c) 2006, Google Inc. 2 * All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions are 6 * met: 7 * 8 * * Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * * Redistributions in binary form must reproduce the above 11 * copyright notice, this list of conditions and the following disclaimer 12 * in the documentation and/or other materials provided with the 13 * distribution. 14 * * Neither the name of Google Inc. nor the names of its 15 * contributors may be used to endorse or promote products derived from 16 * this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31// Implementation of atomic operations for Mac OS X. This file should not 32// be included directly. Clients should instead include 33// "base/atomicops.h". 34 35#ifndef BASE_ATOMICOPS_INTERNALS_MACOSX_H_ 36#define BASE_ATOMICOPS_INTERNALS_MACOSX_H_ 37 38typedef int32_t Atomic32; 39 40// MacOS uses long for intptr_t, AtomicWord and Atomic32 are always different 41// on the Mac, even when they are the same size. Similarly, on __ppc64__, 42// AtomicWord and Atomic64 are always different. Thus, we need explicit 43// casting. 44#ifdef __LP64__ 45#define AtomicWordCastType base::subtle::Atomic64 46#else 47#define AtomicWordCastType Atomic32 48#endif 49 50#if defined(__LP64__) || defined(__i386__) 51#define BASE_HAS_ATOMIC64 1 // Use only in tests and base/atomic* 52#endif 53 54#include <libkern/OSAtomic.h> 55 56namespace base { 57namespace subtle { 58 59#if !defined(__LP64__) && defined(__ppc__) 60 61// The Mac 64-bit OSAtomic implementations are not available for 32-bit PowerPC, 62// while the underlying assembly instructions are available only some 63// implementations of PowerPC. 64 65// The following inline functions will fail with the error message at compile 66// time ONLY IF they are called. So it is safe to use this header if user 67// code only calls AtomicWord and Atomic32 operations. 68// 69// NOTE(vchen): Implementation notes to implement the atomic ops below may 70// be found in "PowerPC Virtual Environment Architecture, Book II, 71// Version 2.02", January 28, 2005, Appendix B, page 46. Unfortunately, 72// extra care must be taken to ensure data are properly 8-byte aligned, and 73// that data are returned correctly according to Mac OS X ABI specs. 74 75inline int64_t OSAtomicCompareAndSwap64( 76 int64_t oldValue, int64_t newValue, int64_t *theValue) { 77 __asm__ __volatile__( 78 "_OSAtomicCompareAndSwap64_not_supported_for_32_bit_ppc\n\t"); 79 return 0; 80} 81 82inline int64_t OSAtomicAdd64(int64_t theAmount, int64_t *theValue) { 83 __asm__ __volatile__( 84 "_OSAtomicAdd64_not_supported_for_32_bit_ppc\n\t"); 85 return 0; 86} 87 88inline int64_t OSAtomicCompareAndSwap64Barrier( 89 int64_t oldValue, int64_t newValue, int64_t *theValue) { 90 int64_t prev = OSAtomicCompareAndSwap64(oldValue, newValue, theValue); 91 OSMemoryBarrier(); 92 return prev; 93} 94 95inline int64_t OSAtomicAdd64Barrier( 96 int64_t theAmount, int64_t *theValue) { 97 int64_t new_val = OSAtomicAdd64(theAmount, theValue); 98 OSMemoryBarrier(); 99 return new_val; 100} 101#endif 102 103typedef int64_t Atomic64; 104 105inline void MemoryBarrier() { 106 OSMemoryBarrier(); 107} 108 109// 32-bit Versions. 110 111inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32 *ptr, 112 Atomic32 old_value, 113 Atomic32 new_value) { 114 Atomic32 prev_value; 115 do { 116 if (OSAtomicCompareAndSwap32(old_value, new_value, 117 const_cast<Atomic32*>(ptr))) { 118 return old_value; 119 } 120 prev_value = *ptr; 121 } while (prev_value == old_value); 122 return prev_value; 123} 124 125inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32 *ptr, 126 Atomic32 new_value) { 127 Atomic32 old_value; 128 do { 129 old_value = *ptr; 130 } while (!OSAtomicCompareAndSwap32(old_value, new_value, 131 const_cast<Atomic32*>(ptr))); 132 return old_value; 133} 134 135inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32 *ptr, 136 Atomic32 increment) { 137 return OSAtomicAdd32(increment, const_cast<Atomic32*>(ptr)); 138} 139 140inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32 *ptr, 141 Atomic32 increment) { 142 return OSAtomicAdd32Barrier(increment, const_cast<Atomic32*>(ptr)); 143} 144 145inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32 *ptr, 146 Atomic32 old_value, 147 Atomic32 new_value) { 148 Atomic32 prev_value; 149 do { 150 if (OSAtomicCompareAndSwap32Barrier(old_value, new_value, 151 const_cast<Atomic32*>(ptr))) { 152 return old_value; 153 } 154 prev_value = *ptr; 155 } while (prev_value == old_value); 156 return prev_value; 157} 158 159inline Atomic32 Release_CompareAndSwap(volatile Atomic32 *ptr, 160 Atomic32 old_value, 161 Atomic32 new_value) { 162 return Acquire_CompareAndSwap(ptr, old_value, new_value); 163} 164 165inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { 166 *ptr = value; 167} 168 169inline void Acquire_Store(volatile Atomic32 *ptr, Atomic32 value) { 170 *ptr = value; 171 MemoryBarrier(); 172} 173 174inline void Release_Store(volatile Atomic32 *ptr, Atomic32 value) { 175 MemoryBarrier(); 176 *ptr = value; 177} 178 179inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { 180 return *ptr; 181} 182 183inline Atomic32 Acquire_Load(volatile const Atomic32 *ptr) { 184 Atomic32 value = *ptr; 185 MemoryBarrier(); 186 return value; 187} 188 189inline Atomic32 Release_Load(volatile const Atomic32 *ptr) { 190 MemoryBarrier(); 191 return *ptr; 192} 193 194// 64-bit version 195 196inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64 *ptr, 197 Atomic64 old_value, 198 Atomic64 new_value) { 199 Atomic64 prev_value; 200 do { 201 if (OSAtomicCompareAndSwap64(old_value, new_value, 202 const_cast<Atomic64*>(ptr))) { 203 return old_value; 204 } 205 prev_value = *ptr; 206 } while (prev_value == old_value); 207 return prev_value; 208} 209 210inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64 *ptr, 211 Atomic64 new_value) { 212 Atomic64 old_value; 213 do { 214 old_value = *ptr; 215 } while (!OSAtomicCompareAndSwap64(old_value, new_value, 216 const_cast<Atomic64*>(ptr))); 217 return old_value; 218} 219 220inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64 *ptr, 221 Atomic64 increment) { 222 return OSAtomicAdd64(increment, const_cast<Atomic64*>(ptr)); 223} 224 225inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64 *ptr, 226 Atomic64 increment) { 227 return OSAtomicAdd64Barrier(increment, const_cast<Atomic64*>(ptr)); 228} 229 230inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64 *ptr, 231 Atomic64 old_value, 232 Atomic64 new_value) { 233 Atomic64 prev_value; 234 do { 235 if (OSAtomicCompareAndSwap64Barrier(old_value, new_value, 236 const_cast<Atomic64*>(ptr))) { 237 return old_value; 238 } 239 prev_value = *ptr; 240 } while (prev_value == old_value); 241 return prev_value; 242} 243 244inline Atomic64 Release_CompareAndSwap(volatile Atomic64 *ptr, 245 Atomic64 old_value, 246 Atomic64 new_value) { 247 // The lib kern interface does not distinguish between 248 // Acquire and Release memory barriers; they are equivalent. 249 return Acquire_CompareAndSwap(ptr, old_value, new_value); 250} 251 252#ifdef __LP64__ 253 254// 64-bit implementation on 64-bit platform 255 256inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { 257 *ptr = value; 258} 259 260inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) { 261 *ptr = value; 262 MemoryBarrier(); 263} 264 265inline void Release_Store(volatile Atomic64 *ptr, Atomic64 value) { 266 MemoryBarrier(); 267 *ptr = value; 268} 269 270inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { 271 return *ptr; 272} 273 274inline Atomic64 Acquire_Load(volatile const Atomic64 *ptr) { 275 Atomic64 value = *ptr; 276 MemoryBarrier(); 277 return value; 278} 279 280inline Atomic64 Release_Load(volatile const Atomic64 *ptr) { 281 MemoryBarrier(); 282 return *ptr; 283} 284 285#else 286 287// 64-bit implementation on 32-bit platform 288 289#if defined(__ppc__) 290 291inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { 292 __asm__ __volatile__( 293 "_NoBarrier_Store_not_supported_for_32_bit_ppc\n\t"); 294} 295 296inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { 297 __asm__ __volatile__( 298 "_NoBarrier_Load_not_supported_for_32_bit_ppc\n\t"); 299 return 0; 300} 301 302#elif defined(__i386__) 303 304inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { 305 __asm__ __volatile__("movq %1, %%mm0\n\t" // Use mmx reg for 64-bit atomic 306 "movq %%mm0, %0\n\t" // moves (ptr could be read-only) 307 "emms\n\t" // Reset FP registers 308 : "=m" (*ptr) 309 : "m" (value) 310 : // mark the FP stack and mmx registers as clobbered 311 "st", "st(1)", "st(2)", "st(3)", "st(4)", 312 "st(5)", "st(6)", "st(7)", "mm0", "mm1", 313 "mm2", "mm3", "mm4", "mm5", "mm6", "mm7"); 314 315} 316 317inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { 318 Atomic64 value; 319 __asm__ __volatile__("movq %1, %%mm0\n\t" // Use mmx reg for 64-bit atomic 320 "movq %%mm0, %0\n\t" // moves (ptr could be read-only) 321 "emms\n\t" // Reset FP registers 322 : "=m" (value) 323 : "m" (*ptr) 324 : // mark the FP stack and mmx registers as clobbered 325 "st", "st(1)", "st(2)", "st(3)", "st(4)", 326 "st(5)", "st(6)", "st(7)", "mm0", "mm1", 327 "mm2", "mm3", "mm4", "mm5", "mm6", "mm7"); 328 329 return value; 330} 331#endif 332 333 334inline void Acquire_Store(volatile Atomic64 *ptr, Atomic64 value) { 335 NoBarrier_Store(ptr, value); 336 MemoryBarrier(); 337} 338 339inline void Release_Store(volatile Atomic64 *ptr, Atomic64 value) { 340 MemoryBarrier(); 341 NoBarrier_Store(ptr, value); 342} 343 344inline Atomic64 Acquire_Load(volatile const Atomic64 *ptr) { 345 Atomic64 value = NoBarrier_Load(ptr); 346 MemoryBarrier(); 347 return value; 348} 349 350inline Atomic64 Release_Load(volatile const Atomic64 *ptr) { 351 MemoryBarrier(); 352 return NoBarrier_Load(ptr); 353} 354#endif // __LP64__ 355 356} // namespace base::subtle 357} // namespace base 358 359#endif // BASE_ATOMICOPS_INTERNALS_MACOSX_H_ 360