1/* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#ifndef ART_RUNTIME_ATOMIC_H_ 18#define ART_RUNTIME_ATOMIC_H_ 19 20#include <stdint.h> 21#include <atomic> 22#include <limits> 23#include <vector> 24 25#include "base/logging.h" 26#include "base/macros.h" 27 28namespace art { 29 30class Mutex; 31 32// QuasiAtomic encapsulates two separate facilities that we are 33// trying to move away from: "quasiatomic" 64 bit operations 34// and custom memory fences. For the time being, they remain 35// exposed. Clients should be converted to use either class Atomic 36// below whenever possible, and should eventually use C++11 atomics. 37// The two facilities that do not have a good C++11 analog are 38// ThreadFenceForConstructor and Atomic::*JavaData. 39// 40// NOTE: Two "quasiatomic" operations on the exact same memory address 41// are guaranteed to operate atomically with respect to each other, 42// but no guarantees are made about quasiatomic operations mixed with 43// non-quasiatomic operations on the same address, nor about 44// quasiatomic operations that are performed on partially-overlapping 45// memory. 46class QuasiAtomic { 47#if defined(__mips__) && !defined(__LP64__) 48 static constexpr bool kNeedSwapMutexes = true; 49#elif defined(__mips__) && defined(__LP64__) 50 // TODO - mips64 still need this for Cas64 ??? 51 static constexpr bool kNeedSwapMutexes = true; 52#else 53 static constexpr bool kNeedSwapMutexes = false; 54#endif 55 56 public: 57 static void Startup(); 58 59 static void Shutdown(); 60 61 // Reads the 64-bit value at "addr" without tearing. 62 static int64_t Read64(volatile const int64_t* addr) { 63 if (!kNeedSwapMutexes) { 64 int64_t value; 65#if defined(__LP64__) 66 value = *addr; 67#else 68#if defined(__arm__) 69#if defined(__ARM_FEATURE_LPAE) 70 // With LPAE support (such as Cortex-A15) then ldrd is defined not to tear. 71 __asm__ __volatile__("@ QuasiAtomic::Read64\n" 72 "ldrd %0, %H0, %1" 73 : "=r" (value) 74 : "m" (*addr)); 75#else 76 // Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary. 77 __asm__ __volatile__("@ QuasiAtomic::Read64\n" 78 "ldrexd %0, %H0, %1" 79 : "=r" (value) 80 : "Q" (*addr)); 81#endif 82#elif defined(__i386__) 83 __asm__ __volatile__( 84 "movq %1, %0\n" 85 : "=x" (value) 86 : "m" (*addr)); 87#else 88 LOG(FATAL) << "Unsupported architecture"; 89#endif 90#endif // defined(__LP64__) 91 return value; 92 } else { 93 return SwapMutexRead64(addr); 94 } 95 } 96 97 // Writes to the 64-bit value at "addr" without tearing. 98 static void Write64(volatile int64_t* addr, int64_t value) { 99 if (!kNeedSwapMutexes) { 100#if defined(__LP64__) 101 *addr = value; 102#else 103#if defined(__arm__) 104#if defined(__ARM_FEATURE_LPAE) 105 // If we know that ARM architecture has LPAE (such as Cortex-A15) strd is defined not to tear. 106 __asm__ __volatile__("@ QuasiAtomic::Write64\n" 107 "strd %1, %H1, %0" 108 : "=m"(*addr) 109 : "r" (value)); 110#else 111 // The write is done as a swap so that the cache-line is in the exclusive state for the store. 112 int64_t prev; 113 int status; 114 do { 115 __asm__ __volatile__("@ QuasiAtomic::Write64\n" 116 "ldrexd %0, %H0, %2\n" 117 "strexd %1, %3, %H3, %2" 118 : "=&r" (prev), "=&r" (status), "+Q"(*addr) 119 : "r" (value) 120 : "cc"); 121 } while (UNLIKELY(status != 0)); 122#endif 123#elif defined(__i386__) 124 __asm__ __volatile__( 125 "movq %1, %0" 126 : "=m" (*addr) 127 : "x" (value)); 128#else 129 LOG(FATAL) << "Unsupported architecture"; 130#endif 131#endif // defined(__LP64__) 132 } else { 133 SwapMutexWrite64(addr, value); 134 } 135 } 136 137 // Atomically compare the value at "addr" to "old_value", if equal replace it with "new_value" 138 // and return true. Otherwise, don't swap, and return false. 139 // This is fully ordered, i.e. it has C++11 memory_order_seq_cst 140 // semantics (assuming all other accesses use a mutex if this one does). 141 // This has "strong" semantics; if it fails then it is guaranteed that 142 // at some point during the execution of Cas64, *addr was not equal to 143 // old_value. 144 static bool Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { 145 if (!kNeedSwapMutexes) { 146 return __sync_bool_compare_and_swap(addr, old_value, new_value); 147 } else { 148 return SwapMutexCas64(old_value, new_value, addr); 149 } 150 } 151 152 // Does the architecture provide reasonable atomic long operations or do we fall back on mutexes? 153 static bool LongAtomicsUseMutexes() { 154 return kNeedSwapMutexes; 155 } 156 157 static void ThreadFenceAcquire() { 158 std::atomic_thread_fence(std::memory_order_acquire); 159 } 160 161 static void ThreadFenceRelease() { 162 std::atomic_thread_fence(std::memory_order_release); 163 } 164 165 static void ThreadFenceForConstructor() { 166 #if defined(__aarch64__) 167 __asm__ __volatile__("dmb ishst" : : : "memory"); 168 #else 169 std::atomic_thread_fence(std::memory_order_release); 170 #endif 171 } 172 173 static void ThreadFenceSequentiallyConsistent() { 174 std::atomic_thread_fence(std::memory_order_seq_cst); 175 } 176 177 private: 178 static Mutex* GetSwapMutex(const volatile int64_t* addr); 179 static int64_t SwapMutexRead64(volatile const int64_t* addr); 180 static void SwapMutexWrite64(volatile int64_t* addr, int64_t val); 181 static bool SwapMutexCas64(int64_t old_value, int64_t new_value, volatile int64_t* addr); 182 183 // We stripe across a bunch of different mutexes to reduce contention. 184 static constexpr size_t kSwapMutexCount = 32; 185 static std::vector<Mutex*>* gSwapMutexes; 186 187 DISALLOW_COPY_AND_ASSIGN(QuasiAtomic); 188}; 189 190template<typename T> 191class PACKED(sizeof(T)) Atomic : public std::atomic<T> { 192 public: 193 Atomic<T>() : std::atomic<T>(0) { } 194 195 explicit Atomic<T>(T value) : std::atomic<T>(value) { } 196 197 // Load from memory without ordering or synchronization constraints. 198 T LoadRelaxed() const { 199 return this->load(std::memory_order_relaxed); 200 } 201 202 // Word tearing allowed, but may race. 203 // TODO: Optimize? 204 // There has been some discussion of eventually disallowing word 205 // tearing for Java data loads. 206 T LoadJavaData() const { 207 return this->load(std::memory_order_relaxed); 208 } 209 210 // Load from memory with a total ordering. 211 // Corresponds exactly to a Java volatile load. 212 T LoadSequentiallyConsistent() const { 213 return this->load(std::memory_order_seq_cst); 214 } 215 216 // Store to memory without ordering or synchronization constraints. 217 void StoreRelaxed(T desired) { 218 this->store(desired, std::memory_order_relaxed); 219 } 220 221 // Word tearing allowed, but may race. 222 void StoreJavaData(T desired) { 223 this->store(desired, std::memory_order_relaxed); 224 } 225 226 // Store to memory with release ordering. 227 void StoreRelease(T desired) { 228 this->store(desired, std::memory_order_release); 229 } 230 231 // Store to memory with a total ordering. 232 void StoreSequentiallyConsistent(T desired) { 233 this->store(desired, std::memory_order_seq_cst); 234 } 235 236 // Atomically replace the value with desired value if it matches the expected value. 237 // Participates in total ordering of atomic operations. 238 bool CompareExchangeStrongSequentiallyConsistent(T expected_value, T desired_value) { 239 return this->compare_exchange_strong(expected_value, desired_value, std::memory_order_seq_cst); 240 } 241 242 // The same, except it may fail spuriously. 243 bool CompareExchangeWeakSequentiallyConsistent(T expected_value, T desired_value) { 244 return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_seq_cst); 245 } 246 247 // Atomically replace the value with desired value if it matches the expected value. Doesn't 248 // imply ordering or synchronization constraints. 249 bool CompareExchangeStrongRelaxed(T expected_value, T desired_value) { 250 return this->compare_exchange_strong(expected_value, desired_value, std::memory_order_relaxed); 251 } 252 253 // The same, except it may fail spuriously. 254 bool CompareExchangeWeakRelaxed(T expected_value, T desired_value) { 255 return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_relaxed); 256 } 257 258 // Atomically replace the value with desired value if it matches the expected value. Prior writes 259 // made to other memory locations by the thread that did the release become visible in this 260 // thread. 261 bool CompareExchangeWeakAcquire(T expected_value, T desired_value) { 262 return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_acquire); 263 } 264 265 // Atomically replace the value with desired value if it matches the expected value. prior writes 266 // to other memory locations become visible to the threads that do a consume or an acquire on the 267 // same location. 268 bool CompareExchangeWeakRelease(T expected_value, T desired_value) { 269 return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_release); 270 } 271 272 T FetchAndAddSequentiallyConsistent(const T value) { 273 return this->fetch_add(value, std::memory_order_seq_cst); // Return old_value. 274 } 275 276 T FetchAndSubSequentiallyConsistent(const T value) { 277 return this->fetch_sub(value, std::memory_order_seq_cst); // Return old value. 278 } 279 280 T FetchAndOrSequentiallyConsistent(const T value) { 281 return this->fetch_or(value, std::memory_order_seq_cst); // Return old_value. 282 } 283 284 T FetchAndAndSequentiallyConsistent(const T value) { 285 return this->fetch_and(value, std::memory_order_seq_cst); // Return old_value. 286 } 287 288 volatile T* Address() { 289 return reinterpret_cast<T*>(this); 290 } 291 292 static T MaxValue() { 293 return std::numeric_limits<T>::max(); 294 } 295}; 296 297typedef Atomic<int32_t> AtomicInteger; 298 299static_assert(sizeof(AtomicInteger) == sizeof(int32_t), "Weird AtomicInteger size"); 300static_assert(alignof(AtomicInteger) == alignof(int32_t), 301 "AtomicInteger alignment differs from that of underlyingtype"); 302static_assert(sizeof(Atomic<int64_t>) == sizeof(int64_t), "Weird Atomic<int64> size"); 303 304// Assert the alignment of 64-bit integers is 64-bit. This isn't true on certain 32-bit 305// architectures (e.g. x86-32) but we know that 64-bit integers here are arranged to be 8-byte 306// aligned. 307#if defined(__LP64__) 308 static_assert(alignof(Atomic<int64_t>) == alignof(int64_t), 309 "Atomic<int64> alignment differs from that of underlying type"); 310#endif 311 312} // namespace art 313 314#endif // ART_RUNTIME_ATOMIC_H_ 315