atomic.h revision 63c5dd056fa20993b35ec5c8548b26c988445763
15ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes/* 25ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Copyright (C) 2008 The Android Open Source Project 35ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 45ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Licensed under the Apache License, Version 2.0 (the "License"); 55ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * you may not use this file except in compliance with the License. 65ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * You may obtain a copy of the License at 75ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 85ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * http://www.apache.org/licenses/LICENSE-2.0 95ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 105ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Unless required by applicable law or agreed to in writing, software 115ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * distributed under the License is distributed on an "AS IS" BASIS, 125ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 135ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * See the License for the specific language governing permissions and 145ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * limitations under the License. 155ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes */ 165ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 17fc0e3219edc9a5bf81b166e82fd5db2796eb6a0dBrian Carlstrom#ifndef ART_RUNTIME_ATOMIC_H_ 18fc0e3219edc9a5bf81b166e82fd5db2796eb6a0dBrian Carlstrom#define ART_RUNTIME_ATOMIC_H_ 195ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 207c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#include <stdint.h> 21b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers#include <vector> 225ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 23a984454098971739a1469d62cba02cda3600268bIan Rogers#include "base/logging.h" 24761600567d73b23324ae0251e871c15d6849ffd8Elliott Hughes#include "base/macros.h" 255ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 267c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesnamespace art { 275ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 28b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogersclass Mutex; 29b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 30ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogerstemplate<typename T> 31ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogersclass Atomic { 32ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers public: 33ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers Atomic<T>() : value_(0) { } 34ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 35ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers explicit Atomic<T>(T value) : value_(value) { } 36ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 37ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers Atomic<T>& operator=(T desired) { 38ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers Store(desired); 39ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return *this; 40ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 41ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 42ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T Load() const { 43ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return value_; 44ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 45ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 46ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers operator T() const { 47ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return Load(); 48ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 49ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 50ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T FetchAndAdd(const T value) { 51ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_fetch_and_add(&value_, value); // Return old_value. 52ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 53ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 54ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T FetchAndSub(const T value) { 55ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_fetch_and_sub(&value_, value); // Return old value. 56ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 57ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 58ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T operator++() { // Prefix operator. 59ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_add_and_fetch(&value_, 1); // Return new value. 60ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 61ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 62ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T operator++(int) { // Postfix operator. 63ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_fetch_and_add(&value_, 1); // Return old value. 64ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 65ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 66ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T operator--() { // Prefix operator. 67ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_sub_and_fetch(&value_, 1); // Return new value. 68ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 69ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 70ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T operator--(int) { // Postfix operator. 71ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_fetch_and_sub(&value_, 1); // Return old value. 72ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 73ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 74ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers bool CompareAndSwap(T expected_value, T desired_value) { 75ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_bool_compare_and_swap(&value_, expected_value, desired_value); 76ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 77ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 78ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers volatile T* Address() { 79ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return &value_; 80ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 81ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 82ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers private: 83ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers // Unsafe = operator for non atomic operations on the integer. 84ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers void Store(T desired) { 85ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers value_ = desired; 86ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 87ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 88ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers volatile T value_; 89ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers}; 90ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 91ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogerstypedef Atomic<int32_t> AtomicInteger; 92ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 937c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// NOTE: Two "quasiatomic" operations on the exact same memory address 947c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// are guaranteed to operate atomically with respect to each other, 957c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// but no guarantees are made about quasiatomic operations mixed with 967c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// non-quasiatomic operations on the same address, nor about 977c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// quasiatomic operations that are performed on partially-overlapping 987c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// memory. 997c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesclass QuasiAtomic { 100936b37f3a7f224d990a36b2ec66782a4462180d6Ian Rogers#if defined(__mips__) && !defined(__LP64__) 101b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static constexpr bool kNeedSwapMutexes = true; 102b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers#else 103b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static constexpr bool kNeedSwapMutexes = false; 104b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers#endif 105b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 1067c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes public: 1077c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes static void Startup(); 1087c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 1097c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes static void Shutdown(); 1107c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 1119adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // Reads the 64-bit value at "addr" without tearing. 112b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static int64_t Read64(volatile const int64_t* addr) { 113b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers if (!kNeedSwapMutexes) { 114a984454098971739a1469d62cba02cda3600268bIan Rogers int64_t value; 115a984454098971739a1469d62cba02cda3600268bIan Rogers#if defined(__LP64__) 116a984454098971739a1469d62cba02cda3600268bIan Rogers value = *addr; 117a984454098971739a1469d62cba02cda3600268bIan Rogers#else 118a984454098971739a1469d62cba02cda3600268bIan Rogers#if defined(__arm__) 119a984454098971739a1469d62cba02cda3600268bIan Rogers#if defined(__ARM_FEATURE_LPAE) 120a984454098971739a1469d62cba02cda3600268bIan Rogers // With LPAE support (such as Cortex-A15) then ldrd is defined not to tear. 121a984454098971739a1469d62cba02cda3600268bIan Rogers __asm__ __volatile__("@ QuasiAtomic::Read64\n" 122a984454098971739a1469d62cba02cda3600268bIan Rogers "ldrd %0, %H0, %1" 123a984454098971739a1469d62cba02cda3600268bIan Rogers : "=r" (value) 124a984454098971739a1469d62cba02cda3600268bIan Rogers : "m" (*addr)); 125a984454098971739a1469d62cba02cda3600268bIan Rogers#else 126a984454098971739a1469d62cba02cda3600268bIan Rogers // Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary. 127a984454098971739a1469d62cba02cda3600268bIan Rogers __asm__ __volatile__("@ QuasiAtomic::Read64\n" 128a984454098971739a1469d62cba02cda3600268bIan Rogers "ldrexd %0, %H0, %1" 129a984454098971739a1469d62cba02cda3600268bIan Rogers : "=r" (value) 130a984454098971739a1469d62cba02cda3600268bIan Rogers : "Q" (*addr)); 131a984454098971739a1469d62cba02cda3600268bIan Rogers#endif 132a984454098971739a1469d62cba02cda3600268bIan Rogers#elif defined(__i386__) 133a984454098971739a1469d62cba02cda3600268bIan Rogers __asm__ __volatile__( 134a984454098971739a1469d62cba02cda3600268bIan Rogers "movq %1, %0\n" 135a984454098971739a1469d62cba02cda3600268bIan Rogers : "=x" (value) 136a984454098971739a1469d62cba02cda3600268bIan Rogers : "m" (*addr)); 137a984454098971739a1469d62cba02cda3600268bIan Rogers#else 138a984454098971739a1469d62cba02cda3600268bIan Rogers LOG(FATAL) << "Unsupported architecture"; 139a984454098971739a1469d62cba02cda3600268bIan Rogers#endif 140a984454098971739a1469d62cba02cda3600268bIan Rogers#endif // defined(__LP64__) 141a984454098971739a1469d62cba02cda3600268bIan Rogers return value; 142b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } else { 143b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers return SwapMutexRead64(addr); 144b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 145b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 1467c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 1479adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // Writes to the 64-bit value at "addr" without tearing. 148a984454098971739a1469d62cba02cda3600268bIan Rogers static void Write64(volatile int64_t* addr, int64_t value) { 149b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers if (!kNeedSwapMutexes) { 150a984454098971739a1469d62cba02cda3600268bIan Rogers#if defined(__LP64__) 151a984454098971739a1469d62cba02cda3600268bIan Rogers *addr = value; 152a984454098971739a1469d62cba02cda3600268bIan Rogers#else 153a984454098971739a1469d62cba02cda3600268bIan Rogers#if defined(__arm__) 154a984454098971739a1469d62cba02cda3600268bIan Rogers#if defined(__ARM_FEATURE_LPAE) 155a984454098971739a1469d62cba02cda3600268bIan Rogers // If we know that ARM architecture has LPAE (such as Cortex-A15) strd is defined not to tear. 156a984454098971739a1469d62cba02cda3600268bIan Rogers __asm__ __volatile__("@ QuasiAtomic::Write64\n" 157a984454098971739a1469d62cba02cda3600268bIan Rogers "strd %1, %H1, %0" 158a984454098971739a1469d62cba02cda3600268bIan Rogers : "=m"(*addr) 159a984454098971739a1469d62cba02cda3600268bIan Rogers : "r" (value)); 160a984454098971739a1469d62cba02cda3600268bIan Rogers#else 161a984454098971739a1469d62cba02cda3600268bIan Rogers // The write is done as a swap so that the cache-line is in the exclusive state for the store. 162a984454098971739a1469d62cba02cda3600268bIan Rogers int64_t prev; 163a984454098971739a1469d62cba02cda3600268bIan Rogers int status; 164a984454098971739a1469d62cba02cda3600268bIan Rogers do { 165a984454098971739a1469d62cba02cda3600268bIan Rogers __asm__ __volatile__("@ QuasiAtomic::Write64\n" 166a984454098971739a1469d62cba02cda3600268bIan Rogers "ldrexd %0, %H0, %2\n" 167a984454098971739a1469d62cba02cda3600268bIan Rogers "strexd %1, %3, %H3, %2" 168a984454098971739a1469d62cba02cda3600268bIan Rogers : "=&r" (prev), "=&r" (status), "+Q"(*addr) 169a984454098971739a1469d62cba02cda3600268bIan Rogers : "r" (value) 170a984454098971739a1469d62cba02cda3600268bIan Rogers : "cc"); 171a984454098971739a1469d62cba02cda3600268bIan Rogers } while (UNLIKELY(status != 0)); 172a984454098971739a1469d62cba02cda3600268bIan Rogers#endif 173a984454098971739a1469d62cba02cda3600268bIan Rogers#elif defined(__i386__) 174a984454098971739a1469d62cba02cda3600268bIan Rogers __asm__ __volatile__( 175a984454098971739a1469d62cba02cda3600268bIan Rogers "movq %1, %0" 176a984454098971739a1469d62cba02cda3600268bIan Rogers : "=m" (*addr) 177a984454098971739a1469d62cba02cda3600268bIan Rogers : "x" (value)); 178a984454098971739a1469d62cba02cda3600268bIan Rogers#else 179a984454098971739a1469d62cba02cda3600268bIan Rogers LOG(FATAL) << "Unsupported architecture"; 180a984454098971739a1469d62cba02cda3600268bIan Rogers#endif 181a984454098971739a1469d62cba02cda3600268bIan Rogers#endif // defined(__LP64__) 182b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } else { 183a984454098971739a1469d62cba02cda3600268bIan Rogers SwapMutexWrite64(addr, value); 184b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 185b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 1867c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 1879adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // Atomically compare the value at "addr" to "old_value", if equal replace it with "new_value" 1889adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // and return true. Otherwise, don't swap, and return false. 189b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static bool Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { 190b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers if (!kNeedSwapMutexes) { 191b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers return __sync_bool_compare_and_swap(addr, old_value, new_value); 192b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } else { 193b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers return SwapMutexCas64(old_value, new_value, addr); 194b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 195b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 1967c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 1979adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // Does the architecture provide reasonable atomic long operations or do we fall back on mutexes? 198b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static bool LongAtomicsUseMutexes() { 19963c5dd056fa20993b35ec5c8548b26c988445763Ian Rogers return kNeedSwapMutexes; 200b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 201b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 202b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static void MembarLoadStore() { 2035817e899c5dd5e82949433d7bfcbe61f63b2a5ffStuart Monteith #if defined(__arm__) || defined(__aarch64__) 204b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("dmb ish" : : : "memory"); 205ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers #elif defined(__i386__) || defined(__x86_64__) 206b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("" : : : "memory"); 207b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #elif defined(__mips__) 208b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("sync" : : : "memory"); 209b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #else 210b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #error Unexpected architecture 211b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #endif 212b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 213b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 214b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static void MembarLoadLoad() { 2155817e899c5dd5e82949433d7bfcbe61f63b2a5ffStuart Monteith #if defined(__arm__) || defined(__aarch64__) 216b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("dmb ish" : : : "memory"); 217ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers #elif defined(__i386__) || defined(__x86_64__) 218b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("" : : : "memory"); 219b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #elif defined(__mips__) 220b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("sync" : : : "memory"); 221b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #else 222b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #error Unexpected architecture 223b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #endif 224b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 225b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 226b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static void MembarStoreStore() { 2275817e899c5dd5e82949433d7bfcbe61f63b2a5ffStuart Monteith #if defined(__arm__) || defined(__aarch64__) 228b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("dmb ishst" : : : "memory"); 229ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers #elif defined(__i386__) || defined(__x86_64__) 230b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("" : : : "memory"); 231b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #elif defined(__mips__) 232b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("sync" : : : "memory"); 233b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #else 234b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #error Unexpected architecture 235b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #endif 236b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 237b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 238b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static void MembarStoreLoad() { 2395817e899c5dd5e82949433d7bfcbe61f63b2a5ffStuart Monteith #if defined(__arm__) || defined(__aarch64__) 240b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("dmb ish" : : : "memory"); 241ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers #elif defined(__i386__) || defined(__x86_64__) 242b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("mfence" : : : "memory"); 243b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #elif defined(__mips__) 244b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("sync" : : : "memory"); 245b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #else 246b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #error Unexpected architecture 247b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #endif 248b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 2497c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 2507c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes private: 251b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static Mutex* GetSwapMutex(const volatile int64_t* addr); 252b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static int64_t SwapMutexRead64(volatile const int64_t* addr); 253b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static void SwapMutexWrite64(volatile int64_t* addr, int64_t val); 254b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static bool SwapMutexCas64(int64_t old_value, int64_t new_value, volatile int64_t* addr); 255b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 256b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers // We stripe across a bunch of different mutexes to reduce contention. 257b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static constexpr size_t kSwapMutexCount = 32; 258b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static std::vector<Mutex*>* gSwapMutexes; 259b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 2607c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes DISALLOW_COPY_AND_ASSIGN(QuasiAtomic); 2617c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes}; 2625ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 2635ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes} // namespace art 2645ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 265fc0e3219edc9a5bf81b166e82fd5db2796eb6a0dBrian Carlstrom#endif // ART_RUNTIME_ATOMIC_H_ 266