atomic.h revision ef7d42fca18c16fbaf103822ad16f23246e2905d
15ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes/* 25ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Copyright (C) 2008 The Android Open Source Project 35ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 45ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Licensed under the Apache License, Version 2.0 (the "License"); 55ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * you may not use this file except in compliance with the License. 65ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * You may obtain a copy of the License at 75ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 85ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * http://www.apache.org/licenses/LICENSE-2.0 95ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 105ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Unless required by applicable law or agreed to in writing, software 115ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * distributed under the License is distributed on an "AS IS" BASIS, 125ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 135ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * See the License for the specific language governing permissions and 145ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * limitations under the License. 155ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes */ 165ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 17fc0e3219edc9a5bf81b166e82fd5db2796eb6a0dBrian Carlstrom#ifndef ART_RUNTIME_ATOMIC_H_ 18fc0e3219edc9a5bf81b166e82fd5db2796eb6a0dBrian Carlstrom#define ART_RUNTIME_ATOMIC_H_ 195ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 207c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#include <stdint.h> 21b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers#include <vector> 225ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 23761600567d73b23324ae0251e871c15d6849ffd8Elliott Hughes#include "base/macros.h" 245ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 257c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesnamespace art { 265ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 27b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogersclass Mutex; 28b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 29ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogerstemplate<typename T> 30ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogersclass Atomic { 31ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers public: 32ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers Atomic<T>() : value_(0) { } 33ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 34ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers explicit Atomic<T>(T value) : value_(value) { } 35ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 36ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers Atomic<T>& operator=(T desired) { 37ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers Store(desired); 38ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return *this; 39ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 40ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 41ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T Load() const { 42ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return value_; 43ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 44ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 45ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers operator T() const { 46ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return Load(); 47ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 48ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 49ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T FetchAndAdd(const T value) { 50ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_fetch_and_add(&value_, value); // Return old_value. 51ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 52ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 53ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T FetchAndSub(const T value) { 54ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_fetch_and_sub(&value_, value); // Return old value. 55ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 56ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 57ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T operator++() { // Prefix operator. 58ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_add_and_fetch(&value_, 1); // Return new value. 59ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 60ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 61ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T operator++(int) { // Postfix operator. 62ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_fetch_and_add(&value_, 1); // Return old value. 63ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 64ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 65ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T operator--() { // Prefix operator. 66ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_sub_and_fetch(&value_, 1); // Return new value. 67ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 68ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 69ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers T operator--(int) { // Postfix operator. 70ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_fetch_and_sub(&value_, 1); // Return old value. 71ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 72ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 73ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers bool CompareAndSwap(T expected_value, T desired_value) { 74ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return __sync_bool_compare_and_swap(&value_, expected_value, desired_value); 75ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 76ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 77ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers volatile T* Address() { 78ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers return &value_; 79ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 80ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 81ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers private: 82ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers // Unsafe = operator for non atomic operations on the integer. 83ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers void Store(T desired) { 84ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers value_ = desired; 85ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers } 86ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 87ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers volatile T value_; 88ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers}; 89ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 90ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogerstypedef Atomic<int32_t> AtomicInteger; 91ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers 927c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// NOTE: Two "quasiatomic" operations on the exact same memory address 937c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// are guaranteed to operate atomically with respect to each other, 947c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// but no guarantees are made about quasiatomic operations mixed with 957c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// non-quasiatomic operations on the same address, nor about 967c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// quasiatomic operations that are performed on partially-overlapping 977c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// memory. 987c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesclass QuasiAtomic { 99b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers#if !defined(__arm__) && !defined(__i386__) 100b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static constexpr bool kNeedSwapMutexes = true; 101b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers#else 102b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static constexpr bool kNeedSwapMutexes = false; 103b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers#endif 104b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 1057c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes public: 1067c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes static void Startup(); 1077c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 1087c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes static void Shutdown(); 1097c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 1109adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // Reads the 64-bit value at "addr" without tearing. 111b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static int64_t Read64(volatile const int64_t* addr) { 112b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers if (!kNeedSwapMutexes) { 113b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers return *addr; 114b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } else { 115b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers return SwapMutexRead64(addr); 116b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 117b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 1187c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 1199adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // Writes to the 64-bit value at "addr" without tearing. 120b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static void Write64(volatile int64_t* addr, int64_t val) { 121b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers if (!kNeedSwapMutexes) { 122b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers *addr = val; 123b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } else { 124b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers SwapMutexWrite64(addr, val); 125b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 126b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 1277c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 1289adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // Atomically compare the value at "addr" to "old_value", if equal replace it with "new_value" 1299adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // and return true. Otherwise, don't swap, and return false. 130b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static bool Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { 131b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers if (!kNeedSwapMutexes) { 132b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers return __sync_bool_compare_and_swap(addr, old_value, new_value); 133b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } else { 134b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers return SwapMutexCas64(old_value, new_value, addr); 135b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 136b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 1377c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 1389adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // Does the architecture provide reasonable atomic long operations or do we fall back on mutexes? 139b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static bool LongAtomicsUseMutexes() { 140b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers return !kNeedSwapMutexes; 141b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 142b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 143b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static void MembarLoadStore() { 144b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #if defined(__arm__) 145b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("dmb ish" : : : "memory"); 146ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers #elif defined(__i386__) || defined(__x86_64__) 147b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("" : : : "memory"); 148b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #elif defined(__mips__) 149b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("sync" : : : "memory"); 150b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #else 151b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #error Unexpected architecture 152b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #endif 153b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 154b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 155b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static void MembarLoadLoad() { 156b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #if defined(__arm__) 157b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("dmb ish" : : : "memory"); 158ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers #elif defined(__i386__) || defined(__x86_64__) 159b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("" : : : "memory"); 160b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #elif defined(__mips__) 161b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("sync" : : : "memory"); 162b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #else 163b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #error Unexpected architecture 164b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #endif 165b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 166b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 167b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static void MembarStoreStore() { 168b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #if defined(__arm__) 169b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("dmb ishst" : : : "memory"); 170ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers #elif defined(__i386__) || defined(__x86_64__) 171b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("" : : : "memory"); 172b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #elif defined(__mips__) 173b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("sync" : : : "memory"); 174b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #else 175b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #error Unexpected architecture 176b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #endif 177b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 178b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 179b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static void MembarStoreLoad() { 180b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #if defined(__arm__) 181b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("dmb ish" : : : "memory"); 182ef7d42fca18c16fbaf103822ad16f23246e2905dIan Rogers #elif defined(__i386__) || defined(__x86_64__) 183b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("mfence" : : : "memory"); 184b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #elif defined(__mips__) 185b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers __asm__ __volatile__("sync" : : : "memory"); 186b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #else 187b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #error Unexpected architecture 188b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers #endif 189b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers } 1907c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 1917c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes private: 192b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static Mutex* GetSwapMutex(const volatile int64_t* addr); 193b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static int64_t SwapMutexRead64(volatile const int64_t* addr); 194b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static void SwapMutexWrite64(volatile int64_t* addr, int64_t val); 195b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static bool SwapMutexCas64(int64_t old_value, int64_t new_value, volatile int64_t* addr); 196b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 197b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers // We stripe across a bunch of different mutexes to reduce contention. 198b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static constexpr size_t kSwapMutexCount = 32; 199b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers static std::vector<Mutex*>* gSwapMutexes; 200b122a4bbed34ab22b4c1541ee25e5cf22f12a926Ian Rogers 2017c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes DISALLOW_COPY_AND_ASSIGN(QuasiAtomic); 2027c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes}; 2035ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 2045ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes} // namespace art 2055ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 206fc0e3219edc9a5bf81b166e82fd5db2796eb6a0dBrian Carlstrom#endif // ART_RUNTIME_ATOMIC_H_ 207