1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "atomic.h" 18 19#define NEED_SWAP_MUTEXES !defined(__arm__) && !defined(__i386__) 20 21#if NEED_SWAP_MUTEXES 22#include <vector> 23#include "base/mutex.h" 24#include "base/stl_util.h" 25#include "base/stringprintf.h" 26#include "thread.h" 27#endif 28 29namespace art { 30 31#if NEED_SWAP_MUTEXES 32// We stripe across a bunch of different mutexes to reduce contention. 33static const size_t kSwapMutexCount = 32; 34static std::vector<Mutex*>* gSwapMutexes; 35 36static Mutex& GetSwapMutex(const volatile int64_t* addr) { 37 return *(*gSwapMutexes)[(reinterpret_cast<unsigned>(addr) >> 3U) % kSwapMutexCount]; 38} 39#endif 40 41void QuasiAtomic::Startup() { 42#if NEED_SWAP_MUTEXES 43 gSwapMutexes = new std::vector<Mutex*>; 44 for (size_t i = 0; i < kSwapMutexCount; ++i) { 45 gSwapMutexes->push_back(new Mutex("QuasiAtomic stripe")); 46 } 47#endif 48} 49 50void QuasiAtomic::Shutdown() { 51#if NEED_SWAP_MUTEXES 52 STLDeleteElements(gSwapMutexes); 53 delete gSwapMutexes; 54#endif 55} 56 57int64_t QuasiAtomic::Read64(volatile const int64_t* addr) { 58 int64_t value; 59#if NEED_SWAP_MUTEXES 60 MutexLock mu(Thread::Current(), GetSwapMutex(addr)); 61 value = *addr; 62#elif defined(__arm__) 63 // Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary. If we 64 // have LPAE (such as Cortex-A15) then ldrd would suffice. 65 __asm__ __volatile__("@ QuasiAtomic::Read64\n" 66 "ldrexd %0, %H0, [%1]" 67 : "=&r" (value) 68 : "r" (addr)); 69#elif defined(__i386__) 70 __asm__ __volatile__( 71 "movq %1, %0\n" 72 : "=x" (value) 73 : "m" (*addr)); 74#else 75#error Unexpected architecture 76#endif 77 return value; 78} 79 80void QuasiAtomic::Write64(volatile int64_t* addr, int64_t value) { 81#if NEED_SWAP_MUTEXES 82 MutexLock mu(Thread::Current(), GetSwapMutex(addr)); 83 *addr = value; 84#elif defined(__arm__) 85 // The write is done as a swap so that the cache-line is in the exclusive state for the store. If 86 // we know that ARM architecture has LPAE (such as Cortex-A15) this isn't necessary and strd will 87 // suffice. 88 int64_t prev; 89 int status; 90 do { 91 __asm__ __volatile__("@ QuasiAtomic::Write64\n" 92 "ldrexd %0, %H0, [%3]\n" 93 "strexd %1, %4, %H4, [%3]" 94 : "=&r" (prev), "=&r" (status), "+m"(*addr) 95 : "r" (addr), "r" (value) 96 : "cc"); 97 } while (__builtin_expect(status != 0, 0)); 98#elif defined(__i386__) 99 __asm__ __volatile__( 100 "movq %1, %0" 101 : "=m" (*addr) 102 : "x" (value)); 103#else 104#error Unexpected architecture 105#endif 106} 107 108 109bool QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { 110#if NEED_SWAP_MUTEXES 111 MutexLock mu(Thread::Current(), GetSwapMutex(addr)); 112 if (*addr == old_value) { 113 *addr = new_value; 114 return true; 115 } 116 return false; 117#elif defined(__arm__) 118 int64_t prev; 119 int status; 120 do { 121 __asm__ __volatile__("@ QuasiAtomic::Cas64\n" 122 "ldrexd %0, %H0, [%3]\n" 123 "mov %1, #0\n" 124 "teq %0, %4\n" 125 "teqeq %H0, %H4\n" 126 "strexdeq %1, %5, %H5, [%3]" 127 : "=&r" (prev), "=&r" (status), "+m"(*addr) 128 : "r" (addr), "Ir" (old_value), "r" (new_value) 129 : "cc"); 130 } while (__builtin_expect(status != 0, 0)); 131 return prev == old_value; 132#elif defined(__i386__) 133 // The compiler does the right job and works better than inline assembly, especially with -O0 134 // compilation. 135 return __sync_bool_compare_and_swap(addr, old_value, new_value); 136#else 137#error Unexpected architecture 138#endif 139} 140 141bool QuasiAtomic::LongAtomicsUseMutexes() { 142#if NEED_SWAP_MUTEXES 143 return true; 144#else 145 return false; 146#endif 147} 148 149} // namespace art 150