atomic.cc revision 1aa246dec5abe212f699de1413a0c4a191ca364a
15ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes/* 25ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Copyright (C) 2010 The Android Open Source Project 35ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 45ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Licensed under the Apache License, Version 2.0 (the "License"); 55ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * you may not use this file except in compliance with the License. 65ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * You may obtain a copy of the License at 75ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 85ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * http://www.apache.org/licenses/LICENSE-2.0 95ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 105ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Unless required by applicable law or agreed to in writing, software 115ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * distributed under the License is distributed on an "AS IS" BASIS, 125ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 135ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * See the License for the specific language governing permissions and 145ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * limitations under the License. 155ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes */ 165ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 175ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#include "atomic.h" 185ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 197c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#include <pthread.h> 2025fd14b87cced64a179dee885573113be5e11944Ian Rogers#include <vector> 215ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 2276b6167407c2b6f5d40ad895b2793a6b037f54b2Elliott Hughes#include "base/mutex.h" 231aa246dec5abe212f699de1413a0c4a191ca364aElliott Hughes#include "base/stl_util.h" 247c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#include "stringprintf.h" 2550b35e2fd1a68cd1240e4a9d9f363e11764957d1Ian Rogers#include "thread.h" 265ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 277c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#if defined(__APPLE__) 287c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#include <libkern/OSAtomic.h> 297c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif 307c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#if defined(__arm__) 317c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#include <machine/cpu-features.h> 327c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif 337c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 347c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesnamespace art { 355ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 365ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#if defined(HAVE_MACOSX_IPC) 377c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#define NEED_MAC_QUASI_ATOMICS 1 385ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 397c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#elif defined(__i386__) || defined(__x86_64__) 407c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#define NEED_PTHREADS_QUASI_ATOMICS 1 415ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 427c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#elif defined(__mips__) 437c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#define NEED_PTHREADS_QUASI_ATOMICS 1 445ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 457c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#elif defined(__arm__) 465ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 477c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#if defined(__ARM_HAVE_LDREXD) 487c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#define NEED_ARM_LDREXD_QUASI_ATOMICS 1 497c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#else 507c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#define NEED_PTHREADS_QUASI_ATOMICS 1 517c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif 52557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes 537c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#else 547c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#error "QuasiAtomic unsupported on this platform" 555ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#endif 565ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 577c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// ***************************************************************************** 585ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 597c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#if NEED_ARM_LDREXD_QUASI_ATOMICS 605ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 61557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughesstatic inline int64_t QuasiAtomicSwap64Impl(int64_t new_value, volatile int64_t* addr) { 625ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes int64_t prev; 635ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes int status; 645ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes do { 657c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes __asm__ __volatile__("@ QuasiAtomic::Swap64\n" 665ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "ldrexd %0, %H0, [%3]\n" 675ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "strexd %1, %4, %H4, [%3]" 685ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "=&r" (prev), "=&r" (status), "+m"(*addr) 695ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "r" (addr), "r" (new_value) 705ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "cc"); 715ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes } while (__builtin_expect(status != 0, 0)); 725ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes return prev; 735ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes} 745ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 757c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesint64_t QuasiAtomic::Swap64(int64_t new_value, volatile int64_t* addr) { 76557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes return QuasiAtomicSwap64Impl(new_value, addr); 77557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes} 78557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes 797c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesint64_t QuasiAtomic::Swap64Sync(int64_t new_value, volatile int64_t* addr) { 80557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes ANDROID_MEMBAR_STORE(); 81557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes int64_t old_value = QuasiAtomicSwap64Impl(new_value, addr); 82557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes ANDROID_MEMBAR_FULL(); 83557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes return old_value; 84557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes} 85557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes 867c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesint64_t QuasiAtomic::Read64(volatile const int64_t* addr) { 877c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes int64_t value; 887c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes __asm__ __volatile__("@ QuasiAtomic::Read64\n" 897c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes "ldrexd %0, %H0, [%1]" 907c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes : "=&r" (value) 917c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes : "r" (addr)); 927c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes return value; 937c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes} 947c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 957c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesint QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { 965ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes int64_t prev; 975ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes int status; 985ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes do { 997c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes __asm__ __volatile__("@ QuasiAtomic::Cas64\n" 1005ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "ldrexd %0, %H0, [%3]\n" 1015ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "mov %1, #0\n" 1025ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "teq %0, %4\n" 1035ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "teqeq %H0, %H4\n" 1045ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "strexdeq %1, %5, %H5, [%3]" 1055ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "=&r" (prev), "=&r" (status), "+m"(*addr) 1065ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "r" (addr), "Ir" (old_value), "r" (new_value) 1075ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "cc"); 1085ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes } while (__builtin_expect(status != 0, 0)); 1095ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes return prev != old_value; 1105ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes} 1115ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1127c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif 1135ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1147c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// ***************************************************************************** 1155ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1167c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#if NEED_MAC_QUASI_ATOMICS 1175ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1187c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesstatic inline int64_t QuasiAtomicSwap64Impl(int64_t value, volatile int64_t* addr) { 1197c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes int64_t old_value; 1207c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes do { 1217c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes old_value = *addr; 1227c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes } while (QuasiAtomic::Cas64(old_value, value, addr)); 123557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes return old_value; 124557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes} 125557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes 1267c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesint64_t QuasiAtomic::Swap64(int64_t value, volatile int64_t* addr) { 1277c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes return QuasiAtomicSwap64Impl(value, addr); 1285ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes} 1295ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1307c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesint64_t QuasiAtomic::Swap64Sync(int64_t value, volatile int64_t* addr) { 1317c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes ANDROID_MEMBAR_STORE(); 1327c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes int64_t old_value = QuasiAtomicSwap64Impl(value, addr); 1337c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes // TUNING: barriers can be avoided on some architectures. 1347c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes ANDROID_MEMBAR_FULL(); 1357c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes return old_value; 1365ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes} 1375ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1387c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesint64_t QuasiAtomic::Read64(volatile const int64_t* addr) { 1397c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes return OSAtomicAdd64Barrier(0, const_cast<volatile int64_t*>(addr)); 1405ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes} 1415ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1427c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesint QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { 1437c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes return OSAtomicCompareAndSwap64Barrier(old_value, new_value, const_cast<int64_t*>(addr)) == 0; 1447c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes} 1455ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1465ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#endif 1475ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1487c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// ***************************************************************************** 1495ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1507c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#if NEED_PTHREADS_QUASI_ATOMICS 1515ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1527c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// In the absence of a better implementation, we implement the 64-bit atomic 1537c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// operations through mutex locking. 1545ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1557c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// We stripe across a bunch of different mutexes to reduce contention. 1567c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesstatic const size_t kSwapLockCount = 32; 1577c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesstatic std::vector<Mutex*>* gSwapLocks; 1585ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1597c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesvoid QuasiAtomic::Startup() { 1607c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes gSwapLocks = new std::vector<Mutex*>; 1617c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes for (size_t i = 0; i < kSwapLockCount; ++i) { 1627c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes gSwapLocks->push_back(new Mutex(StringPrintf("QuasiAtomic stripe %d", i).c_str())); 1635ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes } 1647c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes} 1655ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1667c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesvoid QuasiAtomic::Shutdown() { 1677c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes STLDeleteElements(gSwapLocks); 1687c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes delete gSwapLocks; 1697c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes} 1705ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1717c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesstatic inline Mutex& GetSwapLock(const volatile int64_t* addr) { 1727c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes return *(*gSwapLocks)[((unsigned)(void*)(addr) >> 3U) % kSwapLockCount]; 1737c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes} 1745ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1757c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesint64_t QuasiAtomic::Swap64(int64_t value, volatile int64_t* addr) { 17650b35e2fd1a68cd1240e4a9d9f363e11764957d1Ian Rogers MutexLock mu(Thread::Current(), GetSwapLock(addr)); 1777c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes int64_t old_value = *addr; 1787c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes *addr = value; 1797c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes return old_value; 1805ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes} 1815ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1827c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesint64_t QuasiAtomic::Swap64Sync(int64_t value, volatile int64_t* addr) { 1837c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes // Same as QuasiAtomicSwap64 - mutex handles barrier. 1847c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes return QuasiAtomic::Swap64(value, addr); 1857c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes} 1865ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1877c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesint QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { 18850b35e2fd1a68cd1240e4a9d9f363e11764957d1Ian Rogers MutexLock mu(Thread::Current(), GetSwapLock(addr)); 1897c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes if (*addr == old_value) { 1907c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes *addr = new_value; 1917c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes return 0; 1925ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes } 1937c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes return 1; 1945ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes} 1955ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1967c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesint64_t QuasiAtomic::Read64(volatile const int64_t* addr) { 19750b35e2fd1a68cd1240e4a9d9f363e11764957d1Ian Rogers MutexLock mu(Thread::Current(), GetSwapLock(addr)); 1987c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes return *addr; 1997c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes} 2005ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 2015ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#else 2025ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 2037c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes// The other implementations don't need any special setup. 2047c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesvoid QuasiAtomic::Startup() {} 2057c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesvoid QuasiAtomic::Shutdown() {} 2065ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 2077c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif 2085ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 2095ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes} // namespace art 210