15ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes/* 25ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Copyright (C) 2010 The Android Open Source Project 35ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 45ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Licensed under the Apache License, Version 2.0 (the "License"); 55ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * you may not use this file except in compliance with the License. 65ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * You may obtain a copy of the License at 75ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 85ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * http://www.apache.org/licenses/LICENSE-2.0 95ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 105ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Unless required by applicable law or agreed to in writing, software 115ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * distributed under the License is distributed on an "AS IS" BASIS, 125ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 135ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * See the License for the specific language governing permissions and 145ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * limitations under the License. 155ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes */ 165ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 175ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#include "atomic.h" 185ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 199adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#define NEED_SWAP_MUTEXES !defined(__arm__) && !defined(__i386__) 20e222ee0b794f941af4fb1b32fb8224e32942ea7bElliott Hughes 219adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES 2225fd14b87cced64a179dee885573113be5e11944Ian Rogers#include <vector> 2376b6167407c2b6f5d40ad895b2793a6b037f54b2Elliott Hughes#include "base/mutex.h" 241aa246dec5abe212f699de1413a0c4a191ca364aElliott Hughes#include "base/stl_util.h" 25e222ee0b794f941af4fb1b32fb8224e32942ea7bElliott Hughes#include "base/stringprintf.h" 2650b35e2fd1a68cd1240e4a9d9f363e11764957d1Ian Rogers#include "thread.h" 277c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif 287c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 297c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesnamespace art { 305ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 319adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES 329adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers// We stripe across a bunch of different mutexes to reduce contention. 339adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersstatic const size_t kSwapMutexCount = 32; 349adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersstatic std::vector<Mutex*>* gSwapMutexes; 355ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 369adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersstatic Mutex& GetSwapMutex(const volatile int64_t* addr) { 372d88862f0752a7a0e65145b088f49dabd49d4284Brian Carlstrom return *(*gSwapMutexes)[(reinterpret_cast<unsigned>(addr) >> 3U) % kSwapMutexCount]; 389adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers} 399adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#endif 405ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 419adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersvoid QuasiAtomic::Startup() { 429adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES 439adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers gSwapMutexes = new std::vector<Mutex*>; 449adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers for (size_t i = 0; i < kSwapMutexCount; ++i) { 450fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers gSwapMutexes->push_back(new Mutex("QuasiAtomic stripe")); 469adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers } 479adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#endif 489adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers} 495ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 509adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersvoid QuasiAtomic::Shutdown() { 519adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES 529adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers STLDeleteElements(gSwapMutexes); 539adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers delete gSwapMutexes; 547c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif 559adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers} 56557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes 579adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersint64_t QuasiAtomic::Read64(volatile const int64_t* addr) { 589adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers int64_t value; 590fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers#if NEED_SWAP_MUTEXES 600fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers MutexLock mu(Thread::Current(), GetSwapMutex(addr)); 610fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers value = *addr; 620fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers#elif defined(__arm__) 639adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary. If we 649adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // have LPAE (such as Cortex-A15) then ldrd would suffice. 659adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers __asm__ __volatile__("@ QuasiAtomic::Read64\n" 669adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers "ldrexd %0, %H0, [%1]" 679adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "=&r" (value) 689adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "r" (addr)); 699adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#elif defined(__i386__) 709adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers __asm__ __volatile__( 719adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers "movq %1, %0\n" 729adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "=x" (value) 739adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "m" (*addr)); 747c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#else 750fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers#error Unexpected architecture 765ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#endif 779adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers return value; 789adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers} 795ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 809adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersvoid QuasiAtomic::Write64(volatile int64_t* addr, int64_t value) { 810fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers#if NEED_SWAP_MUTEXES 820fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers MutexLock mu(Thread::Current(), GetSwapMutex(addr)); 830fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers *addr = value; 840fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers#elif defined(__arm__) 859adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // The write is done as a swap so that the cache-line is in the exclusive state for the store. If 869adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // we know that ARM architecture has LPAE (such as Cortex-A15) this isn't necessary and strd will 879adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // suffice. 885ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes int64_t prev; 895ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes int status; 905ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes do { 919adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers __asm__ __volatile__("@ QuasiAtomic::Write64\n" 925ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "ldrexd %0, %H0, [%3]\n" 935ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "strexd %1, %4, %H4, [%3]" 945ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "=&r" (prev), "=&r" (status), "+m"(*addr) 959adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "r" (addr), "r" (value) 965ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "cc"); 975ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes } while (__builtin_expect(status != 0, 0)); 989adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#elif defined(__i386__) 999adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers __asm__ __volatile__( 1009adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers "movq %1, %0" 1019adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "=m" (*addr) 1029adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "x" (value)); 1039adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#else 1040fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers#error Unexpected architecture 1059adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#endif 106557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes} 107557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes 108557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes 1099adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersbool QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { 1100fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers#if NEED_SWAP_MUTEXES 1110fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers MutexLock mu(Thread::Current(), GetSwapMutex(addr)); 1120fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers if (*addr == old_value) { 1130fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers *addr = new_value; 1140fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers return true; 1150fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers } 1160fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers return false; 1170fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers#elif defined(__arm__) 1185ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes int64_t prev; 1195ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes int status; 1205ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes do { 1217c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes __asm__ __volatile__("@ QuasiAtomic::Cas64\n" 1225ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "ldrexd %0, %H0, [%3]\n" 1235ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "mov %1, #0\n" 1245ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "teq %0, %4\n" 1255ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "teqeq %H0, %H4\n" 1265ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "strexdeq %1, %5, %H5, [%3]" 1275ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "=&r" (prev), "=&r" (status), "+m"(*addr) 1285ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "r" (addr), "Ir" (old_value), "r" (new_value) 1295ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "cc"); 1305ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes } while (__builtin_expect(status != 0, 0)); 1310794b6acf40251d25f08defbe8648dbe0a24ba62Ian Rogers return prev == old_value; 1329adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#elif defined(__i386__) 1331c653d545058a1a850fb6d07d74e607fa1ddf48eIan Rogers // The compiler does the right job and works better than inline assembly, especially with -O0 1341c653d545058a1a850fb6d07d74e607fa1ddf48eIan Rogers // compilation. 1351c653d545058a1a850fb6d07d74e607fa1ddf48eIan Rogers return __sync_bool_compare_and_swap(addr, old_value, new_value); 1369adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#else 1370fed328fc9dd21a0b0da3e9b39ee2ce6a4d07117Ian Rogers#error Unexpected architecture 1389adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#endif 1397c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes} 1405ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1419adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersbool QuasiAtomic::LongAtomicsUseMutexes() { 1429adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES 1439adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers return true; 1445ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#else 1459adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers return false; 1467c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif 1479adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers} 1485ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1495ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes} // namespace art 150