atomic.cc revision 9adbff5b85fcae2b3e2443344415f6c17ea3ba0a
15ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes/* 25ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Copyright (C) 2010 The Android Open Source Project 35ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 45ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Licensed under the Apache License, Version 2.0 (the "License"); 55ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * you may not use this file except in compliance with the License. 65ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * You may obtain a copy of the License at 75ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 85ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * http://www.apache.org/licenses/LICENSE-2.0 95ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * 105ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Unless required by applicable law or agreed to in writing, software 115ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * distributed under the License is distributed on an "AS IS" BASIS, 125ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 135ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * See the License for the specific language governing permissions and 145ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * limitations under the License. 155ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes */ 165ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 175ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#include "atomic.h" 185ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 199adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#define NEED_SWAP_MUTEXES !defined(__arm__) && !defined(__i386__) 20e222ee0b794f941af4fb1b32fb8224e32942ea7bElliott Hughes 219adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES 2225fd14b87cced64a179dee885573113be5e11944Ian Rogers#include <vector> 2376b6167407c2b6f5d40ad895b2793a6b037f54b2Elliott Hughes#include "base/mutex.h" 241aa246dec5abe212f699de1413a0c4a191ca364aElliott Hughes#include "base/stl_util.h" 25e222ee0b794f941af4fb1b32fb8224e32942ea7bElliott Hughes#include "base/stringprintf.h" 2650b35e2fd1a68cd1240e4a9d9f363e11764957d1Ian Rogers#include "thread.h" 277c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif 287c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes 297c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesnamespace art { 305ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 319adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES 329adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers// We stripe across a bunch of different mutexes to reduce contention. 339adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersstatic const size_t kSwapMutexCount = 32; 349adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersstatic std::vector<Mutex*>* gSwapMutexes; 355ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 369adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersstatic Mutex& GetSwapMutex(const volatile int64_t* addr) { 379adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers return *(*gSwapMutexes)[((unsigned)(void*)(addr) >> 3U) % kSwapMutexCount]; 389adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers} 399adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#endif 405ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 419adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersvoid QuasiAtomic::Startup() { 429adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES 439adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers gSwapMutexes = new std::vector<Mutex*>; 449adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers for (size_t i = 0; i < kSwapMutexCount; ++i) { 459adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers gSwapMutexes->push_back(new Mutex(StringPrintf("QuasiAtomic stripe %d", i).c_str())); 469adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers } 479adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#endif 489adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers} 495ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 509adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersvoid QuasiAtomic::Shutdown() { 519adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES 529adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers STLDeleteElements(gSwapMutexes); 539adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers delete gSwapMutexes; 547c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif 559adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers} 56557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes 579adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersint64_t QuasiAtomic::Read64(volatile const int64_t* addr) { 589adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers int64_t value; 599adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if defined(__arm__) 609adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary. If we 619adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // have LPAE (such as Cortex-A15) then ldrd would suffice. 629adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers __asm__ __volatile__("@ QuasiAtomic::Read64\n" 639adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers "ldrexd %0, %H0, [%1]" 649adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "=&r" (value) 659adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "r" (addr)); 669adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#elif defined(__i386__) 679adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers __asm__ __volatile__( 689adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers "movq %1, %0\n" 699adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "=x" (value) 709adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "m" (*addr)); 717c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#else 729adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers MutexLock mu(Thread::Current(), GetSwapMutex(addr)); 739adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers return *addr; 745ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#endif 759adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers return value; 769adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers} 775ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 789adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersvoid QuasiAtomic::Write64(volatile int64_t* addr, int64_t value) { 799adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if defined(__arm__) 809adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // The write is done as a swap so that the cache-line is in the exclusive state for the store. If 819adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // we know that ARM architecture has LPAE (such as Cortex-A15) this isn't necessary and strd will 829adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // suffice. 835ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes int64_t prev; 845ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes int status; 855ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes do { 869adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers __asm__ __volatile__("@ QuasiAtomic::Write64\n" 875ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "ldrexd %0, %H0, [%3]\n" 885ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "strexd %1, %4, %H4, [%3]" 895ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "=&r" (prev), "=&r" (status), "+m"(*addr) 909adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "r" (addr), "r" (value) 915ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "cc"); 925ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes } while (__builtin_expect(status != 0, 0)); 939adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#elif defined(__i386__) 949adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers __asm__ __volatile__( 959adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers "movq %1, %0" 969adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "=m" (*addr) 979adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "x" (value)); 989adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#else 999adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers MutexLock mu(Thread::Current(), GetSwapMutex(addr)); 1009adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers *addr = value; 1019adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#endif 102557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes} 103557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes 104557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes 1059adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersbool QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { 1069adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if defined(__arm__) 1075ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes int64_t prev; 1085ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes int status; 1095ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes do { 1107c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes __asm__ __volatile__("@ QuasiAtomic::Cas64\n" 1115ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "ldrexd %0, %H0, [%3]\n" 1125ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "mov %1, #0\n" 1135ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "teq %0, %4\n" 1145ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "teqeq %H0, %H4\n" 1155ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes "strexdeq %1, %5, %H5, [%3]" 1165ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "=&r" (prev), "=&r" (status), "+m"(*addr) 1175ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "r" (addr), "Ir" (old_value), "r" (new_value) 1185ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes : "cc"); 1195ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes } while (__builtin_expect(status != 0, 0)); 1205ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes return prev != old_value; 1219adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#elif defined(__i386__) 1229adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers // cmpxchg8b implicitly uses %ebx which is also the PIC register. 1239adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers int8_t status; 1249adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers __asm__ __volatile__ ( 1259adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers "pushl %%ebx\n" 1269adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers "movl (%3), %%ebx\n" 1279adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers "movl 4(%3), %%ecx\n" 1289adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers "lock cmpxchg8b %1\n" 1299adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers "sete %0\n" 1309adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers "popl %%ebx" 1319adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "=R" (status), "+m" (*addr) 1329adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "A"(old_value), "D" (&new_value) 1339adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers : "%ecx" 1349adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers ); 1359adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers return status != 0; 1369adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#else 1379adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers MutexLock mu(Thread::Current(), GetSwapMutex(addr)); 1387c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes if (*addr == old_value) { 1399adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers *addr = new_value; 1409adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers return true; 1415ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes } 1429adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers return false; 1439adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#endif 1447c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes} 1455ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1469adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersbool QuasiAtomic::LongAtomicsUseMutexes() { 1479adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES 1489adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers return true; 1495ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#else 1509adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers return false; 1517c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif 1529adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers} 1535ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes 1545ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes} // namespace art 155