atomic.cc revision 9adbff5b85fcae2b3e2443344415f6c17ea3ba0a
15ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes/*
25ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Copyright (C) 2010 The Android Open Source Project
35ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes *
45ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Licensed under the Apache License, Version 2.0 (the "License");
55ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * you may not use this file except in compliance with the License.
65ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * You may obtain a copy of the License at
75ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes *
85ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes *      http://www.apache.org/licenses/LICENSE-2.0
95ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes *
105ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * Unless required by applicable law or agreed to in writing, software
115ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * distributed under the License is distributed on an "AS IS" BASIS,
125ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
135ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * See the License for the specific language governing permissions and
145ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes * limitations under the License.
155ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes */
165ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes
175ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#include "atomic.h"
185ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes
199adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#define NEED_SWAP_MUTEXES !defined(__arm__) && !defined(__i386__)
20e222ee0b794f941af4fb1b32fb8224e32942ea7bElliott Hughes
219adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES
2225fd14b87cced64a179dee885573113be5e11944Ian Rogers#include <vector>
2376b6167407c2b6f5d40ad895b2793a6b037f54b2Elliott Hughes#include "base/mutex.h"
241aa246dec5abe212f699de1413a0c4a191ca364aElliott Hughes#include "base/stl_util.h"
25e222ee0b794f941af4fb1b32fb8224e32942ea7bElliott Hughes#include "base/stringprintf.h"
2650b35e2fd1a68cd1240e4a9d9f363e11764957d1Ian Rogers#include "thread.h"
277c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif
287c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes
297c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughesnamespace art {
305ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes
319adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES
329adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers// We stripe across a bunch of different mutexes to reduce contention.
339adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersstatic const size_t kSwapMutexCount = 32;
349adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersstatic std::vector<Mutex*>* gSwapMutexes;
355ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes
369adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersstatic Mutex& GetSwapMutex(const volatile int64_t* addr) {
379adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  return *(*gSwapMutexes)[((unsigned)(void*)(addr) >> 3U) % kSwapMutexCount];
389adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers}
399adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#endif
405ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes
419adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersvoid QuasiAtomic::Startup() {
429adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES
439adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  gSwapMutexes = new std::vector<Mutex*>;
449adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  for (size_t i = 0; i < kSwapMutexCount; ++i) {
459adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers    gSwapMutexes->push_back(new Mutex(StringPrintf("QuasiAtomic stripe %d", i).c_str()));
469adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  }
479adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#endif
489adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers}
495ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes
509adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersvoid QuasiAtomic::Shutdown() {
519adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES
529adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  STLDeleteElements(gSwapMutexes);
539adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  delete gSwapMutexes;
547c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif
559adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers}
56557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes
579adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersint64_t QuasiAtomic::Read64(volatile const int64_t* addr) {
589adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  int64_t value;
599adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if defined(__arm__)
609adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  // Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary. If we
619adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  // have LPAE (such as Cortex-A15) then ldrd would suffice.
629adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  __asm__ __volatile__("@ QuasiAtomic::Read64\n"
639adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      "ldrexd     %0, %H0, [%1]"
649adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      : "=&r" (value)
659adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      : "r" (addr));
669adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#elif defined(__i386__)
679adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  __asm__ __volatile__(
689adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      "movq     %1, %0\n"
699adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      : "=x" (value)
709adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      : "m" (*addr));
717c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#else
729adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  MutexLock mu(Thread::Current(), GetSwapMutex(addr));
739adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  return *addr;
745ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#endif
759adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  return value;
769adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers}
775ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes
789adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersvoid QuasiAtomic::Write64(volatile int64_t* addr, int64_t value) {
799adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if defined(__arm__)
809adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  // The write is done as a swap so that the cache-line is in the exclusive state for the store. If
819adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  // we know that ARM architecture has LPAE (such as Cortex-A15) this isn't necessary and strd will
829adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  // suffice.
835ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes  int64_t prev;
845ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes  int status;
855ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes  do {
869adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers    __asm__ __volatile__("@ QuasiAtomic::Write64\n"
875ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes        "ldrexd     %0, %H0, [%3]\n"
885ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes        "strexd     %1, %4, %H4, [%3]"
895ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes        : "=&r" (prev), "=&r" (status), "+m"(*addr)
909adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers        : "r" (addr), "r" (value)
915ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes        : "cc");
925ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes  } while (__builtin_expect(status != 0, 0));
939adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#elif defined(__i386__)
949adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  __asm__ __volatile__(
959adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      "movq     %1, %0"
969adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      : "=m" (*addr)
979adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      : "x" (value));
989adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#else
999adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  MutexLock mu(Thread::Current(), GetSwapMutex(addr));
1009adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  *addr = value;
1019adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#endif
102557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes}
103557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes
104557e027f86d86f801e1b48055f8116f2d83d3d5cElliott Hughes
1059adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersbool QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) {
1069adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if defined(__arm__)
1075ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes  int64_t prev;
1085ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes  int status;
1095ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes  do {
1107c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes    __asm__ __volatile__("@ QuasiAtomic::Cas64\n"
1115ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes        "ldrexd     %0, %H0, [%3]\n"
1125ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes        "mov        %1, #0\n"
1135ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes        "teq        %0, %4\n"
1145ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes        "teqeq      %H0, %H4\n"
1155ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes        "strexdeq   %1, %5, %H5, [%3]"
1165ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes        : "=&r" (prev), "=&r" (status), "+m"(*addr)
1175ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes        : "r" (addr), "Ir" (old_value), "r" (new_value)
1185ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes        : "cc");
1195ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes  } while (__builtin_expect(status != 0, 0));
1205ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes  return prev != old_value;
1219adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#elif defined(__i386__)
1229adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  // cmpxchg8b implicitly uses %ebx which is also the PIC register.
1239adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  int8_t status;
1249adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  __asm__ __volatile__ (
1259adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      "pushl          %%ebx\n"
1269adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      "movl           (%3), %%ebx\n"
1279adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      "movl           4(%3), %%ecx\n"
1289adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      "lock cmpxchg8b %1\n"
1299adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      "sete           %0\n"
1309adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      "popl           %%ebx"
1319adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      : "=R" (status), "+m" (*addr)
1329adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      : "A"(old_value), "D" (&new_value)
1339adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      : "%ecx"
1349adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers      );
1359adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  return status != 0;
1369adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#else
1379adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  MutexLock mu(Thread::Current(), GetSwapMutex(addr));
1387c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes  if (*addr == old_value) {
1399adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers    *addr = new_value;
1409adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers    return true;
1415ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes  }
1429adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  return false;
1439adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#endif
1447c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes}
1455ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes
1469adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogersbool QuasiAtomic::LongAtomicsUseMutexes() {
1479adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers#if NEED_SWAP_MUTEXES
1489adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  return true;
1495ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes#else
1509adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers  return false;
1517c6169de901fd0a39c8e0c078874dc25207f5b59Elliott Hughes#endif
1529adbff5b85fcae2b3e2443344415f6c17ea3ba0aIan Rogers}
1535ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes
1545ea047b386c5dac78eda62305d14dedf7b5611a8Elliott Hughes}  // namespace art
155