1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "atomic.h"
18
19#define NEED_SWAP_MUTEXES !defined(__arm__) && !defined(__i386__)
20
21#if NEED_SWAP_MUTEXES
22#include <vector>
23#include "base/mutex.h"
24#include "base/stl_util.h"
25#include "base/stringprintf.h"
26#include "thread.h"
27#endif
28
29namespace art {
30
31#if NEED_SWAP_MUTEXES
32// We stripe across a bunch of different mutexes to reduce contention.
33static const size_t kSwapMutexCount = 32;
34static std::vector<Mutex*>* gSwapMutexes;
35
36static Mutex& GetSwapMutex(const volatile int64_t* addr) {
37  return *(*gSwapMutexes)[(reinterpret_cast<unsigned>(addr) >> 3U) % kSwapMutexCount];
38}
39#endif
40
41void QuasiAtomic::Startup() {
42#if NEED_SWAP_MUTEXES
43  gSwapMutexes = new std::vector<Mutex*>;
44  for (size_t i = 0; i < kSwapMutexCount; ++i) {
45    gSwapMutexes->push_back(new Mutex("QuasiAtomic stripe"));
46  }
47#endif
48}
49
50void QuasiAtomic::Shutdown() {
51#if NEED_SWAP_MUTEXES
52  STLDeleteElements(gSwapMutexes);
53  delete gSwapMutexes;
54#endif
55}
56
57int64_t QuasiAtomic::Read64(volatile const int64_t* addr) {
58  int64_t value;
59#if NEED_SWAP_MUTEXES
60  MutexLock mu(Thread::Current(), GetSwapMutex(addr));
61  value = *addr;
62#elif defined(__arm__)
63  // Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary. If we
64  // have LPAE (such as Cortex-A15) then ldrd would suffice.
65  __asm__ __volatile__("@ QuasiAtomic::Read64\n"
66      "ldrexd     %0, %H0, [%1]"
67      : "=&r" (value)
68      : "r" (addr));
69#elif defined(__i386__)
70  __asm__ __volatile__(
71      "movq     %1, %0\n"
72      : "=x" (value)
73      : "m" (*addr));
74#else
75#error Unexpected architecture
76#endif
77  return value;
78}
79
80void QuasiAtomic::Write64(volatile int64_t* addr, int64_t value) {
81#if NEED_SWAP_MUTEXES
82  MutexLock mu(Thread::Current(), GetSwapMutex(addr));
83  *addr = value;
84#elif defined(__arm__)
85  // The write is done as a swap so that the cache-line is in the exclusive state for the store. If
86  // we know that ARM architecture has LPAE (such as Cortex-A15) this isn't necessary and strd will
87  // suffice.
88  int64_t prev;
89  int status;
90  do {
91    __asm__ __volatile__("@ QuasiAtomic::Write64\n"
92        "ldrexd     %0, %H0, [%3]\n"
93        "strexd     %1, %4, %H4, [%3]"
94        : "=&r" (prev), "=&r" (status), "+m"(*addr)
95        : "r" (addr), "r" (value)
96        : "cc");
97  } while (__builtin_expect(status != 0, 0));
98#elif defined(__i386__)
99  __asm__ __volatile__(
100      "movq     %1, %0"
101      : "=m" (*addr)
102      : "x" (value));
103#else
104#error Unexpected architecture
105#endif
106}
107
108
109bool QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) {
110#if NEED_SWAP_MUTEXES
111  MutexLock mu(Thread::Current(), GetSwapMutex(addr));
112  if (*addr == old_value) {
113    *addr = new_value;
114    return true;
115  }
116  return false;
117#elif defined(__arm__)
118  int64_t prev;
119  int status;
120  do {
121    __asm__ __volatile__("@ QuasiAtomic::Cas64\n"
122        "ldrexd     %0, %H0, [%3]\n"
123        "mov        %1, #0\n"
124        "teq        %0, %4\n"
125        "teqeq      %H0, %H4\n"
126        "strexdeq   %1, %5, %H5, [%3]"
127        : "=&r" (prev), "=&r" (status), "+m"(*addr)
128        : "r" (addr), "Ir" (old_value), "r" (new_value)
129        : "cc");
130  } while (__builtin_expect(status != 0, 0));
131  return prev == old_value;
132#elif defined(__i386__)
133  // The compiler does the right job and works better than inline assembly, especially with -O0
134  // compilation.
135  return __sync_bool_compare_and_swap(addr, old_value, new_value);
136#else
137#error Unexpected architecture
138#endif
139}
140
141bool QuasiAtomic::LongAtomicsUseMutexes() {
142#if NEED_SWAP_MUTEXES
143  return true;
144#else
145  return false;
146#endif
147}
148
149}  // namespace art
150