1/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "Dalvik.h"
18
19#include <cutils/atomic.h>
20
21/*
22 * Quasi-atomic 64-bit operations, for platforms that lack the real thing.
23 *
24 * TODO: unify ARMv6/x86/sh implementations using the to-be-written
25 * spin lock implementation.  We don't want to rely on mutex innards,
26 * and it would be great if all platforms were running the same code.
27 */
28
29#if defined(HAVE_MACOSX_IPC)
30
31#include <libkern/OSAtomic.h>
32
33#if defined(__ppc__)        \
34    || defined(__PPC__)     \
35    || defined(__powerpc__) \
36    || defined(__powerpc)   \
37    || defined(__POWERPC__) \
38    || defined(_M_PPC)      \
39    || defined(__PPC)
40#define NEED_QUASIATOMICS 1
41#else
42
43int dvmQuasiAtomicCas64(int64_t oldvalue, int64_t newvalue,
44    volatile int64_t* addr)
45{
46    return OSAtomicCompareAndSwap64Barrier(oldvalue, newvalue,
47            (int64_t*)addr) == 0;
48}
49
50
51static inline int64_t dvmQuasiAtomicSwap64Body(int64_t value,
52                                               volatile int64_t* addr)
53{
54    int64_t oldValue;
55    do {
56        oldValue = *addr;
57    } while (dvmQuasiAtomicCas64(oldValue, value, addr));
58    return oldValue;
59}
60
61int64_t dvmQuasiAtomicSwap64(int64_t value, volatile int64_t* addr)
62{
63    return dvmQuasiAtomicSwap64Body(value, addr);
64}
65
66int64_t dvmQuasiAtomicSwap64Sync(int64_t value, volatile int64_t* addr)
67{
68    int64_t oldValue;
69    ANDROID_MEMBAR_STORE();
70    oldValue = dvmQuasiAtomicSwap64Body(value, addr);
71    /* TUNING: barriers can be avoided on some architectures */
72    ANDROID_MEMBAR_FULL();
73    return oldValue;
74}
75
76int64_t dvmQuasiAtomicRead64(volatile const int64_t* addr)
77{
78    return OSAtomicAdd64Barrier(0, addr);
79}
80#endif
81
82#elif defined(__i386__) || defined(__x86_64__)
83#define NEED_QUASIATOMICS 1
84
85#elif __arm__
86#include <machine/cpu-features.h>
87
88// Clang can not process this assembly at the moment.
89#if defined(__ARM_HAVE_LDREXD) && !defined(__clang__)
90static inline int64_t dvmQuasiAtomicSwap64Body(int64_t newvalue,
91                                               volatile int64_t* addr)
92{
93    int64_t prev;
94    int status;
95    do {
96        __asm__ __volatile__ ("@ dvmQuasiAtomicSwap64\n"
97            "ldrexd     %0, %H0, [%3]\n"
98            "strexd     %1, %4, %H4, [%3]"
99            : "=&r" (prev), "=&r" (status), "+m"(*addr)
100            : "r" (addr), "r" (newvalue)
101            : "cc");
102    } while (__builtin_expect(status != 0, 0));
103    return prev;
104}
105
106int64_t dvmQuasiAtomicSwap64(int64_t newvalue, volatile int64_t* addr)
107{
108    return dvmQuasiAtomicSwap64Body(newvalue, addr);
109}
110
111int64_t dvmQuasiAtomicSwap64Sync(int64_t newvalue, volatile int64_t* addr)
112{
113    int64_t prev;
114    ANDROID_MEMBAR_STORE();
115    prev = dvmQuasiAtomicSwap64Body(newvalue, addr);
116    ANDROID_MEMBAR_FULL();
117    return prev;
118}
119
120int dvmQuasiAtomicCas64(int64_t oldvalue, int64_t newvalue,
121    volatile int64_t* addr)
122{
123    int64_t prev;
124    int status;
125    do {
126        __asm__ __volatile__ ("@ dvmQuasiAtomicCas64\n"
127            "ldrexd     %0, %H0, [%3]\n"
128            "mov        %1, #0\n"
129            "teq        %0, %4\n"
130            "teqeq      %H0, %H4\n"
131            "strexdeq   %1, %5, %H5, [%3]"
132            : "=&r" (prev), "=&r" (status), "+m"(*addr)
133            : "r" (addr), "Ir" (oldvalue), "r" (newvalue)
134            : "cc");
135    } while (__builtin_expect(status != 0, 0));
136    return prev != oldvalue;
137}
138
139int64_t dvmQuasiAtomicRead64(volatile const int64_t* addr)
140{
141    int64_t value;
142    __asm__ __volatile__ ("@ dvmQuasiAtomicRead64\n"
143        "ldrexd     %0, %H0, [%1]"
144        : "=&r" (value)
145        : "r" (addr));
146    return value;
147}
148
149#else
150
151// on the device, we implement the 64-bit atomic operations through
152// mutex locking. normally, this is bad because we must initialize
153// a pthread_mutex_t before being able to use it, and this means
154// having to do an initialization check on each function call, and
155// that's where really ugly things begin...
156//
157// BUT, as a special twist, we take advantage of the fact that in our
158// pthread library, a mutex is simply a volatile word whose value is always
159// initialized to 0. In other words, simply declaring a static mutex
160// object initializes it !
161//
162// another twist is that we use a small array of mutexes to dispatch
163// the contention locks from different memory addresses
164//
165
166#include <pthread.h>
167
168#define  SWAP_LOCK_COUNT  32U
169static pthread_mutex_t  _swap_locks[SWAP_LOCK_COUNT];
170
171#define  SWAP_LOCK(addr)   \
172   &_swap_locks[((unsigned)(void*)(addr) >> 3U) % SWAP_LOCK_COUNT]
173
174
175int64_t dvmQuasiAtomicSwap64(int64_t value, volatile int64_t* addr)
176{
177    int64_t oldValue;
178    pthread_mutex_t*  lock = SWAP_LOCK(addr);
179
180    pthread_mutex_lock(lock);
181
182    oldValue = *addr;
183    *addr    = value;
184
185    pthread_mutex_unlock(lock);
186    return oldValue;
187}
188
189/* Same as dvmQuasiAtomicSwap64 - mutex handles barrier */
190int64_t dvmQuasiAtomicSwap64Sync(int64_t value, volatile int64_t* addr)
191{
192    return dvmQuasiAtomicSwap64(value, addr);
193}
194
195int dvmQuasiAtomicCas64(int64_t oldvalue, int64_t newvalue,
196    volatile int64_t* addr)
197{
198    int result;
199    pthread_mutex_t*  lock = SWAP_LOCK(addr);
200
201    pthread_mutex_lock(lock);
202
203    if (*addr == oldvalue) {
204        *addr  = newvalue;
205        result = 0;
206    } else {
207        result = 1;
208    }
209    pthread_mutex_unlock(lock);
210    return result;
211}
212
213int64_t dvmQuasiAtomicRead64(volatile const int64_t* addr)
214{
215    int64_t result;
216    pthread_mutex_t*  lock = SWAP_LOCK(addr);
217
218    pthread_mutex_lock(lock);
219    result = *addr;
220    pthread_mutex_unlock(lock);
221    return result;
222}
223
224#endif /*__ARM_HAVE_LDREXD*/
225
226/*****************************************************************************/
227#elif __sh__
228#define NEED_QUASIATOMICS 1
229
230#else
231#error "Unsupported atomic operations for this platform"
232#endif
233
234
235#if NEED_QUASIATOMICS
236
237/* Note that a spinlock is *not* a good idea in general
238 * since they can introduce subtle issues. For example,
239 * a real-time thread trying to acquire a spinlock already
240 * acquired by another thread will never yeld, making the
241 * CPU loop endlessly!
242 *
243 * However, this code is only used on the Linux simulator
244 * so it's probably ok for us.
245 *
246 * The alternative is to use a pthread mutex, but
247 * these must be initialized before being used, and
248 * then you have the problem of lazily initializing
249 * a mutex without any other synchronization primitive.
250 *
251 * TODO: these currently use sched_yield(), which is not guaranteed to
252 * do anything at all.  We need to use dvmIterativeSleep or a wait /
253 * notify mechanism if the initial attempt fails.
254 */
255
256/* global spinlock for all 64-bit quasiatomic operations */
257static int32_t quasiatomic_spinlock = 0;
258
259int dvmQuasiAtomicCas64(int64_t oldvalue, int64_t newvalue,
260    volatile int64_t* addr)
261{
262    int result;
263
264    while (android_atomic_acquire_cas(0, 1, &quasiatomic_spinlock)) {
265#ifdef HAVE_WIN32_THREADS
266        Sleep(0);
267#else
268        sched_yield();
269#endif
270    }
271
272    if (*addr == oldvalue) {
273        *addr = newvalue;
274        result = 0;
275    } else {
276        result = 1;
277    }
278
279    android_atomic_release_store(0, &quasiatomic_spinlock);
280
281    return result;
282}
283
284int64_t dvmQuasiAtomicRead64(volatile const int64_t* addr)
285{
286    int64_t result;
287
288    while (android_atomic_acquire_cas(0, 1, &quasiatomic_spinlock)) {
289#ifdef HAVE_WIN32_THREADS
290        Sleep(0);
291#else
292        sched_yield();
293#endif
294    }
295
296    result = *addr;
297    android_atomic_release_store(0, &quasiatomic_spinlock);
298
299    return result;
300}
301
302int64_t dvmQuasiAtomicSwap64(int64_t value, volatile int64_t* addr)
303{
304    int64_t result;
305
306    while (android_atomic_acquire_cas(0, 1, &quasiatomic_spinlock)) {
307#ifdef HAVE_WIN32_THREADS
308        Sleep(0);
309#else
310        sched_yield();
311#endif
312    }
313
314    result = *addr;
315    *addr = value;
316    android_atomic_release_store(0, &quasiatomic_spinlock);
317
318    return result;
319}
320
321/* Same as dvmQuasiAtomicSwap64 - syscall handles barrier */
322int64_t dvmQuasiAtomicSwap64Sync(int64_t value, volatile int64_t* addr)
323{
324    return dvmQuasiAtomicSwap64(value, addr);
325}
326
327#endif /*NEED_QUASIATOMICS*/
328