atomic-arm.h revision 2bf937e6bbe3137cd97596e8f1f26ed45415eb4b
1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#ifndef ANDROID_CUTILS_ATOMIC_ARM_H 18#define ANDROID_CUTILS_ATOMIC_ARM_H 19 20#include <stdint.h> 21#include <machine/cpu-features.h> 22 23extern inline void android_compiler_barrier(void) 24{ 25 __asm__ __volatile__ ("" : : : "memory"); 26} 27 28#if ANDROID_SMP == 0 29extern inline void android_memory_barrier(void) 30{ 31 android_compiler_barrier(); 32} 33extern inline void android_memory_store_barrier(void) 34{ 35 android_compiler_barrier(); 36} 37#elif defined(__ARM_HAVE_DMB) 38extern inline void android_memory_barrier(void) 39{ 40 __asm__ __volatile__ ("dmb" : : : "memory"); 41} 42extern inline void android_memory_store_barrier(void) 43{ 44 __asm__ __volatile__ ("dmb st" : : : "memory"); 45} 46#elif defined(__ARM_HAVE_LDREX_STREX) 47extern inline void android_memory_barrier(void) 48{ 49 __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory"); 50} 51extern inline void android_memory_store_barrier(void) 52{ 53 android_memory_barrier(); 54} 55#else 56extern inline void android_memory_barrier(void) 57{ 58 typedef void (kuser_memory_barrier)(void); 59 (*(kuser_memory_barrier *)0xffff0fa0)(); 60} 61extern inline void android_memory_store_barrier(void) 62{ 63 android_memory_barrier(); 64} 65#endif 66 67extern inline int32_t android_atomic_acquire_load(volatile const int32_t *ptr) 68{ 69 int32_t value = *ptr; 70 android_memory_barrier(); 71 return value; 72} 73 74extern inline int32_t android_atomic_release_load(volatile const int32_t *ptr) 75{ 76 android_memory_barrier(); 77 return *ptr; 78} 79 80extern inline void android_atomic_acquire_store(int32_t value, 81 volatile int32_t *ptr) 82{ 83 *ptr = value; 84 android_memory_barrier(); 85} 86 87extern inline void android_atomic_release_store(int32_t value, 88 volatile int32_t *ptr) 89{ 90 android_memory_barrier(); 91 *ptr = value; 92} 93 94#if defined(__thumb__) 95extern int android_atomic_cas(int32_t old_value, int32_t new_value, 96 volatile int32_t *ptr); 97#elif defined(__ARM_HAVE_LDREX_STREX) 98extern inline int android_atomic_cas(int32_t old_value, int32_t new_value, 99 volatile int32_t *ptr) 100{ 101 int32_t prev, status; 102 do { 103 __asm__ __volatile__ ("ldrex %0, [%3]\n" 104 "mov %1, #0\n" 105 "teq %0, %4\n" 106 "strexeq %1, %5, [%3]" 107 : "=&r" (prev), "=&r" (status), "+m"(*ptr) 108 : "r" (ptr), "Ir" (old_value), "r" (new_value) 109 : "cc"); 110 } while (__builtin_expect(status != 0, 0)); 111 return prev != old_value; 112} 113#else 114extern inline int android_atomic_cas(int32_t old_value, int32_t new_value, 115 volatile int32_t *ptr) 116{ 117 typedef int (kuser_cmpxchg)(int32_t, int32_t, volatile int32_t *); 118 int32_t prev, status; 119 prev = *ptr; 120 do { 121 status = (*(kuser_cmpxchg *)0xffff0fc0)(old_value, new_value, ptr); 122 if (__builtin_expect(status == 0, 1)) 123 return 0; 124 prev = *ptr; 125 } while (prev == old_value); 126 return 1; 127} 128#endif 129 130extern inline int android_atomic_acquire_cas(int32_t old_value, 131 int32_t new_value, 132 volatile int32_t *ptr) 133{ 134 int status = android_atomic_cas(old_value, new_value, ptr); 135 android_memory_barrier(); 136 return status; 137} 138 139extern inline int android_atomic_release_cas(int32_t old_value, 140 int32_t new_value, 141 volatile int32_t *ptr) 142{ 143 android_memory_barrier(); 144 return android_atomic_cas(old_value, new_value, ptr); 145} 146 147 148#if defined(__thumb__) 149extern int32_t android_atomic_swap(int32_t new_value, 150 volatile int32_t *ptr); 151#elif defined(__ARM_HAVE_LDREX_STREX) 152extern inline int32_t android_atomic_swap(int32_t new_value, 153 volatile int32_t *ptr) 154{ 155 int32_t prev, status; 156 do { 157 __asm__ __volatile__ ("ldrex %0, [%3]\n" 158 "strex %1, %4, [%3]" 159 : "=&r" (prev), "=&r" (status), "+m" (*ptr) 160 : "r" (ptr), "r" (new_value) 161 : "cc"); 162 } while (__builtin_expect(status != 0, 0)); 163 android_memory_barrier(); 164 return prev; 165} 166#else 167extern inline int32_t android_atomic_swap(int32_t new_value, 168 volatile int32_t *ptr) 169{ 170 int32_t prev; 171 __asm__ __volatile__ ("swp %0, %2, [%3]" 172 : "=&r" (prev), "+m" (*ptr) 173 : "r" (new_value), "r" (ptr) 174 : "cc"); 175 android_memory_barrier(); 176 return prev; 177} 178#endif 179 180#if defined(__thumb__) 181extern int32_t android_atomic_add(int32_t increment, 182 volatile int32_t *ptr); 183#elif defined(__ARM_HAVE_LDREX_STREX) 184extern inline int32_t android_atomic_add(int32_t increment, 185 volatile int32_t *ptr) 186{ 187 int32_t prev, tmp, status; 188 android_memory_barrier(); 189 do { 190 __asm__ __volatile__ ("ldrex %0, [%4]\n" 191 "add %1, %0, %5\n" 192 "strex %2, %1, [%4]" 193 : "=&r" (prev), "=&r" (tmp), 194 "=&r" (status), "+m" (*ptr) 195 : "r" (ptr), "Ir" (increment) 196 : "cc"); 197 } while (__builtin_expect(status != 0, 0)); 198 return prev; 199} 200#else 201extern inline int32_t android_atomic_add(int32_t increment, 202 volatile int32_t *ptr) 203{ 204 int32_t prev, status; 205 android_memory_barrier(); 206 do { 207 prev = *ptr; 208 status = android_atomic_cas(prev, prev + increment, ptr); 209 } while (__builtin_expect(status != 0, 0)); 210 return prev; 211} 212#endif 213 214extern inline int32_t android_atomic_inc(volatile int32_t *addr) 215{ 216 return android_atomic_add(1, addr); 217} 218 219extern inline int32_t android_atomic_dec(volatile int32_t *addr) 220{ 221 return android_atomic_add(-1, addr); 222} 223 224#if defined(__thumb__) 225extern int32_t android_atomic_and(int32_t value, volatile int32_t *ptr); 226#elif defined(__ARM_HAVE_LDREX_STREX) 227extern inline int32_t android_atomic_and(int32_t value, volatile int32_t *ptr) 228{ 229 int32_t prev, tmp, status; 230 android_memory_barrier(); 231 do { 232 __asm__ __volatile__ ("ldrex %0, [%4]\n" 233 "and %1, %0, %5\n" 234 "strex %2, %1, [%4]" 235 : "=&r" (prev), "=&r" (tmp), 236 "=&r" (status), "+m" (*ptr) 237 : "r" (ptr), "Ir" (value) 238 : "cc"); 239 } while (__builtin_expect(status != 0, 0)); 240 return prev; 241} 242#else 243extern inline int32_t android_atomic_and(int32_t value, volatile int32_t *ptr) 244{ 245 int32_t prev, status; 246 android_memory_barrier(); 247 do { 248 prev = *ptr; 249 status = android_atomic_cas(prev, prev & value, ptr); 250 } while (__builtin_expect(status != 0, 0)); 251 return prev; 252} 253#endif 254 255#if defined(__thumb__) 256extern int32_t android_atomic_or(int32_t value, volatile int32_t *ptr); 257#elif defined(__ARM_HAVE_LDREX_STREX) 258extern inline int32_t android_atomic_or(int32_t value, volatile int32_t *ptr) 259{ 260 int32_t prev, tmp, status; 261 android_memory_barrier(); 262 do { 263 __asm__ __volatile__ ("ldrex %0, [%4]\n" 264 "orr %1, %0, %5\n" 265 "strex %2, %1, [%4]" 266 : "=&r" (prev), "=&r" (tmp), 267 "=&r" (status), "+m" (*ptr) 268 : "r" (ptr), "Ir" (value) 269 : "cc"); 270 } while (__builtin_expect(status != 0, 0)); 271 return prev; 272} 273#else 274extern inline int32_t android_atomic_or(int32_t value, volatile int32_t *ptr) 275{ 276 int32_t prev, status; 277 android_memory_barrier(); 278 do { 279 prev = *ptr; 280 status = android_atomic_cas(prev, prev | value, ptr); 281 } while (__builtin_expect(status != 0, 0)); 282 return prev; 283} 284#endif 285 286#endif /* ANDROID_CUTILS_ATOMIC_ARM_H */ 287