1// Copyright (c) 2011, Google Inc. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above 11// copyright notice, this list of conditions and the following disclaimer 12// in the documentation and/or other materials provided with the 13// distribution. 14// * Neither the name of Google Inc. nor the names of its 15// contributors may be used to endorse or promote products derived from 16// this software without specific prior written permission. 17// 18// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29// --- 30// 31// Author: Sasha Levitskiy 32// based on atomicops-internals by Sanjay Ghemawat 33// 34// This file is an internal atomic implementation, use base/atomicops.h instead. 35// 36// This code implements ARM atomics for architectures V6 and newer. 37 38#ifndef BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ 39#define BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ 40 41#include <stdio.h> 42#include <stdlib.h> 43#include "base/basictypes.h" // For COMPILE_ASSERT 44 45// The LDREXD and STREXD instructions in ARM all v7 variants or above. In v6, 46// only some variants support it. For simplicity, we only use exclusive 47// 64-bit load/store in V7 or above. 48#if defined(ARMV7) 49# define BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD 50#endif 51 52typedef int32_t Atomic32; 53 54namespace base { 55namespace subtle { 56 57typedef int64_t Atomic64; 58 59// 32-bit low-level ops 60 61inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, 62 Atomic32 old_value, 63 Atomic32 new_value) { 64 Atomic32 oldval, res; 65 do { 66 __asm__ __volatile__( 67 "ldrex %1, [%3]\n" 68 "mov %0, #0\n" 69 "teq %1, %4\n" 70 // The following IT (if-then) instruction is needed for the subsequent 71 // conditional instruction STREXEQ when compiling in THUMB mode. 72 // In ARM mode, the compiler/assembler will not generate any code for it. 73 "it eq\n" 74 "strexeq %0, %5, [%3]\n" 75 : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) 76 : "r" (ptr), "Ir" (old_value), "r" (new_value) 77 : "cc"); 78 } while (res); 79 return oldval; 80} 81 82inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, 83 Atomic32 new_value) { 84 Atomic32 tmp, old; 85 __asm__ __volatile__( 86 "1:\n" 87 "ldrex %1, [%2]\n" 88 "strex %0, %3, [%2]\n" 89 "teq %0, #0\n" 90 "bne 1b" 91 : "=&r" (tmp), "=&r" (old) 92 : "r" (ptr), "r" (new_value) 93 : "cc", "memory"); 94 return old; 95} 96 97inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, 98 Atomic32 increment) { 99 Atomic32 tmp, res; 100 __asm__ __volatile__( 101 "1:\n" 102 "ldrex %1, [%2]\n" 103 "add %1, %1, %3\n" 104 "strex %0, %1, [%2]\n" 105 "teq %0, #0\n" 106 "bne 1b" 107 : "=&r" (tmp), "=&r"(res) 108 : "r" (ptr), "r"(increment) 109 : "cc", "memory"); 110 return res; 111} 112 113inline void MemoryBarrier() { 114 __asm__ __volatile__("dmb" : : : "memory"); 115} 116 117inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, 118 Atomic32 increment) { 119 Atomic32 tmp, res; 120 __asm__ __volatile__( 121 "1:\n" 122 "ldrex %1, [%2]\n" 123 "add %1, %1, %3\n" 124 "dmb\n" 125 "strex %0, %1, [%2]\n" 126 "teq %0, #0\n" 127 "bne 1b" 128 : "=&r" (tmp), "=&r"(res) 129 : "r" (ptr), "r"(increment) 130 : "cc", "memory"); 131 return res; 132} 133 134inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, 135 Atomic32 old_value, 136 Atomic32 new_value) { 137 Atomic32 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); 138 MemoryBarrier(); 139 return value; 140} 141 142inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, 143 Atomic32 old_value, 144 Atomic32 new_value) { 145 MemoryBarrier(); 146 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 147} 148 149inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { 150 *ptr = value; 151} 152 153inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { 154 *ptr = value; 155 MemoryBarrier(); 156} 157 158inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { 159 MemoryBarrier(); 160 *ptr = value; 161} 162 163inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { 164 return *ptr; 165} 166 167inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { 168 Atomic32 value = *ptr; 169 MemoryBarrier(); 170 return value; 171} 172 173inline Atomic32 Release_Load(volatile const Atomic32* ptr) { 174 MemoryBarrier(); 175 return *ptr; 176} 177 178// 64-bit versions are only available if LDREXD and STREXD instructions 179// are available. 180#ifdef BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD 181 182#define BASE_HAS_ATOMIC64 1 183 184inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, 185 Atomic64 old_value, 186 Atomic64 new_value) { 187 Atomic64 oldval, res; 188 do { 189 __asm__ __volatile__( 190 "ldrexd %1, [%3]\n" 191 "mov %0, #0\n" 192 "teq %Q1, %Q4\n" 193 // The following IT (if-then) instructions are needed for the subsequent 194 // conditional instructions when compiling in THUMB mode. 195 // In ARM mode, the compiler/assembler will not generate any code for it. 196 "it eq\n" 197 "teqeq %R1, %R4\n" 198 "it eq\n" 199 "strexdeq %0, %5, [%3]\n" 200 : "=&r" (res), "=&r" (oldval), "+Q" (*ptr) 201 : "r" (ptr), "Ir" (old_value), "r" (new_value) 202 : "cc"); 203 } while (res); 204 return oldval; 205} 206 207inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, 208 Atomic64 new_value) { 209 int store_failed; 210 Atomic64 old; 211 __asm__ __volatile__( 212 "1:\n" 213 "ldrexd %1, [%2]\n" 214 "strexd %0, %3, [%2]\n" 215 "teq %0, #0\n" 216 "bne 1b" 217 : "=&r" (store_failed), "=&r" (old) 218 : "r" (ptr), "r" (new_value) 219 : "cc", "memory"); 220 return old; 221} 222 223inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, 224 Atomic64 increment) { 225 int store_failed; 226 Atomic64 res; 227 __asm__ __volatile__( 228 "1:\n" 229 "ldrexd %1, [%2]\n" 230 "adds %Q1, %Q1, %Q3\n" 231 "adc %R1, %R1, %R3\n" 232 "strexd %0, %1, [%2]\n" 233 "teq %0, #0\n" 234 "bne 1b" 235 : "=&r" (store_failed), "=&r"(res) 236 : "r" (ptr), "r"(increment) 237 : "cc", "memory"); 238 return res; 239} 240 241inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, 242 Atomic64 increment) { 243 int store_failed; 244 Atomic64 res; 245 __asm__ __volatile__( 246 "1:\n" 247 "ldrexd %1, [%2]\n" 248 "adds %Q1, %Q1, %Q3\n" 249 "adc %R1, %R1, %R3\n" 250 "dmb\n" 251 "strexd %0, %1, [%2]\n" 252 "teq %0, #0\n" 253 "bne 1b" 254 : "=&r" (store_failed), "=&r"(res) 255 : "r" (ptr), "r"(increment) 256 : "cc", "memory"); 257 return res; 258} 259 260inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { 261 int store_failed; 262 Atomic64 dummy; 263 __asm__ __volatile__( 264 "1:\n" 265 // Dummy load to lock cache line. 266 "ldrexd %1, [%3]\n" 267 "strexd %0, %2, [%3]\n" 268 "teq %0, #0\n" 269 "bne 1b" 270 : "=&r" (store_failed), "=&r"(dummy) 271 : "r"(value), "r" (ptr) 272 : "cc", "memory"); 273} 274 275inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { 276 Atomic64 res; 277 __asm__ __volatile__( 278 "ldrexd %0, [%1]\n" 279 "clrex\n" 280 : "=r" (res) 281 : "r"(ptr), "Q"(*ptr)); 282 return res; 283} 284 285#else // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD 286 287inline void NotImplementedFatalError(const char *function_name) { 288 fprintf(stderr, "64-bit %s() not implemented on this platform\n", 289 function_name); 290 abort(); 291} 292 293inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, 294 Atomic64 old_value, 295 Atomic64 new_value) { 296 NotImplementedFatalError("NoBarrier_CompareAndSwap"); 297 return 0; 298} 299 300inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, 301 Atomic64 new_value) { 302 NotImplementedFatalError("NoBarrier_AtomicExchange"); 303 return 0; 304} 305 306inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, 307 Atomic64 increment) { 308 NotImplementedFatalError("NoBarrier_AtomicIncrement"); 309 return 0; 310} 311 312inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, 313 Atomic64 increment) { 314 NotImplementedFatalError("Barrier_AtomicIncrement"); 315 return 0; 316} 317 318inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { 319 NotImplementedFatalError("NoBarrier_Store"); 320} 321 322inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { 323 NotImplementedFatalError("NoBarrier_Load"); 324 return 0; 325} 326 327#endif // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD 328 329inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { 330 NoBarrier_Store(ptr, value); 331 MemoryBarrier(); 332} 333 334inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { 335 MemoryBarrier(); 336 NoBarrier_Store(ptr, value); 337} 338 339inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { 340 Atomic64 value = NoBarrier_Load(ptr); 341 MemoryBarrier(); 342 return value; 343} 344 345inline Atomic64 Release_Load(volatile const Atomic64* ptr) { 346 MemoryBarrier(); 347 return NoBarrier_Load(ptr); 348} 349 350inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, 351 Atomic64 old_value, 352 Atomic64 new_value) { 353 Atomic64 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); 354 MemoryBarrier(); 355 return value; 356} 357 358inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, 359 Atomic64 old_value, 360 Atomic64 new_value) { 361 MemoryBarrier(); 362 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 363} 364 365} // namespace subtle ends 366} // namespace base ends 367 368#endif // BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ 369