1b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// Copyright 2010 the V8 project authors. All rights reserved. 2b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// Use of this source code is governed by a BSD-style license that can be 3b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// found in the LICENSE file. 4b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 5b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// This file is an internal atomic implementation, use atomicops.h instead. 6b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// 7b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// LinuxKernelCmpxchg and Barrier_AtomicIncrement are from Google Gears. 8b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 9b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#ifndef V8_BASE_ATOMICOPS_INTERNALS_ARM_GCC_H_ 10b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#define V8_BASE_ATOMICOPS_INTERNALS_ARM_GCC_H_ 11b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 12b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#if defined(__QNXNTO__) 13b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include <sys/cpuinline.h> 14b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#endif 15b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 16b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochnamespace v8 { 17b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochnamespace base { 18b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 19b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// Memory barriers on ARM are funky, but the kernel is here to help: 20b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// 21b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// * ARMv5 didn't support SMP, there is no memory barrier instruction at 22b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// all on this architecture, or when targeting its machine code. 23b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// 24b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// * Some ARMv6 CPUs support SMP. A full memory barrier can be produced by 25b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// writing a random value to a very specific coprocessor register. 26b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// 27b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// * On ARMv7, the "dmb" instruction is used to perform a full memory 28b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// barrier (though writing to the co-processor will still work). 29b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// However, on single core devices (e.g. Nexus One, or Nexus S), 30b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// this instruction will take up to 200 ns, which is huge, even though 31b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// it's completely un-needed on these devices. 32b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// 33b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// * There is no easy way to determine at runtime if the device is 34b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// single or multi-core. However, the kernel provides a useful helper 35b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// function at a fixed memory address (0xffff0fa0), which will always 36b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// perform a memory barrier in the most efficient way. I.e. on single 37b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// core devices, this is an empty function that exits immediately. 38b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// On multi-core devices, it implements a full memory barrier. 39b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// 40b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// * This source could be compiled to ARMv5 machine code that runs on a 41b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// multi-core ARMv6 or ARMv7 device. In this case, memory barriers 42b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// are needed for correct execution. Always call the kernel helper, even 43b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// when targeting ARMv5TE. 44b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// 45b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 46b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline void MemoryBarrier() { 47b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#if defined(__linux__) || defined(__ANDROID__) 48b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Note: This is a function call, which is also an implicit compiler barrier. 49b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch typedef void (*KernelMemoryBarrierFunc)(); 50b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ((KernelMemoryBarrierFunc)0xffff0fa0)(); 51b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#elif defined(__QNXNTO__) 52b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch __cpu_membarrier(); 53b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#else 54b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#error MemoryBarrier() is not implemented on this platform. 55b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#endif 56b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 57b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 58b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// An ARM toolchain would only define one of these depending on which 59b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// variant of the target architecture is being used. This tests against 60b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// any known ARMv6 or ARMv7 variant, where it is possible to directly 61b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// use ldrex/strex instructions to implement fast atomic operations. 62b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || \ 63b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || \ 64b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || \ 65b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || \ 66b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch defined(__ARM_ARCH_6KZ__) || defined(__ARM_ARCH_6T2__) 67b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 68b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, 69b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 old_value, 70b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 new_value) { 71b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 prev_value; 72b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int reloop; 73b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch do { 74b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // The following is equivalent to: 75b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // 76b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // prev_value = LDREX(ptr) 77b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // reloop = 0 78b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // if (prev_value != old_value) 79b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // reloop = STREX(ptr, new_value) 80b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch __asm__ __volatile__(" ldrex %0, [%3]\n" 81b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch " mov %1, #0\n" 82b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch " cmp %0, %4\n" 83b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#ifdef __thumb2__ 84b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch " it eq\n" 85b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#endif 86b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch " strexeq %1, %5, [%3]\n" 87b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : "=&r"(prev_value), "=&r"(reloop), "+m"(*ptr) 88b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : "r"(ptr), "r"(old_value), "r"(new_value) 89b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : "cc", "memory"); 90b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } while (reloop != 0); 91b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return prev_value; 92b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 93b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 94b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, 95b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 old_value, 96b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 new_value) { 97b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 result = NoBarrier_CompareAndSwap(ptr, old_value, new_value); 98b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryBarrier(); 99b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return result; 100b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 101b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 102b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, 103b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 old_value, 104b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 new_value) { 105b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryBarrier(); 106b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return NoBarrier_CompareAndSwap(ptr, old_value, new_value); 107b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 108b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 109b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, 110b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 increment) { 111b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 value; 112b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int reloop; 113b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch do { 114b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Equivalent to: 115b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // 116b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // value = LDREX(ptr) 117b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // value += increment 118b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // reloop = STREX(ptr, value) 119b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // 120b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch __asm__ __volatile__(" ldrex %0, [%3]\n" 121b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch " add %0, %0, %4\n" 122b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch " strex %1, %0, [%3]\n" 123b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : "=&r"(value), "=&r"(reloop), "+m"(*ptr) 124b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : "r"(ptr), "r"(increment) 125b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : "cc", "memory"); 126b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } while (reloop); 127b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return value; 128b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 129b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 130b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, 131b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 increment) { 132b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // TODO(digit): Investigate if it's possible to implement this with 133b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // a single MemoryBarrier() operation between the LDREX and STREX. 134b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // See http://crbug.com/246514 135b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryBarrier(); 136b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 result = NoBarrier_AtomicIncrement(ptr, increment); 137b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryBarrier(); 138b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return result; 139b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 140b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 141b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, 142b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 new_value) { 143b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 old_value; 144b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int reloop; 145b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch do { 146b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // old_value = LDREX(ptr) 147b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // reloop = STREX(ptr, new_value) 148b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch __asm__ __volatile__(" ldrex %0, [%3]\n" 149b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch " strex %1, %4, [%3]\n" 150b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : "=&r"(old_value), "=&r"(reloop), "+m"(*ptr) 151b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : "r"(ptr), "r"(new_value) 152b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : "cc", "memory"); 153b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } while (reloop != 0); 154b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return old_value; 155b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 156b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 157b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// This tests against any known ARMv5 variant. 158b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5T__) || \ 159b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch defined(__ARM_ARCH_5TE__) || defined(__ARM_ARCH_5TEJ__) 160b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 161b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// The kernel also provides a helper function to perform an atomic 162b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// compare-and-swap operation at the hard-wired address 0xffff0fc0. 163b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// On ARMv5, this is implemented by a special code path that the kernel 164b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// detects and treats specially when thread pre-emption happens. 165b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// On ARMv6 and higher, it uses LDREX/STREX instructions instead. 166b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// 167b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// Note that this always perform a full memory barrier, there is no 168b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// need to add calls MemoryBarrier() before or after it. It also 169b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// returns 0 on success, and 1 on exit. 170b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// 171b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// Available and reliable since Linux 2.6.24. Both Android and ChromeOS 172b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// use newer kernel revisions, so this should not be a concern. 173b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochnamespace { 174b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 175b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline int LinuxKernelCmpxchg(Atomic32 old_value, 176b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 new_value, 177b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch volatile Atomic32* ptr) { 178b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch typedef int (*KernelCmpxchgFunc)(Atomic32, Atomic32, volatile Atomic32*); 179b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return ((KernelCmpxchgFunc)0xffff0fc0)(old_value, new_value, ptr); 180b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 181b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 182b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} // namespace 183b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 184b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, 185b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 old_value, 186b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 new_value) { 187b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 prev_value; 188b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for (;;) { 189b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch prev_value = *ptr; 190b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (prev_value != old_value) 191b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return prev_value; 192b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!LinuxKernelCmpxchg(old_value, new_value, ptr)) 193b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return old_value; 194b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 195b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 196b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 197b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, 198b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 new_value) { 199b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 old_value; 200b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch do { 201b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch old_value = *ptr; 202b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } while (LinuxKernelCmpxchg(old_value, new_value, ptr)); 203b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return old_value; 204b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 205b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 206b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, 207b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 increment) { 208b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return Barrier_AtomicIncrement(ptr, increment); 209b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 210b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 211b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, 212b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 increment) { 213b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for (;;) { 214b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Atomic exchange the old value with an incremented one. 215b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 old_value = *ptr; 216b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 new_value = old_value + increment; 217b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!LinuxKernelCmpxchg(old_value, new_value, ptr)) { 218b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // The exchange took place as expected. 219b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return new_value; 220b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 221b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Otherwise, *ptr changed mid-loop and we need to retry. 222b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 223b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 224b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 225b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, 226b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 old_value, 227b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 new_value) { 228b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 prev_value; 229b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for (;;) { 230b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch prev_value = *ptr; 231b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (prev_value != old_value) { 232b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Always ensure acquire semantics. 233b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryBarrier(); 234b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return prev_value; 235b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 236b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!LinuxKernelCmpxchg(old_value, new_value, ptr)) 237b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return old_value; 238b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 239b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 240b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 241b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, 242b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 old_value, 243b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 new_value) { 244b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // This could be implemented as: 245b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // MemoryBarrier(); 246b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // return NoBarrier_CompareAndSwap(); 247b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // 248b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // But would use 3 barriers per succesful CAS. To save performance, 249b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // use Acquire_CompareAndSwap(). Its implementation guarantees that: 250b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // - A succesful swap uses only 2 barriers (in the kernel helper). 251b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // - An early return due to (prev_value != old_value) performs 252b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // a memory barrier with no store, which is equivalent to the 253b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // generic implementation above. 254b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return Acquire_CompareAndSwap(ptr, old_value, new_value); 255b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 256b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 257b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#else 258b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# error "Your CPU's ARM architecture is not supported yet" 259b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#endif 260b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 261b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// NOTE: Atomicity of the following load and store operations is only 262b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// guaranteed in case of 32-bit alignement of |ptr| values. 263b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 264b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { 265b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch *ptr = value; 266b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 267b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 268b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { 269b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch *ptr = value; 270b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryBarrier(); 271b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 272b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 273b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { 274b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryBarrier(); 275b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch *ptr = value; 276b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 277b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 278b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { return *ptr; } 279b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 280b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { 281b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Atomic32 value = *ptr; 282b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryBarrier(); 283b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return value; 284b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 285b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 286b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic32 Release_Load(volatile const Atomic32* ptr) { 287b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryBarrier(); 288b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return *ptr; 289b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 290b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 291b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// Byte accessors. 292b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 293b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline void NoBarrier_Store(volatile Atomic8* ptr, Atomic8 value) { 294b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch *ptr = value; 295b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 296b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 297b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochinline Atomic8 NoBarrier_Load(volatile const Atomic8* ptr) { return *ptr; } 298b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 299b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} } // namespace v8::base 300b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 301b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#endif // V8_BASE_ATOMICOPS_INTERNALS_ARM_GCC_H_ 302