1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16#ifndef BIONIC_ATOMIC_ARM_H
17#define BIONIC_ATOMIC_ARM_H
18
19#include <machine/cpu-features.h>
20
21/* Some of the harware instructions used below are not available in Thumb-1
22 * mode (they are if you build in ARM or Thumb-2 mode though). To solve this
23 * problem, we're going to use the same technique than libatomics_ops,
24 * which is to temporarily switch to ARM, do the operation, then switch
25 * back to Thumb-1.
26 *
27 * This results in two 'bx' jumps, just like a normal function call, but
28 * everything is kept inlined, avoids loading or computing the function's
29 * address, and prevents a little I-cache trashing too.
30 *
31 * However, it is highly recommended to avoid compiling any C library source
32 * file that use these functions in Thumb-1 mode.
33 *
34 * Define three helper macros to implement this:
35 */
36#if defined(__thumb__) && !defined(__thumb2__)
37#  define  __ATOMIC_SWITCH_TO_ARM \
38            "adr r3, 5f\n" \
39            "bx  r3\n" \
40            ".align\n" \
41            ".arm\n" \
42        "5:\n"
43/* note: the leading \n below is intentional */
44#  define __ATOMIC_SWITCH_TO_THUMB \
45            "\n" \
46            "adr r3, 6f\n" \
47            "bx  r3\n" \
48            ".thumb" \
49        "6:\n"
50
51#  define __ATOMIC_CLOBBERS   "r3"  /* list of clobbered registers */
52
53/* Warn the user that ARM mode should really be preferred! */
54#  warning Rebuilding this source file in ARM mode is highly recommended for performance!!
55
56#else
57#  define  __ATOMIC_SWITCH_TO_ARM   /* nothing */
58#  define  __ATOMIC_SWITCH_TO_THUMB /* nothing */
59#  define  __ATOMIC_CLOBBERS        /* nothing */
60#endif
61
62
63/* Define a full memory barrier, this is only needed if we build the
64 * platform for a multi-core device. For the record, using a 'dmb'
65 * instruction on a Nexus One device can take up to 180 ns even if
66 * it is completely un-necessary on this device.
67 *
68 * NOTE: This is where the platform and NDK headers atomic headers are
69 *        going to diverge. With the NDK, we don't know if the generated
70 *        code is going to run on a single or multi-core device, so we
71 *        need to be cautious.
72 *
73 *        Fortunately, we can use the kernel helper function that is
74 *        mapped at address 0xffff0fa0 in all user process, and that
75 *        provides a device-specific barrier operation.
76 *
77 *        I.e. on single-core devices, the helper immediately returns,
78 *        on multi-core devices, it uses "dmb" or any other means to
79 *        perform a full-memory barrier.
80 *
81 * There are three cases to consider for the platform:
82 *
83 *    - multi-core ARMv7-A       => use the 'dmb' hardware instruction
84 *    - multi-core ARMv6         => use the coprocessor
85 *    - single core ARMv5TE/6/7  => do not use any hardware barrier
86 */
87#if defined(ANDROID_SMP) && ANDROID_SMP == 1
88
89/* Sanity check, multi-core is only supported starting from ARMv6 */
90#  if __ARM_ARCH__ < 6
91#    error ANDROID_SMP should not be set to 1 for an ARM architecture less than 6
92#  endif
93
94#  ifdef __ARM_HAVE_DMB
95/* For ARMv7-A, we can use the 'dmb' instruction directly */
96__ATOMIC_INLINE__ void
97__bionic_memory_barrier(void)
98{
99    /* Note: we always build in ARM or Thumb-2 on ARMv7-A, so don't
100     * bother with __ATOMIC_SWITCH_TO_ARM */
101    __asm__ __volatile__ ( "dmb" : : : "memory" );
102}
103#  else /* !__ARM_HAVE_DMB */
104/* Otherwise, i.e. for multi-core ARMv6, we need to use the coprocessor,
105 * which requires the use of a general-purpose register, which is slightly
106 * less efficient.
107 */
108__ATOMIC_INLINE__ void
109__bionic_memory_barrier(void)
110{
111    __asm__ __volatile__ (
112        __SWITCH_TO_ARM
113        "mcr p15, 0, %0, c7, c10, 5"
114        __SWITCH_TO_THUMB
115        : : "r" (0) : __ATOMIC_CLOBBERS "memory");
116}
117#  endif /* !__ARM_HAVE_DMB */
118#else /* !ANDROID_SMP */
119__ATOMIC_INLINE__ void
120__bionic_memory_barrier(void)
121{
122    /* A simple compiler barrier */
123    __asm__ __volatile__ ( "" : : : "memory" );
124}
125#endif /* !ANDROID_SMP */
126
127/* Compare-and-swap, without any explicit barriers. Note that this functions
128 * returns 0 on success, and 1 on failure. The opposite convention is typically
129 * used on other platforms.
130 *
131 * There are two cases to consider:
132 *
133 *     - ARMv6+  => use LDREX/STREX instructions
134 *     - < ARMv6 => use kernel helper function mapped at 0xffff0fc0
135 *
136 * LDREX/STREX are only available starting from ARMv6
137 */
138#ifdef __ARM_HAVE_LDREX_STREX
139__ATOMIC_INLINE__ int
140__bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr)
141{
142    int32_t prev, status;
143    do {
144        __asm__ __volatile__ (
145            __ATOMIC_SWITCH_TO_ARM
146            "ldrex %0, [%3]\n"
147            "mov %1, #0\n"
148            "teq %0, %4\n"
149#ifdef __thumb2__
150            "it eq\n"
151#endif
152            "strexeq %1, %5, [%3]"
153            __ATOMIC_SWITCH_TO_THUMB
154            : "=&r" (prev), "=&r" (status), "+m"(*ptr)
155            : "r" (ptr), "Ir" (old_value), "r" (new_value)
156            : __ATOMIC_CLOBBERS "cc");
157    } while (__builtin_expect(status != 0, 0));
158    return prev != old_value;
159}
160#  else /* !__ARM_HAVE_LDREX_STREX */
161
162/* Use the handy kernel helper function mapped at 0xffff0fc0 */
163typedef int (kernel_cmpxchg)(int32_t, int32_t, volatile int32_t *);
164
165__ATOMIC_INLINE__ int
166__kernel_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr)
167{
168    /* Note: the kernel function returns 0 on success too */
169    return (*(kernel_cmpxchg *)0xffff0fc0)(old_value, new_value, ptr);
170}
171
172__ATOMIC_INLINE__ int
173__bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr)
174{
175    return __kernel_cmpxchg(old_value, new_value, ptr);
176}
177#endif /* !__ARM_HAVE_LDREX_STREX */
178
179/* Swap operation, without any explicit barriers.
180 * There are again two similar cases to consider:
181 *
182 *   ARMv6+ => use LDREX/STREX
183 *   < ARMv6 => use SWP instead.
184 */
185#ifdef __ARM_HAVE_LDREX_STREX
186__ATOMIC_INLINE__ int32_t
187__bionic_swap(int32_t new_value, volatile int32_t* ptr)
188{
189    int32_t prev, status;
190    do {
191        __asm__ __volatile__ (
192            __ATOMIC_SWITCH_TO_ARM
193            "ldrex %0, [%3]\n"
194            "strex %1, %4, [%3]"
195            __ATOMIC_SWITCH_TO_THUMB
196            : "=&r" (prev), "=&r" (status), "+m" (*ptr)
197            : "r" (ptr), "r" (new_value)
198            : __ATOMIC_CLOBBERS "cc");
199    } while (__builtin_expect(status != 0, 0));
200    return prev;
201}
202#else /* !__ARM_HAVE_LDREX_STREX */
203__ATOMIC_INLINE__ int32_t
204__bionic_swap(int32_t new_value, volatile int32_t* ptr)
205{
206    int32_t prev;
207    /* NOTE: SWP is available in Thumb-1 too */
208    __asm__ __volatile__ ("swp %0, %2, [%3]"
209                          : "=&r" (prev), "+m" (*ptr)
210                          : "r" (new_value), "r" (ptr)
211                          : "cc");
212    return prev;
213}
214#endif /* !__ARM_HAVE_LDREX_STREX */
215
216/* Atomic increment - without any barriers
217 * This returns the old value
218 */
219#ifdef __ARM_HAVE_LDREX_STREX
220__ATOMIC_INLINE__ int32_t
221__bionic_atomic_inc(volatile int32_t* ptr)
222{
223    int32_t prev, tmp, status;
224    do {
225        __asm__ __volatile__ (
226            __ATOMIC_SWITCH_TO_ARM
227            "ldrex %0, [%4]\n"
228            "add %1, %0, #1\n"
229            "strex %2, %1, [%4]"
230            __ATOMIC_SWITCH_TO_THUMB
231            : "=&r" (prev), "=&r" (tmp), "=&r" (status), "+m"(*ptr)
232            : "r" (ptr)
233            : __ATOMIC_CLOBBERS "cc");
234    } while (__builtin_expect(status != 0, 0));
235    return prev;
236}
237#else
238__ATOMIC_INLINE__ int32_t
239__bionic_atomic_inc(volatile int32_t* ptr)
240{
241    int32_t  prev, status;
242    do {
243        prev = *ptr;
244        status = __kernel_cmpxchg(prev, prev+1, ptr);
245    } while (__builtin_expect(status != 0, 0));
246    return prev;
247}
248#endif
249
250/* Atomic decrement - without any barriers
251 * This returns the old value.
252 */
253#ifdef __ARM_HAVE_LDREX_STREX
254__ATOMIC_INLINE__ int32_t
255__bionic_atomic_dec(volatile int32_t* ptr)
256{
257    int32_t prev, tmp, status;
258    do {
259        __asm__ __volatile__ (
260            __ATOMIC_SWITCH_TO_ARM
261            "ldrex %0, [%4]\n"
262            "sub %1, %0, #1\n"
263            "strex %2, %1, [%4]"
264            __ATOMIC_SWITCH_TO_THUMB
265            : "=&r" (prev), "=&r" (tmp), "=&r" (status), "+m"(*ptr)
266            : "r" (ptr)
267            : __ATOMIC_CLOBBERS "cc");
268    } while (__builtin_expect(status != 0, 0));
269    return prev;
270}
271#else
272__ATOMIC_INLINE__ int32_t
273__bionic_atomic_dec(volatile int32_t* ptr)
274{
275    int32_t  prev, status;
276    do {
277        prev = *ptr;
278        status = __kernel_cmpxchg(prev, prev-1, ptr);
279    } while (__builtin_expect(status != 0, 0));
280    return prev;
281}
282#endif
283
284#endif /* SYS_ATOMICS_ARM_H */
285