1#ifndef __ASM_ARM_CMPXCHG_H
2#define __ASM_ARM_CMPXCHG_H
3
4#include <linux/irqflags.h>
5#include <linux/prefetch.h>
6#include <asm/barrier.h>
7
8#if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110)
9/*
10 * On the StrongARM, "swp" is terminally broken since it bypasses the
11 * cache totally.  This means that the cache becomes inconsistent, and,
12 * since we use normal loads/stores as well, this is really bad.
13 * Typically, this causes oopsen in filp_close, but could have other,
14 * more disastrous effects.  There are two work-arounds:
15 *  1. Disable interrupts and emulate the atomic swap
16 *  2. Clean the cache, perform atomic swap, flush the cache
17 *
18 * We choose (1) since its the "easiest" to achieve here and is not
19 * dependent on the processor type.
20 *
21 * NOTE that this solution won't work on an SMP system, so explcitly
22 * forbid it here.
23 */
24#define swp_is_buggy
25#endif
26
27static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
28{
29	extern void __bad_xchg(volatile void *, int);
30	unsigned long ret;
31#ifdef swp_is_buggy
32	unsigned long flags;
33#endif
34#if __LINUX_ARM_ARCH__ >= 6
35	unsigned int tmp;
36#endif
37
38	smp_mb();
39	prefetchw((const void *)ptr);
40
41	switch (size) {
42#if __LINUX_ARM_ARCH__ >= 6
43	case 1:
44		asm volatile("@	__xchg1\n"
45		"1:	ldrexb	%0, [%3]\n"
46		"	strexb	%1, %2, [%3]\n"
47		"	teq	%1, #0\n"
48		"	bne	1b"
49			: "=&r" (ret), "=&r" (tmp)
50			: "r" (x), "r" (ptr)
51			: "memory", "cc");
52		break;
53	case 4:
54		asm volatile("@	__xchg4\n"
55		"1:	ldrex	%0, [%3]\n"
56		"	strex	%1, %2, [%3]\n"
57		"	teq	%1, #0\n"
58		"	bne	1b"
59			: "=&r" (ret), "=&r" (tmp)
60			: "r" (x), "r" (ptr)
61			: "memory", "cc");
62		break;
63#elif defined(swp_is_buggy)
64#ifdef CONFIG_SMP
65#error SMP is not supported on this platform
66#endif
67	case 1:
68		raw_local_irq_save(flags);
69		ret = *(volatile unsigned char *)ptr;
70		*(volatile unsigned char *)ptr = x;
71		raw_local_irq_restore(flags);
72		break;
73
74	case 4:
75		raw_local_irq_save(flags);
76		ret = *(volatile unsigned long *)ptr;
77		*(volatile unsigned long *)ptr = x;
78		raw_local_irq_restore(flags);
79		break;
80#else
81	case 1:
82		asm volatile("@	__xchg1\n"
83		"	swpb	%0, %1, [%2]"
84			: "=&r" (ret)
85			: "r" (x), "r" (ptr)
86			: "memory", "cc");
87		break;
88	case 4:
89		asm volatile("@	__xchg4\n"
90		"	swp	%0, %1, [%2]"
91			: "=&r" (ret)
92			: "r" (x), "r" (ptr)
93			: "memory", "cc");
94		break;
95#endif
96	default:
97		__bad_xchg(ptr, size), ret = 0;
98		break;
99	}
100	smp_mb();
101
102	return ret;
103}
104
105#define xchg(ptr,x) \
106	((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
107
108#include <asm-generic/cmpxchg-local.h>
109
110#if __LINUX_ARM_ARCH__ < 6
111/* min ARCH < ARMv6 */
112
113#ifdef CONFIG_SMP
114#error "SMP is not supported on this platform"
115#endif
116
117/*
118 * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
119 * them available.
120 */
121#define cmpxchg_local(ptr, o, n)				  	       \
122	((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\
123			(unsigned long)(n), sizeof(*(ptr))))
124#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
125
126#ifndef CONFIG_SMP
127#include <asm-generic/cmpxchg.h>
128#endif
129
130#else	/* min ARCH >= ARMv6 */
131
132extern void __bad_cmpxchg(volatile void *ptr, int size);
133
134/*
135 * cmpxchg only support 32-bits operands on ARMv6.
136 */
137
138static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
139				      unsigned long new, int size)
140{
141	unsigned long oldval, res;
142
143	prefetchw((const void *)ptr);
144
145	switch (size) {
146#ifndef CONFIG_CPU_V6	/* min ARCH >= ARMv6K */
147	case 1:
148		do {
149			asm volatile("@ __cmpxchg1\n"
150			"	ldrexb	%1, [%2]\n"
151			"	mov	%0, #0\n"
152			"	teq	%1, %3\n"
153			"	strexbeq %0, %4, [%2]\n"
154				: "=&r" (res), "=&r" (oldval)
155				: "r" (ptr), "Ir" (old), "r" (new)
156				: "memory", "cc");
157		} while (res);
158		break;
159	case 2:
160		do {
161			asm volatile("@ __cmpxchg1\n"
162			"	ldrexh	%1, [%2]\n"
163			"	mov	%0, #0\n"
164			"	teq	%1, %3\n"
165			"	strexheq %0, %4, [%2]\n"
166				: "=&r" (res), "=&r" (oldval)
167				: "r" (ptr), "Ir" (old), "r" (new)
168				: "memory", "cc");
169		} while (res);
170		break;
171#endif
172	case 4:
173		do {
174			asm volatile("@ __cmpxchg4\n"
175			"	ldrex	%1, [%2]\n"
176			"	mov	%0, #0\n"
177			"	teq	%1, %3\n"
178			"	strexeq %0, %4, [%2]\n"
179				: "=&r" (res), "=&r" (oldval)
180				: "r" (ptr), "Ir" (old), "r" (new)
181				: "memory", "cc");
182		} while (res);
183		break;
184	default:
185		__bad_cmpxchg(ptr, size);
186		oldval = 0;
187	}
188
189	return oldval;
190}
191
192static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
193					 unsigned long new, int size)
194{
195	unsigned long ret;
196
197	smp_mb();
198	ret = __cmpxchg(ptr, old, new, size);
199	smp_mb();
200
201	return ret;
202}
203
204#define cmpxchg(ptr,o,n)						\
205	((__typeof__(*(ptr)))__cmpxchg_mb((ptr),			\
206					  (unsigned long)(o),		\
207					  (unsigned long)(n),		\
208					  sizeof(*(ptr))))
209
210static inline unsigned long __cmpxchg_local(volatile void *ptr,
211					    unsigned long old,
212					    unsigned long new, int size)
213{
214	unsigned long ret;
215
216	switch (size) {
217#ifdef CONFIG_CPU_V6	/* min ARCH == ARMv6 */
218	case 1:
219	case 2:
220		ret = __cmpxchg_local_generic(ptr, old, new, size);
221		break;
222#endif
223	default:
224		ret = __cmpxchg(ptr, old, new, size);
225	}
226
227	return ret;
228}
229
230static inline unsigned long long __cmpxchg64(unsigned long long *ptr,
231					     unsigned long long old,
232					     unsigned long long new)
233{
234	unsigned long long oldval;
235	unsigned long res;
236
237	prefetchw(ptr);
238
239	__asm__ __volatile__(
240"1:	ldrexd		%1, %H1, [%3]\n"
241"	teq		%1, %4\n"
242"	teqeq		%H1, %H4\n"
243"	bne		2f\n"
244"	strexd		%0, %5, %H5, [%3]\n"
245"	teq		%0, #0\n"
246"	bne		1b\n"
247"2:"
248	: "=&r" (res), "=&r" (oldval), "+Qo" (*ptr)
249	: "r" (ptr), "r" (old), "r" (new)
250	: "cc");
251
252	return oldval;
253}
254
255static inline unsigned long long __cmpxchg64_mb(unsigned long long *ptr,
256						unsigned long long old,
257						unsigned long long new)
258{
259	unsigned long long ret;
260
261	smp_mb();
262	ret = __cmpxchg64(ptr, old, new);
263	smp_mb();
264
265	return ret;
266}
267
268#define cmpxchg_local(ptr,o,n)						\
269	((__typeof__(*(ptr)))__cmpxchg_local((ptr),			\
270				       (unsigned long)(o),		\
271				       (unsigned long)(n),		\
272				       sizeof(*(ptr))))
273
274#define cmpxchg64(ptr, o, n)						\
275	((__typeof__(*(ptr)))__cmpxchg64_mb((ptr),			\
276					(unsigned long long)(o),	\
277					(unsigned long long)(n)))
278
279#define cmpxchg64_relaxed(ptr, o, n)					\
280	((__typeof__(*(ptr)))__cmpxchg64((ptr),				\
281					(unsigned long long)(o),	\
282					(unsigned long long)(n)))
283
284#define cmpxchg64_local(ptr, o, n)	cmpxchg64_relaxed((ptr), (o), (n))
285
286#endif	/* __LINUX_ARM_ARCH__ >= 6 */
287
288#endif /* __ASM_ARM_CMPXCHG_H */
289