1/*
2 * Copyright IBM Corp. 1999, 2011
3 *
4 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
5 */
6
7#ifndef __ASM_CMPXCHG_H
8#define __ASM_CMPXCHG_H
9
10#include <linux/mmdebug.h>
11#include <linux/types.h>
12#include <linux/bug.h>
13
14extern void __xchg_called_with_bad_pointer(void);
15
16static inline unsigned long __xchg(unsigned long x, void *ptr, int size)
17{
18	unsigned long addr, old;
19	int shift;
20
21	switch (size) {
22	case 1:
23		addr = (unsigned long) ptr;
24		shift = (3 ^ (addr & 3)) << 3;
25		addr ^= addr & 3;
26		asm volatile(
27			"	l	%0,%4\n"
28			"0:	lr	0,%0\n"
29			"	nr	0,%3\n"
30			"	or	0,%2\n"
31			"	cs	%0,0,%4\n"
32			"	jl	0b\n"
33			: "=&d" (old), "=Q" (*(int *) addr)
34			: "d" ((x & 0xff) << shift), "d" (~(0xff << shift)),
35			  "Q" (*(int *) addr) : "memory", "cc", "0");
36		return old >> shift;
37	case 2:
38		addr = (unsigned long) ptr;
39		shift = (2 ^ (addr & 2)) << 3;
40		addr ^= addr & 2;
41		asm volatile(
42			"	l	%0,%4\n"
43			"0:	lr	0,%0\n"
44			"	nr	0,%3\n"
45			"	or	0,%2\n"
46			"	cs	%0,0,%4\n"
47			"	jl	0b\n"
48			: "=&d" (old), "=Q" (*(int *) addr)
49			: "d" ((x & 0xffff) << shift), "d" (~(0xffff << shift)),
50			  "Q" (*(int *) addr) : "memory", "cc", "0");
51		return old >> shift;
52	case 4:
53		asm volatile(
54			"	l	%0,%3\n"
55			"0:	cs	%0,%2,%3\n"
56			"	jl	0b\n"
57			: "=&d" (old), "=Q" (*(int *) ptr)
58			: "d" (x), "Q" (*(int *) ptr)
59			: "memory", "cc");
60		return old;
61#ifdef CONFIG_64BIT
62	case 8:
63		asm volatile(
64			"	lg	%0,%3\n"
65			"0:	csg	%0,%2,%3\n"
66			"	jl	0b\n"
67			: "=&d" (old), "=m" (*(long *) ptr)
68			: "d" (x), "Q" (*(long *) ptr)
69			: "memory", "cc");
70		return old;
71#endif /* CONFIG_64BIT */
72	}
73	__xchg_called_with_bad_pointer();
74	return x;
75}
76
77#define xchg(ptr, x)							  \
78({									  \
79	__typeof__(*(ptr)) __ret;					  \
80	__ret = (__typeof__(*(ptr)))					  \
81		__xchg((unsigned long)(x), (void *)(ptr), sizeof(*(ptr)));\
82	__ret;								  \
83})
84
85/*
86 * Atomic compare and exchange.	 Compare OLD with MEM, if identical,
87 * store NEW in MEM.  Return the initial value in MEM.	Success is
88 * indicated by comparing RETURN with OLD.
89 */
90
91#define __HAVE_ARCH_CMPXCHG
92
93extern void __cmpxchg_called_with_bad_pointer(void);
94
95static inline unsigned long __cmpxchg(void *ptr, unsigned long old,
96				      unsigned long new, int size)
97{
98	unsigned long addr, prev, tmp;
99	int shift;
100
101	switch (size) {
102	case 1:
103		addr = (unsigned long) ptr;
104		shift = (3 ^ (addr & 3)) << 3;
105		addr ^= addr & 3;
106		asm volatile(
107			"	l	%0,%2\n"
108			"0:	nr	%0,%5\n"
109			"	lr	%1,%0\n"
110			"	or	%0,%3\n"
111			"	or	%1,%4\n"
112			"	cs	%0,%1,%2\n"
113			"	jnl	1f\n"
114			"	xr	%1,%0\n"
115			"	nr	%1,%5\n"
116			"	jnz	0b\n"
117			"1:"
118			: "=&d" (prev), "=&d" (tmp), "+Q" (*(int *) addr)
119			: "d" ((old & 0xff) << shift),
120			  "d" ((new & 0xff) << shift),
121			  "d" (~(0xff << shift))
122			: "memory", "cc");
123		return prev >> shift;
124	case 2:
125		addr = (unsigned long) ptr;
126		shift = (2 ^ (addr & 2)) << 3;
127		addr ^= addr & 2;
128		asm volatile(
129			"	l	%0,%2\n"
130			"0:	nr	%0,%5\n"
131			"	lr	%1,%0\n"
132			"	or	%0,%3\n"
133			"	or	%1,%4\n"
134			"	cs	%0,%1,%2\n"
135			"	jnl	1f\n"
136			"	xr	%1,%0\n"
137			"	nr	%1,%5\n"
138			"	jnz	0b\n"
139			"1:"
140			: "=&d" (prev), "=&d" (tmp), "+Q" (*(int *) addr)
141			: "d" ((old & 0xffff) << shift),
142			  "d" ((new & 0xffff) << shift),
143			  "d" (~(0xffff << shift))
144			: "memory", "cc");
145		return prev >> shift;
146	case 4:
147		asm volatile(
148			"	cs	%0,%3,%1\n"
149			: "=&d" (prev), "=Q" (*(int *) ptr)
150			: "0" (old), "d" (new), "Q" (*(int *) ptr)
151			: "memory", "cc");
152		return prev;
153#ifdef CONFIG_64BIT
154	case 8:
155		asm volatile(
156			"	csg	%0,%3,%1\n"
157			: "=&d" (prev), "=Q" (*(long *) ptr)
158			: "0" (old), "d" (new), "Q" (*(long *) ptr)
159			: "memory", "cc");
160		return prev;
161#endif /* CONFIG_64BIT */
162	}
163	__cmpxchg_called_with_bad_pointer();
164	return old;
165}
166
167#define cmpxchg(ptr, o, n)						 \
168({									 \
169	__typeof__(*(ptr)) __ret;					 \
170	__ret = (__typeof__(*(ptr)))					 \
171		__cmpxchg((ptr), (unsigned long)(o), (unsigned long)(n), \
172			  sizeof(*(ptr)));				 \
173	__ret;								 \
174})
175
176#ifdef CONFIG_64BIT
177#define cmpxchg64(ptr, o, n)						\
178({									\
179	cmpxchg((ptr), (o), (n));					\
180})
181#else /* CONFIG_64BIT */
182static inline unsigned long long __cmpxchg64(void *ptr,
183					     unsigned long long old,
184					     unsigned long long new)
185{
186	register_pair rp_old = {.pair = old};
187	register_pair rp_new = {.pair = new};
188	unsigned long long *ullptr = ptr;
189
190	asm volatile(
191		"	cds	%0,%2,%1"
192		: "+d" (rp_old), "+Q" (*ullptr)
193		: "d" (rp_new)
194		: "memory", "cc");
195	return rp_old.pair;
196}
197
198#define cmpxchg64(ptr, o, n)				\
199({							\
200	__typeof__(*(ptr)) __ret;			\
201	__ret = (__typeof__(*(ptr)))			\
202		__cmpxchg64((ptr),			\
203			    (unsigned long long)(o),	\
204			    (unsigned long long)(n));	\
205	__ret;						\
206})
207#endif /* CONFIG_64BIT */
208
209#define __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, insn)		\
210({									\
211	register __typeof__(*(p1)) __old1 asm("2") = (o1);		\
212	register __typeof__(*(p2)) __old2 asm("3") = (o2);		\
213	register __typeof__(*(p1)) __new1 asm("4") = (n1);		\
214	register __typeof__(*(p2)) __new2 asm("5") = (n2);		\
215	int cc;								\
216	asm volatile(							\
217			insn   " %[old],%[new],%[ptr]\n"		\
218		"	ipm	%[cc]\n"				\
219		"	srl	%[cc],28"				\
220		: [cc] "=d" (cc), [old] "+d" (__old1), "+d" (__old2)	\
221		: [new] "d" (__new1), "d" (__new2),			\
222		  [ptr] "Q" (*(p1)), "Q" (*(p2))			\
223		: "memory", "cc");					\
224	!cc;								\
225})
226
227#define __cmpxchg_double_4(p1, p2, o1, o2, n1, n2) \
228	__cmpxchg_double_op(p1, p2, o1, o2, n1, n2, "cds")
229
230#define __cmpxchg_double_8(p1, p2, o1, o2, n1, n2) \
231	__cmpxchg_double_op(p1, p2, o1, o2, n1, n2, "cdsg")
232
233extern void __cmpxchg_double_called_with_bad_pointer(void);
234
235#define __cmpxchg_double(p1, p2, o1, o2, n1, n2)			\
236({									\
237	int __ret;							\
238	switch (sizeof(*(p1))) {					\
239	case 4:								\
240		__ret = __cmpxchg_double_4(p1, p2, o1, o2, n1, n2);	\
241		break;							\
242	case 8:								\
243		__ret = __cmpxchg_double_8(p1, p2, o1, o2, n1, n2);	\
244		break;							\
245	default:							\
246		__cmpxchg_double_called_with_bad_pointer();		\
247	}								\
248	__ret;								\
249})
250
251#define cmpxchg_double(p1, p2, o1, o2, n1, n2)				\
252({									\
253	__typeof__(p1) __p1 = (p1);					\
254	__typeof__(p2) __p2 = (p2);					\
255	int __ret;							\
256	BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long));			\
257	BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long));			\
258	VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\
259	if (sizeof(long) == 4)						\
260		__ret = __cmpxchg_double_4(__p1, __p2, o1, o2, n1, n2);	\
261	else								\
262		__ret = __cmpxchg_double_8(__p1, __p2, o1, o2, n1, n2);	\
263	__ret;								\
264})
265
266#define system_has_cmpxchg_double()	1
267
268#include <asm-generic/cmpxchg-local.h>
269
270static inline unsigned long __cmpxchg_local(void *ptr,
271					    unsigned long old,
272					    unsigned long new, int size)
273{
274	switch (size) {
275	case 1:
276	case 2:
277	case 4:
278#ifdef CONFIG_64BIT
279	case 8:
280#endif
281		return __cmpxchg(ptr, old, new, size);
282	default:
283		return __cmpxchg_local_generic(ptr, old, new, size);
284	}
285
286	return old;
287}
288
289/*
290 * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
291 * them available.
292 */
293#define cmpxchg_local(ptr, o, n)					\
294({									\
295	__typeof__(*(ptr)) __ret;					\
296	__ret = (__typeof__(*(ptr)))					\
297		__cmpxchg_local((ptr), (unsigned long)(o),		\
298				(unsigned long)(n), sizeof(*(ptr)));	\
299	__ret;								\
300})
301
302#define cmpxchg64_local(ptr, o, n)	cmpxchg64((ptr), (o), (n))
303
304#endif /* __ASM_CMPXCHG_H */
305