atomic_64.S revision caa17d49f9a5cc09b3bbb101dc640f914f3b4ff7 
1/* atomic.S: These things are too big to do inline.
2 *
3 * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net)
4 */
5
6#include <linux/linkage.h>
7#include <asm/asi.h>
8#include <asm/backoff.h>
9
10	.text
11
12	/* Two versions of the atomic routines, one that
13	 * does not return a value and does not perform
14	 * memory barriers, and a second which returns
15	 * a value and does the barriers.
16	 */
17
18#define ATOMIC_OP(op)							\
19ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */		\
20	BACKOFF_SETUP(%o2);						\
211:	lduw	[%o1], %g1;						\
22	op	%g1, %o0, %g7;						\
23	cas	[%o1], %g1, %g7;					\
24	cmp	%g1, %g7;						\
25	bne,pn	%icc, BACKOFF_LABEL(2f, 1b);				\
26	 nop;								\
27	retl;								\
28	 nop;								\
292:	BACKOFF_SPIN(%o2, %o3, 1b);					\
30ENDPROC(atomic_##op);							\
31
32#define ATOMIC_OP_RETURN(op)						\
33ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */	\
34	BACKOFF_SETUP(%o2);						\
351:	lduw	[%o1], %g1;						\
36	op	%g1, %o0, %g7;						\
37	cas	[%o1], %g1, %g7;					\
38	cmp	%g1, %g7;						\
39	bne,pn	%icc, BACKOFF_LABEL(2f, 1b);				\
40	 op	%g1, %o0, %g1;						\
41	retl;								\
42	 sra	%g1, 0, %o0;						\
432:	BACKOFF_SPIN(%o2, %o3, 1b);					\
44ENDPROC(atomic_##op##_return);
45
46#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)
47
48ATOMIC_OPS(add)
49ATOMIC_OPS(sub)
50
51#undef ATOMIC_OPS
52#undef ATOMIC_OP_RETURN
53#undef ATOMIC_OP
54
55#define ATOMIC64_OP(op)							\
56ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */		\
57	BACKOFF_SETUP(%o2);						\
581:	ldx	[%o1], %g1;						\
59	op	%g1, %o0, %g7;						\
60	casx	[%o1], %g1, %g7;					\
61	cmp	%g1, %g7;						\
62	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b);				\
63	 nop;								\
64	retl;								\
65	 nop;								\
662:	BACKOFF_SPIN(%o2, %o3, 1b);					\
67ENDPROC(atomic64_##op);							\
68
69#define ATOMIC64_OP_RETURN(op)						\
70ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */	\
71	BACKOFF_SETUP(%o2);						\
721:	ldx	[%o1], %g1;						\
73	op	%g1, %o0, %g7;						\
74	casx	[%o1], %g1, %g7;					\
75	cmp	%g1, %g7;						\
76	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b);				\
77	 nop;								\
78	retl;								\
79	 op	%g1, %o0, %o0;						\
802:	BACKOFF_SPIN(%o2, %o3, 1b);					\
81ENDPROC(atomic64_##op##_return);
82
83#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op)
84
85ATOMIC64_OPS(add)
86ATOMIC64_OPS(sub)
87
88#undef ATOMIC64_OPS
89#undef ATOMIC64_OP_RETURN
90#undef ATOMIC64_OP
91
92ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */
93	BACKOFF_SETUP(%o2)
941:	ldx	[%o0], %g1
95	brlez,pn %g1, 3f
96	 sub	%g1, 1, %g7
97	casx	[%o0], %g1, %g7
98	cmp	%g1, %g7
99	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
100	 nop
1013:	retl
102	 sub	%g1, 1, %o0
1032:	BACKOFF_SPIN(%o2, %o3, 1b)
104ENDPROC(atomic64_dec_if_positive)
105