1 /*
2  * Based on arch/arm/include/asm/atomic.h
3  *
4  * Copyright (C) 1996 Russell King.
5  * Copyright (C) 2002 Deep Blue Solutions Ltd.
6  * Copyright (C) 2012 ARM Ltd.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 as
10  * published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19  */
20 
21 #ifndef __ASM_ATOMIC_LL_SC_H
22 #define __ASM_ATOMIC_LL_SC_H
23 
24 #ifndef __ARM64_IN_ATOMIC_IMPL
25 #error "please don't include this file directly"
26 #endif
27 
28 /*
29  * AArch64 UP and SMP safe atomic ops.  We use load exclusive and
30  * store exclusive to ensure that these are atomic.  We may loop
31  * to ensure that the update happens.
32  *
33  * NOTE: these functions do *not* follow the PCS and must explicitly
34  * save any clobbered registers other than x0 (regardless of return
35  * value).  This is achieved through -fcall-saved-* compiler flags for
36  * this file, which unfortunately don't work on a per-function basis
37  * (the optimize attribute silently ignores these options).
38  */
39 
40 #define ATOMIC_OP(op, asm_op)						\
41 __LL_SC_INLINE void							\
42 __LL_SC_PREFIX(atomic_##op(int i, atomic_t *v))				\
43 {									\
44 	unsigned long tmp;						\
45 	int result;							\
46 									\
47 	asm volatile("// atomic_" #op "\n"				\
48 "	prfm	pstl1strm, %2\n"					\
49 "1:	ldxr	%w0, %2\n"						\
50 "	" #asm_op "	%w0, %w0, %w3\n"				\
51 "	stxr	%w1, %w0, %2\n"						\
52 "	cbnz	%w1, 1b"						\
53 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
54 	: "Ir" (i));							\
55 }									\
56 __LL_SC_EXPORT(atomic_##op);
57 
58 #define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op)		\
59 __LL_SC_INLINE int							\
60 __LL_SC_PREFIX(atomic_##op##_return##name(int i, atomic_t *v))		\
61 {									\
62 	unsigned long tmp;						\
63 	int result;							\
64 									\
65 	asm volatile("// atomic_" #op "_return" #name "\n"		\
66 "	prfm	pstl1strm, %2\n"					\
67 "1:	ld" #acq "xr	%w0, %2\n"					\
68 "	" #asm_op "	%w0, %w0, %w3\n"				\
69 "	st" #rel "xr	%w1, %w0, %2\n"					\
70 "	cbnz	%w1, 1b\n"						\
71 "	" #mb								\
72 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
73 	: "Ir" (i)							\
74 	: cl);								\
75 									\
76 	return result;							\
77 }									\
78 __LL_SC_EXPORT(atomic_##op##_return##name);
79 
80 #define ATOMIC_OPS(...)							\
81 	ATOMIC_OP(__VA_ARGS__)						\
82 	ATOMIC_OP_RETURN(        , dmb ish,  , l, "memory", __VA_ARGS__)
83 
84 #define ATOMIC_OPS_RLX(...)						\
85 	ATOMIC_OPS(__VA_ARGS__)						\
86 	ATOMIC_OP_RETURN(_relaxed,        ,  ,  ,         , __VA_ARGS__)\
87 	ATOMIC_OP_RETURN(_acquire,        , a,  , "memory", __VA_ARGS__)\
88 	ATOMIC_OP_RETURN(_release,        ,  , l, "memory", __VA_ARGS__)
89 
90 ATOMIC_OPS_RLX(add, add)
91 ATOMIC_OPS_RLX(sub, sub)
92 
93 ATOMIC_OP(and, and)
94 ATOMIC_OP(andnot, bic)
95 ATOMIC_OP(or, orr)
96 ATOMIC_OP(xor, eor)
97 
98 #undef ATOMIC_OPS_RLX
99 #undef ATOMIC_OPS
100 #undef ATOMIC_OP_RETURN
101 #undef ATOMIC_OP
102 
103 #define ATOMIC64_OP(op, asm_op)						\
104 __LL_SC_INLINE void							\
105 __LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v))			\
106 {									\
107 	long result;							\
108 	unsigned long tmp;						\
109 									\
110 	asm volatile("// atomic64_" #op "\n"				\
111 "	prfm	pstl1strm, %2\n"					\
112 "1:	ldxr	%0, %2\n"						\
113 "	" #asm_op "	%0, %0, %3\n"					\
114 "	stxr	%w1, %0, %2\n"						\
115 "	cbnz	%w1, 1b"						\
116 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
117 	: "Ir" (i));							\
118 }									\
119 __LL_SC_EXPORT(atomic64_##op);
120 
121 #define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op)		\
122 __LL_SC_INLINE long							\
123 __LL_SC_PREFIX(atomic64_##op##_return##name(long i, atomic64_t *v))	\
124 {									\
125 	long result;							\
126 	unsigned long tmp;						\
127 									\
128 	asm volatile("// atomic64_" #op "_return" #name "\n"		\
129 "	prfm	pstl1strm, %2\n"					\
130 "1:	ld" #acq "xr	%0, %2\n"					\
131 "	" #asm_op "	%0, %0, %3\n"					\
132 "	st" #rel "xr	%w1, %0, %2\n"					\
133 "	cbnz	%w1, 1b\n"						\
134 "	" #mb								\
135 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
136 	: "Ir" (i)							\
137 	: cl);								\
138 									\
139 	return result;							\
140 }									\
141 __LL_SC_EXPORT(atomic64_##op##_return##name);
142 
143 #define ATOMIC64_OPS(...)						\
144 	ATOMIC64_OP(__VA_ARGS__)					\
145 	ATOMIC64_OP_RETURN(, dmb ish,  , l, "memory", __VA_ARGS__)
146 
147 #define ATOMIC64_OPS_RLX(...)						\
148 	ATOMIC64_OPS(__VA_ARGS__)					\
149 	ATOMIC64_OP_RETURN(_relaxed,,  ,  ,         , __VA_ARGS__)	\
150 	ATOMIC64_OP_RETURN(_acquire,, a,  , "memory", __VA_ARGS__)	\
151 	ATOMIC64_OP_RETURN(_release,,  , l, "memory", __VA_ARGS__)
152 
153 ATOMIC64_OPS_RLX(add, add)
154 ATOMIC64_OPS_RLX(sub, sub)
155 
156 ATOMIC64_OP(and, and)
157 ATOMIC64_OP(andnot, bic)
158 ATOMIC64_OP(or, orr)
159 ATOMIC64_OP(xor, eor)
160 
161 #undef ATOMIC64_OPS_RLX
162 #undef ATOMIC64_OPS
163 #undef ATOMIC64_OP_RETURN
164 #undef ATOMIC64_OP
165 
166 __LL_SC_INLINE long
167 __LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
168 {
169 	long result;
170 	unsigned long tmp;
171 
172 	asm volatile("// atomic64_dec_if_positive\n"
173 "	prfm	pstl1strm, %2\n"
174 "1:	ldxr	%0, %2\n"
175 "	subs	%0, %0, #1\n"
176 "	b.lt	2f\n"
177 "	stlxr	%w1, %0, %2\n"
178 "	cbnz	%w1, 1b\n"
179 "	dmb	ish\n"
180 "2:"
181 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
182 	:
183 	: "cc", "memory");
184 
185 	return result;
186 }
187 __LL_SC_EXPORT(atomic64_dec_if_positive);
188 
189 #define __CMPXCHG_CASE(w, sz, name, mb, acq, rel, cl)			\
190 __LL_SC_INLINE unsigned long						\
191 __LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr,		\
192 				     unsigned long old,			\
193 				     unsigned long new))		\
194 {									\
195 	unsigned long tmp, oldval;					\
196 									\
197 	asm volatile(							\
198 	"	prfm	pstl1strm, %[v]\n"				\
199 	"1:	ld" #acq "xr" #sz "\t%" #w "[oldval], %[v]\n"		\
200 	"	eor	%" #w "[tmp], %" #w "[oldval], %" #w "[old]\n"	\
201 	"	cbnz	%" #w "[tmp], 2f\n"				\
202 	"	st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n"	\
203 	"	cbnz	%w[tmp], 1b\n"					\
204 	"	" #mb "\n"						\
205 	"	mov	%" #w "[oldval], %" #w "[old]\n"		\
206 	"2:"								\
207 	: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval),			\
208 	  [v] "+Q" (*(unsigned long *)ptr)				\
209 	: [old] "Lr" (old), [new] "r" (new)				\
210 	: cl);								\
211 									\
212 	return oldval;							\
213 }									\
214 __LL_SC_EXPORT(__cmpxchg_case_##name);
215 
216 __CMPXCHG_CASE(w, b,     1,        ,  ,  ,         )
217 __CMPXCHG_CASE(w, h,     2,        ,  ,  ,         )
218 __CMPXCHG_CASE(w,  ,     4,        ,  ,  ,         )
219 __CMPXCHG_CASE( ,  ,     8,        ,  ,  ,         )
220 __CMPXCHG_CASE(w, b, acq_1,        , a,  , "memory")
221 __CMPXCHG_CASE(w, h, acq_2,        , a,  , "memory")
222 __CMPXCHG_CASE(w,  , acq_4,        , a,  , "memory")
223 __CMPXCHG_CASE( ,  , acq_8,        , a,  , "memory")
224 __CMPXCHG_CASE(w, b, rel_1,        ,  , l, "memory")
225 __CMPXCHG_CASE(w, h, rel_2,        ,  , l, "memory")
226 __CMPXCHG_CASE(w,  , rel_4,        ,  , l, "memory")
227 __CMPXCHG_CASE( ,  , rel_8,        ,  , l, "memory")
228 __CMPXCHG_CASE(w, b,  mb_1, dmb ish,  , l, "memory")
229 __CMPXCHG_CASE(w, h,  mb_2, dmb ish,  , l, "memory")
230 __CMPXCHG_CASE(w,  ,  mb_4, dmb ish,  , l, "memory")
231 __CMPXCHG_CASE( ,  ,  mb_8, dmb ish,  , l, "memory")
232 
233 #undef __CMPXCHG_CASE
234 
235 #define __CMPXCHG_DBL(name, mb, rel, cl)				\
236 __LL_SC_INLINE long							\
237 __LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1,		\
238 				      unsigned long old2,		\
239 				      unsigned long new1,		\
240 				      unsigned long new2,		\
241 				      volatile void *ptr))		\
242 {									\
243 	unsigned long tmp, ret;						\
244 									\
245 	asm volatile("// __cmpxchg_double" #name "\n"			\
246 	"	prfm	pstl1strm, %2\n"				\
247 	"1:	ldxp	%0, %1, %2\n"					\
248 	"	eor	%0, %0, %3\n"					\
249 	"	eor	%1, %1, %4\n"					\
250 	"	orr	%1, %0, %1\n"					\
251 	"	cbnz	%1, 2f\n"					\
252 	"	st" #rel "xp	%w0, %5, %6, %2\n"			\
253 	"	cbnz	%w0, 1b\n"					\
254 	"	" #mb "\n"						\
255 	"2:"								\
256 	: "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr)	\
257 	: "r" (old1), "r" (old2), "r" (new1), "r" (new2)		\
258 	: cl);								\
259 									\
260 	return ret;							\
261 }									\
262 __LL_SC_EXPORT(__cmpxchg_double##name);
263 
264 __CMPXCHG_DBL(   ,        ,  ,         )
265 __CMPXCHG_DBL(_mb, dmb ish, l, "memory")
266 
267 #undef __CMPXCHG_DBL
268 
269 #endif	/* __ASM_ATOMIC_LL_SC_H */
270