1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Based on arch/arm/include/asm/atomic.h
4  *
5  * Copyright (C) 1996 Russell King.
6  * Copyright (C) 2002 Deep Blue Solutions Ltd.
7  * Copyright (C) 2012 ARM Ltd.
8  */
9 
10 #ifndef __ASM_ATOMIC_LSE_H
11 #define __ASM_ATOMIC_LSE_H
12 
13 #define ATOMIC_OP(op, asm_op)						\
14 static inline void __lse_atomic_##op(int i, atomic_t *v)			\
15 {									\
16 	asm volatile(							\
17 "	" #asm_op "	%w[i], %[v]\n"					\
18 	: [i] "+r" (i), [v] "+Q" (v->counter)				\
19 	: "r" (v));							\
20 }
21 
22 ATOMIC_OP(andnot, stclr)
23 ATOMIC_OP(or, stset)
24 ATOMIC_OP(xor, steor)
25 ATOMIC_OP(add, stadd)
26 
27 #undef ATOMIC_OP
28 
29 #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...)			\
30 static inline int __lse_atomic_fetch_##op##name(int i, atomic_t *v)	\
31 {									\
32 	asm volatile(							\
33 "	" #asm_op #mb "	%w[i], %w[i], %[v]"				\
34 	: [i] "+r" (i), [v] "+Q" (v->counter)				\
35 	: "r" (v)							\
36 	: cl);								\
37 									\
38 	return i;							\
39 }
40 
41 #define ATOMIC_FETCH_OPS(op, asm_op)					\
42 	ATOMIC_FETCH_OP(_relaxed,   , op, asm_op)			\
43 	ATOMIC_FETCH_OP(_acquire,  a, op, asm_op, "memory")		\
44 	ATOMIC_FETCH_OP(_release,  l, op, asm_op, "memory")		\
45 	ATOMIC_FETCH_OP(        , al, op, asm_op, "memory")
46 
47 ATOMIC_FETCH_OPS(andnot, ldclr)
48 ATOMIC_FETCH_OPS(or, ldset)
49 ATOMIC_FETCH_OPS(xor, ldeor)
50 ATOMIC_FETCH_OPS(add, ldadd)
51 
52 #undef ATOMIC_FETCH_OP
53 #undef ATOMIC_FETCH_OPS
54 
55 #define ATOMIC_OP_ADD_RETURN(name, mb, cl...)				\
56 static inline int __lse_atomic_add_return##name(int i, atomic_t *v)	\
57 {									\
58 	u32 tmp;							\
59 									\
60 	asm volatile(							\
61 	"	ldadd" #mb "	%w[i], %w[tmp], %[v]\n"			\
62 	"	add	%w[i], %w[i], %w[tmp]"				\
63 	: [i] "+r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp)	\
64 	: "r" (v)							\
65 	: cl);								\
66 									\
67 	return i;							\
68 }
69 
70 ATOMIC_OP_ADD_RETURN(_relaxed,   )
71 ATOMIC_OP_ADD_RETURN(_acquire,  a, "memory")
72 ATOMIC_OP_ADD_RETURN(_release,  l, "memory")
73 ATOMIC_OP_ADD_RETURN(        , al, "memory")
74 
75 #undef ATOMIC_OP_ADD_RETURN
76 
77 static inline void __lse_atomic_and(int i, atomic_t *v)
78 {
79 	asm volatile(
80 	"	mvn	%w[i], %w[i]\n"
81 	"	stclr	%w[i], %[v]"
82 	: [i] "+&r" (i), [v] "+Q" (v->counter)
83 	: "r" (v));
84 }
85 
86 #define ATOMIC_FETCH_OP_AND(name, mb, cl...)				\
87 static inline int __lse_atomic_fetch_and##name(int i, atomic_t *v)	\
88 {									\
89 	asm volatile(							\
90 	"	mvn	%w[i], %w[i]\n"					\
91 	"	ldclr" #mb "	%w[i], %w[i], %[v]"			\
92 	: [i] "+&r" (i), [v] "+Q" (v->counter)				\
93 	: "r" (v)							\
94 	: cl);								\
95 									\
96 	return i;							\
97 }
98 
99 ATOMIC_FETCH_OP_AND(_relaxed,   )
100 ATOMIC_FETCH_OP_AND(_acquire,  a, "memory")
101 ATOMIC_FETCH_OP_AND(_release,  l, "memory")
102 ATOMIC_FETCH_OP_AND(        , al, "memory")
103 
104 #undef ATOMIC_FETCH_OP_AND
105 
106 static inline void __lse_atomic_sub(int i, atomic_t *v)
107 {
108 	asm volatile(
109 	"	neg	%w[i], %w[i]\n"
110 	"	stadd	%w[i], %[v]"
111 	: [i] "+&r" (i), [v] "+Q" (v->counter)
112 	: "r" (v));
113 }
114 
115 #define ATOMIC_OP_SUB_RETURN(name, mb, cl...)				\
116 static inline int __lse_atomic_sub_return##name(int i, atomic_t *v)	\
117 {									\
118 	u32 tmp;							\
119 									\
120 	asm volatile(							\
121 	"	neg	%w[i], %w[i]\n"					\
122 	"	ldadd" #mb "	%w[i], %w[tmp], %[v]\n"			\
123 	"	add	%w[i], %w[i], %w[tmp]"				\
124 	: [i] "+&r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp)	\
125 	: "r" (v)							\
126 	: cl);							\
127 									\
128 	return i;							\
129 }
130 
131 ATOMIC_OP_SUB_RETURN(_relaxed,   )
132 ATOMIC_OP_SUB_RETURN(_acquire,  a, "memory")
133 ATOMIC_OP_SUB_RETURN(_release,  l, "memory")
134 ATOMIC_OP_SUB_RETURN(        , al, "memory")
135 
136 #undef ATOMIC_OP_SUB_RETURN
137 
138 #define ATOMIC_FETCH_OP_SUB(name, mb, cl...)				\
139 static inline int __lse_atomic_fetch_sub##name(int i, atomic_t *v)	\
140 {									\
141 	asm volatile(							\
142 	"	neg	%w[i], %w[i]\n"					\
143 	"	ldadd" #mb "	%w[i], %w[i], %[v]"			\
144 	: [i] "+&r" (i), [v] "+Q" (v->counter)				\
145 	: "r" (v)							\
146 	: cl);								\
147 									\
148 	return i;							\
149 }
150 
151 ATOMIC_FETCH_OP_SUB(_relaxed,   )
152 ATOMIC_FETCH_OP_SUB(_acquire,  a, "memory")
153 ATOMIC_FETCH_OP_SUB(_release,  l, "memory")
154 ATOMIC_FETCH_OP_SUB(        , al, "memory")
155 
156 #undef ATOMIC_FETCH_OP_SUB
157 
158 #define ATOMIC64_OP(op, asm_op)						\
159 static inline void __lse_atomic64_##op(s64 i, atomic64_t *v)		\
160 {									\
161 	asm volatile(							\
162 "	" #asm_op "	%[i], %[v]\n"					\
163 	: [i] "+r" (i), [v] "+Q" (v->counter)				\
164 	: "r" (v));							\
165 }
166 
167 ATOMIC64_OP(andnot, stclr)
168 ATOMIC64_OP(or, stset)
169 ATOMIC64_OP(xor, steor)
170 ATOMIC64_OP(add, stadd)
171 
172 #undef ATOMIC64_OP
173 
174 #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...)			\
175 static inline long __lse_atomic64_fetch_##op##name(s64 i, atomic64_t *v)\
176 {									\
177 	asm volatile(							\
178 "	" #asm_op #mb "	%[i], %[i], %[v]"				\
179 	: [i] "+r" (i), [v] "+Q" (v->counter)				\
180 	: "r" (v)							\
181 	: cl);								\
182 									\
183 	return i;							\
184 }
185 
186 #define ATOMIC64_FETCH_OPS(op, asm_op)					\
187 	ATOMIC64_FETCH_OP(_relaxed,   , op, asm_op)			\
188 	ATOMIC64_FETCH_OP(_acquire,  a, op, asm_op, "memory")		\
189 	ATOMIC64_FETCH_OP(_release,  l, op, asm_op, "memory")		\
190 	ATOMIC64_FETCH_OP(        , al, op, asm_op, "memory")
191 
192 ATOMIC64_FETCH_OPS(andnot, ldclr)
193 ATOMIC64_FETCH_OPS(or, ldset)
194 ATOMIC64_FETCH_OPS(xor, ldeor)
195 ATOMIC64_FETCH_OPS(add, ldadd)
196 
197 #undef ATOMIC64_FETCH_OP
198 #undef ATOMIC64_FETCH_OPS
199 
200 #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...)				\
201 static inline long __lse_atomic64_add_return##name(s64 i, atomic64_t *v)\
202 {									\
203 	unsigned long tmp;						\
204 									\
205 	asm volatile(							\
206 	"	ldadd" #mb "	%[i], %x[tmp], %[v]\n"			\
207 	"	add	%[i], %[i], %x[tmp]"				\
208 	: [i] "+r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp)	\
209 	: "r" (v)							\
210 	: cl);								\
211 									\
212 	return i;							\
213 }
214 
215 ATOMIC64_OP_ADD_RETURN(_relaxed,   )
216 ATOMIC64_OP_ADD_RETURN(_acquire,  a, "memory")
217 ATOMIC64_OP_ADD_RETURN(_release,  l, "memory")
218 ATOMIC64_OP_ADD_RETURN(        , al, "memory")
219 
220 #undef ATOMIC64_OP_ADD_RETURN
221 
222 static inline void __lse_atomic64_and(s64 i, atomic64_t *v)
223 {
224 	asm volatile(
225 	"	mvn	%[i], %[i]\n"
226 	"	stclr	%[i], %[v]"
227 	: [i] "+&r" (i), [v] "+Q" (v->counter)
228 	: "r" (v));
229 }
230 
231 #define ATOMIC64_FETCH_OP_AND(name, mb, cl...)				\
232 static inline long __lse_atomic64_fetch_and##name(s64 i, atomic64_t *v)	\
233 {									\
234 	asm volatile(							\
235 	"	mvn	%[i], %[i]\n"					\
236 	"	ldclr" #mb "	%[i], %[i], %[v]"			\
237 	: [i] "+&r" (i), [v] "+Q" (v->counter)				\
238 	: "r" (v)							\
239 	: cl);								\
240 									\
241 	return i;							\
242 }
243 
244 ATOMIC64_FETCH_OP_AND(_relaxed,   )
245 ATOMIC64_FETCH_OP_AND(_acquire,  a, "memory")
246 ATOMIC64_FETCH_OP_AND(_release,  l, "memory")
247 ATOMIC64_FETCH_OP_AND(        , al, "memory")
248 
249 #undef ATOMIC64_FETCH_OP_AND
250 
251 static inline void __lse_atomic64_sub(s64 i, atomic64_t *v)
252 {
253 	asm volatile(
254 	"	neg	%[i], %[i]\n"
255 	"	stadd	%[i], %[v]"
256 	: [i] "+&r" (i), [v] "+Q" (v->counter)
257 	: "r" (v));
258 }
259 
260 #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...)				\
261 static inline long __lse_atomic64_sub_return##name(s64 i, atomic64_t *v)	\
262 {									\
263 	unsigned long tmp;						\
264 									\
265 	asm volatile(							\
266 	"	neg	%[i], %[i]\n"					\
267 	"	ldadd" #mb "	%[i], %x[tmp], %[v]\n"			\
268 	"	add	%[i], %[i], %x[tmp]"				\
269 	: [i] "+&r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp)	\
270 	: "r" (v)							\
271 	: cl);								\
272 									\
273 	return i;							\
274 }
275 
276 ATOMIC64_OP_SUB_RETURN(_relaxed,   )
277 ATOMIC64_OP_SUB_RETURN(_acquire,  a, "memory")
278 ATOMIC64_OP_SUB_RETURN(_release,  l, "memory")
279 ATOMIC64_OP_SUB_RETURN(        , al, "memory")
280 
281 #undef ATOMIC64_OP_SUB_RETURN
282 
283 #define ATOMIC64_FETCH_OP_SUB(name, mb, cl...)				\
284 static inline long __lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v)	\
285 {									\
286 	asm volatile(							\
287 	"	neg	%[i], %[i]\n"					\
288 	"	ldadd" #mb "	%[i], %[i], %[v]"			\
289 	: [i] "+&r" (i), [v] "+Q" (v->counter)				\
290 	: "r" (v)							\
291 	: cl);								\
292 									\
293 	return i;							\
294 }
295 
296 ATOMIC64_FETCH_OP_SUB(_relaxed,   )
297 ATOMIC64_FETCH_OP_SUB(_acquire,  a, "memory")
298 ATOMIC64_FETCH_OP_SUB(_release,  l, "memory")
299 ATOMIC64_FETCH_OP_SUB(        , al, "memory")
300 
301 #undef ATOMIC64_FETCH_OP_SUB
302 
303 static inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v)
304 {
305 	unsigned long tmp;
306 
307 	asm volatile(
308 	"1:	ldr	%x[tmp], %[v]\n"
309 	"	subs	%[ret], %x[tmp], #1\n"
310 	"	b.lt	2f\n"
311 	"	casal	%x[tmp], %[ret], %[v]\n"
312 	"	sub	%x[tmp], %x[tmp], #1\n"
313 	"	sub	%x[tmp], %x[tmp], %[ret]\n"
314 	"	cbnz	%x[tmp], 1b\n"
315 	"2:"
316 	: [ret] "+&r" (v), [v] "+Q" (v->counter), [tmp] "=&r" (tmp)
317 	:
318 	: "cc", "memory");
319 
320 	return (long)v;
321 }
322 
323 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...)			\
324 static __always_inline u##sz						\
325 __lse__cmpxchg_case_##name##sz(volatile void *ptr,			\
326 					      u##sz old,		\
327 					      u##sz new)		\
328 {									\
329 	register unsigned long x0 asm ("x0") = (unsigned long)ptr;	\
330 	register u##sz x1 asm ("x1") = old;				\
331 	register u##sz x2 asm ("x2") = new;				\
332 	unsigned long tmp;						\
333 									\
334 	asm volatile(							\
335 	"	mov	%" #w "[tmp], %" #w "[old]\n"			\
336 	"	cas" #mb #sfx "\t%" #w "[tmp], %" #w "[new], %[v]\n"	\
337 	"	mov	%" #w "[ret], %" #w "[tmp]"			\
338 	: [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr),		\
339 	  [tmp] "=&r" (tmp)						\
340 	: [old] "r" (x1), [new] "r" (x2)				\
341 	: cl);								\
342 									\
343 	return x0;							\
344 }
345 
346 __CMPXCHG_CASE(w, b,     ,  8,   )
347 __CMPXCHG_CASE(w, h,     , 16,   )
348 __CMPXCHG_CASE(w,  ,     , 32,   )
349 __CMPXCHG_CASE(x,  ,     , 64,   )
350 __CMPXCHG_CASE(w, b, acq_,  8,  a, "memory")
351 __CMPXCHG_CASE(w, h, acq_, 16,  a, "memory")
352 __CMPXCHG_CASE(w,  , acq_, 32,  a, "memory")
353 __CMPXCHG_CASE(x,  , acq_, 64,  a, "memory")
354 __CMPXCHG_CASE(w, b, rel_,  8,  l, "memory")
355 __CMPXCHG_CASE(w, h, rel_, 16,  l, "memory")
356 __CMPXCHG_CASE(w,  , rel_, 32,  l, "memory")
357 __CMPXCHG_CASE(x,  , rel_, 64,  l, "memory")
358 __CMPXCHG_CASE(w, b,  mb_,  8, al, "memory")
359 __CMPXCHG_CASE(w, h,  mb_, 16, al, "memory")
360 __CMPXCHG_CASE(w,  ,  mb_, 32, al, "memory")
361 __CMPXCHG_CASE(x,  ,  mb_, 64, al, "memory")
362 
363 #undef __CMPXCHG_CASE
364 
365 #define __CMPXCHG_DBL(name, mb, cl...)					\
366 static __always_inline long						\
367 __lse__cmpxchg_double##name(unsigned long old1,				\
368 					 unsigned long old2,		\
369 					 unsigned long new1,		\
370 					 unsigned long new2,		\
371 					 volatile void *ptr)		\
372 {									\
373 	unsigned long oldval1 = old1;					\
374 	unsigned long oldval2 = old2;					\
375 	register unsigned long x0 asm ("x0") = old1;			\
376 	register unsigned long x1 asm ("x1") = old2;			\
377 	register unsigned long x2 asm ("x2") = new1;			\
378 	register unsigned long x3 asm ("x3") = new2;			\
379 	register unsigned long x4 asm ("x4") = (unsigned long)ptr;	\
380 									\
381 	asm volatile(							\
382 	"	casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
383 	"	eor	%[old1], %[old1], %[oldval1]\n"			\
384 	"	eor	%[old2], %[old2], %[oldval2]\n"			\
385 	"	orr	%[old1], %[old1], %[old2]"			\
386 	: [old1] "+&r" (x0), [old2] "+&r" (x1),				\
387 	  [v] "+Q" (*(unsigned long *)ptr)				\
388 	: [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4),		\
389 	  [oldval1] "r" (oldval1), [oldval2] "r" (oldval2)		\
390 	: cl);								\
391 									\
392 	return x0;							\
393 }
394 
395 __CMPXCHG_DBL(   ,   )
396 __CMPXCHG_DBL(_mb, al, "memory")
397 
398 #undef __CMPXCHG_DBL
399 
400 #endif	/* __ASM_ATOMIC_LSE_H */
401