1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Based on arch/arm/include/asm/atomic.h
4  *
5  * Copyright (C) 1996 Russell King.
6  * Copyright (C) 2002 Deep Blue Solutions Ltd.
7  * Copyright (C) 2012 ARM Ltd.
8  */
9 
10 #ifndef __ASM_ATOMIC_LSE_H
11 #define __ASM_ATOMIC_LSE_H
12 
13 #define ATOMIC_OP(op, asm_op)						\
14 static __always_inline void						\
15 __lse_atomic_##op(int i, atomic_t *v)					\
16 {									\
17 	asm volatile(							\
18 	__LSE_PREAMBLE							\
19 	"	" #asm_op "	%w[i], %[v]\n"				\
20 	: [v] "+Q" (v->counter)						\
21 	: [i] "r" (i));							\
22 }
23 
24 ATOMIC_OP(andnot, stclr)
25 ATOMIC_OP(or, stset)
26 ATOMIC_OP(xor, steor)
27 ATOMIC_OP(add, stadd)
28 
29 static __always_inline void __lse_atomic_sub(int i, atomic_t *v)
30 {
31 	__lse_atomic_add(-i, v);
32 }
33 
34 #undef ATOMIC_OP
35 
36 #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...)			\
37 static __always_inline int						\
38 __lse_atomic_fetch_##op##name(int i, atomic_t *v)			\
39 {									\
40 	int old;							\
41 									\
42 	asm volatile(							\
43 	__LSE_PREAMBLE							\
44 	"	" #asm_op #mb "	%w[i], %w[old], %[v]"			\
45 	: [v] "+Q" (v->counter),					\
46 	  [old] "=r" (old)						\
47 	: [i] "r" (i)							\
48 	: cl);								\
49 									\
50 	return old;							\
51 }
52 
53 #define ATOMIC_FETCH_OPS(op, asm_op)					\
54 	ATOMIC_FETCH_OP(_relaxed,   , op, asm_op)			\
55 	ATOMIC_FETCH_OP(_acquire,  a, op, asm_op, "memory")		\
56 	ATOMIC_FETCH_OP(_release,  l, op, asm_op, "memory")		\
57 	ATOMIC_FETCH_OP(        , al, op, asm_op, "memory")
58 
59 ATOMIC_FETCH_OPS(andnot, ldclr)
60 ATOMIC_FETCH_OPS(or, ldset)
61 ATOMIC_FETCH_OPS(xor, ldeor)
62 ATOMIC_FETCH_OPS(add, ldadd)
63 
64 #undef ATOMIC_FETCH_OP
65 #undef ATOMIC_FETCH_OPS
66 
67 #define ATOMIC_FETCH_OP_SUB(name)					\
68 static __always_inline int						\
69 __lse_atomic_fetch_sub##name(int i, atomic_t *v)			\
70 {									\
71 	return __lse_atomic_fetch_add##name(-i, v);			\
72 }
73 
74 ATOMIC_FETCH_OP_SUB(_relaxed)
75 ATOMIC_FETCH_OP_SUB(_acquire)
76 ATOMIC_FETCH_OP_SUB(_release)
77 ATOMIC_FETCH_OP_SUB(        )
78 
79 #undef ATOMIC_FETCH_OP_SUB
80 
81 #define ATOMIC_OP_ADD_SUB_RETURN(name)					\
82 static __always_inline int						\
83 __lse_atomic_add_return##name(int i, atomic_t *v)			\
84 {									\
85 	return __lse_atomic_fetch_add##name(i, v) + i;			\
86 }									\
87 									\
88 static __always_inline int						\
89 __lse_atomic_sub_return##name(int i, atomic_t *v)			\
90 {									\
91 	return __lse_atomic_fetch_sub(i, v) - i;			\
92 }
93 
94 ATOMIC_OP_ADD_SUB_RETURN(_relaxed)
95 ATOMIC_OP_ADD_SUB_RETURN(_acquire)
96 ATOMIC_OP_ADD_SUB_RETURN(_release)
97 ATOMIC_OP_ADD_SUB_RETURN(        )
98 
99 #undef ATOMIC_OP_ADD_SUB_RETURN
100 
101 static __always_inline void __lse_atomic_and(int i, atomic_t *v)
102 {
103 	return __lse_atomic_andnot(~i, v);
104 }
105 
106 #define ATOMIC_FETCH_OP_AND(name, mb, cl...)				\
107 static __always_inline int						\
108 __lse_atomic_fetch_and##name(int i, atomic_t *v)			\
109 {									\
110 	return __lse_atomic_fetch_andnot##name(~i, v);			\
111 }
112 
113 ATOMIC_FETCH_OP_AND(_relaxed,   )
114 ATOMIC_FETCH_OP_AND(_acquire,  a, "memory")
115 ATOMIC_FETCH_OP_AND(_release,  l, "memory")
116 ATOMIC_FETCH_OP_AND(        , al, "memory")
117 
118 #undef ATOMIC_FETCH_OP_AND
119 
120 #define ATOMIC64_OP(op, asm_op)						\
121 static __always_inline void						\
122 __lse_atomic64_##op(s64 i, atomic64_t *v)				\
123 {									\
124 	asm volatile(							\
125 	__LSE_PREAMBLE							\
126 	"	" #asm_op "	%[i], %[v]\n"				\
127 	: [v] "+Q" (v->counter)						\
128 	: [i] "r" (i));							\
129 }
130 
131 ATOMIC64_OP(andnot, stclr)
132 ATOMIC64_OP(or, stset)
133 ATOMIC64_OP(xor, steor)
134 ATOMIC64_OP(add, stadd)
135 
136 static __always_inline void __lse_atomic64_sub(s64 i, atomic64_t *v)
137 {
138 	__lse_atomic64_add(-i, v);
139 }
140 
141 #undef ATOMIC64_OP
142 
143 #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...)			\
144 static __always_inline long						\
145 __lse_atomic64_fetch_##op##name(s64 i, atomic64_t *v)			\
146 {									\
147 	s64 old;							\
148 									\
149 	asm volatile(							\
150 	__LSE_PREAMBLE							\
151 	"	" #asm_op #mb "	%[i], %[old], %[v]"			\
152 	: [v] "+Q" (v->counter),					\
153 	  [old] "=r" (old)						\
154 	: [i] "r" (i) 							\
155 	: cl);								\
156 									\
157 	return old;							\
158 }
159 
160 #define ATOMIC64_FETCH_OPS(op, asm_op)					\
161 	ATOMIC64_FETCH_OP(_relaxed,   , op, asm_op)			\
162 	ATOMIC64_FETCH_OP(_acquire,  a, op, asm_op, "memory")		\
163 	ATOMIC64_FETCH_OP(_release,  l, op, asm_op, "memory")		\
164 	ATOMIC64_FETCH_OP(        , al, op, asm_op, "memory")
165 
166 ATOMIC64_FETCH_OPS(andnot, ldclr)
167 ATOMIC64_FETCH_OPS(or, ldset)
168 ATOMIC64_FETCH_OPS(xor, ldeor)
169 ATOMIC64_FETCH_OPS(add, ldadd)
170 
171 #undef ATOMIC64_FETCH_OP
172 #undef ATOMIC64_FETCH_OPS
173 
174 #define ATOMIC64_FETCH_OP_SUB(name)					\
175 static __always_inline long						\
176 __lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v)			\
177 {									\
178 	return __lse_atomic64_fetch_add##name(-i, v);			\
179 }
180 
181 ATOMIC64_FETCH_OP_SUB(_relaxed)
182 ATOMIC64_FETCH_OP_SUB(_acquire)
183 ATOMIC64_FETCH_OP_SUB(_release)
184 ATOMIC64_FETCH_OP_SUB(        )
185 
186 #undef ATOMIC64_FETCH_OP_SUB
187 
188 #define ATOMIC64_OP_ADD_SUB_RETURN(name)				\
189 static __always_inline long						\
190 __lse_atomic64_add_return##name(s64 i, atomic64_t *v)			\
191 {									\
192 	return __lse_atomic64_fetch_add##name(i, v) + i;		\
193 }									\
194 									\
195 static __always_inline long						\
196 __lse_atomic64_sub_return##name(s64 i, atomic64_t *v)			\
197 {									\
198 	return __lse_atomic64_fetch_sub##name(i, v) - i;		\
199 }
200 
201 ATOMIC64_OP_ADD_SUB_RETURN(_relaxed)
202 ATOMIC64_OP_ADD_SUB_RETURN(_acquire)
203 ATOMIC64_OP_ADD_SUB_RETURN(_release)
204 ATOMIC64_OP_ADD_SUB_RETURN(        )
205 
206 #undef ATOMIC64_OP_ADD_SUB_RETURN
207 
208 static __always_inline void __lse_atomic64_and(s64 i, atomic64_t *v)
209 {
210 	return __lse_atomic64_andnot(~i, v);
211 }
212 
213 #define ATOMIC64_FETCH_OP_AND(name, mb, cl...)				\
214 static __always_inline long						\
215 __lse_atomic64_fetch_and##name(s64 i, atomic64_t *v)			\
216 {									\
217 	return __lse_atomic64_fetch_andnot##name(~i, v);		\
218 }
219 
220 ATOMIC64_FETCH_OP_AND(_relaxed,   )
221 ATOMIC64_FETCH_OP_AND(_acquire,  a, "memory")
222 ATOMIC64_FETCH_OP_AND(_release,  l, "memory")
223 ATOMIC64_FETCH_OP_AND(        , al, "memory")
224 
225 #undef ATOMIC64_FETCH_OP_AND
226 
227 static __always_inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v)
228 {
229 	unsigned long tmp;
230 
231 	asm volatile(
232 	__LSE_PREAMBLE
233 	"1:	ldr	%x[tmp], %[v]\n"
234 	"	subs	%[ret], %x[tmp], #1\n"
235 	"	b.lt	2f\n"
236 	"	casal	%x[tmp], %[ret], %[v]\n"
237 	"	sub	%x[tmp], %x[tmp], #1\n"
238 	"	sub	%x[tmp], %x[tmp], %[ret]\n"
239 	"	cbnz	%x[tmp], 1b\n"
240 	"2:"
241 	: [ret] "+&r" (v), [v] "+Q" (v->counter), [tmp] "=&r" (tmp)
242 	:
243 	: "cc", "memory");
244 
245 	return (long)v;
246 }
247 
248 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...)			\
249 static __always_inline u##sz						\
250 __lse__cmpxchg_case_##name##sz(volatile void *ptr,			\
251 					      u##sz old,		\
252 					      u##sz new)		\
253 {									\
254 	asm volatile(							\
255 	__LSE_PREAMBLE							\
256 	"	cas" #mb #sfx "	%" #w "[old], %" #w "[new], %[v]\n"	\
257 	: [v] "+Q" (*(u##sz *)ptr),					\
258 	  [old] "+r" (old)						\
259 	: [new] "rZ" (new)						\
260 	: cl);								\
261 									\
262 	return old;							\
263 }
264 
265 __CMPXCHG_CASE(w, b,     ,  8,   )
266 __CMPXCHG_CASE(w, h,     , 16,   )
267 __CMPXCHG_CASE(w,  ,     , 32,   )
268 __CMPXCHG_CASE(x,  ,     , 64,   )
269 __CMPXCHG_CASE(w, b, acq_,  8,  a, "memory")
270 __CMPXCHG_CASE(w, h, acq_, 16,  a, "memory")
271 __CMPXCHG_CASE(w,  , acq_, 32,  a, "memory")
272 __CMPXCHG_CASE(x,  , acq_, 64,  a, "memory")
273 __CMPXCHG_CASE(w, b, rel_,  8,  l, "memory")
274 __CMPXCHG_CASE(w, h, rel_, 16,  l, "memory")
275 __CMPXCHG_CASE(w,  , rel_, 32,  l, "memory")
276 __CMPXCHG_CASE(x,  , rel_, 64,  l, "memory")
277 __CMPXCHG_CASE(w, b,  mb_,  8, al, "memory")
278 __CMPXCHG_CASE(w, h,  mb_, 16, al, "memory")
279 __CMPXCHG_CASE(w,  ,  mb_, 32, al, "memory")
280 __CMPXCHG_CASE(x,  ,  mb_, 64, al, "memory")
281 
282 #undef __CMPXCHG_CASE
283 
284 #define __CMPXCHG128(name, mb, cl...)					\
285 static __always_inline u128						\
286 __lse__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new)		\
287 {									\
288 	union __u128_halves r, o = { .full = (old) },			\
289 			       n = { .full = (new) };			\
290 	register unsigned long x0 asm ("x0") = o.low;			\
291 	register unsigned long x1 asm ("x1") = o.high;			\
292 	register unsigned long x2 asm ("x2") = n.low;			\
293 	register unsigned long x3 asm ("x3") = n.high;			\
294 	register unsigned long x4 asm ("x4") = (unsigned long)ptr;	\
295 									\
296 	asm volatile(							\
297 	__LSE_PREAMBLE							\
298 	"	casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
299 	: [old1] "+&r" (x0), [old2] "+&r" (x1),				\
300 	  [v] "+Q" (*(u128 *)ptr)					\
301 	: [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4),		\
302 	  [oldval1] "r" (o.low), [oldval2] "r" (o.high)			\
303 	: cl);								\
304 									\
305 	r.low = x0; r.high = x1;					\
306 									\
307 	return r.full;							\
308 }
309 
310 __CMPXCHG128(   ,   )
311 __CMPXCHG128(_mb, al, "memory")
312 
313 #undef __CMPXCHG128
314 
315 #endif	/* __ASM_ATOMIC_LSE_H */
316