1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Atomic operations.
4  *
5  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
6  */
7 #ifndef _ASM_ATOMIC_H
8 #define _ASM_ATOMIC_H
9 
10 #include <linux/types.h>
11 #include <asm/barrier.h>
12 #include <asm/cmpxchg.h>
13 
14 #if __SIZEOF_LONG__ == 4
15 #define __LL		"ll.w	"
16 #define __SC		"sc.w	"
17 #define __AMADD		"amadd.w	"
18 #define __AMAND_DB	"amand_db.w	"
19 #define __AMOR_DB	"amor_db.w	"
20 #define __AMXOR_DB	"amxor_db.w	"
21 #elif __SIZEOF_LONG__ == 8
22 #define __LL		"ll.d	"
23 #define __SC		"sc.d	"
24 #define __AMADD		"amadd.d	"
25 #define __AMAND_DB	"amand_db.d	"
26 #define __AMOR_DB	"amor_db.d	"
27 #define __AMXOR_DB	"amxor_db.d	"
28 #endif
29 
30 #define ATOMIC_INIT(i)	  { (i) }
31 
32 /*
33  * arch_atomic_read - read atomic variable
34  * @v: pointer of type atomic_t
35  *
36  * Atomically reads the value of @v.
37  */
38 #define arch_atomic_read(v)	READ_ONCE((v)->counter)
39 
40 /*
41  * arch_atomic_set - set atomic variable
42  * @v: pointer of type atomic_t
43  * @i: required value
44  *
45  * Atomically sets the value of @v to @i.
46  */
47 #define arch_atomic_set(v, i)	WRITE_ONCE((v)->counter, (i))
48 
49 #define ATOMIC_OP(op, I, asm_op)					\
50 static inline void arch_atomic_##op(int i, atomic_t *v)			\
51 {									\
52 	__asm__ __volatile__(						\
53 	"am"#asm_op"_db.w" " $zero, %1, %0	\n"			\
54 	: "+ZB" (v->counter)						\
55 	: "r" (I)							\
56 	: "memory");							\
57 }
58 
59 #define ATOMIC_OP_RETURN(op, I, asm_op, c_op)				\
60 static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v)	\
61 {									\
62 	int result;							\
63 									\
64 	__asm__ __volatile__(						\
65 	"am"#asm_op"_db.w" " %1, %2, %0		\n"			\
66 	: "+ZB" (v->counter), "=&r" (result)				\
67 	: "r" (I)							\
68 	: "memory");							\
69 									\
70 	return result c_op I;						\
71 }
72 
73 #define ATOMIC_FETCH_OP(op, I, asm_op)					\
74 static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v)	\
75 {									\
76 	int result;							\
77 									\
78 	__asm__ __volatile__(						\
79 	"am"#asm_op"_db.w" " %1, %2, %0		\n"			\
80 	: "+ZB" (v->counter), "=&r" (result)				\
81 	: "r" (I)							\
82 	: "memory");							\
83 									\
84 	return result;							\
85 }
86 
87 #define ATOMIC_OPS(op, I, asm_op, c_op)					\
88 	ATOMIC_OP(op, I, asm_op)					\
89 	ATOMIC_OP_RETURN(op, I, asm_op, c_op)				\
90 	ATOMIC_FETCH_OP(op, I, asm_op)
91 
92 ATOMIC_OPS(add, i, add, +)
93 ATOMIC_OPS(sub, -i, add, +)
94 
95 #define arch_atomic_add_return_relaxed	arch_atomic_add_return_relaxed
96 #define arch_atomic_sub_return_relaxed	arch_atomic_sub_return_relaxed
97 #define arch_atomic_fetch_add_relaxed	arch_atomic_fetch_add_relaxed
98 #define arch_atomic_fetch_sub_relaxed	arch_atomic_fetch_sub_relaxed
99 
100 #undef ATOMIC_OPS
101 
102 #define ATOMIC_OPS(op, I, asm_op)					\
103 	ATOMIC_OP(op, I, asm_op)					\
104 	ATOMIC_FETCH_OP(op, I, asm_op)
105 
106 ATOMIC_OPS(and, i, and)
107 ATOMIC_OPS(or, i, or)
108 ATOMIC_OPS(xor, i, xor)
109 
110 #define arch_atomic_fetch_and_relaxed	arch_atomic_fetch_and_relaxed
111 #define arch_atomic_fetch_or_relaxed	arch_atomic_fetch_or_relaxed
112 #define arch_atomic_fetch_xor_relaxed	arch_atomic_fetch_xor_relaxed
113 
114 #undef ATOMIC_OPS
115 #undef ATOMIC_FETCH_OP
116 #undef ATOMIC_OP_RETURN
117 #undef ATOMIC_OP
118 
119 static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
120 {
121        int prev, rc;
122 
123 	__asm__ __volatile__ (
124 		"0:	ll.w	%[p],  %[c]\n"
125 		"	beq	%[p],  %[u], 1f\n"
126 		"	add.w	%[rc], %[p], %[a]\n"
127 		"	sc.w	%[rc], %[c]\n"
128 		"	beqz	%[rc], 0b\n"
129 		"	b	2f\n"
130 		"1:\n"
131 		__WEAK_LLSC_MB
132 		"2:\n"
133 		: [p]"=&r" (prev), [rc]"=&r" (rc),
134 		  [c]"=ZB" (v->counter)
135 		: [a]"r" (a), [u]"r" (u)
136 		: "memory");
137 
138 	return prev;
139 }
140 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
141 
142 /*
143  * arch_atomic_sub_if_positive - conditionally subtract integer from atomic variable
144  * @i: integer value to subtract
145  * @v: pointer of type atomic_t
146  *
147  * Atomically test @v and subtract @i if @v is greater or equal than @i.
148  * The function returns the old value of @v minus @i.
149  */
150 static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)
151 {
152 	int result;
153 	int temp;
154 
155 	if (__builtin_constant_p(i)) {
156 		__asm__ __volatile__(
157 		"1:	ll.w	%1, %2		# atomic_sub_if_positive\n"
158 		"	addi.w	%0, %1, %3				\n"
159 		"	move	%1, %0					\n"
160 		"	bltz	%0, 2f					\n"
161 		"	sc.w	%1, %2					\n"
162 		"	beqz	%1, 1b					\n"
163 		"2:							\n"
164 		__WEAK_LLSC_MB
165 		: "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
166 		: "I" (-i));
167 	} else {
168 		__asm__ __volatile__(
169 		"1:	ll.w	%1, %2		# atomic_sub_if_positive\n"
170 		"	sub.w	%0, %1, %3				\n"
171 		"	move	%1, %0					\n"
172 		"	bltz	%0, 2f					\n"
173 		"	sc.w	%1, %2					\n"
174 		"	beqz	%1, 1b					\n"
175 		"2:							\n"
176 		__WEAK_LLSC_MB
177 		: "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
178 		: "r" (i));
179 	}
180 
181 	return result;
182 }
183 
184 #define arch_atomic_cmpxchg(v, o, n) (arch_cmpxchg(&((v)->counter), (o), (n)))
185 #define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), (new)))
186 
187 /*
188  * arch_atomic_dec_if_positive - decrement by 1 if old value positive
189  * @v: pointer of type atomic_t
190  */
191 #define arch_atomic_dec_if_positive(v)	arch_atomic_sub_if_positive(1, v)
192 
193 #ifdef CONFIG_64BIT
194 
195 #define ATOMIC64_INIT(i)    { (i) }
196 
197 /*
198  * arch_atomic64_read - read atomic variable
199  * @v: pointer of type atomic64_t
200  *
201  */
202 #define arch_atomic64_read(v)	READ_ONCE((v)->counter)
203 
204 /*
205  * arch_atomic64_set - set atomic variable
206  * @v: pointer of type atomic64_t
207  * @i: required value
208  */
209 #define arch_atomic64_set(v, i)	WRITE_ONCE((v)->counter, (i))
210 
211 #define ATOMIC64_OP(op, I, asm_op)					\
212 static inline void arch_atomic64_##op(long i, atomic64_t *v)		\
213 {									\
214 	__asm__ __volatile__(						\
215 	"am"#asm_op"_db.d " " $zero, %1, %0	\n"			\
216 	: "+ZB" (v->counter)						\
217 	: "r" (I)							\
218 	: "memory");							\
219 }
220 
221 #define ATOMIC64_OP_RETURN(op, I, asm_op, c_op)					\
222 static inline long arch_atomic64_##op##_return_relaxed(long i, atomic64_t *v)	\
223 {										\
224 	long result;								\
225 	__asm__ __volatile__(							\
226 	"am"#asm_op"_db.d " " %1, %2, %0		\n"			\
227 	: "+ZB" (v->counter), "=&r" (result)					\
228 	: "r" (I)								\
229 	: "memory");								\
230 										\
231 	return result c_op I;							\
232 }
233 
234 #define ATOMIC64_FETCH_OP(op, I, asm_op)					\
235 static inline long arch_atomic64_fetch_##op##_relaxed(long i, atomic64_t *v)	\
236 {										\
237 	long result;								\
238 										\
239 	__asm__ __volatile__(							\
240 	"am"#asm_op"_db.d " " %1, %2, %0		\n"			\
241 	: "+ZB" (v->counter), "=&r" (result)					\
242 	: "r" (I)								\
243 	: "memory");								\
244 										\
245 	return result;								\
246 }
247 
248 #define ATOMIC64_OPS(op, I, asm_op, c_op)				      \
249 	ATOMIC64_OP(op, I, asm_op)					      \
250 	ATOMIC64_OP_RETURN(op, I, asm_op, c_op)				      \
251 	ATOMIC64_FETCH_OP(op, I, asm_op)
252 
253 ATOMIC64_OPS(add, i, add, +)
254 ATOMIC64_OPS(sub, -i, add, +)
255 
256 #define arch_atomic64_add_return_relaxed	arch_atomic64_add_return_relaxed
257 #define arch_atomic64_sub_return_relaxed	arch_atomic64_sub_return_relaxed
258 #define arch_atomic64_fetch_add_relaxed		arch_atomic64_fetch_add_relaxed
259 #define arch_atomic64_fetch_sub_relaxed		arch_atomic64_fetch_sub_relaxed
260 
261 #undef ATOMIC64_OPS
262 
263 #define ATOMIC64_OPS(op, I, asm_op)					      \
264 	ATOMIC64_OP(op, I, asm_op)					      \
265 	ATOMIC64_FETCH_OP(op, I, asm_op)
266 
267 ATOMIC64_OPS(and, i, and)
268 ATOMIC64_OPS(or, i, or)
269 ATOMIC64_OPS(xor, i, xor)
270 
271 #define arch_atomic64_fetch_and_relaxed	arch_atomic64_fetch_and_relaxed
272 #define arch_atomic64_fetch_or_relaxed	arch_atomic64_fetch_or_relaxed
273 #define arch_atomic64_fetch_xor_relaxed	arch_atomic64_fetch_xor_relaxed
274 
275 #undef ATOMIC64_OPS
276 #undef ATOMIC64_FETCH_OP
277 #undef ATOMIC64_OP_RETURN
278 #undef ATOMIC64_OP
279 
280 static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u)
281 {
282        long prev, rc;
283 
284 	__asm__ __volatile__ (
285 		"0:	ll.d	%[p],  %[c]\n"
286 		"	beq	%[p],  %[u], 1f\n"
287 		"	add.d	%[rc], %[p], %[a]\n"
288 		"	sc.d	%[rc], %[c]\n"
289 		"	beqz	%[rc], 0b\n"
290 		"	b	2f\n"
291 		"1:\n"
292 		__WEAK_LLSC_MB
293 		"2:\n"
294 		: [p]"=&r" (prev), [rc]"=&r" (rc),
295 		  [c] "=ZB" (v->counter)
296 		: [a]"r" (a), [u]"r" (u)
297 		: "memory");
298 
299 	return prev;
300 }
301 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
302 
303 /*
304  * arch_atomic64_sub_if_positive - conditionally subtract integer from atomic variable
305  * @i: integer value to subtract
306  * @v: pointer of type atomic64_t
307  *
308  * Atomically test @v and subtract @i if @v is greater or equal than @i.
309  * The function returns the old value of @v minus @i.
310  */
311 static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)
312 {
313 	long result;
314 	long temp;
315 
316 	if (__builtin_constant_p(i)) {
317 		__asm__ __volatile__(
318 		"1:	ll.d	%1, %2	# atomic64_sub_if_positive	\n"
319 		"	addi.d	%0, %1, %3				\n"
320 		"	move	%1, %0					\n"
321 		"	bltz	%0, 2f					\n"
322 		"	sc.d	%1, %2					\n"
323 		"	beqz	%1, 1b					\n"
324 		"2:							\n"
325 		__WEAK_LLSC_MB
326 		: "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
327 		: "I" (-i));
328 	} else {
329 		__asm__ __volatile__(
330 		"1:	ll.d	%1, %2	# atomic64_sub_if_positive	\n"
331 		"	sub.d	%0, %1, %3				\n"
332 		"	move	%1, %0					\n"
333 		"	bltz	%0, 2f					\n"
334 		"	sc.d	%1, %2					\n"
335 		"	beqz	%1, 1b					\n"
336 		"2:							\n"
337 		__WEAK_LLSC_MB
338 		: "=&r" (result), "=&r" (temp), "+ZC" (v->counter)
339 		: "r" (i));
340 	}
341 
342 	return result;
343 }
344 
345 #define arch_atomic64_cmpxchg(v, o, n) \
346 	((__typeof__((v)->counter))arch_cmpxchg(&((v)->counter), (o), (n)))
347 #define arch_atomic64_xchg(v, new) (arch_xchg(&((v)->counter), (new)))
348 
349 /*
350  * arch_atomic64_dec_if_positive - decrement by 1 if old value positive
351  * @v: pointer of type atomic64_t
352  */
353 #define arch_atomic64_dec_if_positive(v)	arch_atomic64_sub_if_positive(1, v)
354 
355 #endif /* CONFIG_64BIT */
356 
357 #endif /* _ASM_ATOMIC_H */
358