1 /*
2  * Based on arch/arm/include/asm/atomic.h
3  *
4  * Copyright (C) 1996 Russell King.
5  * Copyright (C) 2002 Deep Blue Solutions Ltd.
6  * Copyright (C) 2012 ARM Ltd.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 as
10  * published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19  */
20 
21 #ifndef __ASM_ATOMIC_LSE_H
22 #define __ASM_ATOMIC_LSE_H
23 
24 #ifndef __ARM64_IN_ATOMIC_IMPL
25 #error "please don't include this file directly"
26 #endif
27 
28 #define __LL_SC_ATOMIC(op)	__LL_SC_CALL(atomic_##op)
29 #define ATOMIC_OP(op, asm_op)						\
30 static inline void atomic_##op(int i, atomic_t *v)			\
31 {									\
32 	register int w0 asm ("w0") = i;					\
33 	register atomic_t *x1 asm ("x1") = v;				\
34 									\
35 	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(op),		\
36 "	" #asm_op "	%w[i], %[v]\n")					\
37 	: [i] "+r" (w0), [v] "+Q" (v->counter)				\
38 	: "r" (x1)							\
39 	: __LL_SC_CLOBBERS);						\
40 }
41 
42 ATOMIC_OP(andnot, stclr)
43 ATOMIC_OP(or, stset)
44 ATOMIC_OP(xor, steor)
45 ATOMIC_OP(add, stadd)
46 
47 #undef ATOMIC_OP
48 
49 #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...)			\
50 static inline int atomic_fetch_##op##name(int i, atomic_t *v)		\
51 {									\
52 	register int w0 asm ("w0") = i;					\
53 	register atomic_t *x1 asm ("x1") = v;				\
54 									\
55 	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
56 	/* LL/SC */							\
57 	__LL_SC_ATOMIC(fetch_##op##name),				\
58 	/* LSE atomics */						\
59 "	" #asm_op #mb "	%w[i], %w[i], %[v]")				\
60 	: [i] "+r" (w0), [v] "+Q" (v->counter)				\
61 	: "r" (x1)							\
62 	: __LL_SC_CLOBBERS, ##cl);					\
63 									\
64 	return w0;							\
65 }
66 
67 #define ATOMIC_FETCH_OPS(op, asm_op)					\
68 	ATOMIC_FETCH_OP(_relaxed,   , op, asm_op)			\
69 	ATOMIC_FETCH_OP(_acquire,  a, op, asm_op, "memory")		\
70 	ATOMIC_FETCH_OP(_release,  l, op, asm_op, "memory")		\
71 	ATOMIC_FETCH_OP(        , al, op, asm_op, "memory")
72 
73 ATOMIC_FETCH_OPS(andnot, ldclr)
74 ATOMIC_FETCH_OPS(or, ldset)
75 ATOMIC_FETCH_OPS(xor, ldeor)
76 ATOMIC_FETCH_OPS(add, ldadd)
77 
78 #undef ATOMIC_FETCH_OP
79 #undef ATOMIC_FETCH_OPS
80 
81 #define ATOMIC_OP_ADD_RETURN(name, mb, cl...)				\
82 static inline int atomic_add_return##name(int i, atomic_t *v)		\
83 {									\
84 	register int w0 asm ("w0") = i;					\
85 	register atomic_t *x1 asm ("x1") = v;				\
86 									\
87 	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
88 	/* LL/SC */							\
89 	__LL_SC_ATOMIC(add_return##name)				\
90 	__nops(1),							\
91 	/* LSE atomics */						\
92 	"	ldadd" #mb "	%w[i], w30, %[v]\n"			\
93 	"	add	%w[i], %w[i], w30")				\
94 	: [i] "+r" (w0), [v] "+Q" (v->counter)				\
95 	: "r" (x1)							\
96 	: __LL_SC_CLOBBERS, ##cl);					\
97 									\
98 	return w0;							\
99 }
100 
101 ATOMIC_OP_ADD_RETURN(_relaxed,   )
102 ATOMIC_OP_ADD_RETURN(_acquire,  a, "memory")
103 ATOMIC_OP_ADD_RETURN(_release,  l, "memory")
104 ATOMIC_OP_ADD_RETURN(        , al, "memory")
105 
106 #undef ATOMIC_OP_ADD_RETURN
107 
108 static inline void atomic_and(int i, atomic_t *v)
109 {
110 	register int w0 asm ("w0") = i;
111 	register atomic_t *x1 asm ("x1") = v;
112 
113 	asm volatile(ARM64_LSE_ATOMIC_INSN(
114 	/* LL/SC */
115 	__LL_SC_ATOMIC(and)
116 	__nops(1),
117 	/* LSE atomics */
118 	"	mvn	%w[i], %w[i]\n"
119 	"	stclr	%w[i], %[v]")
120 	: [i] "+&r" (w0), [v] "+Q" (v->counter)
121 	: "r" (x1)
122 	: __LL_SC_CLOBBERS);
123 }
124 
125 #define ATOMIC_FETCH_OP_AND(name, mb, cl...)				\
126 static inline int atomic_fetch_and##name(int i, atomic_t *v)		\
127 {									\
128 	register int w0 asm ("w0") = i;					\
129 	register atomic_t *x1 asm ("x1") = v;				\
130 									\
131 	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
132 	/* LL/SC */							\
133 	__LL_SC_ATOMIC(fetch_and##name)					\
134 	__nops(1),							\
135 	/* LSE atomics */						\
136 	"	mvn	%w[i], %w[i]\n"					\
137 	"	ldclr" #mb "	%w[i], %w[i], %[v]")			\
138 	: [i] "+&r" (w0), [v] "+Q" (v->counter)				\
139 	: "r" (x1)							\
140 	: __LL_SC_CLOBBERS, ##cl);					\
141 									\
142 	return w0;							\
143 }
144 
145 ATOMIC_FETCH_OP_AND(_relaxed,   )
146 ATOMIC_FETCH_OP_AND(_acquire,  a, "memory")
147 ATOMIC_FETCH_OP_AND(_release,  l, "memory")
148 ATOMIC_FETCH_OP_AND(        , al, "memory")
149 
150 #undef ATOMIC_FETCH_OP_AND
151 
152 static inline void atomic_sub(int i, atomic_t *v)
153 {
154 	register int w0 asm ("w0") = i;
155 	register atomic_t *x1 asm ("x1") = v;
156 
157 	asm volatile(ARM64_LSE_ATOMIC_INSN(
158 	/* LL/SC */
159 	__LL_SC_ATOMIC(sub)
160 	__nops(1),
161 	/* LSE atomics */
162 	"	neg	%w[i], %w[i]\n"
163 	"	stadd	%w[i], %[v]")
164 	: [i] "+&r" (w0), [v] "+Q" (v->counter)
165 	: "r" (x1)
166 	: __LL_SC_CLOBBERS);
167 }
168 
169 #define ATOMIC_OP_SUB_RETURN(name, mb, cl...)				\
170 static inline int atomic_sub_return##name(int i, atomic_t *v)		\
171 {									\
172 	register int w0 asm ("w0") = i;					\
173 	register atomic_t *x1 asm ("x1") = v;				\
174 									\
175 	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
176 	/* LL/SC */							\
177 	__LL_SC_ATOMIC(sub_return##name)				\
178 	__nops(2),							\
179 	/* LSE atomics */						\
180 	"	neg	%w[i], %w[i]\n"					\
181 	"	ldadd" #mb "	%w[i], w30, %[v]\n"			\
182 	"	add	%w[i], %w[i], w30")				\
183 	: [i] "+&r" (w0), [v] "+Q" (v->counter)				\
184 	: "r" (x1)							\
185 	: __LL_SC_CLOBBERS , ##cl);					\
186 									\
187 	return w0;							\
188 }
189 
190 ATOMIC_OP_SUB_RETURN(_relaxed,   )
191 ATOMIC_OP_SUB_RETURN(_acquire,  a, "memory")
192 ATOMIC_OP_SUB_RETURN(_release,  l, "memory")
193 ATOMIC_OP_SUB_RETURN(        , al, "memory")
194 
195 #undef ATOMIC_OP_SUB_RETURN
196 
197 #define ATOMIC_FETCH_OP_SUB(name, mb, cl...)				\
198 static inline int atomic_fetch_sub##name(int i, atomic_t *v)		\
199 {									\
200 	register int w0 asm ("w0") = i;					\
201 	register atomic_t *x1 asm ("x1") = v;				\
202 									\
203 	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
204 	/* LL/SC */							\
205 	__LL_SC_ATOMIC(fetch_sub##name)					\
206 	__nops(1),							\
207 	/* LSE atomics */						\
208 	"	neg	%w[i], %w[i]\n"					\
209 	"	ldadd" #mb "	%w[i], %w[i], %[v]")			\
210 	: [i] "+&r" (w0), [v] "+Q" (v->counter)				\
211 	: "r" (x1)							\
212 	: __LL_SC_CLOBBERS, ##cl);					\
213 									\
214 	return w0;							\
215 }
216 
217 ATOMIC_FETCH_OP_SUB(_relaxed,   )
218 ATOMIC_FETCH_OP_SUB(_acquire,  a, "memory")
219 ATOMIC_FETCH_OP_SUB(_release,  l, "memory")
220 ATOMIC_FETCH_OP_SUB(        , al, "memory")
221 
222 #undef ATOMIC_FETCH_OP_SUB
223 #undef __LL_SC_ATOMIC
224 
225 #define __LL_SC_ATOMIC64(op)	__LL_SC_CALL(atomic64_##op)
226 #define ATOMIC64_OP(op, asm_op)						\
227 static inline void atomic64_##op(long i, atomic64_t *v)			\
228 {									\
229 	register long x0 asm ("x0") = i;				\
230 	register atomic64_t *x1 asm ("x1") = v;				\
231 									\
232 	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(op),	\
233 "	" #asm_op "	%[i], %[v]\n")					\
234 	: [i] "+r" (x0), [v] "+Q" (v->counter)				\
235 	: "r" (x1)							\
236 	: __LL_SC_CLOBBERS);						\
237 }
238 
239 ATOMIC64_OP(andnot, stclr)
240 ATOMIC64_OP(or, stset)
241 ATOMIC64_OP(xor, steor)
242 ATOMIC64_OP(add, stadd)
243 
244 #undef ATOMIC64_OP
245 
246 #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...)			\
247 static inline long atomic64_fetch_##op##name(long i, atomic64_t *v)	\
248 {									\
249 	register long x0 asm ("x0") = i;				\
250 	register atomic64_t *x1 asm ("x1") = v;				\
251 									\
252 	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
253 	/* LL/SC */							\
254 	__LL_SC_ATOMIC64(fetch_##op##name),				\
255 	/* LSE atomics */						\
256 "	" #asm_op #mb "	%[i], %[i], %[v]")				\
257 	: [i] "+r" (x0), [v] "+Q" (v->counter)				\
258 	: "r" (x1)							\
259 	: __LL_SC_CLOBBERS, ##cl);					\
260 									\
261 	return x0;							\
262 }
263 
264 #define ATOMIC64_FETCH_OPS(op, asm_op)					\
265 	ATOMIC64_FETCH_OP(_relaxed,   , op, asm_op)			\
266 	ATOMIC64_FETCH_OP(_acquire,  a, op, asm_op, "memory")		\
267 	ATOMIC64_FETCH_OP(_release,  l, op, asm_op, "memory")		\
268 	ATOMIC64_FETCH_OP(        , al, op, asm_op, "memory")
269 
270 ATOMIC64_FETCH_OPS(andnot, ldclr)
271 ATOMIC64_FETCH_OPS(or, ldset)
272 ATOMIC64_FETCH_OPS(xor, ldeor)
273 ATOMIC64_FETCH_OPS(add, ldadd)
274 
275 #undef ATOMIC64_FETCH_OP
276 #undef ATOMIC64_FETCH_OPS
277 
278 #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...)				\
279 static inline long atomic64_add_return##name(long i, atomic64_t *v)	\
280 {									\
281 	register long x0 asm ("x0") = i;				\
282 	register atomic64_t *x1 asm ("x1") = v;				\
283 									\
284 	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
285 	/* LL/SC */							\
286 	__LL_SC_ATOMIC64(add_return##name)				\
287 	__nops(1),							\
288 	/* LSE atomics */						\
289 	"	ldadd" #mb "	%[i], x30, %[v]\n"			\
290 	"	add	%[i], %[i], x30")				\
291 	: [i] "+r" (x0), [v] "+Q" (v->counter)				\
292 	: "r" (x1)							\
293 	: __LL_SC_CLOBBERS, ##cl);					\
294 									\
295 	return x0;							\
296 }
297 
298 ATOMIC64_OP_ADD_RETURN(_relaxed,   )
299 ATOMIC64_OP_ADD_RETURN(_acquire,  a, "memory")
300 ATOMIC64_OP_ADD_RETURN(_release,  l, "memory")
301 ATOMIC64_OP_ADD_RETURN(        , al, "memory")
302 
303 #undef ATOMIC64_OP_ADD_RETURN
304 
305 static inline void atomic64_and(long i, atomic64_t *v)
306 {
307 	register long x0 asm ("x0") = i;
308 	register atomic64_t *x1 asm ("x1") = v;
309 
310 	asm volatile(ARM64_LSE_ATOMIC_INSN(
311 	/* LL/SC */
312 	__LL_SC_ATOMIC64(and)
313 	__nops(1),
314 	/* LSE atomics */
315 	"	mvn	%[i], %[i]\n"
316 	"	stclr	%[i], %[v]")
317 	: [i] "+&r" (x0), [v] "+Q" (v->counter)
318 	: "r" (x1)
319 	: __LL_SC_CLOBBERS);
320 }
321 
322 #define ATOMIC64_FETCH_OP_AND(name, mb, cl...)				\
323 static inline long atomic64_fetch_and##name(long i, atomic64_t *v)	\
324 {									\
325 	register long x0 asm ("x0") = i;				\
326 	register atomic64_t *x1 asm ("x1") = v;				\
327 									\
328 	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
329 	/* LL/SC */							\
330 	__LL_SC_ATOMIC64(fetch_and##name)				\
331 	__nops(1),							\
332 	/* LSE atomics */						\
333 	"	mvn	%[i], %[i]\n"					\
334 	"	ldclr" #mb "	%[i], %[i], %[v]")			\
335 	: [i] "+&r" (x0), [v] "+Q" (v->counter)				\
336 	: "r" (x1)							\
337 	: __LL_SC_CLOBBERS, ##cl);					\
338 									\
339 	return x0;							\
340 }
341 
342 ATOMIC64_FETCH_OP_AND(_relaxed,   )
343 ATOMIC64_FETCH_OP_AND(_acquire,  a, "memory")
344 ATOMIC64_FETCH_OP_AND(_release,  l, "memory")
345 ATOMIC64_FETCH_OP_AND(        , al, "memory")
346 
347 #undef ATOMIC64_FETCH_OP_AND
348 
349 static inline void atomic64_sub(long i, atomic64_t *v)
350 {
351 	register long x0 asm ("x0") = i;
352 	register atomic64_t *x1 asm ("x1") = v;
353 
354 	asm volatile(ARM64_LSE_ATOMIC_INSN(
355 	/* LL/SC */
356 	__LL_SC_ATOMIC64(sub)
357 	__nops(1),
358 	/* LSE atomics */
359 	"	neg	%[i], %[i]\n"
360 	"	stadd	%[i], %[v]")
361 	: [i] "+&r" (x0), [v] "+Q" (v->counter)
362 	: "r" (x1)
363 	: __LL_SC_CLOBBERS);
364 }
365 
366 #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...)				\
367 static inline long atomic64_sub_return##name(long i, atomic64_t *v)	\
368 {									\
369 	register long x0 asm ("x0") = i;				\
370 	register atomic64_t *x1 asm ("x1") = v;				\
371 									\
372 	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
373 	/* LL/SC */							\
374 	__LL_SC_ATOMIC64(sub_return##name)				\
375 	__nops(2),							\
376 	/* LSE atomics */						\
377 	"	neg	%[i], %[i]\n"					\
378 	"	ldadd" #mb "	%[i], x30, %[v]\n"			\
379 	"	add	%[i], %[i], x30")				\
380 	: [i] "+&r" (x0), [v] "+Q" (v->counter)				\
381 	: "r" (x1)							\
382 	: __LL_SC_CLOBBERS, ##cl);					\
383 									\
384 	return x0;							\
385 }
386 
387 ATOMIC64_OP_SUB_RETURN(_relaxed,   )
388 ATOMIC64_OP_SUB_RETURN(_acquire,  a, "memory")
389 ATOMIC64_OP_SUB_RETURN(_release,  l, "memory")
390 ATOMIC64_OP_SUB_RETURN(        , al, "memory")
391 
392 #undef ATOMIC64_OP_SUB_RETURN
393 
394 #define ATOMIC64_FETCH_OP_SUB(name, mb, cl...)				\
395 static inline long atomic64_fetch_sub##name(long i, atomic64_t *v)	\
396 {									\
397 	register long x0 asm ("x0") = i;				\
398 	register atomic64_t *x1 asm ("x1") = v;				\
399 									\
400 	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
401 	/* LL/SC */							\
402 	__LL_SC_ATOMIC64(fetch_sub##name)				\
403 	__nops(1),							\
404 	/* LSE atomics */						\
405 	"	neg	%[i], %[i]\n"					\
406 	"	ldadd" #mb "	%[i], %[i], %[v]")			\
407 	: [i] "+&r" (x0), [v] "+Q" (v->counter)				\
408 	: "r" (x1)							\
409 	: __LL_SC_CLOBBERS, ##cl);					\
410 									\
411 	return x0;							\
412 }
413 
414 ATOMIC64_FETCH_OP_SUB(_relaxed,   )
415 ATOMIC64_FETCH_OP_SUB(_acquire,  a, "memory")
416 ATOMIC64_FETCH_OP_SUB(_release,  l, "memory")
417 ATOMIC64_FETCH_OP_SUB(        , al, "memory")
418 
419 #undef ATOMIC64_FETCH_OP_SUB
420 
421 static inline long atomic64_dec_if_positive(atomic64_t *v)
422 {
423 	register long x0 asm ("x0") = (long)v;
424 
425 	asm volatile(ARM64_LSE_ATOMIC_INSN(
426 	/* LL/SC */
427 	__LL_SC_ATOMIC64(dec_if_positive)
428 	__nops(6),
429 	/* LSE atomics */
430 	"1:	ldr	x30, %[v]\n"
431 	"	subs	%[ret], x30, #1\n"
432 	"	b.lt	2f\n"
433 	"	casal	x30, %[ret], %[v]\n"
434 	"	sub	x30, x30, #1\n"
435 	"	sub	x30, x30, %[ret]\n"
436 	"	cbnz	x30, 1b\n"
437 	"2:")
438 	: [ret] "+&r" (x0), [v] "+Q" (v->counter)
439 	:
440 	: __LL_SC_CLOBBERS, "cc", "memory");
441 
442 	return x0;
443 }
444 
445 #undef __LL_SC_ATOMIC64
446 
447 #define __LL_SC_CMPXCHG(op)	__LL_SC_CALL(__cmpxchg_case_##op)
448 
449 #define __CMPXCHG_CASE(w, sz, name, mb, cl...)				\
450 static inline unsigned long __cmpxchg_case_##name(volatile void *ptr,	\
451 						  unsigned long old,	\
452 						  unsigned long new)	\
453 {									\
454 	register unsigned long x0 asm ("x0") = (unsigned long)ptr;	\
455 	register unsigned long x1 asm ("x1") = old;			\
456 	register unsigned long x2 asm ("x2") = new;			\
457 									\
458 	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
459 	/* LL/SC */							\
460 	__LL_SC_CMPXCHG(name)						\
461 	__nops(2),							\
462 	/* LSE atomics */						\
463 	"	mov	" #w "30, %" #w "[old]\n"			\
464 	"	cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n"		\
465 	"	mov	%" #w "[ret], " #w "30")			\
466 	: [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr)		\
467 	: [old] "r" (x1), [new] "r" (x2)				\
468 	: __LL_SC_CLOBBERS, ##cl);					\
469 									\
470 	return x0;							\
471 }
472 
473 __CMPXCHG_CASE(w, b,     1,   )
474 __CMPXCHG_CASE(w, h,     2,   )
475 __CMPXCHG_CASE(w,  ,     4,   )
476 __CMPXCHG_CASE(x,  ,     8,   )
477 __CMPXCHG_CASE(w, b, acq_1,  a, "memory")
478 __CMPXCHG_CASE(w, h, acq_2,  a, "memory")
479 __CMPXCHG_CASE(w,  , acq_4,  a, "memory")
480 __CMPXCHG_CASE(x,  , acq_8,  a, "memory")
481 __CMPXCHG_CASE(w, b, rel_1,  l, "memory")
482 __CMPXCHG_CASE(w, h, rel_2,  l, "memory")
483 __CMPXCHG_CASE(w,  , rel_4,  l, "memory")
484 __CMPXCHG_CASE(x,  , rel_8,  l, "memory")
485 __CMPXCHG_CASE(w, b,  mb_1, al, "memory")
486 __CMPXCHG_CASE(w, h,  mb_2, al, "memory")
487 __CMPXCHG_CASE(w,  ,  mb_4, al, "memory")
488 __CMPXCHG_CASE(x,  ,  mb_8, al, "memory")
489 
490 #undef __LL_SC_CMPXCHG
491 #undef __CMPXCHG_CASE
492 
493 #define __LL_SC_CMPXCHG_DBL(op)	__LL_SC_CALL(__cmpxchg_double##op)
494 
495 #define __CMPXCHG_DBL(name, mb, cl...)					\
496 static inline long __cmpxchg_double##name(unsigned long old1,		\
497 					 unsigned long old2,		\
498 					 unsigned long new1,		\
499 					 unsigned long new2,		\
500 					 volatile void *ptr)		\
501 {									\
502 	unsigned long oldval1 = old1;					\
503 	unsigned long oldval2 = old2;					\
504 	register unsigned long x0 asm ("x0") = old1;			\
505 	register unsigned long x1 asm ("x1") = old2;			\
506 	register unsigned long x2 asm ("x2") = new1;			\
507 	register unsigned long x3 asm ("x3") = new2;			\
508 	register unsigned long x4 asm ("x4") = (unsigned long)ptr;	\
509 									\
510 	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
511 	/* LL/SC */							\
512 	__LL_SC_CMPXCHG_DBL(name)					\
513 	__nops(3),							\
514 	/* LSE atomics */						\
515 	"	casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
516 	"	eor	%[old1], %[old1], %[oldval1]\n"			\
517 	"	eor	%[old2], %[old2], %[oldval2]\n"			\
518 	"	orr	%[old1], %[old1], %[old2]")			\
519 	: [old1] "+&r" (x0), [old2] "+&r" (x1),				\
520 	  [v] "+Q" (*(unsigned long *)ptr)				\
521 	: [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4),		\
522 	  [oldval1] "r" (oldval1), [oldval2] "r" (oldval2)		\
523 	: __LL_SC_CLOBBERS, ##cl);					\
524 									\
525 	return x0;							\
526 }
527 
528 __CMPXCHG_DBL(   ,   )
529 __CMPXCHG_DBL(_mb, al, "memory")
530 
531 #undef __LL_SC_CMPXCHG_DBL
532 #undef __CMPXCHG_DBL
533 
534 #endif	/* __ASM_ATOMIC_LSE_H */
535