1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_POWERPC_CMPXCHG_H_
3 #define _ASM_POWERPC_CMPXCHG_H_
4
5 #ifdef __KERNEL__
6 #include <linux/compiler.h>
7 #include <asm/synch.h>
8 #include <linux/bug.h>
9
10 #ifdef __BIG_ENDIAN
11 #define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE)
12 #else
13 #define BITOFF_CAL(size, off) (off * BITS_PER_BYTE)
14 #endif
15
16 #define XCHG_GEN(type, sfx, cl) \
17 static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \
18 { \
19 unsigned int prev, prev_mask, tmp, bitoff, off; \
20 \
21 off = (unsigned long)p % sizeof(u32); \
22 bitoff = BITOFF_CAL(sizeof(type), off); \
23 p -= off; \
24 val <<= bitoff; \
25 prev_mask = (u32)(type)-1 << bitoff; \
26 \
27 __asm__ __volatile__( \
28 "1: lwarx %0,0,%3\n" \
29 " andc %1,%0,%5\n" \
30 " or %1,%1,%4\n" \
31 " stwcx. %1,0,%3\n" \
32 " bne- 1b\n" \
33 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \
34 : "r" (p), "r" (val), "r" (prev_mask) \
35 : "cc", cl); \
36 \
37 return prev >> bitoff; \
38 }
39
40 #define CMPXCHG_GEN(type, sfx, br, br2, cl) \
41 static inline \
42 u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \
43 { \
44 unsigned int prev, prev_mask, tmp, bitoff, off; \
45 \
46 off = (unsigned long)p % sizeof(u32); \
47 bitoff = BITOFF_CAL(sizeof(type), off); \
48 p -= off; \
49 old <<= bitoff; \
50 new <<= bitoff; \
51 prev_mask = (u32)(type)-1 << bitoff; \
52 \
53 __asm__ __volatile__( \
54 br \
55 "1: lwarx %0,0,%3\n" \
56 " and %1,%0,%6\n" \
57 " cmpw 0,%1,%4\n" \
58 " bne- 2f\n" \
59 " andc %1,%0,%6\n" \
60 " or %1,%1,%5\n" \
61 " stwcx. %1,0,%3\n" \
62 " bne- 1b\n" \
63 br2 \
64 "\n" \
65 "2:" \
66 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \
67 : "r" (p), "r" (old), "r" (new), "r" (prev_mask) \
68 : "cc", cl); \
69 \
70 return prev >> bitoff; \
71 }
72
73 /*
74 * Atomic exchange
75 *
76 * Changes the memory location '*p' to be val and returns
77 * the previous value stored there.
78 */
79
80 #ifndef CONFIG_PPC_HAS_LBARX_LHARX
81 XCHG_GEN(u8, _local, "memory");
82 XCHG_GEN(u8, _relaxed, "cc");
83 XCHG_GEN(u16, _local, "memory");
84 XCHG_GEN(u16, _relaxed, "cc");
85 #else
86 static __always_inline unsigned long
__xchg_u8_local(volatile void * p,unsigned long val)87 __xchg_u8_local(volatile void *p, unsigned long val)
88 {
89 unsigned long prev;
90
91 __asm__ __volatile__(
92 "1: lbarx %0,0,%2 # __xchg_u8_local\n"
93 " stbcx. %3,0,%2 \n"
94 " bne- 1b"
95 : "=&r" (prev), "+m" (*(volatile unsigned char *)p)
96 : "r" (p), "r" (val)
97 : "cc", "memory");
98
99 return prev;
100 }
101
102 static __always_inline unsigned long
__xchg_u8_relaxed(u8 * p,unsigned long val)103 __xchg_u8_relaxed(u8 *p, unsigned long val)
104 {
105 unsigned long prev;
106
107 __asm__ __volatile__(
108 "1: lbarx %0,0,%2 # __xchg_u8_relaxed\n"
109 " stbcx. %3,0,%2\n"
110 " bne- 1b"
111 : "=&r" (prev), "+m" (*p)
112 : "r" (p), "r" (val)
113 : "cc");
114
115 return prev;
116 }
117
118 static __always_inline unsigned long
__xchg_u16_local(volatile void * p,unsigned long val)119 __xchg_u16_local(volatile void *p, unsigned long val)
120 {
121 unsigned long prev;
122
123 __asm__ __volatile__(
124 "1: lharx %0,0,%2 # __xchg_u16_local\n"
125 " sthcx. %3,0,%2\n"
126 " bne- 1b"
127 : "=&r" (prev), "+m" (*(volatile unsigned short *)p)
128 : "r" (p), "r" (val)
129 : "cc", "memory");
130
131 return prev;
132 }
133
134 static __always_inline unsigned long
__xchg_u16_relaxed(u16 * p,unsigned long val)135 __xchg_u16_relaxed(u16 *p, unsigned long val)
136 {
137 unsigned long prev;
138
139 __asm__ __volatile__(
140 "1: lharx %0,0,%2 # __xchg_u16_relaxed\n"
141 " sthcx. %3,0,%2\n"
142 " bne- 1b"
143 : "=&r" (prev), "+m" (*p)
144 : "r" (p), "r" (val)
145 : "cc");
146
147 return prev;
148 }
149 #endif
150
151 static __always_inline unsigned long
__xchg_u32_local(volatile void * p,unsigned long val)152 __xchg_u32_local(volatile void *p, unsigned long val)
153 {
154 unsigned long prev;
155
156 __asm__ __volatile__(
157 "1: lwarx %0,0,%2 \n"
158 " stwcx. %3,0,%2 \n\
159 bne- 1b"
160 : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
161 : "r" (p), "r" (val)
162 : "cc", "memory");
163
164 return prev;
165 }
166
167 static __always_inline unsigned long
__xchg_u32_relaxed(u32 * p,unsigned long val)168 __xchg_u32_relaxed(u32 *p, unsigned long val)
169 {
170 unsigned long prev;
171
172 __asm__ __volatile__(
173 "1: lwarx %0,0,%2\n"
174 " stwcx. %3,0,%2\n"
175 " bne- 1b"
176 : "=&r" (prev), "+m" (*p)
177 : "r" (p), "r" (val)
178 : "cc");
179
180 return prev;
181 }
182
183 #ifdef CONFIG_PPC64
184 static __always_inline unsigned long
__xchg_u64_local(volatile void * p,unsigned long val)185 __xchg_u64_local(volatile void *p, unsigned long val)
186 {
187 unsigned long prev;
188
189 __asm__ __volatile__(
190 "1: ldarx %0,0,%2 \n"
191 " stdcx. %3,0,%2 \n\
192 bne- 1b"
193 : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
194 : "r" (p), "r" (val)
195 : "cc", "memory");
196
197 return prev;
198 }
199
200 static __always_inline unsigned long
__xchg_u64_relaxed(u64 * p,unsigned long val)201 __xchg_u64_relaxed(u64 *p, unsigned long val)
202 {
203 unsigned long prev;
204
205 __asm__ __volatile__(
206 "1: ldarx %0,0,%2\n"
207 " stdcx. %3,0,%2\n"
208 " bne- 1b"
209 : "=&r" (prev), "+m" (*p)
210 : "r" (p), "r" (val)
211 : "cc");
212
213 return prev;
214 }
215 #endif
216
217 static __always_inline unsigned long
__xchg_local(void * ptr,unsigned long x,unsigned int size)218 __xchg_local(void *ptr, unsigned long x, unsigned int size)
219 {
220 switch (size) {
221 case 1:
222 return __xchg_u8_local(ptr, x);
223 case 2:
224 return __xchg_u16_local(ptr, x);
225 case 4:
226 return __xchg_u32_local(ptr, x);
227 #ifdef CONFIG_PPC64
228 case 8:
229 return __xchg_u64_local(ptr, x);
230 #endif
231 }
232 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
233 return x;
234 }
235
236 static __always_inline unsigned long
__xchg_relaxed(void * ptr,unsigned long x,unsigned int size)237 __xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
238 {
239 switch (size) {
240 case 1:
241 return __xchg_u8_relaxed(ptr, x);
242 case 2:
243 return __xchg_u16_relaxed(ptr, x);
244 case 4:
245 return __xchg_u32_relaxed(ptr, x);
246 #ifdef CONFIG_PPC64
247 case 8:
248 return __xchg_u64_relaxed(ptr, x);
249 #endif
250 }
251 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_relaxed");
252 return x;
253 }
254 #define arch_xchg_local(ptr,x) \
255 ({ \
256 __typeof__(*(ptr)) _x_ = (x); \
257 (__typeof__(*(ptr))) __xchg_local((ptr), \
258 (unsigned long)_x_, sizeof(*(ptr))); \
259 })
260
261 #define arch_xchg_relaxed(ptr, x) \
262 ({ \
263 __typeof__(*(ptr)) _x_ = (x); \
264 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \
265 (unsigned long)_x_, sizeof(*(ptr))); \
266 })
267
268 /*
269 * Compare and exchange - if *p == old, set it to new,
270 * and return the old value of *p.
271 */
272 #ifndef CONFIG_PPC_HAS_LBARX_LHARX
273 CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
274 CMPXCHG_GEN(u8, _local, , , "memory");
275 CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
276 CMPXCHG_GEN(u8, _relaxed, , , "cc");
277 CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
278 CMPXCHG_GEN(u16, _local, , , "memory");
279 CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
280 CMPXCHG_GEN(u16, _relaxed, , , "cc");
281 #else
282 static __always_inline unsigned long
__cmpxchg_u8(volatile unsigned char * p,unsigned long old,unsigned long new)283 __cmpxchg_u8(volatile unsigned char *p, unsigned long old, unsigned long new)
284 {
285 unsigned int prev;
286
287 __asm__ __volatile__ (
288 PPC_ATOMIC_ENTRY_BARRIER
289 "1: lbarx %0,0,%2 # __cmpxchg_u8\n"
290 " cmpw 0,%0,%3\n"
291 " bne- 2f\n"
292 " stbcx. %4,0,%2\n"
293 " bne- 1b"
294 PPC_ATOMIC_EXIT_BARRIER
295 "\n\
296 2:"
297 : "=&r" (prev), "+m" (*p)
298 : "r" (p), "r" (old), "r" (new)
299 : "cc", "memory");
300
301 return prev;
302 }
303
304 static __always_inline unsigned long
__cmpxchg_u8_local(volatile unsigned char * p,unsigned long old,unsigned long new)305 __cmpxchg_u8_local(volatile unsigned char *p, unsigned long old,
306 unsigned long new)
307 {
308 unsigned int prev;
309
310 __asm__ __volatile__ (
311 "1: lbarx %0,0,%2 # __cmpxchg_u8_local\n"
312 " cmpw 0,%0,%3\n"
313 " bne- 2f\n"
314 " stbcx. %4,0,%2\n"
315 " bne- 1b\n"
316 "2:"
317 : "=&r" (prev), "+m" (*p)
318 : "r" (p), "r" (old), "r" (new)
319 : "cc", "memory");
320
321 return prev;
322 }
323
324 static __always_inline unsigned long
__cmpxchg_u8_relaxed(u8 * p,unsigned long old,unsigned long new)325 __cmpxchg_u8_relaxed(u8 *p, unsigned long old, unsigned long new)
326 {
327 unsigned long prev;
328
329 __asm__ __volatile__ (
330 "1: lbarx %0,0,%2 # __cmpxchg_u8_relaxed\n"
331 " cmpw 0,%0,%3\n"
332 " bne- 2f\n"
333 " stbcx. %4,0,%2\n"
334 " bne- 1b\n"
335 "2:"
336 : "=&r" (prev), "+m" (*p)
337 : "r" (p), "r" (old), "r" (new)
338 : "cc");
339
340 return prev;
341 }
342
343 static __always_inline unsigned long
__cmpxchg_u8_acquire(u8 * p,unsigned long old,unsigned long new)344 __cmpxchg_u8_acquire(u8 *p, unsigned long old, unsigned long new)
345 {
346 unsigned long prev;
347
348 __asm__ __volatile__ (
349 "1: lbarx %0,0,%2 # __cmpxchg_u8_acquire\n"
350 " cmpw 0,%0,%3\n"
351 " bne- 2f\n"
352 " stbcx. %4,0,%2\n"
353 " bne- 1b\n"
354 PPC_ACQUIRE_BARRIER
355 "2:"
356 : "=&r" (prev), "+m" (*p)
357 : "r" (p), "r" (old), "r" (new)
358 : "cc", "memory");
359
360 return prev;
361 }
362
363 static __always_inline unsigned long
__cmpxchg_u16(volatile unsigned short * p,unsigned long old,unsigned long new)364 __cmpxchg_u16(volatile unsigned short *p, unsigned long old, unsigned long new)
365 {
366 unsigned int prev;
367
368 __asm__ __volatile__ (
369 PPC_ATOMIC_ENTRY_BARRIER
370 "1: lharx %0,0,%2 # __cmpxchg_u16\n"
371 " cmpw 0,%0,%3\n"
372 " bne- 2f\n"
373 " sthcx. %4,0,%2\n"
374 " bne- 1b\n"
375 PPC_ATOMIC_EXIT_BARRIER
376 "2:"
377 : "=&r" (prev), "+m" (*p)
378 : "r" (p), "r" (old), "r" (new)
379 : "cc", "memory");
380
381 return prev;
382 }
383
384 static __always_inline unsigned long
__cmpxchg_u16_local(volatile unsigned short * p,unsigned long old,unsigned long new)385 __cmpxchg_u16_local(volatile unsigned short *p, unsigned long old,
386 unsigned long new)
387 {
388 unsigned int prev;
389
390 __asm__ __volatile__ (
391 "1: lharx %0,0,%2 # __cmpxchg_u16_local\n"
392 " cmpw 0,%0,%3\n"
393 " bne- 2f\n"
394 " sthcx. %4,0,%2\n"
395 " bne- 1b"
396 "2:"
397 : "=&r" (prev), "+m" (*p)
398 : "r" (p), "r" (old), "r" (new)
399 : "cc", "memory");
400
401 return prev;
402 }
403
404 static __always_inline unsigned long
__cmpxchg_u16_relaxed(u16 * p,unsigned long old,unsigned long new)405 __cmpxchg_u16_relaxed(u16 *p, unsigned long old, unsigned long new)
406 {
407 unsigned long prev;
408
409 __asm__ __volatile__ (
410 "1: lharx %0,0,%2 # __cmpxchg_u16_relaxed\n"
411 " cmpw 0,%0,%3\n"
412 " bne- 2f\n"
413 " sthcx. %4,0,%2\n"
414 " bne- 1b\n"
415 "2:"
416 : "=&r" (prev), "+m" (*p)
417 : "r" (p), "r" (old), "r" (new)
418 : "cc");
419
420 return prev;
421 }
422
423 static __always_inline unsigned long
__cmpxchg_u16_acquire(u16 * p,unsigned long old,unsigned long new)424 __cmpxchg_u16_acquire(u16 *p, unsigned long old, unsigned long new)
425 {
426 unsigned long prev;
427
428 __asm__ __volatile__ (
429 "1: lharx %0,0,%2 # __cmpxchg_u16_acquire\n"
430 " cmpw 0,%0,%3\n"
431 " bne- 2f\n"
432 " sthcx. %4,0,%2\n"
433 " bne- 1b\n"
434 PPC_ACQUIRE_BARRIER
435 "2:"
436 : "=&r" (prev), "+m" (*p)
437 : "r" (p), "r" (old), "r" (new)
438 : "cc", "memory");
439
440 return prev;
441 }
442 #endif
443
444 static __always_inline unsigned long
__cmpxchg_u32(volatile unsigned int * p,unsigned long old,unsigned long new)445 __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
446 {
447 unsigned int prev;
448
449 __asm__ __volatile__ (
450 PPC_ATOMIC_ENTRY_BARRIER
451 "1: lwarx %0,0,%2 # __cmpxchg_u32\n\
452 cmpw 0,%0,%3\n\
453 bne- 2f\n"
454 " stwcx. %4,0,%2\n\
455 bne- 1b"
456 PPC_ATOMIC_EXIT_BARRIER
457 "\n\
458 2:"
459 : "=&r" (prev), "+m" (*p)
460 : "r" (p), "r" (old), "r" (new)
461 : "cc", "memory");
462
463 return prev;
464 }
465
466 static __always_inline unsigned long
__cmpxchg_u32_local(volatile unsigned int * p,unsigned long old,unsigned long new)467 __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
468 unsigned long new)
469 {
470 unsigned int prev;
471
472 __asm__ __volatile__ (
473 "1: lwarx %0,0,%2 # __cmpxchg_u32\n\
474 cmpw 0,%0,%3\n\
475 bne- 2f\n"
476 " stwcx. %4,0,%2\n\
477 bne- 1b"
478 "\n\
479 2:"
480 : "=&r" (prev), "+m" (*p)
481 : "r" (p), "r" (old), "r" (new)
482 : "cc", "memory");
483
484 return prev;
485 }
486
487 static __always_inline unsigned long
__cmpxchg_u32_relaxed(u32 * p,unsigned long old,unsigned long new)488 __cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
489 {
490 unsigned long prev;
491
492 __asm__ __volatile__ (
493 "1: lwarx %0,0,%2 # __cmpxchg_u32_relaxed\n"
494 " cmpw 0,%0,%3\n"
495 " bne- 2f\n"
496 " stwcx. %4,0,%2\n"
497 " bne- 1b\n"
498 "2:"
499 : "=&r" (prev), "+m" (*p)
500 : "r" (p), "r" (old), "r" (new)
501 : "cc");
502
503 return prev;
504 }
505
506 /*
507 * cmpxchg family don't have order guarantee if cmp part fails, therefore we
508 * can avoid superfluous barriers if we use assembly code to implement
509 * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
510 * cmpxchg_release() because that will result in putting a barrier in the
511 * middle of a ll/sc loop, which is probably a bad idea. For example, this
512 * might cause the conditional store more likely to fail.
513 */
514 static __always_inline unsigned long
__cmpxchg_u32_acquire(u32 * p,unsigned long old,unsigned long new)515 __cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
516 {
517 unsigned long prev;
518
519 __asm__ __volatile__ (
520 "1: lwarx %0,0,%2 # __cmpxchg_u32_acquire\n"
521 " cmpw 0,%0,%3\n"
522 " bne- 2f\n"
523 " stwcx. %4,0,%2\n"
524 " bne- 1b\n"
525 PPC_ACQUIRE_BARRIER
526 "\n"
527 "2:"
528 : "=&r" (prev), "+m" (*p)
529 : "r" (p), "r" (old), "r" (new)
530 : "cc", "memory");
531
532 return prev;
533 }
534
535 #ifdef CONFIG_PPC64
536 static __always_inline unsigned long
__cmpxchg_u64(volatile unsigned long * p,unsigned long old,unsigned long new)537 __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
538 {
539 unsigned long prev;
540
541 __asm__ __volatile__ (
542 PPC_ATOMIC_ENTRY_BARRIER
543 "1: ldarx %0,0,%2 # __cmpxchg_u64\n\
544 cmpd 0,%0,%3\n\
545 bne- 2f\n\
546 stdcx. %4,0,%2\n\
547 bne- 1b"
548 PPC_ATOMIC_EXIT_BARRIER
549 "\n\
550 2:"
551 : "=&r" (prev), "+m" (*p)
552 : "r" (p), "r" (old), "r" (new)
553 : "cc", "memory");
554
555 return prev;
556 }
557
558 static __always_inline unsigned long
__cmpxchg_u64_local(volatile unsigned long * p,unsigned long old,unsigned long new)559 __cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
560 unsigned long new)
561 {
562 unsigned long prev;
563
564 __asm__ __volatile__ (
565 "1: ldarx %0,0,%2 # __cmpxchg_u64\n\
566 cmpd 0,%0,%3\n\
567 bne- 2f\n\
568 stdcx. %4,0,%2\n\
569 bne- 1b"
570 "\n\
571 2:"
572 : "=&r" (prev), "+m" (*p)
573 : "r" (p), "r" (old), "r" (new)
574 : "cc", "memory");
575
576 return prev;
577 }
578
579 static __always_inline unsigned long
__cmpxchg_u64_relaxed(u64 * p,unsigned long old,unsigned long new)580 __cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
581 {
582 unsigned long prev;
583
584 __asm__ __volatile__ (
585 "1: ldarx %0,0,%2 # __cmpxchg_u64_relaxed\n"
586 " cmpd 0,%0,%3\n"
587 " bne- 2f\n"
588 " stdcx. %4,0,%2\n"
589 " bne- 1b\n"
590 "2:"
591 : "=&r" (prev), "+m" (*p)
592 : "r" (p), "r" (old), "r" (new)
593 : "cc");
594
595 return prev;
596 }
597
598 static __always_inline unsigned long
__cmpxchg_u64_acquire(u64 * p,unsigned long old,unsigned long new)599 __cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
600 {
601 unsigned long prev;
602
603 __asm__ __volatile__ (
604 "1: ldarx %0,0,%2 # __cmpxchg_u64_acquire\n"
605 " cmpd 0,%0,%3\n"
606 " bne- 2f\n"
607 " stdcx. %4,0,%2\n"
608 " bne- 1b\n"
609 PPC_ACQUIRE_BARRIER
610 "\n"
611 "2:"
612 : "=&r" (prev), "+m" (*p)
613 : "r" (p), "r" (old), "r" (new)
614 : "cc", "memory");
615
616 return prev;
617 }
618 #endif
619
620 static __always_inline unsigned long
__cmpxchg(volatile void * ptr,unsigned long old,unsigned long new,unsigned int size)621 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
622 unsigned int size)
623 {
624 switch (size) {
625 case 1:
626 return __cmpxchg_u8(ptr, old, new);
627 case 2:
628 return __cmpxchg_u16(ptr, old, new);
629 case 4:
630 return __cmpxchg_u32(ptr, old, new);
631 #ifdef CONFIG_PPC64
632 case 8:
633 return __cmpxchg_u64(ptr, old, new);
634 #endif
635 }
636 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
637 return old;
638 }
639
640 static __always_inline unsigned long
__cmpxchg_local(void * ptr,unsigned long old,unsigned long new,unsigned int size)641 __cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
642 unsigned int size)
643 {
644 switch (size) {
645 case 1:
646 return __cmpxchg_u8_local(ptr, old, new);
647 case 2:
648 return __cmpxchg_u16_local(ptr, old, new);
649 case 4:
650 return __cmpxchg_u32_local(ptr, old, new);
651 #ifdef CONFIG_PPC64
652 case 8:
653 return __cmpxchg_u64_local(ptr, old, new);
654 #endif
655 }
656 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
657 return old;
658 }
659
660 static __always_inline unsigned long
__cmpxchg_relaxed(void * ptr,unsigned long old,unsigned long new,unsigned int size)661 __cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
662 unsigned int size)
663 {
664 switch (size) {
665 case 1:
666 return __cmpxchg_u8_relaxed(ptr, old, new);
667 case 2:
668 return __cmpxchg_u16_relaxed(ptr, old, new);
669 case 4:
670 return __cmpxchg_u32_relaxed(ptr, old, new);
671 #ifdef CONFIG_PPC64
672 case 8:
673 return __cmpxchg_u64_relaxed(ptr, old, new);
674 #endif
675 }
676 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
677 return old;
678 }
679
680 static __always_inline unsigned long
__cmpxchg_acquire(void * ptr,unsigned long old,unsigned long new,unsigned int size)681 __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
682 unsigned int size)
683 {
684 switch (size) {
685 case 1:
686 return __cmpxchg_u8_acquire(ptr, old, new);
687 case 2:
688 return __cmpxchg_u16_acquire(ptr, old, new);
689 case 4:
690 return __cmpxchg_u32_acquire(ptr, old, new);
691 #ifdef CONFIG_PPC64
692 case 8:
693 return __cmpxchg_u64_acquire(ptr, old, new);
694 #endif
695 }
696 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
697 return old;
698 }
699 #define arch_cmpxchg(ptr, o, n) \
700 ({ \
701 __typeof__(*(ptr)) _o_ = (o); \
702 __typeof__(*(ptr)) _n_ = (n); \
703 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
704 (unsigned long)_n_, sizeof(*(ptr))); \
705 })
706
707
708 #define arch_cmpxchg_local(ptr, o, n) \
709 ({ \
710 __typeof__(*(ptr)) _o_ = (o); \
711 __typeof__(*(ptr)) _n_ = (n); \
712 (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \
713 (unsigned long)_n_, sizeof(*(ptr))); \
714 })
715
716 #define arch_cmpxchg_relaxed(ptr, o, n) \
717 ({ \
718 __typeof__(*(ptr)) _o_ = (o); \
719 __typeof__(*(ptr)) _n_ = (n); \
720 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
721 (unsigned long)_o_, (unsigned long)_n_, \
722 sizeof(*(ptr))); \
723 })
724
725 #define arch_cmpxchg_acquire(ptr, o, n) \
726 ({ \
727 __typeof__(*(ptr)) _o_ = (o); \
728 __typeof__(*(ptr)) _n_ = (n); \
729 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
730 (unsigned long)_o_, (unsigned long)_n_, \
731 sizeof(*(ptr))); \
732 })
733 #ifdef CONFIG_PPC64
734 #define arch_cmpxchg64(ptr, o, n) \
735 ({ \
736 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
737 arch_cmpxchg((ptr), (o), (n)); \
738 })
739 #define arch_cmpxchg64_local(ptr, o, n) \
740 ({ \
741 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
742 arch_cmpxchg_local((ptr), (o), (n)); \
743 })
744 #define arch_cmpxchg64_relaxed(ptr, o, n) \
745 ({ \
746 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
747 arch_cmpxchg_relaxed((ptr), (o), (n)); \
748 })
749 #define arch_cmpxchg64_acquire(ptr, o, n) \
750 ({ \
751 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
752 arch_cmpxchg_acquire((ptr), (o), (n)); \
753 })
754 #else
755 #include <asm-generic/cmpxchg-local.h>
756 #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
757 #endif
758
759 #endif /* __KERNEL__ */
760 #endif /* _ASM_POWERPC_CMPXCHG_H_ */
761