1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_POWERPC_CMPXCHG_H_ 3 #define _ASM_POWERPC_CMPXCHG_H_ 4 5 #ifdef __KERNEL__ 6 #include <linux/compiler.h> 7 #include <asm/synch.h> 8 #include <linux/bug.h> 9 #include <asm/asm-405.h> 10 11 #ifdef __BIG_ENDIAN 12 #define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE) 13 #else 14 #define BITOFF_CAL(size, off) (off * BITS_PER_BYTE) 15 #endif 16 17 #define XCHG_GEN(type, sfx, cl) \ 18 static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \ 19 { \ 20 unsigned int prev, prev_mask, tmp, bitoff, off; \ 21 \ 22 off = (unsigned long)p % sizeof(u32); \ 23 bitoff = BITOFF_CAL(sizeof(type), off); \ 24 p -= off; \ 25 val <<= bitoff; \ 26 prev_mask = (u32)(type)-1 << bitoff; \ 27 \ 28 __asm__ __volatile__( \ 29 "1: lwarx %0,0,%3\n" \ 30 " andc %1,%0,%5\n" \ 31 " or %1,%1,%4\n" \ 32 PPC405_ERR77(0,%3) \ 33 " stwcx. %1,0,%3\n" \ 34 " bne- 1b\n" \ 35 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \ 36 : "r" (p), "r" (val), "r" (prev_mask) \ 37 : "cc", cl); \ 38 \ 39 return prev >> bitoff; \ 40 } 41 42 #define CMPXCHG_GEN(type, sfx, br, br2, cl) \ 43 static inline \ 44 u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \ 45 { \ 46 unsigned int prev, prev_mask, tmp, bitoff, off; \ 47 \ 48 off = (unsigned long)p % sizeof(u32); \ 49 bitoff = BITOFF_CAL(sizeof(type), off); \ 50 p -= off; \ 51 old <<= bitoff; \ 52 new <<= bitoff; \ 53 prev_mask = (u32)(type)-1 << bitoff; \ 54 \ 55 __asm__ __volatile__( \ 56 br \ 57 "1: lwarx %0,0,%3\n" \ 58 " and %1,%0,%6\n" \ 59 " cmpw 0,%1,%4\n" \ 60 " bne- 2f\n" \ 61 " andc %1,%0,%6\n" \ 62 " or %1,%1,%5\n" \ 63 PPC405_ERR77(0,%3) \ 64 " stwcx. %1,0,%3\n" \ 65 " bne- 1b\n" \ 66 br2 \ 67 "\n" \ 68 "2:" \ 69 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \ 70 : "r" (p), "r" (old), "r" (new), "r" (prev_mask) \ 71 : "cc", cl); \ 72 \ 73 return prev >> bitoff; \ 74 } 75 76 /* 77 * Atomic exchange 78 * 79 * Changes the memory location '*p' to be val and returns 80 * the previous value stored there. 81 */ 82 83 XCHG_GEN(u8, _local, "memory"); 84 XCHG_GEN(u8, _relaxed, "cc"); 85 XCHG_GEN(u16, _local, "memory"); 86 XCHG_GEN(u16, _relaxed, "cc"); 87 88 static __always_inline unsigned long 89 __xchg_u32_local(volatile void *p, unsigned long val) 90 { 91 unsigned long prev; 92 93 __asm__ __volatile__( 94 "1: lwarx %0,0,%2 \n" 95 PPC405_ERR77(0,%2) 96 " stwcx. %3,0,%2 \n\ 97 bne- 1b" 98 : "=&r" (prev), "+m" (*(volatile unsigned int *)p) 99 : "r" (p), "r" (val) 100 : "cc", "memory"); 101 102 return prev; 103 } 104 105 static __always_inline unsigned long 106 __xchg_u32_relaxed(u32 *p, unsigned long val) 107 { 108 unsigned long prev; 109 110 __asm__ __volatile__( 111 "1: lwarx %0,0,%2\n" 112 PPC405_ERR77(0, %2) 113 " stwcx. %3,0,%2\n" 114 " bne- 1b" 115 : "=&r" (prev), "+m" (*p) 116 : "r" (p), "r" (val) 117 : "cc"); 118 119 return prev; 120 } 121 122 #ifdef CONFIG_PPC64 123 static __always_inline unsigned long 124 __xchg_u64_local(volatile void *p, unsigned long val) 125 { 126 unsigned long prev; 127 128 __asm__ __volatile__( 129 "1: ldarx %0,0,%2 \n" 130 PPC405_ERR77(0,%2) 131 " stdcx. %3,0,%2 \n\ 132 bne- 1b" 133 : "=&r" (prev), "+m" (*(volatile unsigned long *)p) 134 : "r" (p), "r" (val) 135 : "cc", "memory"); 136 137 return prev; 138 } 139 140 static __always_inline unsigned long 141 __xchg_u64_relaxed(u64 *p, unsigned long val) 142 { 143 unsigned long prev; 144 145 __asm__ __volatile__( 146 "1: ldarx %0,0,%2\n" 147 PPC405_ERR77(0, %2) 148 " stdcx. %3,0,%2\n" 149 " bne- 1b" 150 : "=&r" (prev), "+m" (*p) 151 : "r" (p), "r" (val) 152 : "cc"); 153 154 return prev; 155 } 156 #endif 157 158 static __always_inline unsigned long 159 __xchg_local(void *ptr, unsigned long x, unsigned int size) 160 { 161 switch (size) { 162 case 1: 163 return __xchg_u8_local(ptr, x); 164 case 2: 165 return __xchg_u16_local(ptr, x); 166 case 4: 167 return __xchg_u32_local(ptr, x); 168 #ifdef CONFIG_PPC64 169 case 8: 170 return __xchg_u64_local(ptr, x); 171 #endif 172 } 173 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg"); 174 return x; 175 } 176 177 static __always_inline unsigned long 178 __xchg_relaxed(void *ptr, unsigned long x, unsigned int size) 179 { 180 switch (size) { 181 case 1: 182 return __xchg_u8_relaxed(ptr, x); 183 case 2: 184 return __xchg_u16_relaxed(ptr, x); 185 case 4: 186 return __xchg_u32_relaxed(ptr, x); 187 #ifdef CONFIG_PPC64 188 case 8: 189 return __xchg_u64_relaxed(ptr, x); 190 #endif 191 } 192 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local"); 193 return x; 194 } 195 #define xchg_local(ptr,x) \ 196 ({ \ 197 __typeof__(*(ptr)) _x_ = (x); \ 198 (__typeof__(*(ptr))) __xchg_local((ptr), \ 199 (unsigned long)_x_, sizeof(*(ptr))); \ 200 }) 201 202 #define xchg_relaxed(ptr, x) \ 203 ({ \ 204 __typeof__(*(ptr)) _x_ = (x); \ 205 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \ 206 (unsigned long)_x_, sizeof(*(ptr))); \ 207 }) 208 /* 209 * Compare and exchange - if *p == old, set it to new, 210 * and return the old value of *p. 211 */ 212 213 CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory"); 214 CMPXCHG_GEN(u8, _local, , , "memory"); 215 CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory"); 216 CMPXCHG_GEN(u8, _relaxed, , , "cc"); 217 CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory"); 218 CMPXCHG_GEN(u16, _local, , , "memory"); 219 CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory"); 220 CMPXCHG_GEN(u16, _relaxed, , , "cc"); 221 222 static __always_inline unsigned long 223 __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new) 224 { 225 unsigned int prev; 226 227 __asm__ __volatile__ ( 228 PPC_ATOMIC_ENTRY_BARRIER 229 "1: lwarx %0,0,%2 # __cmpxchg_u32\n\ 230 cmpw 0,%0,%3\n\ 231 bne- 2f\n" 232 PPC405_ERR77(0,%2) 233 " stwcx. %4,0,%2\n\ 234 bne- 1b" 235 PPC_ATOMIC_EXIT_BARRIER 236 "\n\ 237 2:" 238 : "=&r" (prev), "+m" (*p) 239 : "r" (p), "r" (old), "r" (new) 240 : "cc", "memory"); 241 242 return prev; 243 } 244 245 static __always_inline unsigned long 246 __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old, 247 unsigned long new) 248 { 249 unsigned int prev; 250 251 __asm__ __volatile__ ( 252 "1: lwarx %0,0,%2 # __cmpxchg_u32\n\ 253 cmpw 0,%0,%3\n\ 254 bne- 2f\n" 255 PPC405_ERR77(0,%2) 256 " stwcx. %4,0,%2\n\ 257 bne- 1b" 258 "\n\ 259 2:" 260 : "=&r" (prev), "+m" (*p) 261 : "r" (p), "r" (old), "r" (new) 262 : "cc", "memory"); 263 264 return prev; 265 } 266 267 static __always_inline unsigned long 268 __cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new) 269 { 270 unsigned long prev; 271 272 __asm__ __volatile__ ( 273 "1: lwarx %0,0,%2 # __cmpxchg_u32_relaxed\n" 274 " cmpw 0,%0,%3\n" 275 " bne- 2f\n" 276 PPC405_ERR77(0, %2) 277 " stwcx. %4,0,%2\n" 278 " bne- 1b\n" 279 "2:" 280 : "=&r" (prev), "+m" (*p) 281 : "r" (p), "r" (old), "r" (new) 282 : "cc"); 283 284 return prev; 285 } 286 287 /* 288 * cmpxchg family don't have order guarantee if cmp part fails, therefore we 289 * can avoid superfluous barriers if we use assembly code to implement 290 * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for 291 * cmpxchg_release() because that will result in putting a barrier in the 292 * middle of a ll/sc loop, which is probably a bad idea. For example, this 293 * might cause the conditional store more likely to fail. 294 */ 295 static __always_inline unsigned long 296 __cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new) 297 { 298 unsigned long prev; 299 300 __asm__ __volatile__ ( 301 "1: lwarx %0,0,%2 # __cmpxchg_u32_acquire\n" 302 " cmpw 0,%0,%3\n" 303 " bne- 2f\n" 304 PPC405_ERR77(0, %2) 305 " stwcx. %4,0,%2\n" 306 " bne- 1b\n" 307 PPC_ACQUIRE_BARRIER 308 "\n" 309 "2:" 310 : "=&r" (prev), "+m" (*p) 311 : "r" (p), "r" (old), "r" (new) 312 : "cc", "memory"); 313 314 return prev; 315 } 316 317 #ifdef CONFIG_PPC64 318 static __always_inline unsigned long 319 __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new) 320 { 321 unsigned long prev; 322 323 __asm__ __volatile__ ( 324 PPC_ATOMIC_ENTRY_BARRIER 325 "1: ldarx %0,0,%2 # __cmpxchg_u64\n\ 326 cmpd 0,%0,%3\n\ 327 bne- 2f\n\ 328 stdcx. %4,0,%2\n\ 329 bne- 1b" 330 PPC_ATOMIC_EXIT_BARRIER 331 "\n\ 332 2:" 333 : "=&r" (prev), "+m" (*p) 334 : "r" (p), "r" (old), "r" (new) 335 : "cc", "memory"); 336 337 return prev; 338 } 339 340 static __always_inline unsigned long 341 __cmpxchg_u64_local(volatile unsigned long *p, unsigned long old, 342 unsigned long new) 343 { 344 unsigned long prev; 345 346 __asm__ __volatile__ ( 347 "1: ldarx %0,0,%2 # __cmpxchg_u64\n\ 348 cmpd 0,%0,%3\n\ 349 bne- 2f\n\ 350 stdcx. %4,0,%2\n\ 351 bne- 1b" 352 "\n\ 353 2:" 354 : "=&r" (prev), "+m" (*p) 355 : "r" (p), "r" (old), "r" (new) 356 : "cc", "memory"); 357 358 return prev; 359 } 360 361 static __always_inline unsigned long 362 __cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new) 363 { 364 unsigned long prev; 365 366 __asm__ __volatile__ ( 367 "1: ldarx %0,0,%2 # __cmpxchg_u64_relaxed\n" 368 " cmpd 0,%0,%3\n" 369 " bne- 2f\n" 370 " stdcx. %4,0,%2\n" 371 " bne- 1b\n" 372 "2:" 373 : "=&r" (prev), "+m" (*p) 374 : "r" (p), "r" (old), "r" (new) 375 : "cc"); 376 377 return prev; 378 } 379 380 static __always_inline unsigned long 381 __cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new) 382 { 383 unsigned long prev; 384 385 __asm__ __volatile__ ( 386 "1: ldarx %0,0,%2 # __cmpxchg_u64_acquire\n" 387 " cmpd 0,%0,%3\n" 388 " bne- 2f\n" 389 " stdcx. %4,0,%2\n" 390 " bne- 1b\n" 391 PPC_ACQUIRE_BARRIER 392 "\n" 393 "2:" 394 : "=&r" (prev), "+m" (*p) 395 : "r" (p), "r" (old), "r" (new) 396 : "cc", "memory"); 397 398 return prev; 399 } 400 #endif 401 402 static __always_inline unsigned long 403 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, 404 unsigned int size) 405 { 406 switch (size) { 407 case 1: 408 return __cmpxchg_u8(ptr, old, new); 409 case 2: 410 return __cmpxchg_u16(ptr, old, new); 411 case 4: 412 return __cmpxchg_u32(ptr, old, new); 413 #ifdef CONFIG_PPC64 414 case 8: 415 return __cmpxchg_u64(ptr, old, new); 416 #endif 417 } 418 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg"); 419 return old; 420 } 421 422 static __always_inline unsigned long 423 __cmpxchg_local(void *ptr, unsigned long old, unsigned long new, 424 unsigned int size) 425 { 426 switch (size) { 427 case 1: 428 return __cmpxchg_u8_local(ptr, old, new); 429 case 2: 430 return __cmpxchg_u16_local(ptr, old, new); 431 case 4: 432 return __cmpxchg_u32_local(ptr, old, new); 433 #ifdef CONFIG_PPC64 434 case 8: 435 return __cmpxchg_u64_local(ptr, old, new); 436 #endif 437 } 438 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local"); 439 return old; 440 } 441 442 static __always_inline unsigned long 443 __cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new, 444 unsigned int size) 445 { 446 switch (size) { 447 case 1: 448 return __cmpxchg_u8_relaxed(ptr, old, new); 449 case 2: 450 return __cmpxchg_u16_relaxed(ptr, old, new); 451 case 4: 452 return __cmpxchg_u32_relaxed(ptr, old, new); 453 #ifdef CONFIG_PPC64 454 case 8: 455 return __cmpxchg_u64_relaxed(ptr, old, new); 456 #endif 457 } 458 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed"); 459 return old; 460 } 461 462 static __always_inline unsigned long 463 __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new, 464 unsigned int size) 465 { 466 switch (size) { 467 case 1: 468 return __cmpxchg_u8_acquire(ptr, old, new); 469 case 2: 470 return __cmpxchg_u16_acquire(ptr, old, new); 471 case 4: 472 return __cmpxchg_u32_acquire(ptr, old, new); 473 #ifdef CONFIG_PPC64 474 case 8: 475 return __cmpxchg_u64_acquire(ptr, old, new); 476 #endif 477 } 478 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire"); 479 return old; 480 } 481 #define cmpxchg(ptr, o, n) \ 482 ({ \ 483 __typeof__(*(ptr)) _o_ = (o); \ 484 __typeof__(*(ptr)) _n_ = (n); \ 485 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ 486 (unsigned long)_n_, sizeof(*(ptr))); \ 487 }) 488 489 490 #define cmpxchg_local(ptr, o, n) \ 491 ({ \ 492 __typeof__(*(ptr)) _o_ = (o); \ 493 __typeof__(*(ptr)) _n_ = (n); \ 494 (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \ 495 (unsigned long)_n_, sizeof(*(ptr))); \ 496 }) 497 498 #define cmpxchg_relaxed(ptr, o, n) \ 499 ({ \ 500 __typeof__(*(ptr)) _o_ = (o); \ 501 __typeof__(*(ptr)) _n_ = (n); \ 502 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \ 503 (unsigned long)_o_, (unsigned long)_n_, \ 504 sizeof(*(ptr))); \ 505 }) 506 507 #define cmpxchg_acquire(ptr, o, n) \ 508 ({ \ 509 __typeof__(*(ptr)) _o_ = (o); \ 510 __typeof__(*(ptr)) _n_ = (n); \ 511 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \ 512 (unsigned long)_o_, (unsigned long)_n_, \ 513 sizeof(*(ptr))); \ 514 }) 515 #ifdef CONFIG_PPC64 516 #define cmpxchg64(ptr, o, n) \ 517 ({ \ 518 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 519 cmpxchg((ptr), (o), (n)); \ 520 }) 521 #define cmpxchg64_local(ptr, o, n) \ 522 ({ \ 523 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 524 cmpxchg_local((ptr), (o), (n)); \ 525 }) 526 #define cmpxchg64_relaxed(ptr, o, n) \ 527 ({ \ 528 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 529 cmpxchg_relaxed((ptr), (o), (n)); \ 530 }) 531 #define cmpxchg64_acquire(ptr, o, n) \ 532 ({ \ 533 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 534 cmpxchg_acquire((ptr), (o), (n)); \ 535 }) 536 #else 537 #include <asm-generic/cmpxchg-local.h> 538 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) 539 #endif 540 541 #endif /* __KERNEL__ */ 542 #endif /* _ASM_POWERPC_CMPXCHG_H_ */ 543