1 #ifndef _ASM_POWERPC_CMPXCHG_H_ 2 #define _ASM_POWERPC_CMPXCHG_H_ 3 4 #ifdef __KERNEL__ 5 #include <linux/compiler.h> 6 #include <asm/synch.h> 7 #include <asm/asm-compat.h> 8 #include <linux/bug.h> 9 10 #ifdef __BIG_ENDIAN 11 #define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE) 12 #else 13 #define BITOFF_CAL(size, off) (off * BITS_PER_BYTE) 14 #endif 15 16 #define XCHG_GEN(type, sfx, cl) \ 17 static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \ 18 { \ 19 unsigned int prev, prev_mask, tmp, bitoff, off; \ 20 \ 21 off = (unsigned long)p % sizeof(u32); \ 22 bitoff = BITOFF_CAL(sizeof(type), off); \ 23 p -= off; \ 24 val <<= bitoff; \ 25 prev_mask = (u32)(type)-1 << bitoff; \ 26 \ 27 __asm__ __volatile__( \ 28 "1: lwarx %0,0,%3\n" \ 29 " andc %1,%0,%5\n" \ 30 " or %1,%1,%4\n" \ 31 PPC405_ERR77(0,%3) \ 32 " stwcx. %1,0,%3\n" \ 33 " bne- 1b\n" \ 34 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \ 35 : "r" (p), "r" (val), "r" (prev_mask) \ 36 : "cc", cl); \ 37 \ 38 return prev >> bitoff; \ 39 } 40 41 #define CMPXCHG_GEN(type, sfx, br, br2, cl) \ 42 static inline \ 43 u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \ 44 { \ 45 unsigned int prev, prev_mask, tmp, bitoff, off; \ 46 \ 47 off = (unsigned long)p % sizeof(u32); \ 48 bitoff = BITOFF_CAL(sizeof(type), off); \ 49 p -= off; \ 50 old <<= bitoff; \ 51 new <<= bitoff; \ 52 prev_mask = (u32)(type)-1 << bitoff; \ 53 \ 54 __asm__ __volatile__( \ 55 br \ 56 "1: lwarx %0,0,%3\n" \ 57 " and %1,%0,%6\n" \ 58 " cmpw 0,%1,%4\n" \ 59 " bne- 2f\n" \ 60 " andc %1,%0,%6\n" \ 61 " or %1,%1,%5\n" \ 62 PPC405_ERR77(0,%3) \ 63 " stwcx. %1,0,%3\n" \ 64 " bne- 1b\n" \ 65 br2 \ 66 "\n" \ 67 "2:" \ 68 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \ 69 : "r" (p), "r" (old), "r" (new), "r" (prev_mask) \ 70 : "cc", cl); \ 71 \ 72 return prev >> bitoff; \ 73 } 74 75 /* 76 * Atomic exchange 77 * 78 * Changes the memory location '*p' to be val and returns 79 * the previous value stored there. 80 */ 81 82 XCHG_GEN(u8, _local, "memory"); 83 XCHG_GEN(u8, _relaxed, "cc"); 84 XCHG_GEN(u16, _local, "memory"); 85 XCHG_GEN(u16, _relaxed, "cc"); 86 87 static __always_inline unsigned long 88 __xchg_u32_local(volatile void *p, unsigned long val) 89 { 90 unsigned long prev; 91 92 __asm__ __volatile__( 93 "1: lwarx %0,0,%2 \n" 94 PPC405_ERR77(0,%2) 95 " stwcx. %3,0,%2 \n\ 96 bne- 1b" 97 : "=&r" (prev), "+m" (*(volatile unsigned int *)p) 98 : "r" (p), "r" (val) 99 : "cc", "memory"); 100 101 return prev; 102 } 103 104 static __always_inline unsigned long 105 __xchg_u32_relaxed(u32 *p, unsigned long val) 106 { 107 unsigned long prev; 108 109 __asm__ __volatile__( 110 "1: lwarx %0,0,%2\n" 111 PPC405_ERR77(0, %2) 112 " stwcx. %3,0,%2\n" 113 " bne- 1b" 114 : "=&r" (prev), "+m" (*p) 115 : "r" (p), "r" (val) 116 : "cc"); 117 118 return prev; 119 } 120 121 #ifdef CONFIG_PPC64 122 static __always_inline unsigned long 123 __xchg_u64_local(volatile void *p, unsigned long val) 124 { 125 unsigned long prev; 126 127 __asm__ __volatile__( 128 "1: ldarx %0,0,%2 \n" 129 PPC405_ERR77(0,%2) 130 " stdcx. %3,0,%2 \n\ 131 bne- 1b" 132 : "=&r" (prev), "+m" (*(volatile unsigned long *)p) 133 : "r" (p), "r" (val) 134 : "cc", "memory"); 135 136 return prev; 137 } 138 139 static __always_inline unsigned long 140 __xchg_u64_relaxed(u64 *p, unsigned long val) 141 { 142 unsigned long prev; 143 144 __asm__ __volatile__( 145 "1: ldarx %0,0,%2\n" 146 PPC405_ERR77(0, %2) 147 " stdcx. %3,0,%2\n" 148 " bne- 1b" 149 : "=&r" (prev), "+m" (*p) 150 : "r" (p), "r" (val) 151 : "cc"); 152 153 return prev; 154 } 155 #endif 156 157 static __always_inline unsigned long 158 __xchg_local(void *ptr, unsigned long x, unsigned int size) 159 { 160 switch (size) { 161 case 1: 162 return __xchg_u8_local(ptr, x); 163 case 2: 164 return __xchg_u16_local(ptr, x); 165 case 4: 166 return __xchg_u32_local(ptr, x); 167 #ifdef CONFIG_PPC64 168 case 8: 169 return __xchg_u64_local(ptr, x); 170 #endif 171 } 172 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg"); 173 return x; 174 } 175 176 static __always_inline unsigned long 177 __xchg_relaxed(void *ptr, unsigned long x, unsigned int size) 178 { 179 switch (size) { 180 case 1: 181 return __xchg_u8_relaxed(ptr, x); 182 case 2: 183 return __xchg_u16_relaxed(ptr, x); 184 case 4: 185 return __xchg_u32_relaxed(ptr, x); 186 #ifdef CONFIG_PPC64 187 case 8: 188 return __xchg_u64_relaxed(ptr, x); 189 #endif 190 } 191 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local"); 192 return x; 193 } 194 #define xchg_local(ptr,x) \ 195 ({ \ 196 __typeof__(*(ptr)) _x_ = (x); \ 197 (__typeof__(*(ptr))) __xchg_local((ptr), \ 198 (unsigned long)_x_, sizeof(*(ptr))); \ 199 }) 200 201 #define xchg_relaxed(ptr, x) \ 202 ({ \ 203 __typeof__(*(ptr)) _x_ = (x); \ 204 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \ 205 (unsigned long)_x_, sizeof(*(ptr))); \ 206 }) 207 /* 208 * Compare and exchange - if *p == old, set it to new, 209 * and return the old value of *p. 210 */ 211 212 CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory"); 213 CMPXCHG_GEN(u8, _local, , , "memory"); 214 CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory"); 215 CMPXCHG_GEN(u8, _relaxed, , , "cc"); 216 CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory"); 217 CMPXCHG_GEN(u16, _local, , , "memory"); 218 CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory"); 219 CMPXCHG_GEN(u16, _relaxed, , , "cc"); 220 221 static __always_inline unsigned long 222 __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new) 223 { 224 unsigned int prev; 225 226 __asm__ __volatile__ ( 227 PPC_ATOMIC_ENTRY_BARRIER 228 "1: lwarx %0,0,%2 # __cmpxchg_u32\n\ 229 cmpw 0,%0,%3\n\ 230 bne- 2f\n" 231 PPC405_ERR77(0,%2) 232 " stwcx. %4,0,%2\n\ 233 bne- 1b" 234 PPC_ATOMIC_EXIT_BARRIER 235 "\n\ 236 2:" 237 : "=&r" (prev), "+m" (*p) 238 : "r" (p), "r" (old), "r" (new) 239 : "cc", "memory"); 240 241 return prev; 242 } 243 244 static __always_inline unsigned long 245 __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old, 246 unsigned long new) 247 { 248 unsigned int prev; 249 250 __asm__ __volatile__ ( 251 "1: lwarx %0,0,%2 # __cmpxchg_u32\n\ 252 cmpw 0,%0,%3\n\ 253 bne- 2f\n" 254 PPC405_ERR77(0,%2) 255 " stwcx. %4,0,%2\n\ 256 bne- 1b" 257 "\n\ 258 2:" 259 : "=&r" (prev), "+m" (*p) 260 : "r" (p), "r" (old), "r" (new) 261 : "cc", "memory"); 262 263 return prev; 264 } 265 266 static __always_inline unsigned long 267 __cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new) 268 { 269 unsigned long prev; 270 271 __asm__ __volatile__ ( 272 "1: lwarx %0,0,%2 # __cmpxchg_u32_relaxed\n" 273 " cmpw 0,%0,%3\n" 274 " bne- 2f\n" 275 PPC405_ERR77(0, %2) 276 " stwcx. %4,0,%2\n" 277 " bne- 1b\n" 278 "2:" 279 : "=&r" (prev), "+m" (*p) 280 : "r" (p), "r" (old), "r" (new) 281 : "cc"); 282 283 return prev; 284 } 285 286 /* 287 * cmpxchg family don't have order guarantee if cmp part fails, therefore we 288 * can avoid superfluous barriers if we use assembly code to implement 289 * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for 290 * cmpxchg_release() because that will result in putting a barrier in the 291 * middle of a ll/sc loop, which is probably a bad idea. For example, this 292 * might cause the conditional store more likely to fail. 293 */ 294 static __always_inline unsigned long 295 __cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new) 296 { 297 unsigned long prev; 298 299 __asm__ __volatile__ ( 300 "1: lwarx %0,0,%2 # __cmpxchg_u32_acquire\n" 301 " cmpw 0,%0,%3\n" 302 " bne- 2f\n" 303 PPC405_ERR77(0, %2) 304 " stwcx. %4,0,%2\n" 305 " bne- 1b\n" 306 PPC_ACQUIRE_BARRIER 307 "\n" 308 "2:" 309 : "=&r" (prev), "+m" (*p) 310 : "r" (p), "r" (old), "r" (new) 311 : "cc", "memory"); 312 313 return prev; 314 } 315 316 #ifdef CONFIG_PPC64 317 static __always_inline unsigned long 318 __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new) 319 { 320 unsigned long prev; 321 322 __asm__ __volatile__ ( 323 PPC_ATOMIC_ENTRY_BARRIER 324 "1: ldarx %0,0,%2 # __cmpxchg_u64\n\ 325 cmpd 0,%0,%3\n\ 326 bne- 2f\n\ 327 stdcx. %4,0,%2\n\ 328 bne- 1b" 329 PPC_ATOMIC_EXIT_BARRIER 330 "\n\ 331 2:" 332 : "=&r" (prev), "+m" (*p) 333 : "r" (p), "r" (old), "r" (new) 334 : "cc", "memory"); 335 336 return prev; 337 } 338 339 static __always_inline unsigned long 340 __cmpxchg_u64_local(volatile unsigned long *p, unsigned long old, 341 unsigned long new) 342 { 343 unsigned long prev; 344 345 __asm__ __volatile__ ( 346 "1: ldarx %0,0,%2 # __cmpxchg_u64\n\ 347 cmpd 0,%0,%3\n\ 348 bne- 2f\n\ 349 stdcx. %4,0,%2\n\ 350 bne- 1b" 351 "\n\ 352 2:" 353 : "=&r" (prev), "+m" (*p) 354 : "r" (p), "r" (old), "r" (new) 355 : "cc", "memory"); 356 357 return prev; 358 } 359 360 static __always_inline unsigned long 361 __cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new) 362 { 363 unsigned long prev; 364 365 __asm__ __volatile__ ( 366 "1: ldarx %0,0,%2 # __cmpxchg_u64_relaxed\n" 367 " cmpd 0,%0,%3\n" 368 " bne- 2f\n" 369 " stdcx. %4,0,%2\n" 370 " bne- 1b\n" 371 "2:" 372 : "=&r" (prev), "+m" (*p) 373 : "r" (p), "r" (old), "r" (new) 374 : "cc"); 375 376 return prev; 377 } 378 379 static __always_inline unsigned long 380 __cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new) 381 { 382 unsigned long prev; 383 384 __asm__ __volatile__ ( 385 "1: ldarx %0,0,%2 # __cmpxchg_u64_acquire\n" 386 " cmpd 0,%0,%3\n" 387 " bne- 2f\n" 388 " stdcx. %4,0,%2\n" 389 " bne- 1b\n" 390 PPC_ACQUIRE_BARRIER 391 "\n" 392 "2:" 393 : "=&r" (prev), "+m" (*p) 394 : "r" (p), "r" (old), "r" (new) 395 : "cc", "memory"); 396 397 return prev; 398 } 399 #endif 400 401 static __always_inline unsigned long 402 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, 403 unsigned int size) 404 { 405 switch (size) { 406 case 1: 407 return __cmpxchg_u8(ptr, old, new); 408 case 2: 409 return __cmpxchg_u16(ptr, old, new); 410 case 4: 411 return __cmpxchg_u32(ptr, old, new); 412 #ifdef CONFIG_PPC64 413 case 8: 414 return __cmpxchg_u64(ptr, old, new); 415 #endif 416 } 417 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg"); 418 return old; 419 } 420 421 static __always_inline unsigned long 422 __cmpxchg_local(void *ptr, unsigned long old, unsigned long new, 423 unsigned int size) 424 { 425 switch (size) { 426 case 1: 427 return __cmpxchg_u8_local(ptr, old, new); 428 case 2: 429 return __cmpxchg_u16_local(ptr, old, new); 430 case 4: 431 return __cmpxchg_u32_local(ptr, old, new); 432 #ifdef CONFIG_PPC64 433 case 8: 434 return __cmpxchg_u64_local(ptr, old, new); 435 #endif 436 } 437 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local"); 438 return old; 439 } 440 441 static __always_inline unsigned long 442 __cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new, 443 unsigned int size) 444 { 445 switch (size) { 446 case 1: 447 return __cmpxchg_u8_relaxed(ptr, old, new); 448 case 2: 449 return __cmpxchg_u16_relaxed(ptr, old, new); 450 case 4: 451 return __cmpxchg_u32_relaxed(ptr, old, new); 452 #ifdef CONFIG_PPC64 453 case 8: 454 return __cmpxchg_u64_relaxed(ptr, old, new); 455 #endif 456 } 457 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed"); 458 return old; 459 } 460 461 static __always_inline unsigned long 462 __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new, 463 unsigned int size) 464 { 465 switch (size) { 466 case 1: 467 return __cmpxchg_u8_acquire(ptr, old, new); 468 case 2: 469 return __cmpxchg_u16_acquire(ptr, old, new); 470 case 4: 471 return __cmpxchg_u32_acquire(ptr, old, new); 472 #ifdef CONFIG_PPC64 473 case 8: 474 return __cmpxchg_u64_acquire(ptr, old, new); 475 #endif 476 } 477 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire"); 478 return old; 479 } 480 #define cmpxchg(ptr, o, n) \ 481 ({ \ 482 __typeof__(*(ptr)) _o_ = (o); \ 483 __typeof__(*(ptr)) _n_ = (n); \ 484 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ 485 (unsigned long)_n_, sizeof(*(ptr))); \ 486 }) 487 488 489 #define cmpxchg_local(ptr, o, n) \ 490 ({ \ 491 __typeof__(*(ptr)) _o_ = (o); \ 492 __typeof__(*(ptr)) _n_ = (n); \ 493 (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \ 494 (unsigned long)_n_, sizeof(*(ptr))); \ 495 }) 496 497 #define cmpxchg_relaxed(ptr, o, n) \ 498 ({ \ 499 __typeof__(*(ptr)) _o_ = (o); \ 500 __typeof__(*(ptr)) _n_ = (n); \ 501 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \ 502 (unsigned long)_o_, (unsigned long)_n_, \ 503 sizeof(*(ptr))); \ 504 }) 505 506 #define cmpxchg_acquire(ptr, o, n) \ 507 ({ \ 508 __typeof__(*(ptr)) _o_ = (o); \ 509 __typeof__(*(ptr)) _n_ = (n); \ 510 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \ 511 (unsigned long)_o_, (unsigned long)_n_, \ 512 sizeof(*(ptr))); \ 513 }) 514 #ifdef CONFIG_PPC64 515 #define cmpxchg64(ptr, o, n) \ 516 ({ \ 517 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 518 cmpxchg((ptr), (o), (n)); \ 519 }) 520 #define cmpxchg64_local(ptr, o, n) \ 521 ({ \ 522 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 523 cmpxchg_local((ptr), (o), (n)); \ 524 }) 525 #define cmpxchg64_relaxed(ptr, o, n) \ 526 ({ \ 527 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 528 cmpxchg_relaxed((ptr), (o), (n)); \ 529 }) 530 #define cmpxchg64_acquire(ptr, o, n) \ 531 ({ \ 532 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 533 cmpxchg_acquire((ptr), (o), (n)); \ 534 }) 535 #else 536 #include <asm-generic/cmpxchg-local.h> 537 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) 538 #endif 539 540 #endif /* __KERNEL__ */ 541 #endif /* _ASM_POWERPC_CMPXCHG_H_ */ 542