1 #ifndef _ASM_POWERPC_ATOMIC_H_ 2 #define _ASM_POWERPC_ATOMIC_H_ 3 4 /* 5 * PowerPC atomic operations 6 */ 7 8 #ifdef __KERNEL__ 9 #include <linux/types.h> 10 #include <asm/cmpxchg.h> 11 #include <asm/barrier.h> 12 13 #define ATOMIC_INIT(i) { (i) } 14 15 /* 16 * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with 17 * a "bne-" instruction at the end, so an isync is enough as a acquire barrier 18 * on the platform without lwsync. 19 */ 20 #define __atomic_op_acquire(op, args...) \ 21 ({ \ 22 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \ 23 __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory"); \ 24 __ret; \ 25 }) 26 27 #define __atomic_op_release(op, args...) \ 28 ({ \ 29 __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory"); \ 30 op##_relaxed(args); \ 31 }) 32 33 static __inline__ int atomic_read(const atomic_t *v) 34 { 35 int t; 36 37 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 38 39 return t; 40 } 41 42 static __inline__ void atomic_set(atomic_t *v, int i) 43 { 44 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 45 } 46 47 #define ATOMIC_OP(op, asm_op) \ 48 static __inline__ void atomic_##op(int a, atomic_t *v) \ 49 { \ 50 int t; \ 51 \ 52 __asm__ __volatile__( \ 53 "1: lwarx %0,0,%3 # atomic_" #op "\n" \ 54 #asm_op " %0,%2,%0\n" \ 55 PPC405_ERR77(0,%3) \ 56 " stwcx. %0,0,%3 \n" \ 57 " bne- 1b\n" \ 58 : "=&r" (t), "+m" (v->counter) \ 59 : "r" (a), "r" (&v->counter) \ 60 : "cc"); \ 61 } \ 62 63 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ 64 static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \ 65 { \ 66 int t; \ 67 \ 68 __asm__ __volatile__( \ 69 "1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \ 70 #asm_op " %0,%2,%0\n" \ 71 PPC405_ERR77(0, %3) \ 72 " stwcx. %0,0,%3\n" \ 73 " bne- 1b\n" \ 74 : "=&r" (t), "+m" (v->counter) \ 75 : "r" (a), "r" (&v->counter) \ 76 : "cc"); \ 77 \ 78 return t; \ 79 } 80 81 #define ATOMIC_FETCH_OP_RELAXED(op, asm_op) \ 82 static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ 83 { \ 84 int res, t; \ 85 \ 86 __asm__ __volatile__( \ 87 "1: lwarx %0,0,%4 # atomic_fetch_" #op "_relaxed\n" \ 88 #asm_op " %1,%3,%0\n" \ 89 PPC405_ERR77(0, %4) \ 90 " stwcx. %1,0,%4\n" \ 91 " bne- 1b\n" \ 92 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 93 : "r" (a), "r" (&v->counter) \ 94 : "cc"); \ 95 \ 96 return res; \ 97 } 98 99 #define ATOMIC_OPS(op, asm_op) \ 100 ATOMIC_OP(op, asm_op) \ 101 ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ 102 ATOMIC_FETCH_OP_RELAXED(op, asm_op) 103 104 ATOMIC_OPS(add, add) 105 ATOMIC_OPS(sub, subf) 106 107 #define atomic_add_return_relaxed atomic_add_return_relaxed 108 #define atomic_sub_return_relaxed atomic_sub_return_relaxed 109 110 #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed 111 #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed 112 113 #undef ATOMIC_OPS 114 #define ATOMIC_OPS(op, asm_op) \ 115 ATOMIC_OP(op, asm_op) \ 116 ATOMIC_FETCH_OP_RELAXED(op, asm_op) 117 118 ATOMIC_OPS(and, and) 119 ATOMIC_OPS(or, or) 120 ATOMIC_OPS(xor, xor) 121 122 #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed 123 #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed 124 #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed 125 126 #undef ATOMIC_OPS 127 #undef ATOMIC_FETCH_OP_RELAXED 128 #undef ATOMIC_OP_RETURN_RELAXED 129 #undef ATOMIC_OP 130 131 #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) 132 133 static __inline__ void atomic_inc(atomic_t *v) 134 { 135 int t; 136 137 __asm__ __volatile__( 138 "1: lwarx %0,0,%2 # atomic_inc\n\ 139 addic %0,%0,1\n" 140 PPC405_ERR77(0,%2) 141 " stwcx. %0,0,%2 \n\ 142 bne- 1b" 143 : "=&r" (t), "+m" (v->counter) 144 : "r" (&v->counter) 145 : "cc", "xer"); 146 } 147 148 static __inline__ int atomic_inc_return_relaxed(atomic_t *v) 149 { 150 int t; 151 152 __asm__ __volatile__( 153 "1: lwarx %0,0,%2 # atomic_inc_return_relaxed\n" 154 " addic %0,%0,1\n" 155 PPC405_ERR77(0, %2) 156 " stwcx. %0,0,%2\n" 157 " bne- 1b" 158 : "=&r" (t), "+m" (v->counter) 159 : "r" (&v->counter) 160 : "cc", "xer"); 161 162 return t; 163 } 164 165 /* 166 * atomic_inc_and_test - increment and test 167 * @v: pointer of type atomic_t 168 * 169 * Atomically increments @v by 1 170 * and returns true if the result is zero, or false for all 171 * other cases. 172 */ 173 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) 174 175 static __inline__ void atomic_dec(atomic_t *v) 176 { 177 int t; 178 179 __asm__ __volatile__( 180 "1: lwarx %0,0,%2 # atomic_dec\n\ 181 addic %0,%0,-1\n" 182 PPC405_ERR77(0,%2)\ 183 " stwcx. %0,0,%2\n\ 184 bne- 1b" 185 : "=&r" (t), "+m" (v->counter) 186 : "r" (&v->counter) 187 : "cc", "xer"); 188 } 189 190 static __inline__ int atomic_dec_return_relaxed(atomic_t *v) 191 { 192 int t; 193 194 __asm__ __volatile__( 195 "1: lwarx %0,0,%2 # atomic_dec_return_relaxed\n" 196 " addic %0,%0,-1\n" 197 PPC405_ERR77(0, %2) 198 " stwcx. %0,0,%2\n" 199 " bne- 1b" 200 : "=&r" (t), "+m" (v->counter) 201 : "r" (&v->counter) 202 : "cc", "xer"); 203 204 return t; 205 } 206 207 #define atomic_inc_return_relaxed atomic_inc_return_relaxed 208 #define atomic_dec_return_relaxed atomic_dec_return_relaxed 209 210 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 211 #define atomic_cmpxchg_relaxed(v, o, n) \ 212 cmpxchg_relaxed(&((v)->counter), (o), (n)) 213 #define atomic_cmpxchg_acquire(v, o, n) \ 214 cmpxchg_acquire(&((v)->counter), (o), (n)) 215 216 #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 217 #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) 218 219 /** 220 * __atomic_add_unless - add unless the number is a given value 221 * @v: pointer of type atomic_t 222 * @a: the amount to add to v... 223 * @u: ...unless v is equal to u. 224 * 225 * Atomically adds @a to @v, so long as it was not @u. 226 * Returns the old value of @v. 227 */ 228 static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) 229 { 230 int t; 231 232 __asm__ __volatile__ ( 233 PPC_ATOMIC_ENTRY_BARRIER 234 "1: lwarx %0,0,%1 # __atomic_add_unless\n\ 235 cmpw 0,%0,%3 \n\ 236 beq 2f \n\ 237 add %0,%2,%0 \n" 238 PPC405_ERR77(0,%2) 239 " stwcx. %0,0,%1 \n\ 240 bne- 1b \n" 241 PPC_ATOMIC_EXIT_BARRIER 242 " subf %0,%2,%0 \n\ 243 2:" 244 : "=&r" (t) 245 : "r" (&v->counter), "r" (a), "r" (u) 246 : "cc", "memory"); 247 248 return t; 249 } 250 251 /** 252 * atomic_inc_not_zero - increment unless the number is zero 253 * @v: pointer of type atomic_t 254 * 255 * Atomically increments @v by 1, so long as @v is non-zero. 256 * Returns non-zero if @v was non-zero, and zero otherwise. 257 */ 258 static __inline__ int atomic_inc_not_zero(atomic_t *v) 259 { 260 int t1, t2; 261 262 __asm__ __volatile__ ( 263 PPC_ATOMIC_ENTRY_BARRIER 264 "1: lwarx %0,0,%2 # atomic_inc_not_zero\n\ 265 cmpwi 0,%0,0\n\ 266 beq- 2f\n\ 267 addic %1,%0,1\n" 268 PPC405_ERR77(0,%2) 269 " stwcx. %1,0,%2\n\ 270 bne- 1b\n" 271 PPC_ATOMIC_EXIT_BARRIER 272 "\n\ 273 2:" 274 : "=&r" (t1), "=&r" (t2) 275 : "r" (&v->counter) 276 : "cc", "xer", "memory"); 277 278 return t1; 279 } 280 #define atomic_inc_not_zero(v) atomic_inc_not_zero((v)) 281 282 #define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0) 283 #define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0) 284 285 /* 286 * Atomically test *v and decrement if it is greater than 0. 287 * The function returns the old value of *v minus 1, even if 288 * the atomic variable, v, was not decremented. 289 */ 290 static __inline__ int atomic_dec_if_positive(atomic_t *v) 291 { 292 int t; 293 294 __asm__ __volatile__( 295 PPC_ATOMIC_ENTRY_BARRIER 296 "1: lwarx %0,0,%1 # atomic_dec_if_positive\n\ 297 cmpwi %0,1\n\ 298 addi %0,%0,-1\n\ 299 blt- 2f\n" 300 PPC405_ERR77(0,%1) 301 " stwcx. %0,0,%1\n\ 302 bne- 1b" 303 PPC_ATOMIC_EXIT_BARRIER 304 "\n\ 305 2:" : "=&b" (t) 306 : "r" (&v->counter) 307 : "cc", "memory"); 308 309 return t; 310 } 311 #define atomic_dec_if_positive atomic_dec_if_positive 312 313 #ifdef __powerpc64__ 314 315 #define ATOMIC64_INIT(i) { (i) } 316 317 static __inline__ long atomic64_read(const atomic64_t *v) 318 { 319 long t; 320 321 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 322 323 return t; 324 } 325 326 static __inline__ void atomic64_set(atomic64_t *v, long i) 327 { 328 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 329 } 330 331 #define ATOMIC64_OP(op, asm_op) \ 332 static __inline__ void atomic64_##op(long a, atomic64_t *v) \ 333 { \ 334 long t; \ 335 \ 336 __asm__ __volatile__( \ 337 "1: ldarx %0,0,%3 # atomic64_" #op "\n" \ 338 #asm_op " %0,%2,%0\n" \ 339 " stdcx. %0,0,%3 \n" \ 340 " bne- 1b\n" \ 341 : "=&r" (t), "+m" (v->counter) \ 342 : "r" (a), "r" (&v->counter) \ 343 : "cc"); \ 344 } 345 346 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 347 static inline long \ 348 atomic64_##op##_return_relaxed(long a, atomic64_t *v) \ 349 { \ 350 long t; \ 351 \ 352 __asm__ __volatile__( \ 353 "1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \ 354 #asm_op " %0,%2,%0\n" \ 355 " stdcx. %0,0,%3\n" \ 356 " bne- 1b\n" \ 357 : "=&r" (t), "+m" (v->counter) \ 358 : "r" (a), "r" (&v->counter) \ 359 : "cc"); \ 360 \ 361 return t; \ 362 } 363 364 #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \ 365 static inline long \ 366 atomic64_fetch_##op##_relaxed(long a, atomic64_t *v) \ 367 { \ 368 long res, t; \ 369 \ 370 __asm__ __volatile__( \ 371 "1: ldarx %0,0,%4 # atomic64_fetch_" #op "_relaxed\n" \ 372 #asm_op " %1,%3,%0\n" \ 373 " stdcx. %1,0,%4\n" \ 374 " bne- 1b\n" \ 375 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 376 : "r" (a), "r" (&v->counter) \ 377 : "cc"); \ 378 \ 379 return res; \ 380 } 381 382 #define ATOMIC64_OPS(op, asm_op) \ 383 ATOMIC64_OP(op, asm_op) \ 384 ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 385 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 386 387 ATOMIC64_OPS(add, add) 388 ATOMIC64_OPS(sub, subf) 389 390 #define atomic64_add_return_relaxed atomic64_add_return_relaxed 391 #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed 392 393 #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed 394 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed 395 396 #undef ATOMIC64_OPS 397 #define ATOMIC64_OPS(op, asm_op) \ 398 ATOMIC64_OP(op, asm_op) \ 399 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 400 401 ATOMIC64_OPS(and, and) 402 ATOMIC64_OPS(or, or) 403 ATOMIC64_OPS(xor, xor) 404 405 #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed 406 #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed 407 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed 408 409 #undef ATOPIC64_OPS 410 #undef ATOMIC64_FETCH_OP_RELAXED 411 #undef ATOMIC64_OP_RETURN_RELAXED 412 #undef ATOMIC64_OP 413 414 #define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) 415 416 static __inline__ void atomic64_inc(atomic64_t *v) 417 { 418 long t; 419 420 __asm__ __volatile__( 421 "1: ldarx %0,0,%2 # atomic64_inc\n\ 422 addic %0,%0,1\n\ 423 stdcx. %0,0,%2 \n\ 424 bne- 1b" 425 : "=&r" (t), "+m" (v->counter) 426 : "r" (&v->counter) 427 : "cc", "xer"); 428 } 429 430 static __inline__ long atomic64_inc_return_relaxed(atomic64_t *v) 431 { 432 long t; 433 434 __asm__ __volatile__( 435 "1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n" 436 " addic %0,%0,1\n" 437 " stdcx. %0,0,%2\n" 438 " bne- 1b" 439 : "=&r" (t), "+m" (v->counter) 440 : "r" (&v->counter) 441 : "cc", "xer"); 442 443 return t; 444 } 445 446 /* 447 * atomic64_inc_and_test - increment and test 448 * @v: pointer of type atomic64_t 449 * 450 * Atomically increments @v by 1 451 * and returns true if the result is zero, or false for all 452 * other cases. 453 */ 454 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) 455 456 static __inline__ void atomic64_dec(atomic64_t *v) 457 { 458 long t; 459 460 __asm__ __volatile__( 461 "1: ldarx %0,0,%2 # atomic64_dec\n\ 462 addic %0,%0,-1\n\ 463 stdcx. %0,0,%2\n\ 464 bne- 1b" 465 : "=&r" (t), "+m" (v->counter) 466 : "r" (&v->counter) 467 : "cc", "xer"); 468 } 469 470 static __inline__ long atomic64_dec_return_relaxed(atomic64_t *v) 471 { 472 long t; 473 474 __asm__ __volatile__( 475 "1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n" 476 " addic %0,%0,-1\n" 477 " stdcx. %0,0,%2\n" 478 " bne- 1b" 479 : "=&r" (t), "+m" (v->counter) 480 : "r" (&v->counter) 481 : "cc", "xer"); 482 483 return t; 484 } 485 486 #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed 487 #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed 488 489 #define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0) 490 #define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) 491 492 /* 493 * Atomically test *v and decrement if it is greater than 0. 494 * The function returns the old value of *v minus 1. 495 */ 496 static __inline__ long atomic64_dec_if_positive(atomic64_t *v) 497 { 498 long t; 499 500 __asm__ __volatile__( 501 PPC_ATOMIC_ENTRY_BARRIER 502 "1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\ 503 addic. %0,%0,-1\n\ 504 blt- 2f\n\ 505 stdcx. %0,0,%1\n\ 506 bne- 1b" 507 PPC_ATOMIC_EXIT_BARRIER 508 "\n\ 509 2:" : "=&r" (t) 510 : "r" (&v->counter) 511 : "cc", "xer", "memory"); 512 513 return t; 514 } 515 516 #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 517 #define atomic64_cmpxchg_relaxed(v, o, n) \ 518 cmpxchg_relaxed(&((v)->counter), (o), (n)) 519 #define atomic64_cmpxchg_acquire(v, o, n) \ 520 cmpxchg_acquire(&((v)->counter), (o), (n)) 521 522 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) 523 #define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) 524 525 /** 526 * atomic64_add_unless - add unless the number is a given value 527 * @v: pointer of type atomic64_t 528 * @a: the amount to add to v... 529 * @u: ...unless v is equal to u. 530 * 531 * Atomically adds @a to @v, so long as it was not @u. 532 * Returns the old value of @v. 533 */ 534 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) 535 { 536 long t; 537 538 __asm__ __volatile__ ( 539 PPC_ATOMIC_ENTRY_BARRIER 540 "1: ldarx %0,0,%1 # __atomic_add_unless\n\ 541 cmpd 0,%0,%3 \n\ 542 beq 2f \n\ 543 add %0,%2,%0 \n" 544 " stdcx. %0,0,%1 \n\ 545 bne- 1b \n" 546 PPC_ATOMIC_EXIT_BARRIER 547 " subf %0,%2,%0 \n\ 548 2:" 549 : "=&r" (t) 550 : "r" (&v->counter), "r" (a), "r" (u) 551 : "cc", "memory"); 552 553 return t != u; 554 } 555 556 /** 557 * atomic_inc64_not_zero - increment unless the number is zero 558 * @v: pointer of type atomic64_t 559 * 560 * Atomically increments @v by 1, so long as @v is non-zero. 561 * Returns non-zero if @v was non-zero, and zero otherwise. 562 */ 563 static __inline__ int atomic64_inc_not_zero(atomic64_t *v) 564 { 565 long t1, t2; 566 567 __asm__ __volatile__ ( 568 PPC_ATOMIC_ENTRY_BARRIER 569 "1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\ 570 cmpdi 0,%0,0\n\ 571 beq- 2f\n\ 572 addic %1,%0,1\n\ 573 stdcx. %1,0,%2\n\ 574 bne- 1b\n" 575 PPC_ATOMIC_EXIT_BARRIER 576 "\n\ 577 2:" 578 : "=&r" (t1), "=&r" (t2) 579 : "r" (&v->counter) 580 : "cc", "xer", "memory"); 581 582 return t1 != 0; 583 } 584 585 #endif /* __powerpc64__ */ 586 587 #endif /* __KERNEL__ */ 588 #endif /* _ASM_POWERPC_ATOMIC_H_ */ 589