1 #ifndef _ASM_POWERPC_ATOMIC_H_ 2 #define _ASM_POWERPC_ATOMIC_H_ 3 4 /* 5 * PowerPC atomic operations 6 */ 7 8 #ifdef __KERNEL__ 9 #include <linux/types.h> 10 #include <asm/cmpxchg.h> 11 #include <asm/barrier.h> 12 13 #define ATOMIC_INIT(i) { (i) } 14 15 /* 16 * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with 17 * a "bne-" instruction at the end, so an isync is enough as a acquire barrier 18 * on the platform without lwsync. 19 */ 20 #define __atomic_op_acquire(op, args...) \ 21 ({ \ 22 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \ 23 __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory"); \ 24 __ret; \ 25 }) 26 27 #define __atomic_op_release(op, args...) \ 28 ({ \ 29 __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory"); \ 30 op##_relaxed(args); \ 31 }) 32 33 static __inline__ int atomic_read(const atomic_t *v) 34 { 35 int t; 36 37 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 38 39 return t; 40 } 41 42 static __inline__ void atomic_set(atomic_t *v, int i) 43 { 44 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 45 } 46 47 #define ATOMIC_OP(op, asm_op) \ 48 static __inline__ void atomic_##op(int a, atomic_t *v) \ 49 { \ 50 int t; \ 51 \ 52 __asm__ __volatile__( \ 53 "1: lwarx %0,0,%3 # atomic_" #op "\n" \ 54 #asm_op " %0,%2,%0\n" \ 55 PPC405_ERR77(0,%3) \ 56 " stwcx. %0,0,%3 \n" \ 57 " bne- 1b\n" \ 58 : "=&r" (t), "+m" (v->counter) \ 59 : "r" (a), "r" (&v->counter) \ 60 : "cc"); \ 61 } \ 62 63 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ 64 static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \ 65 { \ 66 int t; \ 67 \ 68 __asm__ __volatile__( \ 69 "1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \ 70 #asm_op " %0,%2,%0\n" \ 71 PPC405_ERR77(0, %3) \ 72 " stwcx. %0,0,%3\n" \ 73 " bne- 1b\n" \ 74 : "=&r" (t), "+m" (v->counter) \ 75 : "r" (a), "r" (&v->counter) \ 76 : "cc"); \ 77 \ 78 return t; \ 79 } 80 81 #define ATOMIC_OPS(op, asm_op) \ 82 ATOMIC_OP(op, asm_op) \ 83 ATOMIC_OP_RETURN_RELAXED(op, asm_op) 84 85 ATOMIC_OPS(add, add) 86 ATOMIC_OPS(sub, subf) 87 88 ATOMIC_OP(and, and) 89 ATOMIC_OP(or, or) 90 ATOMIC_OP(xor, xor) 91 92 #define atomic_add_return_relaxed atomic_add_return_relaxed 93 #define atomic_sub_return_relaxed atomic_sub_return_relaxed 94 95 #undef ATOMIC_OPS 96 #undef ATOMIC_OP_RETURN_RELAXED 97 #undef ATOMIC_OP 98 99 #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) 100 101 static __inline__ void atomic_inc(atomic_t *v) 102 { 103 int t; 104 105 __asm__ __volatile__( 106 "1: lwarx %0,0,%2 # atomic_inc\n\ 107 addic %0,%0,1\n" 108 PPC405_ERR77(0,%2) 109 " stwcx. %0,0,%2 \n\ 110 bne- 1b" 111 : "=&r" (t), "+m" (v->counter) 112 : "r" (&v->counter) 113 : "cc", "xer"); 114 } 115 116 static __inline__ int atomic_inc_return_relaxed(atomic_t *v) 117 { 118 int t; 119 120 __asm__ __volatile__( 121 "1: lwarx %0,0,%2 # atomic_inc_return_relaxed\n" 122 " addic %0,%0,1\n" 123 PPC405_ERR77(0, %2) 124 " stwcx. %0,0,%2\n" 125 " bne- 1b" 126 : "=&r" (t), "+m" (v->counter) 127 : "r" (&v->counter) 128 : "cc", "xer"); 129 130 return t; 131 } 132 133 /* 134 * atomic_inc_and_test - increment and test 135 * @v: pointer of type atomic_t 136 * 137 * Atomically increments @v by 1 138 * and returns true if the result is zero, or false for all 139 * other cases. 140 */ 141 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) 142 143 static __inline__ void atomic_dec(atomic_t *v) 144 { 145 int t; 146 147 __asm__ __volatile__( 148 "1: lwarx %0,0,%2 # atomic_dec\n\ 149 addic %0,%0,-1\n" 150 PPC405_ERR77(0,%2)\ 151 " stwcx. %0,0,%2\n\ 152 bne- 1b" 153 : "=&r" (t), "+m" (v->counter) 154 : "r" (&v->counter) 155 : "cc", "xer"); 156 } 157 158 static __inline__ int atomic_dec_return_relaxed(atomic_t *v) 159 { 160 int t; 161 162 __asm__ __volatile__( 163 "1: lwarx %0,0,%2 # atomic_dec_return_relaxed\n" 164 " addic %0,%0,-1\n" 165 PPC405_ERR77(0, %2) 166 " stwcx. %0,0,%2\n" 167 " bne- 1b" 168 : "=&r" (t), "+m" (v->counter) 169 : "r" (&v->counter) 170 : "cc", "xer"); 171 172 return t; 173 } 174 175 #define atomic_inc_return_relaxed atomic_inc_return_relaxed 176 #define atomic_dec_return_relaxed atomic_dec_return_relaxed 177 178 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 179 #define atomic_cmpxchg_relaxed(v, o, n) \ 180 cmpxchg_relaxed(&((v)->counter), (o), (n)) 181 #define atomic_cmpxchg_acquire(v, o, n) \ 182 cmpxchg_acquire(&((v)->counter), (o), (n)) 183 184 #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 185 #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) 186 187 /** 188 * __atomic_add_unless - add unless the number is a given value 189 * @v: pointer of type atomic_t 190 * @a: the amount to add to v... 191 * @u: ...unless v is equal to u. 192 * 193 * Atomically adds @a to @v, so long as it was not @u. 194 * Returns the old value of @v. 195 */ 196 static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) 197 { 198 int t; 199 200 __asm__ __volatile__ ( 201 PPC_ATOMIC_ENTRY_BARRIER 202 "1: lwarx %0,0,%1 # __atomic_add_unless\n\ 203 cmpw 0,%0,%3 \n\ 204 beq- 2f \n\ 205 add %0,%2,%0 \n" 206 PPC405_ERR77(0,%2) 207 " stwcx. %0,0,%1 \n\ 208 bne- 1b \n" 209 PPC_ATOMIC_EXIT_BARRIER 210 " subf %0,%2,%0 \n\ 211 2:" 212 : "=&r" (t) 213 : "r" (&v->counter), "r" (a), "r" (u) 214 : "cc", "memory"); 215 216 return t; 217 } 218 219 /** 220 * atomic_inc_not_zero - increment unless the number is zero 221 * @v: pointer of type atomic_t 222 * 223 * Atomically increments @v by 1, so long as @v is non-zero. 224 * Returns non-zero if @v was non-zero, and zero otherwise. 225 */ 226 static __inline__ int atomic_inc_not_zero(atomic_t *v) 227 { 228 int t1, t2; 229 230 __asm__ __volatile__ ( 231 PPC_ATOMIC_ENTRY_BARRIER 232 "1: lwarx %0,0,%2 # atomic_inc_not_zero\n\ 233 cmpwi 0,%0,0\n\ 234 beq- 2f\n\ 235 addic %1,%0,1\n" 236 PPC405_ERR77(0,%2) 237 " stwcx. %1,0,%2\n\ 238 bne- 1b\n" 239 PPC_ATOMIC_EXIT_BARRIER 240 "\n\ 241 2:" 242 : "=&r" (t1), "=&r" (t2) 243 : "r" (&v->counter) 244 : "cc", "xer", "memory"); 245 246 return t1; 247 } 248 #define atomic_inc_not_zero(v) atomic_inc_not_zero((v)) 249 250 #define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0) 251 #define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0) 252 253 /* 254 * Atomically test *v and decrement if it is greater than 0. 255 * The function returns the old value of *v minus 1, even if 256 * the atomic variable, v, was not decremented. 257 */ 258 static __inline__ int atomic_dec_if_positive(atomic_t *v) 259 { 260 int t; 261 262 __asm__ __volatile__( 263 PPC_ATOMIC_ENTRY_BARRIER 264 "1: lwarx %0,0,%1 # atomic_dec_if_positive\n\ 265 cmpwi %0,1\n\ 266 addi %0,%0,-1\n\ 267 blt- 2f\n" 268 PPC405_ERR77(0,%1) 269 " stwcx. %0,0,%1\n\ 270 bne- 1b" 271 PPC_ATOMIC_EXIT_BARRIER 272 "\n\ 273 2:" : "=&b" (t) 274 : "r" (&v->counter) 275 : "cc", "memory"); 276 277 return t; 278 } 279 #define atomic_dec_if_positive atomic_dec_if_positive 280 281 #ifdef __powerpc64__ 282 283 #define ATOMIC64_INIT(i) { (i) } 284 285 static __inline__ long atomic64_read(const atomic64_t *v) 286 { 287 long t; 288 289 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 290 291 return t; 292 } 293 294 static __inline__ void atomic64_set(atomic64_t *v, long i) 295 { 296 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 297 } 298 299 #define ATOMIC64_OP(op, asm_op) \ 300 static __inline__ void atomic64_##op(long a, atomic64_t *v) \ 301 { \ 302 long t; \ 303 \ 304 __asm__ __volatile__( \ 305 "1: ldarx %0,0,%3 # atomic64_" #op "\n" \ 306 #asm_op " %0,%2,%0\n" \ 307 " stdcx. %0,0,%3 \n" \ 308 " bne- 1b\n" \ 309 : "=&r" (t), "+m" (v->counter) \ 310 : "r" (a), "r" (&v->counter) \ 311 : "cc"); \ 312 } 313 314 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 315 static inline long \ 316 atomic64_##op##_return_relaxed(long a, atomic64_t *v) \ 317 { \ 318 long t; \ 319 \ 320 __asm__ __volatile__( \ 321 "1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \ 322 #asm_op " %0,%2,%0\n" \ 323 " stdcx. %0,0,%3\n" \ 324 " bne- 1b\n" \ 325 : "=&r" (t), "+m" (v->counter) \ 326 : "r" (a), "r" (&v->counter) \ 327 : "cc"); \ 328 \ 329 return t; \ 330 } 331 332 #define ATOMIC64_OPS(op, asm_op) \ 333 ATOMIC64_OP(op, asm_op) \ 334 ATOMIC64_OP_RETURN_RELAXED(op, asm_op) 335 336 ATOMIC64_OPS(add, add) 337 ATOMIC64_OPS(sub, subf) 338 ATOMIC64_OP(and, and) 339 ATOMIC64_OP(or, or) 340 ATOMIC64_OP(xor, xor) 341 342 #define atomic64_add_return_relaxed atomic64_add_return_relaxed 343 #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed 344 345 #undef ATOPIC64_OPS 346 #undef ATOMIC64_OP_RETURN_RELAXED 347 #undef ATOMIC64_OP 348 349 #define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) 350 351 static __inline__ void atomic64_inc(atomic64_t *v) 352 { 353 long t; 354 355 __asm__ __volatile__( 356 "1: ldarx %0,0,%2 # atomic64_inc\n\ 357 addic %0,%0,1\n\ 358 stdcx. %0,0,%2 \n\ 359 bne- 1b" 360 : "=&r" (t), "+m" (v->counter) 361 : "r" (&v->counter) 362 : "cc", "xer"); 363 } 364 365 static __inline__ long atomic64_inc_return_relaxed(atomic64_t *v) 366 { 367 long t; 368 369 __asm__ __volatile__( 370 "1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n" 371 " addic %0,%0,1\n" 372 " stdcx. %0,0,%2\n" 373 " bne- 1b" 374 : "=&r" (t), "+m" (v->counter) 375 : "r" (&v->counter) 376 : "cc", "xer"); 377 378 return t; 379 } 380 381 /* 382 * atomic64_inc_and_test - increment and test 383 * @v: pointer of type atomic64_t 384 * 385 * Atomically increments @v by 1 386 * and returns true if the result is zero, or false for all 387 * other cases. 388 */ 389 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) 390 391 static __inline__ void atomic64_dec(atomic64_t *v) 392 { 393 long t; 394 395 __asm__ __volatile__( 396 "1: ldarx %0,0,%2 # atomic64_dec\n\ 397 addic %0,%0,-1\n\ 398 stdcx. %0,0,%2\n\ 399 bne- 1b" 400 : "=&r" (t), "+m" (v->counter) 401 : "r" (&v->counter) 402 : "cc", "xer"); 403 } 404 405 static __inline__ long atomic64_dec_return_relaxed(atomic64_t *v) 406 { 407 long t; 408 409 __asm__ __volatile__( 410 "1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n" 411 " addic %0,%0,-1\n" 412 " stdcx. %0,0,%2\n" 413 " bne- 1b" 414 : "=&r" (t), "+m" (v->counter) 415 : "r" (&v->counter) 416 : "cc", "xer"); 417 418 return t; 419 } 420 421 #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed 422 #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed 423 424 #define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0) 425 #define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) 426 427 /* 428 * Atomically test *v and decrement if it is greater than 0. 429 * The function returns the old value of *v minus 1. 430 */ 431 static __inline__ long atomic64_dec_if_positive(atomic64_t *v) 432 { 433 long t; 434 435 __asm__ __volatile__( 436 PPC_ATOMIC_ENTRY_BARRIER 437 "1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\ 438 addic. %0,%0,-1\n\ 439 blt- 2f\n\ 440 stdcx. %0,0,%1\n\ 441 bne- 1b" 442 PPC_ATOMIC_EXIT_BARRIER 443 "\n\ 444 2:" : "=&r" (t) 445 : "r" (&v->counter) 446 : "cc", "xer", "memory"); 447 448 return t; 449 } 450 451 #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 452 #define atomic64_cmpxchg_relaxed(v, o, n) \ 453 cmpxchg_relaxed(&((v)->counter), (o), (n)) 454 #define atomic64_cmpxchg_acquire(v, o, n) \ 455 cmpxchg_acquire(&((v)->counter), (o), (n)) 456 457 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) 458 #define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) 459 460 /** 461 * atomic64_add_unless - add unless the number is a given value 462 * @v: pointer of type atomic64_t 463 * @a: the amount to add to v... 464 * @u: ...unless v is equal to u. 465 * 466 * Atomically adds @a to @v, so long as it was not @u. 467 * Returns the old value of @v. 468 */ 469 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) 470 { 471 long t; 472 473 __asm__ __volatile__ ( 474 PPC_ATOMIC_ENTRY_BARRIER 475 "1: ldarx %0,0,%1 # __atomic_add_unless\n\ 476 cmpd 0,%0,%3 \n\ 477 beq- 2f \n\ 478 add %0,%2,%0 \n" 479 " stdcx. %0,0,%1 \n\ 480 bne- 1b \n" 481 PPC_ATOMIC_EXIT_BARRIER 482 " subf %0,%2,%0 \n\ 483 2:" 484 : "=&r" (t) 485 : "r" (&v->counter), "r" (a), "r" (u) 486 : "cc", "memory"); 487 488 return t != u; 489 } 490 491 /** 492 * atomic_inc64_not_zero - increment unless the number is zero 493 * @v: pointer of type atomic64_t 494 * 495 * Atomically increments @v by 1, so long as @v is non-zero. 496 * Returns non-zero if @v was non-zero, and zero otherwise. 497 */ 498 static __inline__ long atomic64_inc_not_zero(atomic64_t *v) 499 { 500 long t1, t2; 501 502 __asm__ __volatile__ ( 503 PPC_ATOMIC_ENTRY_BARRIER 504 "1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\ 505 cmpdi 0,%0,0\n\ 506 beq- 2f\n\ 507 addic %1,%0,1\n\ 508 stdcx. %1,0,%2\n\ 509 bne- 1b\n" 510 PPC_ATOMIC_EXIT_BARRIER 511 "\n\ 512 2:" 513 : "=&r" (t1), "=&r" (t2) 514 : "r" (&v->counter) 515 : "cc", "xer", "memory"); 516 517 return t1; 518 } 519 520 #endif /* __powerpc64__ */ 521 522 #endif /* __KERNEL__ */ 523 #endif /* _ASM_POWERPC_ATOMIC_H_ */ 524