1 #ifndef _ASM_POWERPC_ATOMIC_H_ 2 #define _ASM_POWERPC_ATOMIC_H_ 3 4 /* 5 * PowerPC atomic operations 6 */ 7 8 #ifdef __KERNEL__ 9 #include <linux/types.h> 10 #include <asm/cmpxchg.h> 11 #include <asm/barrier.h> 12 13 #define ATOMIC_INIT(i) { (i) } 14 15 static __inline__ int atomic_read(const atomic_t *v) 16 { 17 int t; 18 19 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 20 21 return t; 22 } 23 24 static __inline__ void atomic_set(atomic_t *v, int i) 25 { 26 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 27 } 28 29 static __inline__ void atomic_add(int a, atomic_t *v) 30 { 31 int t; 32 33 __asm__ __volatile__( 34 "1: lwarx %0,0,%3 # atomic_add\n\ 35 add %0,%2,%0\n" 36 PPC405_ERR77(0,%3) 37 " stwcx. %0,0,%3 \n\ 38 bne- 1b" 39 : "=&r" (t), "+m" (v->counter) 40 : "r" (a), "r" (&v->counter) 41 : "cc"); 42 } 43 44 static __inline__ int atomic_add_return(int a, atomic_t *v) 45 { 46 int t; 47 48 __asm__ __volatile__( 49 PPC_ATOMIC_ENTRY_BARRIER 50 "1: lwarx %0,0,%2 # atomic_add_return\n\ 51 add %0,%1,%0\n" 52 PPC405_ERR77(0,%2) 53 " stwcx. %0,0,%2 \n\ 54 bne- 1b" 55 PPC_ATOMIC_EXIT_BARRIER 56 : "=&r" (t) 57 : "r" (a), "r" (&v->counter) 58 : "cc", "memory"); 59 60 return t; 61 } 62 63 #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) 64 65 static __inline__ void atomic_sub(int a, atomic_t *v) 66 { 67 int t; 68 69 __asm__ __volatile__( 70 "1: lwarx %0,0,%3 # atomic_sub\n\ 71 subf %0,%2,%0\n" 72 PPC405_ERR77(0,%3) 73 " stwcx. %0,0,%3 \n\ 74 bne- 1b" 75 : "=&r" (t), "+m" (v->counter) 76 : "r" (a), "r" (&v->counter) 77 : "cc"); 78 } 79 80 static __inline__ int atomic_sub_return(int a, atomic_t *v) 81 { 82 int t; 83 84 __asm__ __volatile__( 85 PPC_ATOMIC_ENTRY_BARRIER 86 "1: lwarx %0,0,%2 # atomic_sub_return\n\ 87 subf %0,%1,%0\n" 88 PPC405_ERR77(0,%2) 89 " stwcx. %0,0,%2 \n\ 90 bne- 1b" 91 PPC_ATOMIC_EXIT_BARRIER 92 : "=&r" (t) 93 : "r" (a), "r" (&v->counter) 94 : "cc", "memory"); 95 96 return t; 97 } 98 99 static __inline__ void atomic_inc(atomic_t *v) 100 { 101 int t; 102 103 __asm__ __volatile__( 104 "1: lwarx %0,0,%2 # atomic_inc\n\ 105 addic %0,%0,1\n" 106 PPC405_ERR77(0,%2) 107 " stwcx. %0,0,%2 \n\ 108 bne- 1b" 109 : "=&r" (t), "+m" (v->counter) 110 : "r" (&v->counter) 111 : "cc", "xer"); 112 } 113 114 static __inline__ int atomic_inc_return(atomic_t *v) 115 { 116 int t; 117 118 __asm__ __volatile__( 119 PPC_ATOMIC_ENTRY_BARRIER 120 "1: lwarx %0,0,%1 # atomic_inc_return\n\ 121 addic %0,%0,1\n" 122 PPC405_ERR77(0,%1) 123 " stwcx. %0,0,%1 \n\ 124 bne- 1b" 125 PPC_ATOMIC_EXIT_BARRIER 126 : "=&r" (t) 127 : "r" (&v->counter) 128 : "cc", "xer", "memory"); 129 130 return t; 131 } 132 133 /* 134 * atomic_inc_and_test - increment and test 135 * @v: pointer of type atomic_t 136 * 137 * Atomically increments @v by 1 138 * and returns true if the result is zero, or false for all 139 * other cases. 140 */ 141 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) 142 143 static __inline__ void atomic_dec(atomic_t *v) 144 { 145 int t; 146 147 __asm__ __volatile__( 148 "1: lwarx %0,0,%2 # atomic_dec\n\ 149 addic %0,%0,-1\n" 150 PPC405_ERR77(0,%2)\ 151 " stwcx. %0,0,%2\n\ 152 bne- 1b" 153 : "=&r" (t), "+m" (v->counter) 154 : "r" (&v->counter) 155 : "cc", "xer"); 156 } 157 158 static __inline__ int atomic_dec_return(atomic_t *v) 159 { 160 int t; 161 162 __asm__ __volatile__( 163 PPC_ATOMIC_ENTRY_BARRIER 164 "1: lwarx %0,0,%1 # atomic_dec_return\n\ 165 addic %0,%0,-1\n" 166 PPC405_ERR77(0,%1) 167 " stwcx. %0,0,%1\n\ 168 bne- 1b" 169 PPC_ATOMIC_EXIT_BARRIER 170 : "=&r" (t) 171 : "r" (&v->counter) 172 : "cc", "xer", "memory"); 173 174 return t; 175 } 176 177 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 178 #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 179 180 /** 181 * __atomic_add_unless - add unless the number is a given value 182 * @v: pointer of type atomic_t 183 * @a: the amount to add to v... 184 * @u: ...unless v is equal to u. 185 * 186 * Atomically adds @a to @v, so long as it was not @u. 187 * Returns the old value of @v. 188 */ 189 static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) 190 { 191 int t; 192 193 __asm__ __volatile__ ( 194 PPC_ATOMIC_ENTRY_BARRIER 195 "1: lwarx %0,0,%1 # __atomic_add_unless\n\ 196 cmpw 0,%0,%3 \n\ 197 beq- 2f \n\ 198 add %0,%2,%0 \n" 199 PPC405_ERR77(0,%2) 200 " stwcx. %0,0,%1 \n\ 201 bne- 1b \n" 202 PPC_ATOMIC_EXIT_BARRIER 203 " subf %0,%2,%0 \n\ 204 2:" 205 : "=&r" (t) 206 : "r" (&v->counter), "r" (a), "r" (u) 207 : "cc", "memory"); 208 209 return t; 210 } 211 212 /** 213 * atomic_inc_not_zero - increment unless the number is zero 214 * @v: pointer of type atomic_t 215 * 216 * Atomically increments @v by 1, so long as @v is non-zero. 217 * Returns non-zero if @v was non-zero, and zero otherwise. 218 */ 219 static __inline__ int atomic_inc_not_zero(atomic_t *v) 220 { 221 int t1, t2; 222 223 __asm__ __volatile__ ( 224 PPC_ATOMIC_ENTRY_BARRIER 225 "1: lwarx %0,0,%2 # atomic_inc_not_zero\n\ 226 cmpwi 0,%0,0\n\ 227 beq- 2f\n\ 228 addic %1,%0,1\n" 229 PPC405_ERR77(0,%2) 230 " stwcx. %1,0,%2\n\ 231 bne- 1b\n" 232 PPC_ATOMIC_EXIT_BARRIER 233 "\n\ 234 2:" 235 : "=&r" (t1), "=&r" (t2) 236 : "r" (&v->counter) 237 : "cc", "xer", "memory"); 238 239 return t1; 240 } 241 #define atomic_inc_not_zero(v) atomic_inc_not_zero((v)) 242 243 #define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0) 244 #define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0) 245 246 /* 247 * Atomically test *v and decrement if it is greater than 0. 248 * The function returns the old value of *v minus 1, even if 249 * the atomic variable, v, was not decremented. 250 */ 251 static __inline__ int atomic_dec_if_positive(atomic_t *v) 252 { 253 int t; 254 255 __asm__ __volatile__( 256 PPC_ATOMIC_ENTRY_BARRIER 257 "1: lwarx %0,0,%1 # atomic_dec_if_positive\n\ 258 cmpwi %0,1\n\ 259 addi %0,%0,-1\n\ 260 blt- 2f\n" 261 PPC405_ERR77(0,%1) 262 " stwcx. %0,0,%1\n\ 263 bne- 1b" 264 PPC_ATOMIC_EXIT_BARRIER 265 "\n\ 266 2:" : "=&b" (t) 267 : "r" (&v->counter) 268 : "cc", "memory"); 269 270 return t; 271 } 272 #define atomic_dec_if_positive atomic_dec_if_positive 273 274 #ifdef __powerpc64__ 275 276 #define ATOMIC64_INIT(i) { (i) } 277 278 static __inline__ long atomic64_read(const atomic64_t *v) 279 { 280 long t; 281 282 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 283 284 return t; 285 } 286 287 static __inline__ void atomic64_set(atomic64_t *v, long i) 288 { 289 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 290 } 291 292 static __inline__ void atomic64_add(long a, atomic64_t *v) 293 { 294 long t; 295 296 __asm__ __volatile__( 297 "1: ldarx %0,0,%3 # atomic64_add\n\ 298 add %0,%2,%0\n\ 299 stdcx. %0,0,%3 \n\ 300 bne- 1b" 301 : "=&r" (t), "+m" (v->counter) 302 : "r" (a), "r" (&v->counter) 303 : "cc"); 304 } 305 306 static __inline__ long atomic64_add_return(long a, atomic64_t *v) 307 { 308 long t; 309 310 __asm__ __volatile__( 311 PPC_ATOMIC_ENTRY_BARRIER 312 "1: ldarx %0,0,%2 # atomic64_add_return\n\ 313 add %0,%1,%0\n\ 314 stdcx. %0,0,%2 \n\ 315 bne- 1b" 316 PPC_ATOMIC_EXIT_BARRIER 317 : "=&r" (t) 318 : "r" (a), "r" (&v->counter) 319 : "cc", "memory"); 320 321 return t; 322 } 323 324 #define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) 325 326 static __inline__ void atomic64_sub(long a, atomic64_t *v) 327 { 328 long t; 329 330 __asm__ __volatile__( 331 "1: ldarx %0,0,%3 # atomic64_sub\n\ 332 subf %0,%2,%0\n\ 333 stdcx. %0,0,%3 \n\ 334 bne- 1b" 335 : "=&r" (t), "+m" (v->counter) 336 : "r" (a), "r" (&v->counter) 337 : "cc"); 338 } 339 340 static __inline__ long atomic64_sub_return(long a, atomic64_t *v) 341 { 342 long t; 343 344 __asm__ __volatile__( 345 PPC_ATOMIC_ENTRY_BARRIER 346 "1: ldarx %0,0,%2 # atomic64_sub_return\n\ 347 subf %0,%1,%0\n\ 348 stdcx. %0,0,%2 \n\ 349 bne- 1b" 350 PPC_ATOMIC_EXIT_BARRIER 351 : "=&r" (t) 352 : "r" (a), "r" (&v->counter) 353 : "cc", "memory"); 354 355 return t; 356 } 357 358 static __inline__ void atomic64_inc(atomic64_t *v) 359 { 360 long t; 361 362 __asm__ __volatile__( 363 "1: ldarx %0,0,%2 # atomic64_inc\n\ 364 addic %0,%0,1\n\ 365 stdcx. %0,0,%2 \n\ 366 bne- 1b" 367 : "=&r" (t), "+m" (v->counter) 368 : "r" (&v->counter) 369 : "cc", "xer"); 370 } 371 372 static __inline__ long atomic64_inc_return(atomic64_t *v) 373 { 374 long t; 375 376 __asm__ __volatile__( 377 PPC_ATOMIC_ENTRY_BARRIER 378 "1: ldarx %0,0,%1 # atomic64_inc_return\n\ 379 addic %0,%0,1\n\ 380 stdcx. %0,0,%1 \n\ 381 bne- 1b" 382 PPC_ATOMIC_EXIT_BARRIER 383 : "=&r" (t) 384 : "r" (&v->counter) 385 : "cc", "xer", "memory"); 386 387 return t; 388 } 389 390 /* 391 * atomic64_inc_and_test - increment and test 392 * @v: pointer of type atomic64_t 393 * 394 * Atomically increments @v by 1 395 * and returns true if the result is zero, or false for all 396 * other cases. 397 */ 398 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) 399 400 static __inline__ void atomic64_dec(atomic64_t *v) 401 { 402 long t; 403 404 __asm__ __volatile__( 405 "1: ldarx %0,0,%2 # atomic64_dec\n\ 406 addic %0,%0,-1\n\ 407 stdcx. %0,0,%2\n\ 408 bne- 1b" 409 : "=&r" (t), "+m" (v->counter) 410 : "r" (&v->counter) 411 : "cc", "xer"); 412 } 413 414 static __inline__ long atomic64_dec_return(atomic64_t *v) 415 { 416 long t; 417 418 __asm__ __volatile__( 419 PPC_ATOMIC_ENTRY_BARRIER 420 "1: ldarx %0,0,%1 # atomic64_dec_return\n\ 421 addic %0,%0,-1\n\ 422 stdcx. %0,0,%1\n\ 423 bne- 1b" 424 PPC_ATOMIC_EXIT_BARRIER 425 : "=&r" (t) 426 : "r" (&v->counter) 427 : "cc", "xer", "memory"); 428 429 return t; 430 } 431 432 #define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0) 433 #define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) 434 435 /* 436 * Atomically test *v and decrement if it is greater than 0. 437 * The function returns the old value of *v minus 1. 438 */ 439 static __inline__ long atomic64_dec_if_positive(atomic64_t *v) 440 { 441 long t; 442 443 __asm__ __volatile__( 444 PPC_ATOMIC_ENTRY_BARRIER 445 "1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\ 446 addic. %0,%0,-1\n\ 447 blt- 2f\n\ 448 stdcx. %0,0,%1\n\ 449 bne- 1b" 450 PPC_ATOMIC_EXIT_BARRIER 451 "\n\ 452 2:" : "=&r" (t) 453 : "r" (&v->counter) 454 : "cc", "xer", "memory"); 455 456 return t; 457 } 458 459 #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 460 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) 461 462 /** 463 * atomic64_add_unless - add unless the number is a given value 464 * @v: pointer of type atomic64_t 465 * @a: the amount to add to v... 466 * @u: ...unless v is equal to u. 467 * 468 * Atomically adds @a to @v, so long as it was not @u. 469 * Returns the old value of @v. 470 */ 471 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) 472 { 473 long t; 474 475 __asm__ __volatile__ ( 476 PPC_ATOMIC_ENTRY_BARRIER 477 "1: ldarx %0,0,%1 # __atomic_add_unless\n\ 478 cmpd 0,%0,%3 \n\ 479 beq- 2f \n\ 480 add %0,%2,%0 \n" 481 " stdcx. %0,0,%1 \n\ 482 bne- 1b \n" 483 PPC_ATOMIC_EXIT_BARRIER 484 " subf %0,%2,%0 \n\ 485 2:" 486 : "=&r" (t) 487 : "r" (&v->counter), "r" (a), "r" (u) 488 : "cc", "memory"); 489 490 return t != u; 491 } 492 493 /** 494 * atomic_inc64_not_zero - increment unless the number is zero 495 * @v: pointer of type atomic64_t 496 * 497 * Atomically increments @v by 1, so long as @v is non-zero. 498 * Returns non-zero if @v was non-zero, and zero otherwise. 499 */ 500 static __inline__ long atomic64_inc_not_zero(atomic64_t *v) 501 { 502 long t1, t2; 503 504 __asm__ __volatile__ ( 505 PPC_ATOMIC_ENTRY_BARRIER 506 "1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\ 507 cmpdi 0,%0,0\n\ 508 beq- 2f\n\ 509 addic %1,%0,1\n\ 510 stdcx. %1,0,%2\n\ 511 bne- 1b\n" 512 PPC_ATOMIC_EXIT_BARRIER 513 "\n\ 514 2:" 515 : "=&r" (t1), "=&r" (t2) 516 : "r" (&v->counter) 517 : "cc", "xer", "memory"); 518 519 return t1; 520 } 521 522 #endif /* __powerpc64__ */ 523 524 #endif /* __KERNEL__ */ 525 #endif /* _ASM_POWERPC_ATOMIC_H_ */ 526