1 #ifndef _ASM_POWERPC_ATOMIC_H_ 2 #define _ASM_POWERPC_ATOMIC_H_ 3 4 /* 5 * PowerPC atomic operations 6 */ 7 8 #include <linux/types.h> 9 10 #ifdef __KERNEL__ 11 #include <linux/compiler.h> 12 #include <asm/synch.h> 13 #include <asm/asm-compat.h> 14 #include <asm/system.h> 15 16 #define ATOMIC_INIT(i) { (i) } 17 18 static __inline__ int atomic_read(const atomic_t *v) 19 { 20 int t; 21 22 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 23 24 return t; 25 } 26 27 static __inline__ void atomic_set(atomic_t *v, int i) 28 { 29 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 30 } 31 32 static __inline__ void atomic_add(int a, atomic_t *v) 33 { 34 int t; 35 36 __asm__ __volatile__( 37 "1: lwarx %0,0,%3 # atomic_add\n\ 38 add %0,%2,%0\n" 39 PPC405_ERR77(0,%3) 40 " stwcx. %0,0,%3 \n\ 41 bne- 1b" 42 : "=&r" (t), "+m" (v->counter) 43 : "r" (a), "r" (&v->counter) 44 : "cc"); 45 } 46 47 static __inline__ int atomic_add_return(int a, atomic_t *v) 48 { 49 int t; 50 51 __asm__ __volatile__( 52 PPC_RELEASE_BARRIER 53 "1: lwarx %0,0,%2 # atomic_add_return\n\ 54 add %0,%1,%0\n" 55 PPC405_ERR77(0,%2) 56 " stwcx. %0,0,%2 \n\ 57 bne- 1b" 58 PPC_ACQUIRE_BARRIER 59 : "=&r" (t) 60 : "r" (a), "r" (&v->counter) 61 : "cc", "memory"); 62 63 return t; 64 } 65 66 #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) 67 68 static __inline__ void atomic_sub(int a, atomic_t *v) 69 { 70 int t; 71 72 __asm__ __volatile__( 73 "1: lwarx %0,0,%3 # atomic_sub\n\ 74 subf %0,%2,%0\n" 75 PPC405_ERR77(0,%3) 76 " stwcx. %0,0,%3 \n\ 77 bne- 1b" 78 : "=&r" (t), "+m" (v->counter) 79 : "r" (a), "r" (&v->counter) 80 : "cc"); 81 } 82 83 static __inline__ int atomic_sub_return(int a, atomic_t *v) 84 { 85 int t; 86 87 __asm__ __volatile__( 88 PPC_RELEASE_BARRIER 89 "1: lwarx %0,0,%2 # atomic_sub_return\n\ 90 subf %0,%1,%0\n" 91 PPC405_ERR77(0,%2) 92 " stwcx. %0,0,%2 \n\ 93 bne- 1b" 94 PPC_ACQUIRE_BARRIER 95 : "=&r" (t) 96 : "r" (a), "r" (&v->counter) 97 : "cc", "memory"); 98 99 return t; 100 } 101 102 static __inline__ void atomic_inc(atomic_t *v) 103 { 104 int t; 105 106 __asm__ __volatile__( 107 "1: lwarx %0,0,%2 # atomic_inc\n\ 108 addic %0,%0,1\n" 109 PPC405_ERR77(0,%2) 110 " stwcx. %0,0,%2 \n\ 111 bne- 1b" 112 : "=&r" (t), "+m" (v->counter) 113 : "r" (&v->counter) 114 : "cc", "xer"); 115 } 116 117 static __inline__ int atomic_inc_return(atomic_t *v) 118 { 119 int t; 120 121 __asm__ __volatile__( 122 PPC_RELEASE_BARRIER 123 "1: lwarx %0,0,%1 # atomic_inc_return\n\ 124 addic %0,%0,1\n" 125 PPC405_ERR77(0,%1) 126 " stwcx. %0,0,%1 \n\ 127 bne- 1b" 128 PPC_ACQUIRE_BARRIER 129 : "=&r" (t) 130 : "r" (&v->counter) 131 : "cc", "xer", "memory"); 132 133 return t; 134 } 135 136 /* 137 * atomic_inc_and_test - increment and test 138 * @v: pointer of type atomic_t 139 * 140 * Atomically increments @v by 1 141 * and returns true if the result is zero, or false for all 142 * other cases. 143 */ 144 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) 145 146 static __inline__ void atomic_dec(atomic_t *v) 147 { 148 int t; 149 150 __asm__ __volatile__( 151 "1: lwarx %0,0,%2 # atomic_dec\n\ 152 addic %0,%0,-1\n" 153 PPC405_ERR77(0,%2)\ 154 " stwcx. %0,0,%2\n\ 155 bne- 1b" 156 : "=&r" (t), "+m" (v->counter) 157 : "r" (&v->counter) 158 : "cc", "xer"); 159 } 160 161 static __inline__ int atomic_dec_return(atomic_t *v) 162 { 163 int t; 164 165 __asm__ __volatile__( 166 PPC_RELEASE_BARRIER 167 "1: lwarx %0,0,%1 # atomic_dec_return\n\ 168 addic %0,%0,-1\n" 169 PPC405_ERR77(0,%1) 170 " stwcx. %0,0,%1\n\ 171 bne- 1b" 172 PPC_ACQUIRE_BARRIER 173 : "=&r" (t) 174 : "r" (&v->counter) 175 : "cc", "xer", "memory"); 176 177 return t; 178 } 179 180 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 181 #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 182 183 /** 184 * atomic_add_unless - add unless the number is a given value 185 * @v: pointer of type atomic_t 186 * @a: the amount to add to v... 187 * @u: ...unless v is equal to u. 188 * 189 * Atomically adds @a to @v, so long as it was not @u. 190 * Returns non-zero if @v was not @u, and zero otherwise. 191 */ 192 static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) 193 { 194 int t; 195 196 __asm__ __volatile__ ( 197 PPC_RELEASE_BARRIER 198 "1: lwarx %0,0,%1 # atomic_add_unless\n\ 199 cmpw 0,%0,%3 \n\ 200 beq- 2f \n\ 201 add %0,%2,%0 \n" 202 PPC405_ERR77(0,%2) 203 " stwcx. %0,0,%1 \n\ 204 bne- 1b \n" 205 PPC_ACQUIRE_BARRIER 206 " subf %0,%2,%0 \n\ 207 2:" 208 : "=&r" (t) 209 : "r" (&v->counter), "r" (a), "r" (u) 210 : "cc", "memory"); 211 212 return t != u; 213 } 214 215 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 216 217 #define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0) 218 #define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0) 219 220 /* 221 * Atomically test *v and decrement if it is greater than 0. 222 * The function returns the old value of *v minus 1, even if 223 * the atomic variable, v, was not decremented. 224 */ 225 static __inline__ int atomic_dec_if_positive(atomic_t *v) 226 { 227 int t; 228 229 __asm__ __volatile__( 230 PPC_RELEASE_BARRIER 231 "1: lwarx %0,0,%1 # atomic_dec_if_positive\n\ 232 cmpwi %0,1\n\ 233 addi %0,%0,-1\n\ 234 blt- 2f\n" 235 PPC405_ERR77(0,%1) 236 " stwcx. %0,0,%1\n\ 237 bne- 1b" 238 PPC_ACQUIRE_BARRIER 239 "\n\ 240 2:" : "=&b" (t) 241 : "r" (&v->counter) 242 : "cc", "memory"); 243 244 return t; 245 } 246 247 #define smp_mb__before_atomic_dec() smp_mb() 248 #define smp_mb__after_atomic_dec() smp_mb() 249 #define smp_mb__before_atomic_inc() smp_mb() 250 #define smp_mb__after_atomic_inc() smp_mb() 251 252 #ifdef __powerpc64__ 253 254 #define ATOMIC64_INIT(i) { (i) } 255 256 static __inline__ long atomic64_read(const atomic64_t *v) 257 { 258 long t; 259 260 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 261 262 return t; 263 } 264 265 static __inline__ void atomic64_set(atomic64_t *v, long i) 266 { 267 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 268 } 269 270 static __inline__ void atomic64_add(long a, atomic64_t *v) 271 { 272 long t; 273 274 __asm__ __volatile__( 275 "1: ldarx %0,0,%3 # atomic64_add\n\ 276 add %0,%2,%0\n\ 277 stdcx. %0,0,%3 \n\ 278 bne- 1b" 279 : "=&r" (t), "+m" (v->counter) 280 : "r" (a), "r" (&v->counter) 281 : "cc"); 282 } 283 284 static __inline__ long atomic64_add_return(long a, atomic64_t *v) 285 { 286 long t; 287 288 __asm__ __volatile__( 289 PPC_RELEASE_BARRIER 290 "1: ldarx %0,0,%2 # atomic64_add_return\n\ 291 add %0,%1,%0\n\ 292 stdcx. %0,0,%2 \n\ 293 bne- 1b" 294 PPC_ACQUIRE_BARRIER 295 : "=&r" (t) 296 : "r" (a), "r" (&v->counter) 297 : "cc", "memory"); 298 299 return t; 300 } 301 302 #define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) 303 304 static __inline__ void atomic64_sub(long a, atomic64_t *v) 305 { 306 long t; 307 308 __asm__ __volatile__( 309 "1: ldarx %0,0,%3 # atomic64_sub\n\ 310 subf %0,%2,%0\n\ 311 stdcx. %0,0,%3 \n\ 312 bne- 1b" 313 : "=&r" (t), "+m" (v->counter) 314 : "r" (a), "r" (&v->counter) 315 : "cc"); 316 } 317 318 static __inline__ long atomic64_sub_return(long a, atomic64_t *v) 319 { 320 long t; 321 322 __asm__ __volatile__( 323 PPC_RELEASE_BARRIER 324 "1: ldarx %0,0,%2 # atomic64_sub_return\n\ 325 subf %0,%1,%0\n\ 326 stdcx. %0,0,%2 \n\ 327 bne- 1b" 328 PPC_ACQUIRE_BARRIER 329 : "=&r" (t) 330 : "r" (a), "r" (&v->counter) 331 : "cc", "memory"); 332 333 return t; 334 } 335 336 static __inline__ void atomic64_inc(atomic64_t *v) 337 { 338 long t; 339 340 __asm__ __volatile__( 341 "1: ldarx %0,0,%2 # atomic64_inc\n\ 342 addic %0,%0,1\n\ 343 stdcx. %0,0,%2 \n\ 344 bne- 1b" 345 : "=&r" (t), "+m" (v->counter) 346 : "r" (&v->counter) 347 : "cc", "xer"); 348 } 349 350 static __inline__ long atomic64_inc_return(atomic64_t *v) 351 { 352 long t; 353 354 __asm__ __volatile__( 355 PPC_RELEASE_BARRIER 356 "1: ldarx %0,0,%1 # atomic64_inc_return\n\ 357 addic %0,%0,1\n\ 358 stdcx. %0,0,%1 \n\ 359 bne- 1b" 360 PPC_ACQUIRE_BARRIER 361 : "=&r" (t) 362 : "r" (&v->counter) 363 : "cc", "xer", "memory"); 364 365 return t; 366 } 367 368 /* 369 * atomic64_inc_and_test - increment and test 370 * @v: pointer of type atomic64_t 371 * 372 * Atomically increments @v by 1 373 * and returns true if the result is zero, or false for all 374 * other cases. 375 */ 376 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) 377 378 static __inline__ void atomic64_dec(atomic64_t *v) 379 { 380 long t; 381 382 __asm__ __volatile__( 383 "1: ldarx %0,0,%2 # atomic64_dec\n\ 384 addic %0,%0,-1\n\ 385 stdcx. %0,0,%2\n\ 386 bne- 1b" 387 : "=&r" (t), "+m" (v->counter) 388 : "r" (&v->counter) 389 : "cc", "xer"); 390 } 391 392 static __inline__ long atomic64_dec_return(atomic64_t *v) 393 { 394 long t; 395 396 __asm__ __volatile__( 397 PPC_RELEASE_BARRIER 398 "1: ldarx %0,0,%1 # atomic64_dec_return\n\ 399 addic %0,%0,-1\n\ 400 stdcx. %0,0,%1\n\ 401 bne- 1b" 402 PPC_ACQUIRE_BARRIER 403 : "=&r" (t) 404 : "r" (&v->counter) 405 : "cc", "xer", "memory"); 406 407 return t; 408 } 409 410 #define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0) 411 #define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) 412 413 /* 414 * Atomically test *v and decrement if it is greater than 0. 415 * The function returns the old value of *v minus 1. 416 */ 417 static __inline__ long atomic64_dec_if_positive(atomic64_t *v) 418 { 419 long t; 420 421 __asm__ __volatile__( 422 PPC_RELEASE_BARRIER 423 "1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\ 424 addic. %0,%0,-1\n\ 425 blt- 2f\n\ 426 stdcx. %0,0,%1\n\ 427 bne- 1b" 428 PPC_ACQUIRE_BARRIER 429 "\n\ 430 2:" : "=&r" (t) 431 : "r" (&v->counter) 432 : "cc", "xer", "memory"); 433 434 return t; 435 } 436 437 #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 438 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) 439 440 /** 441 * atomic64_add_unless - add unless the number is a given value 442 * @v: pointer of type atomic64_t 443 * @a: the amount to add to v... 444 * @u: ...unless v is equal to u. 445 * 446 * Atomically adds @a to @v, so long as it was not @u. 447 * Returns non-zero if @v was not @u, and zero otherwise. 448 */ 449 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) 450 { 451 long t; 452 453 __asm__ __volatile__ ( 454 PPC_RELEASE_BARRIER 455 "1: ldarx %0,0,%1 # atomic_add_unless\n\ 456 cmpd 0,%0,%3 \n\ 457 beq- 2f \n\ 458 add %0,%2,%0 \n" 459 " stdcx. %0,0,%1 \n\ 460 bne- 1b \n" 461 PPC_ACQUIRE_BARRIER 462 " subf %0,%2,%0 \n\ 463 2:" 464 : "=&r" (t) 465 : "r" (&v->counter), "r" (a), "r" (u) 466 : "cc", "memory"); 467 468 return t != u; 469 } 470 471 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) 472 473 #else /* __powerpc64__ */ 474 #include <asm-generic/atomic64.h> 475 476 #endif /* __powerpc64__ */ 477 478 #include <asm-generic/atomic-long.h> 479 #endif /* __KERNEL__ */ 480 #endif /* _ASM_POWERPC_ATOMIC_H_ */ 481