1 /* 2 * Atomic operations that C can't guarantee us. Useful for 3 * resource counting etc.. 4 * 5 * But use these as seldom as possible since they are much more slower 6 * than regular operations. 7 * 8 * This file is subject to the terms and conditions of the GNU General Public 9 * License. See the file "COPYING" in the main directory of this archive 10 * for more details. 11 * 12 * Copyright (C) 1996, 97, 99, 2000, 03, 04, 06 by Ralf Baechle 13 */ 14 #ifndef _ASM_ATOMIC_H 15 #define _ASM_ATOMIC_H 16 17 #include <linux/irqflags.h> 18 #include <linux/types.h> 19 #include <asm/barrier.h> 20 #include <asm/compiler.h> 21 #include <asm/cpu-features.h> 22 #include <asm/cmpxchg.h> 23 #include <asm/war.h> 24 25 #define ATOMIC_INIT(i) { (i) } 26 27 /* 28 * atomic_read - read atomic variable 29 * @v: pointer of type atomic_t 30 * 31 * Atomically reads the value of @v. 32 */ 33 #define atomic_read(v) ACCESS_ONCE((v)->counter) 34 35 /* 36 * atomic_set - set atomic variable 37 * @v: pointer of type atomic_t 38 * @i: required value 39 * 40 * Atomically sets the value of @v to @i. 41 */ 42 #define atomic_set(v, i) ((v)->counter = (i)) 43 44 #define ATOMIC_OP(op, c_op, asm_op) \ 45 static __inline__ void atomic_##op(int i, atomic_t * v) \ 46 { \ 47 if (kernel_uses_llsc && R10000_LLSC_WAR) { \ 48 int temp; \ 49 \ 50 __asm__ __volatile__( \ 51 " .set arch=r4000 \n" \ 52 "1: ll %0, %1 # atomic_" #op " \n" \ 53 " " #asm_op " %0, %2 \n" \ 54 " sc %0, %1 \n" \ 55 " beqzl %0, 1b \n" \ 56 " .set mips0 \n" \ 57 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ 58 : "Ir" (i)); \ 59 } else if (kernel_uses_llsc) { \ 60 int temp; \ 61 \ 62 do { \ 63 __asm__ __volatile__( \ 64 " .set "MIPS_ISA_LEVEL" \n" \ 65 " ll %0, %1 # atomic_" #op "\n" \ 66 " " #asm_op " %0, %2 \n" \ 67 " sc %0, %1 \n" \ 68 " .set mips0 \n" \ 69 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ 70 : "Ir" (i)); \ 71 } while (unlikely(!temp)); \ 72 } else { \ 73 unsigned long flags; \ 74 \ 75 raw_local_irq_save(flags); \ 76 v->counter c_op i; \ 77 raw_local_irq_restore(flags); \ 78 } \ 79 } 80 81 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 82 static __inline__ int atomic_##op##_return(int i, atomic_t * v) \ 83 { \ 84 int result; \ 85 \ 86 smp_mb__before_llsc(); \ 87 \ 88 if (kernel_uses_llsc && R10000_LLSC_WAR) { \ 89 int temp; \ 90 \ 91 __asm__ __volatile__( \ 92 " .set arch=r4000 \n" \ 93 "1: ll %1, %2 # atomic_" #op "_return \n" \ 94 " " #asm_op " %0, %1, %3 \n" \ 95 " sc %0, %2 \n" \ 96 " beqzl %0, 1b \n" \ 97 " " #asm_op " %0, %1, %3 \n" \ 98 " .set mips0 \n" \ 99 : "=&r" (result), "=&r" (temp), \ 100 "+" GCC_OFF_SMALL_ASM() (v->counter) \ 101 : "Ir" (i)); \ 102 } else if (kernel_uses_llsc) { \ 103 int temp; \ 104 \ 105 do { \ 106 __asm__ __volatile__( \ 107 " .set "MIPS_ISA_LEVEL" \n" \ 108 " ll %1, %2 # atomic_" #op "_return \n" \ 109 " " #asm_op " %0, %1, %3 \n" \ 110 " sc %0, %2 \n" \ 111 " .set mips0 \n" \ 112 : "=&r" (result), "=&r" (temp), \ 113 "+" GCC_OFF_SMALL_ASM() (v->counter) \ 114 : "Ir" (i)); \ 115 } while (unlikely(!result)); \ 116 \ 117 result = temp; result c_op i; \ 118 } else { \ 119 unsigned long flags; \ 120 \ 121 raw_local_irq_save(flags); \ 122 result = v->counter; \ 123 result c_op i; \ 124 v->counter = result; \ 125 raw_local_irq_restore(flags); \ 126 } \ 127 \ 128 smp_llsc_mb(); \ 129 \ 130 return result; \ 131 } 132 133 #define ATOMIC_OPS(op, c_op, asm_op) \ 134 ATOMIC_OP(op, c_op, asm_op) \ 135 ATOMIC_OP_RETURN(op, c_op, asm_op) 136 137 ATOMIC_OPS(add, +=, addu) 138 ATOMIC_OPS(sub, -=, subu) 139 140 #undef ATOMIC_OPS 141 #undef ATOMIC_OP_RETURN 142 #undef ATOMIC_OP 143 144 /* 145 * atomic_sub_if_positive - conditionally subtract integer from atomic variable 146 * @i: integer value to subtract 147 * @v: pointer of type atomic_t 148 * 149 * Atomically test @v and subtract @i if @v is greater or equal than @i. 150 * The function returns the old value of @v minus @i. 151 */ 152 static __inline__ int atomic_sub_if_positive(int i, atomic_t * v) 153 { 154 int result; 155 156 smp_mb__before_llsc(); 157 158 if (kernel_uses_llsc && R10000_LLSC_WAR) { 159 int temp; 160 161 __asm__ __volatile__( 162 " .set arch=r4000 \n" 163 "1: ll %1, %2 # atomic_sub_if_positive\n" 164 " subu %0, %1, %3 \n" 165 " bltz %0, 1f \n" 166 " sc %0, %2 \n" 167 " .set noreorder \n" 168 " beqzl %0, 1b \n" 169 " subu %0, %1, %3 \n" 170 " .set reorder \n" 171 "1: \n" 172 " .set mips0 \n" 173 : "=&r" (result), "=&r" (temp), 174 "+" GCC_OFF_SMALL_ASM() (v->counter) 175 : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter) 176 : "memory"); 177 } else if (kernel_uses_llsc) { 178 int temp; 179 180 __asm__ __volatile__( 181 " .set "MIPS_ISA_LEVEL" \n" 182 "1: ll %1, %2 # atomic_sub_if_positive\n" 183 " subu %0, %1, %3 \n" 184 " bltz %0, 1f \n" 185 " sc %0, %2 \n" 186 " .set noreorder \n" 187 " beqz %0, 1b \n" 188 " subu %0, %1, %3 \n" 189 " .set reorder \n" 190 "1: \n" 191 " .set mips0 \n" 192 : "=&r" (result), "=&r" (temp), 193 "+" GCC_OFF_SMALL_ASM() (v->counter) 194 : "Ir" (i)); 195 } else { 196 unsigned long flags; 197 198 raw_local_irq_save(flags); 199 result = v->counter; 200 result -= i; 201 if (result >= 0) 202 v->counter = result; 203 raw_local_irq_restore(flags); 204 } 205 206 smp_llsc_mb(); 207 208 return result; 209 } 210 211 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 212 #define atomic_xchg(v, new) (xchg(&((v)->counter), (new))) 213 214 /** 215 * __atomic_add_unless - add unless the number is a given value 216 * @v: pointer of type atomic_t 217 * @a: the amount to add to v... 218 * @u: ...unless v is equal to u. 219 * 220 * Atomically adds @a to @v, so long as it was not @u. 221 * Returns the old value of @v. 222 */ 223 static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) 224 { 225 int c, old; 226 c = atomic_read(v); 227 for (;;) { 228 if (unlikely(c == (u))) 229 break; 230 old = atomic_cmpxchg((v), c, c + (a)); 231 if (likely(old == c)) 232 break; 233 c = old; 234 } 235 return c; 236 } 237 238 #define atomic_dec_return(v) atomic_sub_return(1, (v)) 239 #define atomic_inc_return(v) atomic_add_return(1, (v)) 240 241 /* 242 * atomic_sub_and_test - subtract value from variable and test result 243 * @i: integer value to subtract 244 * @v: pointer of type atomic_t 245 * 246 * Atomically subtracts @i from @v and returns 247 * true if the result is zero, or false for all 248 * other cases. 249 */ 250 #define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0) 251 252 /* 253 * atomic_inc_and_test - increment and test 254 * @v: pointer of type atomic_t 255 * 256 * Atomically increments @v by 1 257 * and returns true if the result is zero, or false for all 258 * other cases. 259 */ 260 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) 261 262 /* 263 * atomic_dec_and_test - decrement by 1 and test 264 * @v: pointer of type atomic_t 265 * 266 * Atomically decrements @v by 1 and 267 * returns true if the result is 0, or false for all other 268 * cases. 269 */ 270 #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) 271 272 /* 273 * atomic_dec_if_positive - decrement by 1 if old value positive 274 * @v: pointer of type atomic_t 275 */ 276 #define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v) 277 278 /* 279 * atomic_inc - increment atomic variable 280 * @v: pointer of type atomic_t 281 * 282 * Atomically increments @v by 1. 283 */ 284 #define atomic_inc(v) atomic_add(1, (v)) 285 286 /* 287 * atomic_dec - decrement and test 288 * @v: pointer of type atomic_t 289 * 290 * Atomically decrements @v by 1. 291 */ 292 #define atomic_dec(v) atomic_sub(1, (v)) 293 294 /* 295 * atomic_add_negative - add and test if negative 296 * @v: pointer of type atomic_t 297 * @i: integer value to add 298 * 299 * Atomically adds @i to @v and returns true 300 * if the result is negative, or false when 301 * result is greater than or equal to zero. 302 */ 303 #define atomic_add_negative(i, v) (atomic_add_return(i, (v)) < 0) 304 305 #ifdef CONFIG_64BIT 306 307 #define ATOMIC64_INIT(i) { (i) } 308 309 /* 310 * atomic64_read - read atomic variable 311 * @v: pointer of type atomic64_t 312 * 313 */ 314 #define atomic64_read(v) ACCESS_ONCE((v)->counter) 315 316 /* 317 * atomic64_set - set atomic variable 318 * @v: pointer of type atomic64_t 319 * @i: required value 320 */ 321 #define atomic64_set(v, i) ((v)->counter = (i)) 322 323 #define ATOMIC64_OP(op, c_op, asm_op) \ 324 static __inline__ void atomic64_##op(long i, atomic64_t * v) \ 325 { \ 326 if (kernel_uses_llsc && R10000_LLSC_WAR) { \ 327 long temp; \ 328 \ 329 __asm__ __volatile__( \ 330 " .set arch=r4000 \n" \ 331 "1: lld %0, %1 # atomic64_" #op " \n" \ 332 " " #asm_op " %0, %2 \n" \ 333 " scd %0, %1 \n" \ 334 " beqzl %0, 1b \n" \ 335 " .set mips0 \n" \ 336 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ 337 : "Ir" (i)); \ 338 } else if (kernel_uses_llsc) { \ 339 long temp; \ 340 \ 341 do { \ 342 __asm__ __volatile__( \ 343 " .set "MIPS_ISA_LEVEL" \n" \ 344 " lld %0, %1 # atomic64_" #op "\n" \ 345 " " #asm_op " %0, %2 \n" \ 346 " scd %0, %1 \n" \ 347 " .set mips0 \n" \ 348 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (v->counter) \ 349 : "Ir" (i)); \ 350 } while (unlikely(!temp)); \ 351 } else { \ 352 unsigned long flags; \ 353 \ 354 raw_local_irq_save(flags); \ 355 v->counter c_op i; \ 356 raw_local_irq_restore(flags); \ 357 } \ 358 } 359 360 #define ATOMIC64_OP_RETURN(op, c_op, asm_op) \ 361 static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ 362 { \ 363 long result; \ 364 \ 365 smp_mb__before_llsc(); \ 366 \ 367 if (kernel_uses_llsc && R10000_LLSC_WAR) { \ 368 long temp; \ 369 \ 370 __asm__ __volatile__( \ 371 " .set arch=r4000 \n" \ 372 "1: lld %1, %2 # atomic64_" #op "_return\n" \ 373 " " #asm_op " %0, %1, %3 \n" \ 374 " scd %0, %2 \n" \ 375 " beqzl %0, 1b \n" \ 376 " " #asm_op " %0, %1, %3 \n" \ 377 " .set mips0 \n" \ 378 : "=&r" (result), "=&r" (temp), \ 379 "+" GCC_OFF_SMALL_ASM() (v->counter) \ 380 : "Ir" (i)); \ 381 } else if (kernel_uses_llsc) { \ 382 long temp; \ 383 \ 384 do { \ 385 __asm__ __volatile__( \ 386 " .set "MIPS_ISA_LEVEL" \n" \ 387 " lld %1, %2 # atomic64_" #op "_return\n" \ 388 " " #asm_op " %0, %1, %3 \n" \ 389 " scd %0, %2 \n" \ 390 " .set mips0 \n" \ 391 : "=&r" (result), "=&r" (temp), \ 392 "=" GCC_OFF_SMALL_ASM() (v->counter) \ 393 : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter) \ 394 : "memory"); \ 395 } while (unlikely(!result)); \ 396 \ 397 result = temp; result c_op i; \ 398 } else { \ 399 unsigned long flags; \ 400 \ 401 raw_local_irq_save(flags); \ 402 result = v->counter; \ 403 result c_op i; \ 404 v->counter = result; \ 405 raw_local_irq_restore(flags); \ 406 } \ 407 \ 408 smp_llsc_mb(); \ 409 \ 410 return result; \ 411 } 412 413 #define ATOMIC64_OPS(op, c_op, asm_op) \ 414 ATOMIC64_OP(op, c_op, asm_op) \ 415 ATOMIC64_OP_RETURN(op, c_op, asm_op) 416 417 ATOMIC64_OPS(add, +=, daddu) 418 ATOMIC64_OPS(sub, -=, dsubu) 419 420 #undef ATOMIC64_OPS 421 #undef ATOMIC64_OP_RETURN 422 #undef ATOMIC64_OP 423 424 /* 425 * atomic64_sub_if_positive - conditionally subtract integer from atomic 426 * variable 427 * @i: integer value to subtract 428 * @v: pointer of type atomic64_t 429 * 430 * Atomically test @v and subtract @i if @v is greater or equal than @i. 431 * The function returns the old value of @v minus @i. 432 */ 433 static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v) 434 { 435 long result; 436 437 smp_mb__before_llsc(); 438 439 if (kernel_uses_llsc && R10000_LLSC_WAR) { 440 long temp; 441 442 __asm__ __volatile__( 443 " .set arch=r4000 \n" 444 "1: lld %1, %2 # atomic64_sub_if_positive\n" 445 " dsubu %0, %1, %3 \n" 446 " bltz %0, 1f \n" 447 " scd %0, %2 \n" 448 " .set noreorder \n" 449 " beqzl %0, 1b \n" 450 " dsubu %0, %1, %3 \n" 451 " .set reorder \n" 452 "1: \n" 453 " .set mips0 \n" 454 : "=&r" (result), "=&r" (temp), 455 "=" GCC_OFF_SMALL_ASM() (v->counter) 456 : "Ir" (i), GCC_OFF_SMALL_ASM() (v->counter) 457 : "memory"); 458 } else if (kernel_uses_llsc) { 459 long temp; 460 461 __asm__ __volatile__( 462 " .set "MIPS_ISA_LEVEL" \n" 463 "1: lld %1, %2 # atomic64_sub_if_positive\n" 464 " dsubu %0, %1, %3 \n" 465 " bltz %0, 1f \n" 466 " scd %0, %2 \n" 467 " .set noreorder \n" 468 " beqz %0, 1b \n" 469 " dsubu %0, %1, %3 \n" 470 " .set reorder \n" 471 "1: \n" 472 " .set mips0 \n" 473 : "=&r" (result), "=&r" (temp), 474 "+" GCC_OFF_SMALL_ASM() (v->counter) 475 : "Ir" (i)); 476 } else { 477 unsigned long flags; 478 479 raw_local_irq_save(flags); 480 result = v->counter; 481 result -= i; 482 if (result >= 0) 483 v->counter = result; 484 raw_local_irq_restore(flags); 485 } 486 487 smp_llsc_mb(); 488 489 return result; 490 } 491 492 #define atomic64_cmpxchg(v, o, n) \ 493 ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) 494 #define atomic64_xchg(v, new) (xchg(&((v)->counter), (new))) 495 496 /** 497 * atomic64_add_unless - add unless the number is a given value 498 * @v: pointer of type atomic64_t 499 * @a: the amount to add to v... 500 * @u: ...unless v is equal to u. 501 * 502 * Atomically adds @a to @v, so long as it was not @u. 503 * Returns the old value of @v. 504 */ 505 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) 506 { 507 long c, old; 508 c = atomic64_read(v); 509 for (;;) { 510 if (unlikely(c == (u))) 511 break; 512 old = atomic64_cmpxchg((v), c, c + (a)); 513 if (likely(old == c)) 514 break; 515 c = old; 516 } 517 return c != (u); 518 } 519 520 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) 521 522 #define atomic64_dec_return(v) atomic64_sub_return(1, (v)) 523 #define atomic64_inc_return(v) atomic64_add_return(1, (v)) 524 525 /* 526 * atomic64_sub_and_test - subtract value from variable and test result 527 * @i: integer value to subtract 528 * @v: pointer of type atomic64_t 529 * 530 * Atomically subtracts @i from @v and returns 531 * true if the result is zero, or false for all 532 * other cases. 533 */ 534 #define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0) 535 536 /* 537 * atomic64_inc_and_test - increment and test 538 * @v: pointer of type atomic64_t 539 * 540 * Atomically increments @v by 1 541 * and returns true if the result is zero, or false for all 542 * other cases. 543 */ 544 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) 545 546 /* 547 * atomic64_dec_and_test - decrement by 1 and test 548 * @v: pointer of type atomic64_t 549 * 550 * Atomically decrements @v by 1 and 551 * returns true if the result is 0, or false for all other 552 * cases. 553 */ 554 #define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0) 555 556 /* 557 * atomic64_dec_if_positive - decrement by 1 if old value positive 558 * @v: pointer of type atomic64_t 559 */ 560 #define atomic64_dec_if_positive(v) atomic64_sub_if_positive(1, v) 561 562 /* 563 * atomic64_inc - increment atomic variable 564 * @v: pointer of type atomic64_t 565 * 566 * Atomically increments @v by 1. 567 */ 568 #define atomic64_inc(v) atomic64_add(1, (v)) 569 570 /* 571 * atomic64_dec - decrement and test 572 * @v: pointer of type atomic64_t 573 * 574 * Atomically decrements @v by 1. 575 */ 576 #define atomic64_dec(v) atomic64_sub(1, (v)) 577 578 /* 579 * atomic64_add_negative - add and test if negative 580 * @v: pointer of type atomic64_t 581 * @i: integer value to add 582 * 583 * Atomically adds @i to @v and returns true 584 * if the result is negative, or false when 585 * result is greater than or equal to zero. 586 */ 587 #define atomic64_add_negative(i, v) (atomic64_add_return(i, (v)) < 0) 588 589 #endif /* CONFIG_64BIT */ 590 591 #endif /* _ASM_ATOMIC_H */ 592