1 /* 2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #ifndef _ASM_ARC_ATOMIC_H 10 #define _ASM_ARC_ATOMIC_H 11 12 #ifndef __ASSEMBLY__ 13 14 #include <linux/types.h> 15 #include <linux/compiler.h> 16 #include <asm/cmpxchg.h> 17 #include <asm/barrier.h> 18 #include <asm/smp.h> 19 20 #define ATOMIC_INIT(i) { (i) } 21 22 #ifndef CONFIG_ARC_PLAT_EZNPS 23 24 #define atomic_read(v) READ_ONCE((v)->counter) 25 26 #ifdef CONFIG_ARC_HAS_LLSC 27 28 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) 29 30 #define ATOMIC_OP(op, c_op, asm_op) \ 31 static inline void atomic_##op(int i, atomic_t *v) \ 32 { \ 33 unsigned int val; \ 34 \ 35 __asm__ __volatile__( \ 36 "1: llock %[val], [%[ctr]] \n" \ 37 " " #asm_op " %[val], %[val], %[i] \n" \ 38 " scond %[val], [%[ctr]] \n" \ 39 " bnz 1b \n" \ 40 : [val] "=&r" (val) /* Early clobber to prevent reg reuse */ \ 41 : [ctr] "r" (&v->counter), /* Not "m": llock only supports reg direct addr mode */ \ 42 [i] "ir" (i) \ 43 : "cc"); \ 44 } \ 45 46 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 47 static inline int atomic_##op##_return(int i, atomic_t *v) \ 48 { \ 49 unsigned int val; \ 50 \ 51 /* \ 52 * Explicit full memory barrier needed before/after as \ 53 * LLOCK/SCOND thmeselves don't provide any such semantics \ 54 */ \ 55 smp_mb(); \ 56 \ 57 __asm__ __volatile__( \ 58 "1: llock %[val], [%[ctr]] \n" \ 59 " " #asm_op " %[val], %[val], %[i] \n" \ 60 " scond %[val], [%[ctr]] \n" \ 61 " bnz 1b \n" \ 62 : [val] "=&r" (val) \ 63 : [ctr] "r" (&v->counter), \ 64 [i] "ir" (i) \ 65 : "cc"); \ 66 \ 67 smp_mb(); \ 68 \ 69 return val; \ 70 } 71 72 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 73 static inline int atomic_fetch_##op(int i, atomic_t *v) \ 74 { \ 75 unsigned int val, orig; \ 76 \ 77 /* \ 78 * Explicit full memory barrier needed before/after as \ 79 * LLOCK/SCOND thmeselves don't provide any such semantics \ 80 */ \ 81 smp_mb(); \ 82 \ 83 __asm__ __volatile__( \ 84 "1: llock %[orig], [%[ctr]] \n" \ 85 " " #asm_op " %[val], %[orig], %[i] \n" \ 86 " scond %[val], [%[ctr]] \n" \ 87 " \n" \ 88 : [val] "=&r" (val), \ 89 [orig] "=&r" (orig) \ 90 : [ctr] "r" (&v->counter), \ 91 [i] "ir" (i) \ 92 : "cc"); \ 93 \ 94 smp_mb(); \ 95 \ 96 return orig; \ 97 } 98 99 #else /* !CONFIG_ARC_HAS_LLSC */ 100 101 #ifndef CONFIG_SMP 102 103 /* violating atomic_xxx API locking protocol in UP for optimization sake */ 104 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) 105 106 #else 107 108 static inline void atomic_set(atomic_t *v, int i) 109 { 110 /* 111 * Independent of hardware support, all of the atomic_xxx() APIs need 112 * to follow the same locking rules to make sure that a "hardware" 113 * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn 114 * sequence 115 * 116 * Thus atomic_set() despite being 1 insn (and seemingly atomic) 117 * requires the locking. 118 */ 119 unsigned long flags; 120 121 atomic_ops_lock(flags); 122 WRITE_ONCE(v->counter, i); 123 atomic_ops_unlock(flags); 124 } 125 126 #define atomic_set_release(v, i) atomic_set((v), (i)) 127 128 #endif 129 130 /* 131 * Non hardware assisted Atomic-R-M-W 132 * Locking would change to irq-disabling only (UP) and spinlocks (SMP) 133 */ 134 135 #define ATOMIC_OP(op, c_op, asm_op) \ 136 static inline void atomic_##op(int i, atomic_t *v) \ 137 { \ 138 unsigned long flags; \ 139 \ 140 atomic_ops_lock(flags); \ 141 v->counter c_op i; \ 142 atomic_ops_unlock(flags); \ 143 } 144 145 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 146 static inline int atomic_##op##_return(int i, atomic_t *v) \ 147 { \ 148 unsigned long flags; \ 149 unsigned long temp; \ 150 \ 151 /* \ 152 * spin lock/unlock provides the needed smp_mb() before/after \ 153 */ \ 154 atomic_ops_lock(flags); \ 155 temp = v->counter; \ 156 temp c_op i; \ 157 v->counter = temp; \ 158 atomic_ops_unlock(flags); \ 159 \ 160 return temp; \ 161 } 162 163 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 164 static inline int atomic_fetch_##op(int i, atomic_t *v) \ 165 { \ 166 unsigned long flags; \ 167 unsigned long orig; \ 168 \ 169 /* \ 170 * spin lock/unlock provides the needed smp_mb() before/after \ 171 */ \ 172 atomic_ops_lock(flags); \ 173 orig = v->counter; \ 174 v->counter c_op i; \ 175 atomic_ops_unlock(flags); \ 176 \ 177 return orig; \ 178 } 179 180 #endif /* !CONFIG_ARC_HAS_LLSC */ 181 182 #define ATOMIC_OPS(op, c_op, asm_op) \ 183 ATOMIC_OP(op, c_op, asm_op) \ 184 ATOMIC_OP_RETURN(op, c_op, asm_op) \ 185 ATOMIC_FETCH_OP(op, c_op, asm_op) 186 187 ATOMIC_OPS(add, +=, add) 188 ATOMIC_OPS(sub, -=, sub) 189 190 #define atomic_andnot atomic_andnot 191 192 #undef ATOMIC_OPS 193 #define ATOMIC_OPS(op, c_op, asm_op) \ 194 ATOMIC_OP(op, c_op, asm_op) \ 195 ATOMIC_FETCH_OP(op, c_op, asm_op) 196 197 ATOMIC_OPS(and, &=, and) 198 ATOMIC_OPS(andnot, &= ~, bic) 199 ATOMIC_OPS(or, |=, or) 200 ATOMIC_OPS(xor, ^=, xor) 201 202 #else /* CONFIG_ARC_PLAT_EZNPS */ 203 204 static inline int atomic_read(const atomic_t *v) 205 { 206 int temp; 207 208 __asm__ __volatile__( 209 " ld.di %0, [%1]" 210 : "=r"(temp) 211 : "r"(&v->counter) 212 : "memory"); 213 return temp; 214 } 215 216 static inline void atomic_set(atomic_t *v, int i) 217 { 218 __asm__ __volatile__( 219 " st.di %0,[%1]" 220 : 221 : "r"(i), "r"(&v->counter) 222 : "memory"); 223 } 224 225 #define ATOMIC_OP(op, c_op, asm_op) \ 226 static inline void atomic_##op(int i, atomic_t *v) \ 227 { \ 228 __asm__ __volatile__( \ 229 " mov r2, %0\n" \ 230 " mov r3, %1\n" \ 231 " .word %2\n" \ 232 : \ 233 : "r"(i), "r"(&v->counter), "i"(asm_op) \ 234 : "r2", "r3", "memory"); \ 235 } \ 236 237 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 238 static inline int atomic_##op##_return(int i, atomic_t *v) \ 239 { \ 240 unsigned int temp = i; \ 241 \ 242 /* Explicit full memory barrier needed before/after */ \ 243 smp_mb(); \ 244 \ 245 __asm__ __volatile__( \ 246 " mov r2, %0\n" \ 247 " mov r3, %1\n" \ 248 " .word %2\n" \ 249 " mov %0, r2" \ 250 : "+r"(temp) \ 251 : "r"(&v->counter), "i"(asm_op) \ 252 : "r2", "r3", "memory"); \ 253 \ 254 smp_mb(); \ 255 \ 256 temp c_op i; \ 257 \ 258 return temp; \ 259 } 260 261 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 262 static inline int atomic_fetch_##op(int i, atomic_t *v) \ 263 { \ 264 unsigned int temp = i; \ 265 \ 266 /* Explicit full memory barrier needed before/after */ \ 267 smp_mb(); \ 268 \ 269 __asm__ __volatile__( \ 270 " mov r2, %0\n" \ 271 " mov r3, %1\n" \ 272 " .word %2\n" \ 273 " mov %0, r2" \ 274 : "+r"(temp) \ 275 : "r"(&v->counter), "i"(asm_op) \ 276 : "r2", "r3", "memory"); \ 277 \ 278 smp_mb(); \ 279 \ 280 return temp; \ 281 } 282 283 #define ATOMIC_OPS(op, c_op, asm_op) \ 284 ATOMIC_OP(op, c_op, asm_op) \ 285 ATOMIC_OP_RETURN(op, c_op, asm_op) \ 286 ATOMIC_FETCH_OP(op, c_op, asm_op) 287 288 ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3) 289 #define atomic_sub(i, v) atomic_add(-(i), (v)) 290 #define atomic_sub_return(i, v) atomic_add_return(-(i), (v)) 291 #define atomic_fetch_sub(i, v) atomic_fetch_add(-(i), (v)) 292 293 #undef ATOMIC_OPS 294 #define ATOMIC_OPS(op, c_op, asm_op) \ 295 ATOMIC_OP(op, c_op, asm_op) \ 296 ATOMIC_FETCH_OP(op, c_op, asm_op) 297 298 ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3) 299 #define atomic_andnot(mask, v) atomic_and(~(mask), (v)) 300 #define atomic_fetch_andnot(mask, v) atomic_fetch_and(~(mask), (v)) 301 ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3) 302 ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3) 303 304 #endif /* CONFIG_ARC_PLAT_EZNPS */ 305 306 #undef ATOMIC_OPS 307 #undef ATOMIC_FETCH_OP 308 #undef ATOMIC_OP_RETURN 309 #undef ATOMIC_OP 310 311 /** 312 * __atomic_add_unless - add unless the number is a given value 313 * @v: pointer of type atomic_t 314 * @a: the amount to add to v... 315 * @u: ...unless v is equal to u. 316 * 317 * Atomically adds @a to @v, so long as it was not @u. 318 * Returns the old value of @v 319 */ 320 #define __atomic_add_unless(v, a, u) \ 321 ({ \ 322 int c, old; \ 323 \ 324 /* \ 325 * Explicit full memory barrier needed before/after as \ 326 * LLOCK/SCOND thmeselves don't provide any such semantics \ 327 */ \ 328 smp_mb(); \ 329 \ 330 c = atomic_read(v); \ 331 while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c)\ 332 c = old; \ 333 \ 334 smp_mb(); \ 335 \ 336 c; \ 337 }) 338 339 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 340 341 #define atomic_inc(v) atomic_add(1, v) 342 #define atomic_dec(v) atomic_sub(1, v) 343 344 #define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0) 345 #define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0) 346 #define atomic_inc_return(v) atomic_add_return(1, (v)) 347 #define atomic_dec_return(v) atomic_sub_return(1, (v)) 348 #define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) 349 350 #define atomic_add_negative(i, v) (atomic_add_return(i, v) < 0) 351 352 353 #ifdef CONFIG_GENERIC_ATOMIC64 354 355 #include <asm-generic/atomic64.h> 356 357 #else /* Kconfig ensures this is only enabled with needed h/w assist */ 358 359 /* 360 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD) 361 * - The address HAS to be 64-bit aligned 362 * - There are 2 semantics involved here: 363 * = exclusive implies no interim update between load/store to same addr 364 * = both words are observed/updated together: this is guaranteed even 365 * for regular 64-bit load (LDD) / store (STD). Thus atomic64_set() 366 * is NOT required to use LLOCKD+SCONDD, STD suffices 367 */ 368 369 typedef struct { 370 aligned_u64 counter; 371 } atomic64_t; 372 373 #define ATOMIC64_INIT(a) { (a) } 374 375 static inline long long atomic64_read(const atomic64_t *v) 376 { 377 unsigned long long val; 378 379 __asm__ __volatile__( 380 " ldd %0, [%1] \n" 381 : "=r"(val) 382 : "r"(&v->counter)); 383 384 return val; 385 } 386 387 static inline void atomic64_set(atomic64_t *v, long long a) 388 { 389 /* 390 * This could have been a simple assignment in "C" but would need 391 * explicit volatile. Otherwise gcc optimizers could elide the store 392 * which borked atomic64 self-test 393 * In the inline asm version, memory clobber needed for exact same 394 * reason, to tell gcc about the store. 395 * 396 * This however is not needed for sibling atomic64_add() etc since both 397 * load/store are explicitly done in inline asm. As long as API is used 398 * for each access, gcc has no way to optimize away any load/store 399 */ 400 __asm__ __volatile__( 401 " std %0, [%1] \n" 402 : 403 : "r"(a), "r"(&v->counter) 404 : "memory"); 405 } 406 407 #define ATOMIC64_OP(op, op1, op2) \ 408 static inline void atomic64_##op(long long a, atomic64_t *v) \ 409 { \ 410 unsigned long long val; \ 411 \ 412 __asm__ __volatile__( \ 413 "1: \n" \ 414 " llockd %0, [%1] \n" \ 415 " " #op1 " %L0, %L0, %L2 \n" \ 416 " " #op2 " %H0, %H0, %H2 \n" \ 417 " scondd %0, [%1] \n" \ 418 " bnz 1b \n" \ 419 : "=&r"(val) \ 420 : "r"(&v->counter), "ir"(a) \ 421 : "cc"); \ 422 } \ 423 424 #define ATOMIC64_OP_RETURN(op, op1, op2) \ 425 static inline long long atomic64_##op##_return(long long a, atomic64_t *v) \ 426 { \ 427 unsigned long long val; \ 428 \ 429 smp_mb(); \ 430 \ 431 __asm__ __volatile__( \ 432 "1: \n" \ 433 " llockd %0, [%1] \n" \ 434 " " #op1 " %L0, %L0, %L2 \n" \ 435 " " #op2 " %H0, %H0, %H2 \n" \ 436 " scondd %0, [%1] \n" \ 437 " bnz 1b \n" \ 438 : [val] "=&r"(val) \ 439 : "r"(&v->counter), "ir"(a) \ 440 : "cc"); /* memory clobber comes from smp_mb() */ \ 441 \ 442 smp_mb(); \ 443 \ 444 return val; \ 445 } 446 447 #define ATOMIC64_FETCH_OP(op, op1, op2) \ 448 static inline long long atomic64_fetch_##op(long long a, atomic64_t *v) \ 449 { \ 450 unsigned long long val, orig; \ 451 \ 452 smp_mb(); \ 453 \ 454 __asm__ __volatile__( \ 455 "1: \n" \ 456 " llockd %0, [%2] \n" \ 457 " " #op1 " %L1, %L0, %L3 \n" \ 458 " " #op2 " %H1, %H0, %H3 \n" \ 459 " scondd %1, [%2] \n" \ 460 " bnz 1b \n" \ 461 : "=&r"(orig), "=&r"(val) \ 462 : "r"(&v->counter), "ir"(a) \ 463 : "cc"); /* memory clobber comes from smp_mb() */ \ 464 \ 465 smp_mb(); \ 466 \ 467 return orig; \ 468 } 469 470 #define ATOMIC64_OPS(op, op1, op2) \ 471 ATOMIC64_OP(op, op1, op2) \ 472 ATOMIC64_OP_RETURN(op, op1, op2) \ 473 ATOMIC64_FETCH_OP(op, op1, op2) 474 475 #define atomic64_andnot atomic64_andnot 476 477 ATOMIC64_OPS(add, add.f, adc) 478 ATOMIC64_OPS(sub, sub.f, sbc) 479 ATOMIC64_OPS(and, and, and) 480 ATOMIC64_OPS(andnot, bic, bic) 481 ATOMIC64_OPS(or, or, or) 482 ATOMIC64_OPS(xor, xor, xor) 483 484 #undef ATOMIC64_OPS 485 #undef ATOMIC64_FETCH_OP 486 #undef ATOMIC64_OP_RETURN 487 #undef ATOMIC64_OP 488 489 static inline long long 490 atomic64_cmpxchg(atomic64_t *ptr, long long expected, long long new) 491 { 492 long long prev; 493 494 smp_mb(); 495 496 __asm__ __volatile__( 497 "1: llockd %0, [%1] \n" 498 " brne %L0, %L2, 2f \n" 499 " brne %H0, %H2, 2f \n" 500 " scondd %3, [%1] \n" 501 " bnz 1b \n" 502 "2: \n" 503 : "=&r"(prev) 504 : "r"(ptr), "ir"(expected), "r"(new) 505 : "cc"); /* memory clobber comes from smp_mb() */ 506 507 smp_mb(); 508 509 return prev; 510 } 511 512 static inline long long atomic64_xchg(atomic64_t *ptr, long long new) 513 { 514 long long prev; 515 516 smp_mb(); 517 518 __asm__ __volatile__( 519 "1: llockd %0, [%1] \n" 520 " scondd %2, [%1] \n" 521 " bnz 1b \n" 522 "2: \n" 523 : "=&r"(prev) 524 : "r"(ptr), "r"(new) 525 : "cc"); /* memory clobber comes from smp_mb() */ 526 527 smp_mb(); 528 529 return prev; 530 } 531 532 /** 533 * atomic64_dec_if_positive - decrement by 1 if old value positive 534 * @v: pointer of type atomic64_t 535 * 536 * The function returns the old value of *v minus 1, even if 537 * the atomic variable, v, was not decremented. 538 */ 539 540 static inline long long atomic64_dec_if_positive(atomic64_t *v) 541 { 542 long long val; 543 544 smp_mb(); 545 546 __asm__ __volatile__( 547 "1: llockd %0, [%1] \n" 548 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n" 549 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n" 550 " brlt %H0, 0, 2f \n" 551 " scondd %0, [%1] \n" 552 " bnz 1b \n" 553 "2: \n" 554 : "=&r"(val) 555 : "r"(&v->counter) 556 : "cc"); /* memory clobber comes from smp_mb() */ 557 558 smp_mb(); 559 560 return val; 561 } 562 563 /** 564 * atomic64_add_unless - add unless the number is a given value 565 * @v: pointer of type atomic64_t 566 * @a: the amount to add to v... 567 * @u: ...unless v is equal to u. 568 * 569 * if (v != u) { v += a; ret = 1} else {ret = 0} 570 * Returns 1 iff @v was not @u (i.e. if add actually happened) 571 */ 572 static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) 573 { 574 long long val; 575 int op_done; 576 577 smp_mb(); 578 579 __asm__ __volatile__( 580 "1: llockd %0, [%2] \n" 581 " mov %1, 1 \n" 582 " brne %L0, %L4, 2f # continue to add since v != u \n" 583 " breq.d %H0, %H4, 3f # return since v == u \n" 584 " mov %1, 0 \n" 585 "2: \n" 586 " add.f %L0, %L0, %L3 \n" 587 " adc %H0, %H0, %H3 \n" 588 " scondd %0, [%2] \n" 589 " bnz 1b \n" 590 "3: \n" 591 : "=&r"(val), "=&r" (op_done) 592 : "r"(&v->counter), "r"(a), "r"(u) 593 : "cc"); /* memory clobber comes from smp_mb() */ 594 595 smp_mb(); 596 597 return op_done; 598 } 599 600 #define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) 601 #define atomic64_inc(v) atomic64_add(1LL, (v)) 602 #define atomic64_inc_return(v) atomic64_add_return(1LL, (v)) 603 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) 604 #define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0) 605 #define atomic64_dec(v) atomic64_sub(1LL, (v)) 606 #define atomic64_dec_return(v) atomic64_sub_return(1LL, (v)) 607 #define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) 608 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL) 609 610 #endif /* !CONFIG_GENERIC_ATOMIC64 */ 611 612 #endif /* !__ASSEMBLY__ */ 613 614 #endif 615