1 /* 2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #ifndef _ASM_ARC_ATOMIC_H 10 #define _ASM_ARC_ATOMIC_H 11 12 #ifndef __ASSEMBLY__ 13 14 #include <linux/types.h> 15 #include <linux/compiler.h> 16 #include <asm/cmpxchg.h> 17 #include <asm/barrier.h> 18 #include <asm/smp.h> 19 20 #define ATOMIC_INIT(i) { (i) } 21 22 #ifndef CONFIG_ARC_PLAT_EZNPS 23 24 #define atomic_read(v) READ_ONCE((v)->counter) 25 26 #ifdef CONFIG_ARC_HAS_LLSC 27 28 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) 29 30 #define ATOMIC_OP(op, c_op, asm_op) \ 31 static inline void atomic_##op(int i, atomic_t *v) \ 32 { \ 33 unsigned int val; \ 34 \ 35 __asm__ __volatile__( \ 36 "1: llock %[val], [%[ctr]] \n" \ 37 " " #asm_op " %[val], %[val], %[i] \n" \ 38 " scond %[val], [%[ctr]] \n" \ 39 " bnz 1b \n" \ 40 : [val] "=&r" (val) /* Early clobber to prevent reg reuse */ \ 41 : [ctr] "r" (&v->counter), /* Not "m": llock only supports reg direct addr mode */ \ 42 [i] "ir" (i) \ 43 : "cc"); \ 44 } \ 45 46 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 47 static inline int atomic_##op##_return(int i, atomic_t *v) \ 48 { \ 49 unsigned int val; \ 50 \ 51 /* \ 52 * Explicit full memory barrier needed before/after as \ 53 * LLOCK/SCOND thmeselves don't provide any such semantics \ 54 */ \ 55 smp_mb(); \ 56 \ 57 __asm__ __volatile__( \ 58 "1: llock %[val], [%[ctr]] \n" \ 59 " " #asm_op " %[val], %[val], %[i] \n" \ 60 " scond %[val], [%[ctr]] \n" \ 61 " bnz 1b \n" \ 62 : [val] "=&r" (val) \ 63 : [ctr] "r" (&v->counter), \ 64 [i] "ir" (i) \ 65 : "cc"); \ 66 \ 67 smp_mb(); \ 68 \ 69 return val; \ 70 } 71 72 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 73 static inline int atomic_fetch_##op(int i, atomic_t *v) \ 74 { \ 75 unsigned int val, orig; \ 76 \ 77 /* \ 78 * Explicit full memory barrier needed before/after as \ 79 * LLOCK/SCOND thmeselves don't provide any such semantics \ 80 */ \ 81 smp_mb(); \ 82 \ 83 __asm__ __volatile__( \ 84 "1: llock %[orig], [%[ctr]] \n" \ 85 " " #asm_op " %[val], %[orig], %[i] \n" \ 86 " scond %[val], [%[ctr]] \n" \ 87 " bnz 1b \n" \ 88 : [val] "=&r" (val), \ 89 [orig] "=&r" (orig) \ 90 : [ctr] "r" (&v->counter), \ 91 [i] "ir" (i) \ 92 : "cc"); \ 93 \ 94 smp_mb(); \ 95 \ 96 return orig; \ 97 } 98 99 #else /* !CONFIG_ARC_HAS_LLSC */ 100 101 #ifndef CONFIG_SMP 102 103 /* violating atomic_xxx API locking protocol in UP for optimization sake */ 104 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) 105 106 #else 107 108 static inline void atomic_set(atomic_t *v, int i) 109 { 110 /* 111 * Independent of hardware support, all of the atomic_xxx() APIs need 112 * to follow the same locking rules to make sure that a "hardware" 113 * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn 114 * sequence 115 * 116 * Thus atomic_set() despite being 1 insn (and seemingly atomic) 117 * requires the locking. 118 */ 119 unsigned long flags; 120 121 atomic_ops_lock(flags); 122 WRITE_ONCE(v->counter, i); 123 atomic_ops_unlock(flags); 124 } 125 126 #define atomic_set_release(v, i) atomic_set((v), (i)) 127 128 #endif 129 130 /* 131 * Non hardware assisted Atomic-R-M-W 132 * Locking would change to irq-disabling only (UP) and spinlocks (SMP) 133 */ 134 135 #define ATOMIC_OP(op, c_op, asm_op) \ 136 static inline void atomic_##op(int i, atomic_t *v) \ 137 { \ 138 unsigned long flags; \ 139 \ 140 atomic_ops_lock(flags); \ 141 v->counter c_op i; \ 142 atomic_ops_unlock(flags); \ 143 } 144 145 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 146 static inline int atomic_##op##_return(int i, atomic_t *v) \ 147 { \ 148 unsigned long flags; \ 149 unsigned long temp; \ 150 \ 151 /* \ 152 * spin lock/unlock provides the needed smp_mb() before/after \ 153 */ \ 154 atomic_ops_lock(flags); \ 155 temp = v->counter; \ 156 temp c_op i; \ 157 v->counter = temp; \ 158 atomic_ops_unlock(flags); \ 159 \ 160 return temp; \ 161 } 162 163 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 164 static inline int atomic_fetch_##op(int i, atomic_t *v) \ 165 { \ 166 unsigned long flags; \ 167 unsigned long orig; \ 168 \ 169 /* \ 170 * spin lock/unlock provides the needed smp_mb() before/after \ 171 */ \ 172 atomic_ops_lock(flags); \ 173 orig = v->counter; \ 174 v->counter c_op i; \ 175 atomic_ops_unlock(flags); \ 176 \ 177 return orig; \ 178 } 179 180 #endif /* !CONFIG_ARC_HAS_LLSC */ 181 182 #define ATOMIC_OPS(op, c_op, asm_op) \ 183 ATOMIC_OP(op, c_op, asm_op) \ 184 ATOMIC_OP_RETURN(op, c_op, asm_op) \ 185 ATOMIC_FETCH_OP(op, c_op, asm_op) 186 187 ATOMIC_OPS(add, +=, add) 188 ATOMIC_OPS(sub, -=, sub) 189 190 #define atomic_andnot atomic_andnot 191 #define atomic_fetch_andnot atomic_fetch_andnot 192 193 #undef ATOMIC_OPS 194 #define ATOMIC_OPS(op, c_op, asm_op) \ 195 ATOMIC_OP(op, c_op, asm_op) \ 196 ATOMIC_FETCH_OP(op, c_op, asm_op) 197 198 ATOMIC_OPS(and, &=, and) 199 ATOMIC_OPS(andnot, &= ~, bic) 200 ATOMIC_OPS(or, |=, or) 201 ATOMIC_OPS(xor, ^=, xor) 202 203 #else /* CONFIG_ARC_PLAT_EZNPS */ 204 205 static inline int atomic_read(const atomic_t *v) 206 { 207 int temp; 208 209 __asm__ __volatile__( 210 " ld.di %0, [%1]" 211 : "=r"(temp) 212 : "r"(&v->counter) 213 : "memory"); 214 return temp; 215 } 216 217 static inline void atomic_set(atomic_t *v, int i) 218 { 219 __asm__ __volatile__( 220 " st.di %0,[%1]" 221 : 222 : "r"(i), "r"(&v->counter) 223 : "memory"); 224 } 225 226 #define ATOMIC_OP(op, c_op, asm_op) \ 227 static inline void atomic_##op(int i, atomic_t *v) \ 228 { \ 229 __asm__ __volatile__( \ 230 " mov r2, %0\n" \ 231 " mov r3, %1\n" \ 232 " .word %2\n" \ 233 : \ 234 : "r"(i), "r"(&v->counter), "i"(asm_op) \ 235 : "r2", "r3", "memory"); \ 236 } \ 237 238 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 239 static inline int atomic_##op##_return(int i, atomic_t *v) \ 240 { \ 241 unsigned int temp = i; \ 242 \ 243 /* Explicit full memory barrier needed before/after */ \ 244 smp_mb(); \ 245 \ 246 __asm__ __volatile__( \ 247 " mov r2, %0\n" \ 248 " mov r3, %1\n" \ 249 " .word %2\n" \ 250 " mov %0, r2" \ 251 : "+r"(temp) \ 252 : "r"(&v->counter), "i"(asm_op) \ 253 : "r2", "r3", "memory"); \ 254 \ 255 smp_mb(); \ 256 \ 257 temp c_op i; \ 258 \ 259 return temp; \ 260 } 261 262 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 263 static inline int atomic_fetch_##op(int i, atomic_t *v) \ 264 { \ 265 unsigned int temp = i; \ 266 \ 267 /* Explicit full memory barrier needed before/after */ \ 268 smp_mb(); \ 269 \ 270 __asm__ __volatile__( \ 271 " mov r2, %0\n" \ 272 " mov r3, %1\n" \ 273 " .word %2\n" \ 274 " mov %0, r2" \ 275 : "+r"(temp) \ 276 : "r"(&v->counter), "i"(asm_op) \ 277 : "r2", "r3", "memory"); \ 278 \ 279 smp_mb(); \ 280 \ 281 return temp; \ 282 } 283 284 #define ATOMIC_OPS(op, c_op, asm_op) \ 285 ATOMIC_OP(op, c_op, asm_op) \ 286 ATOMIC_OP_RETURN(op, c_op, asm_op) \ 287 ATOMIC_FETCH_OP(op, c_op, asm_op) 288 289 ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3) 290 #define atomic_sub(i, v) atomic_add(-(i), (v)) 291 #define atomic_sub_return(i, v) atomic_add_return(-(i), (v)) 292 #define atomic_fetch_sub(i, v) atomic_fetch_add(-(i), (v)) 293 294 #undef ATOMIC_OPS 295 #define ATOMIC_OPS(op, c_op, asm_op) \ 296 ATOMIC_OP(op, c_op, asm_op) \ 297 ATOMIC_FETCH_OP(op, c_op, asm_op) 298 299 ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3) 300 ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3) 301 ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3) 302 303 #endif /* CONFIG_ARC_PLAT_EZNPS */ 304 305 #undef ATOMIC_OPS 306 #undef ATOMIC_FETCH_OP 307 #undef ATOMIC_OP_RETURN 308 #undef ATOMIC_OP 309 310 #ifdef CONFIG_GENERIC_ATOMIC64 311 312 #include <asm-generic/atomic64.h> 313 314 #else /* Kconfig ensures this is only enabled with needed h/w assist */ 315 316 /* 317 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD) 318 * - The address HAS to be 64-bit aligned 319 * - There are 2 semantics involved here: 320 * = exclusive implies no interim update between load/store to same addr 321 * = both words are observed/updated together: this is guaranteed even 322 * for regular 64-bit load (LDD) / store (STD). Thus atomic64_set() 323 * is NOT required to use LLOCKD+SCONDD, STD suffices 324 */ 325 326 typedef struct { 327 aligned_u64 counter; 328 } atomic64_t; 329 330 #define ATOMIC64_INIT(a) { (a) } 331 332 static inline long long atomic64_read(const atomic64_t *v) 333 { 334 unsigned long long val; 335 336 __asm__ __volatile__( 337 " ldd %0, [%1] \n" 338 : "=r"(val) 339 : "r"(&v->counter)); 340 341 return val; 342 } 343 344 static inline void atomic64_set(atomic64_t *v, long long a) 345 { 346 /* 347 * This could have been a simple assignment in "C" but would need 348 * explicit volatile. Otherwise gcc optimizers could elide the store 349 * which borked atomic64 self-test 350 * In the inline asm version, memory clobber needed for exact same 351 * reason, to tell gcc about the store. 352 * 353 * This however is not needed for sibling atomic64_add() etc since both 354 * load/store are explicitly done in inline asm. As long as API is used 355 * for each access, gcc has no way to optimize away any load/store 356 */ 357 __asm__ __volatile__( 358 " std %0, [%1] \n" 359 : 360 : "r"(a), "r"(&v->counter) 361 : "memory"); 362 } 363 364 #define ATOMIC64_OP(op, op1, op2) \ 365 static inline void atomic64_##op(long long a, atomic64_t *v) \ 366 { \ 367 unsigned long long val; \ 368 \ 369 __asm__ __volatile__( \ 370 "1: \n" \ 371 " llockd %0, [%1] \n" \ 372 " " #op1 " %L0, %L0, %L2 \n" \ 373 " " #op2 " %H0, %H0, %H2 \n" \ 374 " scondd %0, [%1] \n" \ 375 " bnz 1b \n" \ 376 : "=&r"(val) \ 377 : "r"(&v->counter), "ir"(a) \ 378 : "cc"); \ 379 } \ 380 381 #define ATOMIC64_OP_RETURN(op, op1, op2) \ 382 static inline long long atomic64_##op##_return(long long a, atomic64_t *v) \ 383 { \ 384 unsigned long long val; \ 385 \ 386 smp_mb(); \ 387 \ 388 __asm__ __volatile__( \ 389 "1: \n" \ 390 " llockd %0, [%1] \n" \ 391 " " #op1 " %L0, %L0, %L2 \n" \ 392 " " #op2 " %H0, %H0, %H2 \n" \ 393 " scondd %0, [%1] \n" \ 394 " bnz 1b \n" \ 395 : [val] "=&r"(val) \ 396 : "r"(&v->counter), "ir"(a) \ 397 : "cc"); /* memory clobber comes from smp_mb() */ \ 398 \ 399 smp_mb(); \ 400 \ 401 return val; \ 402 } 403 404 #define ATOMIC64_FETCH_OP(op, op1, op2) \ 405 static inline long long atomic64_fetch_##op(long long a, atomic64_t *v) \ 406 { \ 407 unsigned long long val, orig; \ 408 \ 409 smp_mb(); \ 410 \ 411 __asm__ __volatile__( \ 412 "1: \n" \ 413 " llockd %0, [%2] \n" \ 414 " " #op1 " %L1, %L0, %L3 \n" \ 415 " " #op2 " %H1, %H0, %H3 \n" \ 416 " scondd %1, [%2] \n" \ 417 " bnz 1b \n" \ 418 : "=&r"(orig), "=&r"(val) \ 419 : "r"(&v->counter), "ir"(a) \ 420 : "cc"); /* memory clobber comes from smp_mb() */ \ 421 \ 422 smp_mb(); \ 423 \ 424 return orig; \ 425 } 426 427 #define ATOMIC64_OPS(op, op1, op2) \ 428 ATOMIC64_OP(op, op1, op2) \ 429 ATOMIC64_OP_RETURN(op, op1, op2) \ 430 ATOMIC64_FETCH_OP(op, op1, op2) 431 432 #define atomic64_andnot atomic64_andnot 433 #define atomic64_fetch_andnot atomic64_fetch_andnot 434 435 ATOMIC64_OPS(add, add.f, adc) 436 ATOMIC64_OPS(sub, sub.f, sbc) 437 ATOMIC64_OPS(and, and, and) 438 ATOMIC64_OPS(andnot, bic, bic) 439 ATOMIC64_OPS(or, or, or) 440 ATOMIC64_OPS(xor, xor, xor) 441 442 #undef ATOMIC64_OPS 443 #undef ATOMIC64_FETCH_OP 444 #undef ATOMIC64_OP_RETURN 445 #undef ATOMIC64_OP 446 447 static inline long long 448 atomic64_cmpxchg(atomic64_t *ptr, long long expected, long long new) 449 { 450 long long prev; 451 452 smp_mb(); 453 454 __asm__ __volatile__( 455 "1: llockd %0, [%1] \n" 456 " brne %L0, %L2, 2f \n" 457 " brne %H0, %H2, 2f \n" 458 " scondd %3, [%1] \n" 459 " bnz 1b \n" 460 "2: \n" 461 : "=&r"(prev) 462 : "r"(ptr), "ir"(expected), "r"(new) 463 : "cc"); /* memory clobber comes from smp_mb() */ 464 465 smp_mb(); 466 467 return prev; 468 } 469 470 static inline long long atomic64_xchg(atomic64_t *ptr, long long new) 471 { 472 long long prev; 473 474 smp_mb(); 475 476 __asm__ __volatile__( 477 "1: llockd %0, [%1] \n" 478 " scondd %2, [%1] \n" 479 " bnz 1b \n" 480 "2: \n" 481 : "=&r"(prev) 482 : "r"(ptr), "r"(new) 483 : "cc"); /* memory clobber comes from smp_mb() */ 484 485 smp_mb(); 486 487 return prev; 488 } 489 490 /** 491 * atomic64_dec_if_positive - decrement by 1 if old value positive 492 * @v: pointer of type atomic64_t 493 * 494 * The function returns the old value of *v minus 1, even if 495 * the atomic variable, v, was not decremented. 496 */ 497 498 static inline long long atomic64_dec_if_positive(atomic64_t *v) 499 { 500 long long val; 501 502 smp_mb(); 503 504 __asm__ __volatile__( 505 "1: llockd %0, [%1] \n" 506 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n" 507 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n" 508 " brlt %H0, 0, 2f \n" 509 " scondd %0, [%1] \n" 510 " bnz 1b \n" 511 "2: \n" 512 : "=&r"(val) 513 : "r"(&v->counter) 514 : "cc"); /* memory clobber comes from smp_mb() */ 515 516 smp_mb(); 517 518 return val; 519 } 520 #define atomic64_dec_if_positive atomic64_dec_if_positive 521 522 /** 523 * atomic64_fetch_add_unless - add unless the number is a given value 524 * @v: pointer of type atomic64_t 525 * @a: the amount to add to v... 526 * @u: ...unless v is equal to u. 527 * 528 * Atomically adds @a to @v, if it was not @u. 529 * Returns the old value of @v 530 */ 531 static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a, 532 long long u) 533 { 534 long long old, temp; 535 536 smp_mb(); 537 538 __asm__ __volatile__( 539 "1: llockd %0, [%2] \n" 540 " brne %L0, %L4, 2f # continue to add since v != u \n" 541 " breq.d %H0, %H4, 3f # return since v == u \n" 542 "2: \n" 543 " add.f %L1, %L0, %L3 \n" 544 " adc %H1, %H0, %H3 \n" 545 " scondd %1, [%2] \n" 546 " bnz 1b \n" 547 "3: \n" 548 : "=&r"(old), "=&r" (temp) 549 : "r"(&v->counter), "r"(a), "r"(u) 550 : "cc"); /* memory clobber comes from smp_mb() */ 551 552 smp_mb(); 553 554 return old; 555 } 556 #define atomic64_fetch_add_unless atomic64_fetch_add_unless 557 558 #endif /* !CONFIG_GENERIC_ATOMIC64 */ 559 560 #endif /* !__ASSEMBLY__ */ 561 562 #endif 563