1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 4 */ 5 6 #ifndef _ASM_ARC_ATOMIC_H 7 #define _ASM_ARC_ATOMIC_H 8 9 #ifndef __ASSEMBLY__ 10 11 #include <linux/types.h> 12 #include <linux/compiler.h> 13 #include <asm/cmpxchg.h> 14 #include <asm/barrier.h> 15 #include <asm/smp.h> 16 17 #define ATOMIC_INIT(i) { (i) } 18 19 #ifndef CONFIG_ARC_PLAT_EZNPS 20 21 #define atomic_read(v) READ_ONCE((v)->counter) 22 23 #ifdef CONFIG_ARC_HAS_LLSC 24 25 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) 26 27 #define ATOMIC_OP(op, c_op, asm_op) \ 28 static inline void atomic_##op(int i, atomic_t *v) \ 29 { \ 30 unsigned int val; \ 31 \ 32 __asm__ __volatile__( \ 33 "1: llock %[val], [%[ctr]] \n" \ 34 " " #asm_op " %[val], %[val], %[i] \n" \ 35 " scond %[val], [%[ctr]] \n" \ 36 " bnz 1b \n" \ 37 : [val] "=&r" (val) /* Early clobber to prevent reg reuse */ \ 38 : [ctr] "r" (&v->counter), /* Not "m": llock only supports reg direct addr mode */ \ 39 [i] "ir" (i) \ 40 : "cc"); \ 41 } \ 42 43 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 44 static inline int atomic_##op##_return(int i, atomic_t *v) \ 45 { \ 46 unsigned int val; \ 47 \ 48 /* \ 49 * Explicit full memory barrier needed before/after as \ 50 * LLOCK/SCOND thmeselves don't provide any such semantics \ 51 */ \ 52 smp_mb(); \ 53 \ 54 __asm__ __volatile__( \ 55 "1: llock %[val], [%[ctr]] \n" \ 56 " " #asm_op " %[val], %[val], %[i] \n" \ 57 " scond %[val], [%[ctr]] \n" \ 58 " bnz 1b \n" \ 59 : [val] "=&r" (val) \ 60 : [ctr] "r" (&v->counter), \ 61 [i] "ir" (i) \ 62 : "cc"); \ 63 \ 64 smp_mb(); \ 65 \ 66 return val; \ 67 } 68 69 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 70 static inline int atomic_fetch_##op(int i, atomic_t *v) \ 71 { \ 72 unsigned int val, orig; \ 73 \ 74 /* \ 75 * Explicit full memory barrier needed before/after as \ 76 * LLOCK/SCOND thmeselves don't provide any such semantics \ 77 */ \ 78 smp_mb(); \ 79 \ 80 __asm__ __volatile__( \ 81 "1: llock %[orig], [%[ctr]] \n" \ 82 " " #asm_op " %[val], %[orig], %[i] \n" \ 83 " scond %[val], [%[ctr]] \n" \ 84 " bnz 1b \n" \ 85 : [val] "=&r" (val), \ 86 [orig] "=&r" (orig) \ 87 : [ctr] "r" (&v->counter), \ 88 [i] "ir" (i) \ 89 : "cc"); \ 90 \ 91 smp_mb(); \ 92 \ 93 return orig; \ 94 } 95 96 #else /* !CONFIG_ARC_HAS_LLSC */ 97 98 #ifndef CONFIG_SMP 99 100 /* violating atomic_xxx API locking protocol in UP for optimization sake */ 101 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) 102 103 #else 104 105 static inline void atomic_set(atomic_t *v, int i) 106 { 107 /* 108 * Independent of hardware support, all of the atomic_xxx() APIs need 109 * to follow the same locking rules to make sure that a "hardware" 110 * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn 111 * sequence 112 * 113 * Thus atomic_set() despite being 1 insn (and seemingly atomic) 114 * requires the locking. 115 */ 116 unsigned long flags; 117 118 atomic_ops_lock(flags); 119 WRITE_ONCE(v->counter, i); 120 atomic_ops_unlock(flags); 121 } 122 123 #define atomic_set_release(v, i) atomic_set((v), (i)) 124 125 #endif 126 127 /* 128 * Non hardware assisted Atomic-R-M-W 129 * Locking would change to irq-disabling only (UP) and spinlocks (SMP) 130 */ 131 132 #define ATOMIC_OP(op, c_op, asm_op) \ 133 static inline void atomic_##op(int i, atomic_t *v) \ 134 { \ 135 unsigned long flags; \ 136 \ 137 atomic_ops_lock(flags); \ 138 v->counter c_op i; \ 139 atomic_ops_unlock(flags); \ 140 } 141 142 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 143 static inline int atomic_##op##_return(int i, atomic_t *v) \ 144 { \ 145 unsigned long flags; \ 146 unsigned long temp; \ 147 \ 148 /* \ 149 * spin lock/unlock provides the needed smp_mb() before/after \ 150 */ \ 151 atomic_ops_lock(flags); \ 152 temp = v->counter; \ 153 temp c_op i; \ 154 v->counter = temp; \ 155 atomic_ops_unlock(flags); \ 156 \ 157 return temp; \ 158 } 159 160 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 161 static inline int atomic_fetch_##op(int i, atomic_t *v) \ 162 { \ 163 unsigned long flags; \ 164 unsigned long orig; \ 165 \ 166 /* \ 167 * spin lock/unlock provides the needed smp_mb() before/after \ 168 */ \ 169 atomic_ops_lock(flags); \ 170 orig = v->counter; \ 171 v->counter c_op i; \ 172 atomic_ops_unlock(flags); \ 173 \ 174 return orig; \ 175 } 176 177 #endif /* !CONFIG_ARC_HAS_LLSC */ 178 179 #define ATOMIC_OPS(op, c_op, asm_op) \ 180 ATOMIC_OP(op, c_op, asm_op) \ 181 ATOMIC_OP_RETURN(op, c_op, asm_op) \ 182 ATOMIC_FETCH_OP(op, c_op, asm_op) 183 184 ATOMIC_OPS(add, +=, add) 185 ATOMIC_OPS(sub, -=, sub) 186 187 #define atomic_andnot atomic_andnot 188 #define atomic_fetch_andnot atomic_fetch_andnot 189 190 #undef ATOMIC_OPS 191 #define ATOMIC_OPS(op, c_op, asm_op) \ 192 ATOMIC_OP(op, c_op, asm_op) \ 193 ATOMIC_FETCH_OP(op, c_op, asm_op) 194 195 ATOMIC_OPS(and, &=, and) 196 ATOMIC_OPS(andnot, &= ~, bic) 197 ATOMIC_OPS(or, |=, or) 198 ATOMIC_OPS(xor, ^=, xor) 199 200 #else /* CONFIG_ARC_PLAT_EZNPS */ 201 202 static inline int atomic_read(const atomic_t *v) 203 { 204 int temp; 205 206 __asm__ __volatile__( 207 " ld.di %0, [%1]" 208 : "=r"(temp) 209 : "r"(&v->counter) 210 : "memory"); 211 return temp; 212 } 213 214 static inline void atomic_set(atomic_t *v, int i) 215 { 216 __asm__ __volatile__( 217 " st.di %0,[%1]" 218 : 219 : "r"(i), "r"(&v->counter) 220 : "memory"); 221 } 222 223 #define ATOMIC_OP(op, c_op, asm_op) \ 224 static inline void atomic_##op(int i, atomic_t *v) \ 225 { \ 226 __asm__ __volatile__( \ 227 " mov r2, %0\n" \ 228 " mov r3, %1\n" \ 229 " .word %2\n" \ 230 : \ 231 : "r"(i), "r"(&v->counter), "i"(asm_op) \ 232 : "r2", "r3", "memory"); \ 233 } \ 234 235 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 236 static inline int atomic_##op##_return(int i, atomic_t *v) \ 237 { \ 238 unsigned int temp = i; \ 239 \ 240 /* Explicit full memory barrier needed before/after */ \ 241 smp_mb(); \ 242 \ 243 __asm__ __volatile__( \ 244 " mov r2, %0\n" \ 245 " mov r3, %1\n" \ 246 " .word %2\n" \ 247 " mov %0, r2" \ 248 : "+r"(temp) \ 249 : "r"(&v->counter), "i"(asm_op) \ 250 : "r2", "r3", "memory"); \ 251 \ 252 smp_mb(); \ 253 \ 254 temp c_op i; \ 255 \ 256 return temp; \ 257 } 258 259 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 260 static inline int atomic_fetch_##op(int i, atomic_t *v) \ 261 { \ 262 unsigned int temp = i; \ 263 \ 264 /* Explicit full memory barrier needed before/after */ \ 265 smp_mb(); \ 266 \ 267 __asm__ __volatile__( \ 268 " mov r2, %0\n" \ 269 " mov r3, %1\n" \ 270 " .word %2\n" \ 271 " mov %0, r2" \ 272 : "+r"(temp) \ 273 : "r"(&v->counter), "i"(asm_op) \ 274 : "r2", "r3", "memory"); \ 275 \ 276 smp_mb(); \ 277 \ 278 return temp; \ 279 } 280 281 #define ATOMIC_OPS(op, c_op, asm_op) \ 282 ATOMIC_OP(op, c_op, asm_op) \ 283 ATOMIC_OP_RETURN(op, c_op, asm_op) \ 284 ATOMIC_FETCH_OP(op, c_op, asm_op) 285 286 ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3) 287 #define atomic_sub(i, v) atomic_add(-(i), (v)) 288 #define atomic_sub_return(i, v) atomic_add_return(-(i), (v)) 289 #define atomic_fetch_sub(i, v) atomic_fetch_add(-(i), (v)) 290 291 #undef ATOMIC_OPS 292 #define ATOMIC_OPS(op, c_op, asm_op) \ 293 ATOMIC_OP(op, c_op, asm_op) \ 294 ATOMIC_FETCH_OP(op, c_op, asm_op) 295 296 ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3) 297 ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3) 298 ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3) 299 300 #endif /* CONFIG_ARC_PLAT_EZNPS */ 301 302 #undef ATOMIC_OPS 303 #undef ATOMIC_FETCH_OP 304 #undef ATOMIC_OP_RETURN 305 #undef ATOMIC_OP 306 307 #ifdef CONFIG_GENERIC_ATOMIC64 308 309 #include <asm-generic/atomic64.h> 310 311 #else /* Kconfig ensures this is only enabled with needed h/w assist */ 312 313 /* 314 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD) 315 * - The address HAS to be 64-bit aligned 316 * - There are 2 semantics involved here: 317 * = exclusive implies no interim update between load/store to same addr 318 * = both words are observed/updated together: this is guaranteed even 319 * for regular 64-bit load (LDD) / store (STD). Thus atomic64_set() 320 * is NOT required to use LLOCKD+SCONDD, STD suffices 321 */ 322 323 typedef struct { 324 s64 __aligned(8) counter; 325 } atomic64_t; 326 327 #define ATOMIC64_INIT(a) { (a) } 328 329 static inline s64 atomic64_read(const atomic64_t *v) 330 { 331 s64 val; 332 333 __asm__ __volatile__( 334 " ldd %0, [%1] \n" 335 : "=r"(val) 336 : "r"(&v->counter)); 337 338 return val; 339 } 340 341 static inline void atomic64_set(atomic64_t *v, s64 a) 342 { 343 /* 344 * This could have been a simple assignment in "C" but would need 345 * explicit volatile. Otherwise gcc optimizers could elide the store 346 * which borked atomic64 self-test 347 * In the inline asm version, memory clobber needed for exact same 348 * reason, to tell gcc about the store. 349 * 350 * This however is not needed for sibling atomic64_add() etc since both 351 * load/store are explicitly done in inline asm. As long as API is used 352 * for each access, gcc has no way to optimize away any load/store 353 */ 354 __asm__ __volatile__( 355 " std %0, [%1] \n" 356 : 357 : "r"(a), "r"(&v->counter) 358 : "memory"); 359 } 360 361 #define ATOMIC64_OP(op, op1, op2) \ 362 static inline void atomic64_##op(s64 a, atomic64_t *v) \ 363 { \ 364 s64 val; \ 365 \ 366 __asm__ __volatile__( \ 367 "1: \n" \ 368 " llockd %0, [%1] \n" \ 369 " " #op1 " %L0, %L0, %L2 \n" \ 370 " " #op2 " %H0, %H0, %H2 \n" \ 371 " scondd %0, [%1] \n" \ 372 " bnz 1b \n" \ 373 : "=&r"(val) \ 374 : "r"(&v->counter), "ir"(a) \ 375 : "cc"); \ 376 } \ 377 378 #define ATOMIC64_OP_RETURN(op, op1, op2) \ 379 static inline s64 atomic64_##op##_return(s64 a, atomic64_t *v) \ 380 { \ 381 s64 val; \ 382 \ 383 smp_mb(); \ 384 \ 385 __asm__ __volatile__( \ 386 "1: \n" \ 387 " llockd %0, [%1] \n" \ 388 " " #op1 " %L0, %L0, %L2 \n" \ 389 " " #op2 " %H0, %H0, %H2 \n" \ 390 " scondd %0, [%1] \n" \ 391 " bnz 1b \n" \ 392 : [val] "=&r"(val) \ 393 : "r"(&v->counter), "ir"(a) \ 394 : "cc"); /* memory clobber comes from smp_mb() */ \ 395 \ 396 smp_mb(); \ 397 \ 398 return val; \ 399 } 400 401 #define ATOMIC64_FETCH_OP(op, op1, op2) \ 402 static inline s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \ 403 { \ 404 s64 val, orig; \ 405 \ 406 smp_mb(); \ 407 \ 408 __asm__ __volatile__( \ 409 "1: \n" \ 410 " llockd %0, [%2] \n" \ 411 " " #op1 " %L1, %L0, %L3 \n" \ 412 " " #op2 " %H1, %H0, %H3 \n" \ 413 " scondd %1, [%2] \n" \ 414 " bnz 1b \n" \ 415 : "=&r"(orig), "=&r"(val) \ 416 : "r"(&v->counter), "ir"(a) \ 417 : "cc"); /* memory clobber comes from smp_mb() */ \ 418 \ 419 smp_mb(); \ 420 \ 421 return orig; \ 422 } 423 424 #define ATOMIC64_OPS(op, op1, op2) \ 425 ATOMIC64_OP(op, op1, op2) \ 426 ATOMIC64_OP_RETURN(op, op1, op2) \ 427 ATOMIC64_FETCH_OP(op, op1, op2) 428 429 #define atomic64_andnot atomic64_andnot 430 #define atomic64_fetch_andnot atomic64_fetch_andnot 431 432 ATOMIC64_OPS(add, add.f, adc) 433 ATOMIC64_OPS(sub, sub.f, sbc) 434 ATOMIC64_OPS(and, and, and) 435 ATOMIC64_OPS(andnot, bic, bic) 436 ATOMIC64_OPS(or, or, or) 437 ATOMIC64_OPS(xor, xor, xor) 438 439 #undef ATOMIC64_OPS 440 #undef ATOMIC64_FETCH_OP 441 #undef ATOMIC64_OP_RETURN 442 #undef ATOMIC64_OP 443 444 static inline s64 445 atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new) 446 { 447 s64 prev; 448 449 smp_mb(); 450 451 __asm__ __volatile__( 452 "1: llockd %0, [%1] \n" 453 " brne %L0, %L2, 2f \n" 454 " brne %H0, %H2, 2f \n" 455 " scondd %3, [%1] \n" 456 " bnz 1b \n" 457 "2: \n" 458 : "=&r"(prev) 459 : "r"(ptr), "ir"(expected), "r"(new) 460 : "cc"); /* memory clobber comes from smp_mb() */ 461 462 smp_mb(); 463 464 return prev; 465 } 466 467 static inline s64 atomic64_xchg(atomic64_t *ptr, s64 new) 468 { 469 s64 prev; 470 471 smp_mb(); 472 473 __asm__ __volatile__( 474 "1: llockd %0, [%1] \n" 475 " scondd %2, [%1] \n" 476 " bnz 1b \n" 477 "2: \n" 478 : "=&r"(prev) 479 : "r"(ptr), "r"(new) 480 : "cc"); /* memory clobber comes from smp_mb() */ 481 482 smp_mb(); 483 484 return prev; 485 } 486 487 /** 488 * atomic64_dec_if_positive - decrement by 1 if old value positive 489 * @v: pointer of type atomic64_t 490 * 491 * The function returns the old value of *v minus 1, even if 492 * the atomic variable, v, was not decremented. 493 */ 494 495 static inline s64 atomic64_dec_if_positive(atomic64_t *v) 496 { 497 s64 val; 498 499 smp_mb(); 500 501 __asm__ __volatile__( 502 "1: llockd %0, [%1] \n" 503 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n" 504 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n" 505 " brlt %H0, 0, 2f \n" 506 " scondd %0, [%1] \n" 507 " bnz 1b \n" 508 "2: \n" 509 : "=&r"(val) 510 : "r"(&v->counter) 511 : "cc"); /* memory clobber comes from smp_mb() */ 512 513 smp_mb(); 514 515 return val; 516 } 517 #define atomic64_dec_if_positive atomic64_dec_if_positive 518 519 /** 520 * atomic64_fetch_add_unless - add unless the number is a given value 521 * @v: pointer of type atomic64_t 522 * @a: the amount to add to v... 523 * @u: ...unless v is equal to u. 524 * 525 * Atomically adds @a to @v, if it was not @u. 526 * Returns the old value of @v 527 */ 528 static inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 529 { 530 s64 old, temp; 531 532 smp_mb(); 533 534 __asm__ __volatile__( 535 "1: llockd %0, [%2] \n" 536 " brne %L0, %L4, 2f # continue to add since v != u \n" 537 " breq.d %H0, %H4, 3f # return since v == u \n" 538 "2: \n" 539 " add.f %L1, %L0, %L3 \n" 540 " adc %H1, %H0, %H3 \n" 541 " scondd %1, [%2] \n" 542 " bnz 1b \n" 543 "3: \n" 544 : "=&r"(old), "=&r" (temp) 545 : "r"(&v->counter), "r"(a), "r"(u) 546 : "cc"); /* memory clobber comes from smp_mb() */ 547 548 smp_mb(); 549 550 return old; 551 } 552 #define atomic64_fetch_add_unless atomic64_fetch_add_unless 553 554 #endif /* !CONFIG_GENERIC_ATOMIC64 */ 555 556 #endif /* !__ASSEMBLY__ */ 557 558 #endif 559