1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 4 */ 5 6 #ifndef _ASM_ARC_ATOMIC_H 7 #define _ASM_ARC_ATOMIC_H 8 9 #ifndef __ASSEMBLY__ 10 11 #include <linux/types.h> 12 #include <linux/compiler.h> 13 #include <asm/cmpxchg.h> 14 #include <asm/barrier.h> 15 #include <asm/smp.h> 16 17 #define arch_atomic_read(v) READ_ONCE((v)->counter) 18 19 #ifdef CONFIG_ARC_HAS_LLSC 20 21 #define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) 22 23 #define ATOMIC_OP(op, c_op, asm_op) \ 24 static inline void arch_atomic_##op(int i, atomic_t *v) \ 25 { \ 26 unsigned int val; \ 27 \ 28 __asm__ __volatile__( \ 29 "1: llock %[val], [%[ctr]] \n" \ 30 " " #asm_op " %[val], %[val], %[i] \n" \ 31 " scond %[val], [%[ctr]] \n" \ 32 " bnz 1b \n" \ 33 : [val] "=&r" (val) /* Early clobber to prevent reg reuse */ \ 34 : [ctr] "r" (&v->counter), /* Not "m": llock only supports reg direct addr mode */ \ 35 [i] "ir" (i) \ 36 : "cc"); \ 37 } \ 38 39 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 40 static inline int arch_atomic_##op##_return(int i, atomic_t *v) \ 41 { \ 42 unsigned int val; \ 43 \ 44 /* \ 45 * Explicit full memory barrier needed before/after as \ 46 * LLOCK/SCOND themselves don't provide any such semantics \ 47 */ \ 48 smp_mb(); \ 49 \ 50 __asm__ __volatile__( \ 51 "1: llock %[val], [%[ctr]] \n" \ 52 " " #asm_op " %[val], %[val], %[i] \n" \ 53 " scond %[val], [%[ctr]] \n" \ 54 " bnz 1b \n" \ 55 : [val] "=&r" (val) \ 56 : [ctr] "r" (&v->counter), \ 57 [i] "ir" (i) \ 58 : "cc"); \ 59 \ 60 smp_mb(); \ 61 \ 62 return val; \ 63 } 64 65 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 66 static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ 67 { \ 68 unsigned int val, orig; \ 69 \ 70 /* \ 71 * Explicit full memory barrier needed before/after as \ 72 * LLOCK/SCOND themselves don't provide any such semantics \ 73 */ \ 74 smp_mb(); \ 75 \ 76 __asm__ __volatile__( \ 77 "1: llock %[orig], [%[ctr]] \n" \ 78 " " #asm_op " %[val], %[orig], %[i] \n" \ 79 " scond %[val], [%[ctr]] \n" \ 80 " bnz 1b \n" \ 81 : [val] "=&r" (val), \ 82 [orig] "=&r" (orig) \ 83 : [ctr] "r" (&v->counter), \ 84 [i] "ir" (i) \ 85 : "cc"); \ 86 \ 87 smp_mb(); \ 88 \ 89 return orig; \ 90 } 91 92 #else /* !CONFIG_ARC_HAS_LLSC */ 93 94 #ifndef CONFIG_SMP 95 96 /* violating atomic_xxx API locking protocol in UP for optimization sake */ 97 #define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) 98 99 #else 100 101 static inline void arch_atomic_set(atomic_t *v, int i) 102 { 103 /* 104 * Independent of hardware support, all of the atomic_xxx() APIs need 105 * to follow the same locking rules to make sure that a "hardware" 106 * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn 107 * sequence 108 * 109 * Thus atomic_set() despite being 1 insn (and seemingly atomic) 110 * requires the locking. 111 */ 112 unsigned long flags; 113 114 atomic_ops_lock(flags); 115 WRITE_ONCE(v->counter, i); 116 atomic_ops_unlock(flags); 117 } 118 119 #define arch_atomic_set_release(v, i) arch_atomic_set((v), (i)) 120 121 #endif 122 123 /* 124 * Non hardware assisted Atomic-R-M-W 125 * Locking would change to irq-disabling only (UP) and spinlocks (SMP) 126 */ 127 128 #define ATOMIC_OP(op, c_op, asm_op) \ 129 static inline void arch_atomic_##op(int i, atomic_t *v) \ 130 { \ 131 unsigned long flags; \ 132 \ 133 atomic_ops_lock(flags); \ 134 v->counter c_op i; \ 135 atomic_ops_unlock(flags); \ 136 } 137 138 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 139 static inline int arch_atomic_##op##_return(int i, atomic_t *v) \ 140 { \ 141 unsigned long flags; \ 142 unsigned long temp; \ 143 \ 144 /* \ 145 * spin lock/unlock provides the needed smp_mb() before/after \ 146 */ \ 147 atomic_ops_lock(flags); \ 148 temp = v->counter; \ 149 temp c_op i; \ 150 v->counter = temp; \ 151 atomic_ops_unlock(flags); \ 152 \ 153 return temp; \ 154 } 155 156 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 157 static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ 158 { \ 159 unsigned long flags; \ 160 unsigned long orig; \ 161 \ 162 /* \ 163 * spin lock/unlock provides the needed smp_mb() before/after \ 164 */ \ 165 atomic_ops_lock(flags); \ 166 orig = v->counter; \ 167 v->counter c_op i; \ 168 atomic_ops_unlock(flags); \ 169 \ 170 return orig; \ 171 } 172 173 #endif /* !CONFIG_ARC_HAS_LLSC */ 174 175 #define ATOMIC_OPS(op, c_op, asm_op) \ 176 ATOMIC_OP(op, c_op, asm_op) \ 177 ATOMIC_OP_RETURN(op, c_op, asm_op) \ 178 ATOMIC_FETCH_OP(op, c_op, asm_op) 179 180 ATOMIC_OPS(add, +=, add) 181 ATOMIC_OPS(sub, -=, sub) 182 183 #undef ATOMIC_OPS 184 #define ATOMIC_OPS(op, c_op, asm_op) \ 185 ATOMIC_OP(op, c_op, asm_op) \ 186 ATOMIC_FETCH_OP(op, c_op, asm_op) 187 188 ATOMIC_OPS(and, &=, and) 189 ATOMIC_OPS(andnot, &= ~, bic) 190 ATOMIC_OPS(or, |=, or) 191 ATOMIC_OPS(xor, ^=, xor) 192 193 #define arch_atomic_andnot arch_atomic_andnot 194 #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot 195 196 #undef ATOMIC_OPS 197 #undef ATOMIC_FETCH_OP 198 #undef ATOMIC_OP_RETURN 199 #undef ATOMIC_OP 200 201 #ifdef CONFIG_GENERIC_ATOMIC64 202 203 #include <asm-generic/atomic64.h> 204 205 #else /* Kconfig ensures this is only enabled with needed h/w assist */ 206 207 /* 208 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD) 209 * - The address HAS to be 64-bit aligned 210 * - There are 2 semantics involved here: 211 * = exclusive implies no interim update between load/store to same addr 212 * = both words are observed/updated together: this is guaranteed even 213 * for regular 64-bit load (LDD) / store (STD). Thus atomic64_set() 214 * is NOT required to use LLOCKD+SCONDD, STD suffices 215 */ 216 217 typedef struct { 218 s64 __aligned(8) counter; 219 } atomic64_t; 220 221 #define ATOMIC64_INIT(a) { (a) } 222 223 static inline s64 arch_atomic64_read(const atomic64_t *v) 224 { 225 s64 val; 226 227 __asm__ __volatile__( 228 " ldd %0, [%1] \n" 229 : "=r"(val) 230 : "r"(&v->counter)); 231 232 return val; 233 } 234 235 static inline void arch_atomic64_set(atomic64_t *v, s64 a) 236 { 237 /* 238 * This could have been a simple assignment in "C" but would need 239 * explicit volatile. Otherwise gcc optimizers could elide the store 240 * which borked atomic64 self-test 241 * In the inline asm version, memory clobber needed for exact same 242 * reason, to tell gcc about the store. 243 * 244 * This however is not needed for sibling atomic64_add() etc since both 245 * load/store are explicitly done in inline asm. As long as API is used 246 * for each access, gcc has no way to optimize away any load/store 247 */ 248 __asm__ __volatile__( 249 " std %0, [%1] \n" 250 : 251 : "r"(a), "r"(&v->counter) 252 : "memory"); 253 } 254 255 #define ATOMIC64_OP(op, op1, op2) \ 256 static inline void arch_atomic64_##op(s64 a, atomic64_t *v) \ 257 { \ 258 s64 val; \ 259 \ 260 __asm__ __volatile__( \ 261 "1: \n" \ 262 " llockd %0, [%1] \n" \ 263 " " #op1 " %L0, %L0, %L2 \n" \ 264 " " #op2 " %H0, %H0, %H2 \n" \ 265 " scondd %0, [%1] \n" \ 266 " bnz 1b \n" \ 267 : "=&r"(val) \ 268 : "r"(&v->counter), "ir"(a) \ 269 : "cc"); \ 270 } \ 271 272 #define ATOMIC64_OP_RETURN(op, op1, op2) \ 273 static inline s64 arch_atomic64_##op##_return(s64 a, atomic64_t *v) \ 274 { \ 275 s64 val; \ 276 \ 277 smp_mb(); \ 278 \ 279 __asm__ __volatile__( \ 280 "1: \n" \ 281 " llockd %0, [%1] \n" \ 282 " " #op1 " %L0, %L0, %L2 \n" \ 283 " " #op2 " %H0, %H0, %H2 \n" \ 284 " scondd %0, [%1] \n" \ 285 " bnz 1b \n" \ 286 : [val] "=&r"(val) \ 287 : "r"(&v->counter), "ir"(a) \ 288 : "cc"); /* memory clobber comes from smp_mb() */ \ 289 \ 290 smp_mb(); \ 291 \ 292 return val; \ 293 } 294 295 #define ATOMIC64_FETCH_OP(op, op1, op2) \ 296 static inline s64 arch_atomic64_fetch_##op(s64 a, atomic64_t *v) \ 297 { \ 298 s64 val, orig; \ 299 \ 300 smp_mb(); \ 301 \ 302 __asm__ __volatile__( \ 303 "1: \n" \ 304 " llockd %0, [%2] \n" \ 305 " " #op1 " %L1, %L0, %L3 \n" \ 306 " " #op2 " %H1, %H0, %H3 \n" \ 307 " scondd %1, [%2] \n" \ 308 " bnz 1b \n" \ 309 : "=&r"(orig), "=&r"(val) \ 310 : "r"(&v->counter), "ir"(a) \ 311 : "cc"); /* memory clobber comes from smp_mb() */ \ 312 \ 313 smp_mb(); \ 314 \ 315 return orig; \ 316 } 317 318 #define ATOMIC64_OPS(op, op1, op2) \ 319 ATOMIC64_OP(op, op1, op2) \ 320 ATOMIC64_OP_RETURN(op, op1, op2) \ 321 ATOMIC64_FETCH_OP(op, op1, op2) 322 323 ATOMIC64_OPS(add, add.f, adc) 324 ATOMIC64_OPS(sub, sub.f, sbc) 325 ATOMIC64_OPS(and, and, and) 326 ATOMIC64_OPS(andnot, bic, bic) 327 ATOMIC64_OPS(or, or, or) 328 ATOMIC64_OPS(xor, xor, xor) 329 330 #define arch_atomic64_andnot arch_atomic64_andnot 331 #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot 332 333 #undef ATOMIC64_OPS 334 #undef ATOMIC64_FETCH_OP 335 #undef ATOMIC64_OP_RETURN 336 #undef ATOMIC64_OP 337 338 static inline s64 339 arch_atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new) 340 { 341 s64 prev; 342 343 smp_mb(); 344 345 __asm__ __volatile__( 346 "1: llockd %0, [%1] \n" 347 " brne %L0, %L2, 2f \n" 348 " brne %H0, %H2, 2f \n" 349 " scondd %3, [%1] \n" 350 " bnz 1b \n" 351 "2: \n" 352 : "=&r"(prev) 353 : "r"(ptr), "ir"(expected), "r"(new) 354 : "cc"); /* memory clobber comes from smp_mb() */ 355 356 smp_mb(); 357 358 return prev; 359 } 360 361 static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new) 362 { 363 s64 prev; 364 365 smp_mb(); 366 367 __asm__ __volatile__( 368 "1: llockd %0, [%1] \n" 369 " scondd %2, [%1] \n" 370 " bnz 1b \n" 371 "2: \n" 372 : "=&r"(prev) 373 : "r"(ptr), "r"(new) 374 : "cc"); /* memory clobber comes from smp_mb() */ 375 376 smp_mb(); 377 378 return prev; 379 } 380 381 /** 382 * arch_atomic64_dec_if_positive - decrement by 1 if old value positive 383 * @v: pointer of type atomic64_t 384 * 385 * The function returns the old value of *v minus 1, even if 386 * the atomic variable, v, was not decremented. 387 */ 388 389 static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 390 { 391 s64 val; 392 393 smp_mb(); 394 395 __asm__ __volatile__( 396 "1: llockd %0, [%1] \n" 397 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n" 398 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n" 399 " brlt %H0, 0, 2f \n" 400 " scondd %0, [%1] \n" 401 " bnz 1b \n" 402 "2: \n" 403 : "=&r"(val) 404 : "r"(&v->counter) 405 : "cc"); /* memory clobber comes from smp_mb() */ 406 407 smp_mb(); 408 409 return val; 410 } 411 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 412 413 /** 414 * arch_atomic64_fetch_add_unless - add unless the number is a given value 415 * @v: pointer of type atomic64_t 416 * @a: the amount to add to v... 417 * @u: ...unless v is equal to u. 418 * 419 * Atomically adds @a to @v, if it was not @u. 420 * Returns the old value of @v 421 */ 422 static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 423 { 424 s64 old, temp; 425 426 smp_mb(); 427 428 __asm__ __volatile__( 429 "1: llockd %0, [%2] \n" 430 " brne %L0, %L4, 2f # continue to add since v != u \n" 431 " breq.d %H0, %H4, 3f # return since v == u \n" 432 "2: \n" 433 " add.f %L1, %L0, %L3 \n" 434 " adc %H1, %H0, %H3 \n" 435 " scondd %1, [%2] \n" 436 " bnz 1b \n" 437 "3: \n" 438 : "=&r"(old), "=&r" (temp) 439 : "r"(&v->counter), "r"(a), "r"(u) 440 : "cc"); /* memory clobber comes from smp_mb() */ 441 442 smp_mb(); 443 444 return old; 445 } 446 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 447 448 #endif /* !CONFIG_GENERIC_ATOMIC64 */ 449 450 #endif /* !__ASSEMBLY__ */ 451 452 #endif 453