1 /* 2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #ifndef _ASM_ARC_ATOMIC_H 10 #define _ASM_ARC_ATOMIC_H 11 12 #ifndef __ASSEMBLY__ 13 14 #include <linux/types.h> 15 #include <linux/compiler.h> 16 #include <asm/cmpxchg.h> 17 #include <asm/barrier.h> 18 #include <asm/smp.h> 19 20 #ifndef CONFIG_ARC_PLAT_EZNPS 21 22 #define atomic_read(v) READ_ONCE((v)->counter) 23 24 #ifdef CONFIG_ARC_HAS_LLSC 25 26 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) 27 28 #define ATOMIC_OP(op, c_op, asm_op) \ 29 static inline void atomic_##op(int i, atomic_t *v) \ 30 { \ 31 unsigned int val; \ 32 \ 33 __asm__ __volatile__( \ 34 "1: llock %[val], [%[ctr]] \n" \ 35 " " #asm_op " %[val], %[val], %[i] \n" \ 36 " scond %[val], [%[ctr]] \n" \ 37 " bnz 1b \n" \ 38 : [val] "=&r" (val) /* Early clobber to prevent reg reuse */ \ 39 : [ctr] "r" (&v->counter), /* Not "m": llock only supports reg direct addr mode */ \ 40 [i] "ir" (i) \ 41 : "cc"); \ 42 } \ 43 44 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 45 static inline int atomic_##op##_return(int i, atomic_t *v) \ 46 { \ 47 unsigned int val; \ 48 \ 49 /* \ 50 * Explicit full memory barrier needed before/after as \ 51 * LLOCK/SCOND thmeselves don't provide any such semantics \ 52 */ \ 53 smp_mb(); \ 54 \ 55 __asm__ __volatile__( \ 56 "1: llock %[val], [%[ctr]] \n" \ 57 " " #asm_op " %[val], %[val], %[i] \n" \ 58 " scond %[val], [%[ctr]] \n" \ 59 " bnz 1b \n" \ 60 : [val] "=&r" (val) \ 61 : [ctr] "r" (&v->counter), \ 62 [i] "ir" (i) \ 63 : "cc"); \ 64 \ 65 smp_mb(); \ 66 \ 67 return val; \ 68 } 69 70 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 71 static inline int atomic_fetch_##op(int i, atomic_t *v) \ 72 { \ 73 unsigned int val, orig; \ 74 \ 75 /* \ 76 * Explicit full memory barrier needed before/after as \ 77 * LLOCK/SCOND thmeselves don't provide any such semantics \ 78 */ \ 79 smp_mb(); \ 80 \ 81 __asm__ __volatile__( \ 82 "1: llock %[orig], [%[ctr]] \n" \ 83 " " #asm_op " %[val], %[orig], %[i] \n" \ 84 " scond %[val], [%[ctr]] \n" \ 85 " \n" \ 86 : [val] "=&r" (val), \ 87 [orig] "=&r" (orig) \ 88 : [ctr] "r" (&v->counter), \ 89 [i] "ir" (i) \ 90 : "cc"); \ 91 \ 92 smp_mb(); \ 93 \ 94 return orig; \ 95 } 96 97 #else /* !CONFIG_ARC_HAS_LLSC */ 98 99 #ifndef CONFIG_SMP 100 101 /* violating atomic_xxx API locking protocol in UP for optimization sake */ 102 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) 103 104 #else 105 106 static inline void atomic_set(atomic_t *v, int i) 107 { 108 /* 109 * Independent of hardware support, all of the atomic_xxx() APIs need 110 * to follow the same locking rules to make sure that a "hardware" 111 * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn 112 * sequence 113 * 114 * Thus atomic_set() despite being 1 insn (and seemingly atomic) 115 * requires the locking. 116 */ 117 unsigned long flags; 118 119 atomic_ops_lock(flags); 120 WRITE_ONCE(v->counter, i); 121 atomic_ops_unlock(flags); 122 } 123 124 #endif 125 126 /* 127 * Non hardware assisted Atomic-R-M-W 128 * Locking would change to irq-disabling only (UP) and spinlocks (SMP) 129 */ 130 131 #define ATOMIC_OP(op, c_op, asm_op) \ 132 static inline void atomic_##op(int i, atomic_t *v) \ 133 { \ 134 unsigned long flags; \ 135 \ 136 atomic_ops_lock(flags); \ 137 v->counter c_op i; \ 138 atomic_ops_unlock(flags); \ 139 } 140 141 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 142 static inline int atomic_##op##_return(int i, atomic_t *v) \ 143 { \ 144 unsigned long flags; \ 145 unsigned long temp; \ 146 \ 147 /* \ 148 * spin lock/unlock provides the needed smp_mb() before/after \ 149 */ \ 150 atomic_ops_lock(flags); \ 151 temp = v->counter; \ 152 temp c_op i; \ 153 v->counter = temp; \ 154 atomic_ops_unlock(flags); \ 155 \ 156 return temp; \ 157 } 158 159 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 160 static inline int atomic_fetch_##op(int i, atomic_t *v) \ 161 { \ 162 unsigned long flags; \ 163 unsigned long orig; \ 164 \ 165 /* \ 166 * spin lock/unlock provides the needed smp_mb() before/after \ 167 */ \ 168 atomic_ops_lock(flags); \ 169 orig = v->counter; \ 170 v->counter c_op i; \ 171 atomic_ops_unlock(flags); \ 172 \ 173 return orig; \ 174 } 175 176 #endif /* !CONFIG_ARC_HAS_LLSC */ 177 178 #define ATOMIC_OPS(op, c_op, asm_op) \ 179 ATOMIC_OP(op, c_op, asm_op) \ 180 ATOMIC_OP_RETURN(op, c_op, asm_op) \ 181 ATOMIC_FETCH_OP(op, c_op, asm_op) 182 183 ATOMIC_OPS(add, +=, add) 184 ATOMIC_OPS(sub, -=, sub) 185 186 #define atomic_andnot atomic_andnot 187 188 #undef ATOMIC_OPS 189 #define ATOMIC_OPS(op, c_op, asm_op) \ 190 ATOMIC_OP(op, c_op, asm_op) \ 191 ATOMIC_FETCH_OP(op, c_op, asm_op) 192 193 ATOMIC_OPS(and, &=, and) 194 ATOMIC_OPS(andnot, &= ~, bic) 195 ATOMIC_OPS(or, |=, or) 196 ATOMIC_OPS(xor, ^=, xor) 197 198 #else /* CONFIG_ARC_PLAT_EZNPS */ 199 200 static inline int atomic_read(const atomic_t *v) 201 { 202 int temp; 203 204 __asm__ __volatile__( 205 " ld.di %0, [%1]" 206 : "=r"(temp) 207 : "r"(&v->counter) 208 : "memory"); 209 return temp; 210 } 211 212 static inline void atomic_set(atomic_t *v, int i) 213 { 214 __asm__ __volatile__( 215 " st.di %0,[%1]" 216 : 217 : "r"(i), "r"(&v->counter) 218 : "memory"); 219 } 220 221 #define ATOMIC_OP(op, c_op, asm_op) \ 222 static inline void atomic_##op(int i, atomic_t *v) \ 223 { \ 224 __asm__ __volatile__( \ 225 " mov r2, %0\n" \ 226 " mov r3, %1\n" \ 227 " .word %2\n" \ 228 : \ 229 : "r"(i), "r"(&v->counter), "i"(asm_op) \ 230 : "r2", "r3", "memory"); \ 231 } \ 232 233 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ 234 static inline int atomic_##op##_return(int i, atomic_t *v) \ 235 { \ 236 unsigned int temp = i; \ 237 \ 238 /* Explicit full memory barrier needed before/after */ \ 239 smp_mb(); \ 240 \ 241 __asm__ __volatile__( \ 242 " mov r2, %0\n" \ 243 " mov r3, %1\n" \ 244 " .word %2\n" \ 245 " mov %0, r2" \ 246 : "+r"(temp) \ 247 : "r"(&v->counter), "i"(asm_op) \ 248 : "r2", "r3", "memory"); \ 249 \ 250 smp_mb(); \ 251 \ 252 temp c_op i; \ 253 \ 254 return temp; \ 255 } 256 257 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \ 258 static inline int atomic_fetch_##op(int i, atomic_t *v) \ 259 { \ 260 unsigned int temp = i; \ 261 \ 262 /* Explicit full memory barrier needed before/after */ \ 263 smp_mb(); \ 264 \ 265 __asm__ __volatile__( \ 266 " mov r2, %0\n" \ 267 " mov r3, %1\n" \ 268 " .word %2\n" \ 269 " mov %0, r2" \ 270 : "+r"(temp) \ 271 : "r"(&v->counter), "i"(asm_op) \ 272 : "r2", "r3", "memory"); \ 273 \ 274 smp_mb(); \ 275 \ 276 return temp; \ 277 } 278 279 #define ATOMIC_OPS(op, c_op, asm_op) \ 280 ATOMIC_OP(op, c_op, asm_op) \ 281 ATOMIC_OP_RETURN(op, c_op, asm_op) \ 282 ATOMIC_FETCH_OP(op, c_op, asm_op) 283 284 ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3) 285 #define atomic_sub(i, v) atomic_add(-(i), (v)) 286 #define atomic_sub_return(i, v) atomic_add_return(-(i), (v)) 287 288 #undef ATOMIC_OPS 289 #define ATOMIC_OPS(op, c_op, asm_op) \ 290 ATOMIC_OP(op, c_op, asm_op) \ 291 ATOMIC_FETCH_OP(op, c_op, asm_op) 292 293 ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3) 294 #define atomic_andnot(mask, v) atomic_and(~(mask), (v)) 295 ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3) 296 ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3) 297 298 #endif /* CONFIG_ARC_PLAT_EZNPS */ 299 300 #undef ATOMIC_OPS 301 #undef ATOMIC_FETCH_OP 302 #undef ATOMIC_OP_RETURN 303 #undef ATOMIC_OP 304 305 /** 306 * __atomic_add_unless - add unless the number is a given value 307 * @v: pointer of type atomic_t 308 * @a: the amount to add to v... 309 * @u: ...unless v is equal to u. 310 * 311 * Atomically adds @a to @v, so long as it was not @u. 312 * Returns the old value of @v 313 */ 314 #define __atomic_add_unless(v, a, u) \ 315 ({ \ 316 int c, old; \ 317 \ 318 /* \ 319 * Explicit full memory barrier needed before/after as \ 320 * LLOCK/SCOND thmeselves don't provide any such semantics \ 321 */ \ 322 smp_mb(); \ 323 \ 324 c = atomic_read(v); \ 325 while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c)\ 326 c = old; \ 327 \ 328 smp_mb(); \ 329 \ 330 c; \ 331 }) 332 333 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 334 335 #define atomic_inc(v) atomic_add(1, v) 336 #define atomic_dec(v) atomic_sub(1, v) 337 338 #define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0) 339 #define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0) 340 #define atomic_inc_return(v) atomic_add_return(1, (v)) 341 #define atomic_dec_return(v) atomic_sub_return(1, (v)) 342 #define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) 343 344 #define atomic_add_negative(i, v) (atomic_add_return(i, v) < 0) 345 346 #define ATOMIC_INIT(i) { (i) } 347 348 #include <asm-generic/atomic64.h> 349 350 #endif 351 352 #endif 353