1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ALPHA_ATOMIC_H 3 #define _ALPHA_ATOMIC_H 4 5 #include <linux/types.h> 6 #include <asm/barrier.h> 7 #include <asm/cmpxchg.h> 8 9 /* 10 * Atomic operations that C can't guarantee us. Useful for 11 * resource counting etc... 12 * 13 * But use these as seldom as possible since they are much slower 14 * than regular operations. 15 */ 16 17 /* 18 * To ensure dependency ordering is preserved for the _relaxed and 19 * _release atomics, an smp_read_barrier_depends() is unconditionally 20 * inserted into the _relaxed variants, which are used to build the 21 * barriered versions. To avoid redundant back-to-back fences, we can 22 * define the _acquire and _fence versions explicitly. 23 */ 24 #define __atomic_op_acquire(op, args...) op##_relaxed(args) 25 #define __atomic_op_fence __atomic_op_release 26 27 #define ATOMIC_INIT(i) { (i) } 28 #define ATOMIC64_INIT(i) { (i) } 29 30 #define atomic_read(v) READ_ONCE((v)->counter) 31 #define atomic64_read(v) READ_ONCE((v)->counter) 32 33 #define atomic_set(v,i) WRITE_ONCE((v)->counter, (i)) 34 #define atomic64_set(v,i) WRITE_ONCE((v)->counter, (i)) 35 36 /* 37 * To get proper branch prediction for the main line, we must branch 38 * forward to code at the end of this object's .text section, then 39 * branch back to restart the operation. 40 */ 41 42 #define ATOMIC_OP(op, asm_op) \ 43 static __inline__ void atomic_##op(int i, atomic_t * v) \ 44 { \ 45 unsigned long temp; \ 46 __asm__ __volatile__( \ 47 "1: ldl_l %0,%1\n" \ 48 " " #asm_op " %0,%2,%0\n" \ 49 " stl_c %0,%1\n" \ 50 " beq %0,2f\n" \ 51 ".subsection 2\n" \ 52 "2: br 1b\n" \ 53 ".previous" \ 54 :"=&r" (temp), "=m" (v->counter) \ 55 :"Ir" (i), "m" (v->counter)); \ 56 } \ 57 58 #define ATOMIC_OP_RETURN(op, asm_op) \ 59 static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ 60 { \ 61 long temp, result; \ 62 __asm__ __volatile__( \ 63 "1: ldl_l %0,%1\n" \ 64 " " #asm_op " %0,%3,%2\n" \ 65 " " #asm_op " %0,%3,%0\n" \ 66 " stl_c %0,%1\n" \ 67 " beq %0,2f\n" \ 68 ".subsection 2\n" \ 69 "2: br 1b\n" \ 70 ".previous" \ 71 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 72 :"Ir" (i), "m" (v->counter) : "memory"); \ 73 smp_read_barrier_depends(); \ 74 return result; \ 75 } 76 77 #define ATOMIC_FETCH_OP(op, asm_op) \ 78 static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ 79 { \ 80 long temp, result; \ 81 __asm__ __volatile__( \ 82 "1: ldl_l %2,%1\n" \ 83 " " #asm_op " %2,%3,%0\n" \ 84 " stl_c %0,%1\n" \ 85 " beq %0,2f\n" \ 86 ".subsection 2\n" \ 87 "2: br 1b\n" \ 88 ".previous" \ 89 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 90 :"Ir" (i), "m" (v->counter) : "memory"); \ 91 smp_read_barrier_depends(); \ 92 return result; \ 93 } 94 95 #define ATOMIC64_OP(op, asm_op) \ 96 static __inline__ void atomic64_##op(long i, atomic64_t * v) \ 97 { \ 98 unsigned long temp; \ 99 __asm__ __volatile__( \ 100 "1: ldq_l %0,%1\n" \ 101 " " #asm_op " %0,%2,%0\n" \ 102 " stq_c %0,%1\n" \ 103 " beq %0,2f\n" \ 104 ".subsection 2\n" \ 105 "2: br 1b\n" \ 106 ".previous" \ 107 :"=&r" (temp), "=m" (v->counter) \ 108 :"Ir" (i), "m" (v->counter)); \ 109 } \ 110 111 #define ATOMIC64_OP_RETURN(op, asm_op) \ 112 static __inline__ long atomic64_##op##_return_relaxed(long i, atomic64_t * v) \ 113 { \ 114 long temp, result; \ 115 __asm__ __volatile__( \ 116 "1: ldq_l %0,%1\n" \ 117 " " #asm_op " %0,%3,%2\n" \ 118 " " #asm_op " %0,%3,%0\n" \ 119 " stq_c %0,%1\n" \ 120 " beq %0,2f\n" \ 121 ".subsection 2\n" \ 122 "2: br 1b\n" \ 123 ".previous" \ 124 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 125 :"Ir" (i), "m" (v->counter) : "memory"); \ 126 smp_read_barrier_depends(); \ 127 return result; \ 128 } 129 130 #define ATOMIC64_FETCH_OP(op, asm_op) \ 131 static __inline__ long atomic64_fetch_##op##_relaxed(long i, atomic64_t * v) \ 132 { \ 133 long temp, result; \ 134 __asm__ __volatile__( \ 135 "1: ldq_l %2,%1\n" \ 136 " " #asm_op " %2,%3,%0\n" \ 137 " stq_c %0,%1\n" \ 138 " beq %0,2f\n" \ 139 ".subsection 2\n" \ 140 "2: br 1b\n" \ 141 ".previous" \ 142 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 143 :"Ir" (i), "m" (v->counter) : "memory"); \ 144 smp_read_barrier_depends(); \ 145 return result; \ 146 } 147 148 #define ATOMIC_OPS(op) \ 149 ATOMIC_OP(op, op##l) \ 150 ATOMIC_OP_RETURN(op, op##l) \ 151 ATOMIC_FETCH_OP(op, op##l) \ 152 ATOMIC64_OP(op, op##q) \ 153 ATOMIC64_OP_RETURN(op, op##q) \ 154 ATOMIC64_FETCH_OP(op, op##q) 155 156 ATOMIC_OPS(add) 157 ATOMIC_OPS(sub) 158 159 #define atomic_add_return_relaxed atomic_add_return_relaxed 160 #define atomic_sub_return_relaxed atomic_sub_return_relaxed 161 #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed 162 #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed 163 164 #define atomic64_add_return_relaxed atomic64_add_return_relaxed 165 #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed 166 #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed 167 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed 168 169 #define atomic_andnot atomic_andnot 170 #define atomic64_andnot atomic64_andnot 171 172 #undef ATOMIC_OPS 173 #define ATOMIC_OPS(op, asm) \ 174 ATOMIC_OP(op, asm) \ 175 ATOMIC_FETCH_OP(op, asm) \ 176 ATOMIC64_OP(op, asm) \ 177 ATOMIC64_FETCH_OP(op, asm) 178 179 ATOMIC_OPS(and, and) 180 ATOMIC_OPS(andnot, bic) 181 ATOMIC_OPS(or, bis) 182 ATOMIC_OPS(xor, xor) 183 184 #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed 185 #define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed 186 #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed 187 #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed 188 189 #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed 190 #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed 191 #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed 192 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed 193 194 #undef ATOMIC_OPS 195 #undef ATOMIC64_FETCH_OP 196 #undef ATOMIC64_OP_RETURN 197 #undef ATOMIC64_OP 198 #undef ATOMIC_FETCH_OP 199 #undef ATOMIC_OP_RETURN 200 #undef ATOMIC_OP 201 202 #define atomic64_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) 203 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) 204 205 #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) 206 #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 207 208 /** 209 * __atomic_add_unless - add unless the number is a given value 210 * @v: pointer of type atomic_t 211 * @a: the amount to add to v... 212 * @u: ...unless v is equal to u. 213 * 214 * Atomically adds @a to @v, so long as it was not @u. 215 * Returns the old value of @v. 216 */ 217 static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) 218 { 219 int c, new, old; 220 smp_mb(); 221 __asm__ __volatile__( 222 "1: ldl_l %[old],%[mem]\n" 223 " cmpeq %[old],%[u],%[c]\n" 224 " addl %[old],%[a],%[new]\n" 225 " bne %[c],2f\n" 226 " stl_c %[new],%[mem]\n" 227 " beq %[new],3f\n" 228 "2:\n" 229 ".subsection 2\n" 230 "3: br 1b\n" 231 ".previous" 232 : [old] "=&r"(old), [new] "=&r"(new), [c] "=&r"(c) 233 : [mem] "m"(*v), [a] "rI"(a), [u] "rI"((long)u) 234 : "memory"); 235 smp_mb(); 236 return old; 237 } 238 239 240 /** 241 * atomic64_add_unless - add unless the number is a given value 242 * @v: pointer of type atomic64_t 243 * @a: the amount to add to v... 244 * @u: ...unless v is equal to u. 245 * 246 * Atomically adds @a to @v, so long as it was not @u. 247 * Returns true iff @v was not @u. 248 */ 249 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u) 250 { 251 long c, tmp; 252 smp_mb(); 253 __asm__ __volatile__( 254 "1: ldq_l %[tmp],%[mem]\n" 255 " cmpeq %[tmp],%[u],%[c]\n" 256 " addq %[tmp],%[a],%[tmp]\n" 257 " bne %[c],2f\n" 258 " stq_c %[tmp],%[mem]\n" 259 " beq %[tmp],3f\n" 260 "2:\n" 261 ".subsection 2\n" 262 "3: br 1b\n" 263 ".previous" 264 : [tmp] "=&r"(tmp), [c] "=&r"(c) 265 : [mem] "m"(*v), [a] "rI"(a), [u] "rI"(u) 266 : "memory"); 267 smp_mb(); 268 return !c; 269 } 270 271 /* 272 * atomic64_dec_if_positive - decrement by 1 if old value positive 273 * @v: pointer of type atomic_t 274 * 275 * The function returns the old value of *v minus 1, even if 276 * the atomic variable, v, was not decremented. 277 */ 278 static inline long atomic64_dec_if_positive(atomic64_t *v) 279 { 280 long old, tmp; 281 smp_mb(); 282 __asm__ __volatile__( 283 "1: ldq_l %[old],%[mem]\n" 284 " subq %[old],1,%[tmp]\n" 285 " ble %[old],2f\n" 286 " stq_c %[tmp],%[mem]\n" 287 " beq %[tmp],3f\n" 288 "2:\n" 289 ".subsection 2\n" 290 "3: br 1b\n" 291 ".previous" 292 : [old] "=&r"(old), [tmp] "=&r"(tmp) 293 : [mem] "m"(*v) 294 : "memory"); 295 smp_mb(); 296 return old - 1; 297 } 298 299 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) 300 301 #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) 302 #define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) 303 304 #define atomic_dec_return(v) atomic_sub_return(1,(v)) 305 #define atomic64_dec_return(v) atomic64_sub_return(1,(v)) 306 307 #define atomic_inc_return(v) atomic_add_return(1,(v)) 308 #define atomic64_inc_return(v) atomic64_add_return(1,(v)) 309 310 #define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0) 311 #define atomic64_sub_and_test(i,v) (atomic64_sub_return((i), (v)) == 0) 312 313 #define atomic_inc_and_test(v) (atomic_add_return(1, (v)) == 0) 314 #define atomic64_inc_and_test(v) (atomic64_add_return(1, (v)) == 0) 315 316 #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) 317 #define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0) 318 319 #define atomic_inc(v) atomic_add(1,(v)) 320 #define atomic64_inc(v) atomic64_add(1,(v)) 321 322 #define atomic_dec(v) atomic_sub(1,(v)) 323 #define atomic64_dec(v) atomic64_sub(1,(v)) 324 325 #endif /* _ALPHA_ATOMIC_H */ 326