1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ALPHA_ATOMIC_H 3 #define _ALPHA_ATOMIC_H 4 5 #include <linux/types.h> 6 #include <asm/barrier.h> 7 #include <asm/cmpxchg.h> 8 9 /* 10 * Atomic operations that C can't guarantee us. Useful for 11 * resource counting etc... 12 * 13 * But use these as seldom as possible since they are much slower 14 * than regular operations. 15 */ 16 17 /* 18 * To ensure dependency ordering is preserved for the _relaxed and 19 * _release atomics, an smp_mb() is unconditionally inserted into the 20 * _relaxed variants, which are used to build the barriered versions. 21 * Avoid redundant back-to-back fences in the _acquire and _fence 22 * versions. 23 */ 24 #define __atomic_acquire_fence() 25 #define __atomic_post_full_fence() 26 27 #define ATOMIC64_INIT(i) { (i) } 28 29 #define arch_atomic_read(v) READ_ONCE((v)->counter) 30 #define arch_atomic64_read(v) READ_ONCE((v)->counter) 31 32 #define arch_atomic_set(v,i) WRITE_ONCE((v)->counter, (i)) 33 #define arch_atomic64_set(v,i) WRITE_ONCE((v)->counter, (i)) 34 35 /* 36 * To get proper branch prediction for the main line, we must branch 37 * forward to code at the end of this object's .text section, then 38 * branch back to restart the operation. 39 */ 40 41 #define ATOMIC_OP(op, asm_op) \ 42 static __inline__ void arch_atomic_##op(int i, atomic_t * v) \ 43 { \ 44 unsigned long temp; \ 45 __asm__ __volatile__( \ 46 "1: ldl_l %0,%1\n" \ 47 " " #asm_op " %0,%2,%0\n" \ 48 " stl_c %0,%1\n" \ 49 " beq %0,2f\n" \ 50 ".subsection 2\n" \ 51 "2: br 1b\n" \ 52 ".previous" \ 53 :"=&r" (temp), "=m" (v->counter) \ 54 :"Ir" (i), "m" (v->counter)); \ 55 } \ 56 57 #define ATOMIC_OP_RETURN(op, asm_op) \ 58 static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \ 59 { \ 60 long temp, result; \ 61 __asm__ __volatile__( \ 62 "1: ldl_l %0,%1\n" \ 63 " " #asm_op " %0,%3,%2\n" \ 64 " " #asm_op " %0,%3,%0\n" \ 65 " stl_c %0,%1\n" \ 66 " beq %0,2f\n" \ 67 ".subsection 2\n" \ 68 "2: br 1b\n" \ 69 ".previous" \ 70 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 71 :"Ir" (i), "m" (v->counter) : "memory"); \ 72 smp_mb(); \ 73 return result; \ 74 } 75 76 #define ATOMIC_FETCH_OP(op, asm_op) \ 77 static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ 78 { \ 79 long temp, result; \ 80 __asm__ __volatile__( \ 81 "1: ldl_l %2,%1\n" \ 82 " " #asm_op " %2,%3,%0\n" \ 83 " stl_c %0,%1\n" \ 84 " beq %0,2f\n" \ 85 ".subsection 2\n" \ 86 "2: br 1b\n" \ 87 ".previous" \ 88 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 89 :"Ir" (i), "m" (v->counter) : "memory"); \ 90 smp_mb(); \ 91 return result; \ 92 } 93 94 #define ATOMIC64_OP(op, asm_op) \ 95 static __inline__ void arch_atomic64_##op(s64 i, atomic64_t * v) \ 96 { \ 97 s64 temp; \ 98 __asm__ __volatile__( \ 99 "1: ldq_l %0,%1\n" \ 100 " " #asm_op " %0,%2,%0\n" \ 101 " stq_c %0,%1\n" \ 102 " beq %0,2f\n" \ 103 ".subsection 2\n" \ 104 "2: br 1b\n" \ 105 ".previous" \ 106 :"=&r" (temp), "=m" (v->counter) \ 107 :"Ir" (i), "m" (v->counter)); \ 108 } \ 109 110 #define ATOMIC64_OP_RETURN(op, asm_op) \ 111 static __inline__ s64 \ 112 arch_atomic64_##op##_return_relaxed(s64 i, atomic64_t * v) \ 113 { \ 114 s64 temp, result; \ 115 __asm__ __volatile__( \ 116 "1: ldq_l %0,%1\n" \ 117 " " #asm_op " %0,%3,%2\n" \ 118 " " #asm_op " %0,%3,%0\n" \ 119 " stq_c %0,%1\n" \ 120 " beq %0,2f\n" \ 121 ".subsection 2\n" \ 122 "2: br 1b\n" \ 123 ".previous" \ 124 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 125 :"Ir" (i), "m" (v->counter) : "memory"); \ 126 smp_mb(); \ 127 return result; \ 128 } 129 130 #define ATOMIC64_FETCH_OP(op, asm_op) \ 131 static __inline__ s64 \ 132 arch_atomic64_fetch_##op##_relaxed(s64 i, atomic64_t * v) \ 133 { \ 134 s64 temp, result; \ 135 __asm__ __volatile__( \ 136 "1: ldq_l %2,%1\n" \ 137 " " #asm_op " %2,%3,%0\n" \ 138 " stq_c %0,%1\n" \ 139 " beq %0,2f\n" \ 140 ".subsection 2\n" \ 141 "2: br 1b\n" \ 142 ".previous" \ 143 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 144 :"Ir" (i), "m" (v->counter) : "memory"); \ 145 smp_mb(); \ 146 return result; \ 147 } 148 149 #define ATOMIC_OPS(op) \ 150 ATOMIC_OP(op, op##l) \ 151 ATOMIC_OP_RETURN(op, op##l) \ 152 ATOMIC_FETCH_OP(op, op##l) \ 153 ATOMIC64_OP(op, op##q) \ 154 ATOMIC64_OP_RETURN(op, op##q) \ 155 ATOMIC64_FETCH_OP(op, op##q) 156 157 ATOMIC_OPS(add) 158 ATOMIC_OPS(sub) 159 160 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed 161 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed 162 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed 163 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed 164 165 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 166 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 167 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 168 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 169 170 #define arch_atomic_andnot arch_atomic_andnot 171 #define arch_atomic64_andnot arch_atomic64_andnot 172 173 #undef ATOMIC_OPS 174 #define ATOMIC_OPS(op, asm) \ 175 ATOMIC_OP(op, asm) \ 176 ATOMIC_FETCH_OP(op, asm) \ 177 ATOMIC64_OP(op, asm) \ 178 ATOMIC64_FETCH_OP(op, asm) 179 180 ATOMIC_OPS(and, and) 181 ATOMIC_OPS(andnot, bic) 182 ATOMIC_OPS(or, bis) 183 ATOMIC_OPS(xor, xor) 184 185 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed 186 #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed 187 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed 188 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed 189 190 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 191 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed 192 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 193 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 194 195 #undef ATOMIC_OPS 196 #undef ATOMIC64_FETCH_OP 197 #undef ATOMIC64_OP_RETURN 198 #undef ATOMIC64_OP 199 #undef ATOMIC_FETCH_OP 200 #undef ATOMIC_OP_RETURN 201 #undef ATOMIC_OP 202 203 #define arch_atomic64_cmpxchg(v, old, new) \ 204 (arch_cmpxchg(&((v)->counter), old, new)) 205 #define arch_atomic64_xchg(v, new) \ 206 (arch_xchg(&((v)->counter), new)) 207 208 #define arch_atomic_cmpxchg(v, old, new) \ 209 (arch_cmpxchg(&((v)->counter), old, new)) 210 #define arch_atomic_xchg(v, new) \ 211 (arch_xchg(&((v)->counter), new)) 212 213 /** 214 * arch_atomic_fetch_add_unless - add unless the number is a given value 215 * @v: pointer of type atomic_t 216 * @a: the amount to add to v... 217 * @u: ...unless v is equal to u. 218 * 219 * Atomically adds @a to @v, so long as it was not @u. 220 * Returns the old value of @v. 221 */ 222 static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 223 { 224 int c, new, old; 225 smp_mb(); 226 __asm__ __volatile__( 227 "1: ldl_l %[old],%[mem]\n" 228 " cmpeq %[old],%[u],%[c]\n" 229 " addl %[old],%[a],%[new]\n" 230 " bne %[c],2f\n" 231 " stl_c %[new],%[mem]\n" 232 " beq %[new],3f\n" 233 "2:\n" 234 ".subsection 2\n" 235 "3: br 1b\n" 236 ".previous" 237 : [old] "=&r"(old), [new] "=&r"(new), [c] "=&r"(c) 238 : [mem] "m"(*v), [a] "rI"(a), [u] "rI"((long)u) 239 : "memory"); 240 smp_mb(); 241 return old; 242 } 243 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 244 245 /** 246 * arch_atomic64_fetch_add_unless - add unless the number is a given value 247 * @v: pointer of type atomic64_t 248 * @a: the amount to add to v... 249 * @u: ...unless v is equal to u. 250 * 251 * Atomically adds @a to @v, so long as it was not @u. 252 * Returns the old value of @v. 253 */ 254 static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 255 { 256 s64 c, new, old; 257 smp_mb(); 258 __asm__ __volatile__( 259 "1: ldq_l %[old],%[mem]\n" 260 " cmpeq %[old],%[u],%[c]\n" 261 " addq %[old],%[a],%[new]\n" 262 " bne %[c],2f\n" 263 " stq_c %[new],%[mem]\n" 264 " beq %[new],3f\n" 265 "2:\n" 266 ".subsection 2\n" 267 "3: br 1b\n" 268 ".previous" 269 : [old] "=&r"(old), [new] "=&r"(new), [c] "=&r"(c) 270 : [mem] "m"(*v), [a] "rI"(a), [u] "rI"(u) 271 : "memory"); 272 smp_mb(); 273 return old; 274 } 275 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 276 277 /* 278 * arch_atomic64_dec_if_positive - decrement by 1 if old value positive 279 * @v: pointer of type atomic_t 280 * 281 * The function returns the old value of *v minus 1, even if 282 * the atomic variable, v, was not decremented. 283 */ 284 static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 285 { 286 s64 old, tmp; 287 smp_mb(); 288 __asm__ __volatile__( 289 "1: ldq_l %[old],%[mem]\n" 290 " subq %[old],1,%[tmp]\n" 291 " ble %[old],2f\n" 292 " stq_c %[tmp],%[mem]\n" 293 " beq %[tmp],3f\n" 294 "2:\n" 295 ".subsection 2\n" 296 "3: br 1b\n" 297 ".previous" 298 : [old] "=&r"(old), [tmp] "=&r"(tmp) 299 : [mem] "m"(*v) 300 : "memory"); 301 smp_mb(); 302 return old - 1; 303 } 304 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 305 306 #endif /* _ALPHA_ATOMIC_H */ 307