1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_IA64_ATOMIC_H 3 #define _ASM_IA64_ATOMIC_H 4 5 /* 6 * Atomic operations that C can't guarantee us. Useful for 7 * resource counting etc.. 8 * 9 * NOTE: don't mess with the types below! The "unsigned long" and 10 * "int" types were carefully placed so as to ensure proper operation 11 * of the macros. 12 * 13 * Copyright (C) 1998, 1999, 2002-2003 Hewlett-Packard Co 14 * David Mosberger-Tang <davidm@hpl.hp.com> 15 */ 16 #include <linux/types.h> 17 18 #include <asm/intrinsics.h> 19 #include <asm/barrier.h> 20 21 22 #define ATOMIC_INIT(i) { (i) } 23 #define ATOMIC64_INIT(i) { (i) } 24 25 #define atomic_read(v) READ_ONCE((v)->counter) 26 #define atomic64_read(v) READ_ONCE((v)->counter) 27 28 #define atomic_set(v,i) WRITE_ONCE(((v)->counter), (i)) 29 #define atomic64_set(v,i) WRITE_ONCE(((v)->counter), (i)) 30 31 #define ATOMIC_OP(op, c_op) \ 32 static __inline__ int \ 33 ia64_atomic_##op (int i, atomic_t *v) \ 34 { \ 35 __s32 old, new; \ 36 CMPXCHG_BUGCHECK_DECL \ 37 \ 38 do { \ 39 CMPXCHG_BUGCHECK(v); \ 40 old = atomic_read(v); \ 41 new = old c_op i; \ 42 } while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic_t)) != old); \ 43 return new; \ 44 } 45 46 #define ATOMIC_FETCH_OP(op, c_op) \ 47 static __inline__ int \ 48 ia64_atomic_fetch_##op (int i, atomic_t *v) \ 49 { \ 50 __s32 old, new; \ 51 CMPXCHG_BUGCHECK_DECL \ 52 \ 53 do { \ 54 CMPXCHG_BUGCHECK(v); \ 55 old = atomic_read(v); \ 56 new = old c_op i; \ 57 } while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic_t)) != old); \ 58 return old; \ 59 } 60 61 #define ATOMIC_OPS(op, c_op) \ 62 ATOMIC_OP(op, c_op) \ 63 ATOMIC_FETCH_OP(op, c_op) 64 65 ATOMIC_OPS(add, +) 66 ATOMIC_OPS(sub, -) 67 68 #define atomic_add_return(i,v) \ 69 ({ \ 70 int __ia64_aar_i = (i); \ 71 (__builtin_constant_p(i) \ 72 && ( (__ia64_aar_i == 1) || (__ia64_aar_i == 4) \ 73 || (__ia64_aar_i == 8) || (__ia64_aar_i == 16) \ 74 || (__ia64_aar_i == -1) || (__ia64_aar_i == -4) \ 75 || (__ia64_aar_i == -8) || (__ia64_aar_i == -16))) \ 76 ? ia64_fetch_and_add(__ia64_aar_i, &(v)->counter) \ 77 : ia64_atomic_add(__ia64_aar_i, v); \ 78 }) 79 80 #define atomic_sub_return(i,v) \ 81 ({ \ 82 int __ia64_asr_i = (i); \ 83 (__builtin_constant_p(i) \ 84 && ( (__ia64_asr_i == 1) || (__ia64_asr_i == 4) \ 85 || (__ia64_asr_i == 8) || (__ia64_asr_i == 16) \ 86 || (__ia64_asr_i == -1) || (__ia64_asr_i == -4) \ 87 || (__ia64_asr_i == -8) || (__ia64_asr_i == -16))) \ 88 ? ia64_fetch_and_add(-__ia64_asr_i, &(v)->counter) \ 89 : ia64_atomic_sub(__ia64_asr_i, v); \ 90 }) 91 92 #define atomic_fetch_add(i,v) \ 93 ({ \ 94 int __ia64_aar_i = (i); \ 95 (__builtin_constant_p(i) \ 96 && ( (__ia64_aar_i == 1) || (__ia64_aar_i == 4) \ 97 || (__ia64_aar_i == 8) || (__ia64_aar_i == 16) \ 98 || (__ia64_aar_i == -1) || (__ia64_aar_i == -4) \ 99 || (__ia64_aar_i == -8) || (__ia64_aar_i == -16))) \ 100 ? ia64_fetchadd(__ia64_aar_i, &(v)->counter, acq) \ 101 : ia64_atomic_fetch_add(__ia64_aar_i, v); \ 102 }) 103 104 #define atomic_fetch_sub(i,v) \ 105 ({ \ 106 int __ia64_asr_i = (i); \ 107 (__builtin_constant_p(i) \ 108 && ( (__ia64_asr_i == 1) || (__ia64_asr_i == 4) \ 109 || (__ia64_asr_i == 8) || (__ia64_asr_i == 16) \ 110 || (__ia64_asr_i == -1) || (__ia64_asr_i == -4) \ 111 || (__ia64_asr_i == -8) || (__ia64_asr_i == -16))) \ 112 ? ia64_fetchadd(-__ia64_asr_i, &(v)->counter, acq) \ 113 : ia64_atomic_fetch_sub(__ia64_asr_i, v); \ 114 }) 115 116 ATOMIC_FETCH_OP(and, &) 117 ATOMIC_FETCH_OP(or, |) 118 ATOMIC_FETCH_OP(xor, ^) 119 120 #define atomic_and(i,v) (void)ia64_atomic_fetch_and(i,v) 121 #define atomic_or(i,v) (void)ia64_atomic_fetch_or(i,v) 122 #define atomic_xor(i,v) (void)ia64_atomic_fetch_xor(i,v) 123 124 #define atomic_fetch_and(i,v) ia64_atomic_fetch_and(i,v) 125 #define atomic_fetch_or(i,v) ia64_atomic_fetch_or(i,v) 126 #define atomic_fetch_xor(i,v) ia64_atomic_fetch_xor(i,v) 127 128 #undef ATOMIC_OPS 129 #undef ATOMIC_FETCH_OP 130 #undef ATOMIC_OP 131 132 #define ATOMIC64_OP(op, c_op) \ 133 static __inline__ long \ 134 ia64_atomic64_##op (__s64 i, atomic64_t *v) \ 135 { \ 136 __s64 old, new; \ 137 CMPXCHG_BUGCHECK_DECL \ 138 \ 139 do { \ 140 CMPXCHG_BUGCHECK(v); \ 141 old = atomic64_read(v); \ 142 new = old c_op i; \ 143 } while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic64_t)) != old); \ 144 return new; \ 145 } 146 147 #define ATOMIC64_FETCH_OP(op, c_op) \ 148 static __inline__ long \ 149 ia64_atomic64_fetch_##op (__s64 i, atomic64_t *v) \ 150 { \ 151 __s64 old, new; \ 152 CMPXCHG_BUGCHECK_DECL \ 153 \ 154 do { \ 155 CMPXCHG_BUGCHECK(v); \ 156 old = atomic64_read(v); \ 157 new = old c_op i; \ 158 } while (ia64_cmpxchg(acq, v, old, new, sizeof(atomic64_t)) != old); \ 159 return old; \ 160 } 161 162 #define ATOMIC64_OPS(op, c_op) \ 163 ATOMIC64_OP(op, c_op) \ 164 ATOMIC64_FETCH_OP(op, c_op) 165 166 ATOMIC64_OPS(add, +) 167 ATOMIC64_OPS(sub, -) 168 169 #define atomic64_add_return(i,v) \ 170 ({ \ 171 long __ia64_aar_i = (i); \ 172 (__builtin_constant_p(i) \ 173 && ( (__ia64_aar_i == 1) || (__ia64_aar_i == 4) \ 174 || (__ia64_aar_i == 8) || (__ia64_aar_i == 16) \ 175 || (__ia64_aar_i == -1) || (__ia64_aar_i == -4) \ 176 || (__ia64_aar_i == -8) || (__ia64_aar_i == -16))) \ 177 ? ia64_fetch_and_add(__ia64_aar_i, &(v)->counter) \ 178 : ia64_atomic64_add(__ia64_aar_i, v); \ 179 }) 180 181 #define atomic64_sub_return(i,v) \ 182 ({ \ 183 long __ia64_asr_i = (i); \ 184 (__builtin_constant_p(i) \ 185 && ( (__ia64_asr_i == 1) || (__ia64_asr_i == 4) \ 186 || (__ia64_asr_i == 8) || (__ia64_asr_i == 16) \ 187 || (__ia64_asr_i == -1) || (__ia64_asr_i == -4) \ 188 || (__ia64_asr_i == -8) || (__ia64_asr_i == -16))) \ 189 ? ia64_fetch_and_add(-__ia64_asr_i, &(v)->counter) \ 190 : ia64_atomic64_sub(__ia64_asr_i, v); \ 191 }) 192 193 #define atomic64_fetch_add(i,v) \ 194 ({ \ 195 long __ia64_aar_i = (i); \ 196 (__builtin_constant_p(i) \ 197 && ( (__ia64_aar_i == 1) || (__ia64_aar_i == 4) \ 198 || (__ia64_aar_i == 8) || (__ia64_aar_i == 16) \ 199 || (__ia64_aar_i == -1) || (__ia64_aar_i == -4) \ 200 || (__ia64_aar_i == -8) || (__ia64_aar_i == -16))) \ 201 ? ia64_fetchadd(__ia64_aar_i, &(v)->counter, acq) \ 202 : ia64_atomic64_fetch_add(__ia64_aar_i, v); \ 203 }) 204 205 #define atomic64_fetch_sub(i,v) \ 206 ({ \ 207 long __ia64_asr_i = (i); \ 208 (__builtin_constant_p(i) \ 209 && ( (__ia64_asr_i == 1) || (__ia64_asr_i == 4) \ 210 || (__ia64_asr_i == 8) || (__ia64_asr_i == 16) \ 211 || (__ia64_asr_i == -1) || (__ia64_asr_i == -4) \ 212 || (__ia64_asr_i == -8) || (__ia64_asr_i == -16))) \ 213 ? ia64_fetchadd(-__ia64_asr_i, &(v)->counter, acq) \ 214 : ia64_atomic64_fetch_sub(__ia64_asr_i, v); \ 215 }) 216 217 ATOMIC64_FETCH_OP(and, &) 218 ATOMIC64_FETCH_OP(or, |) 219 ATOMIC64_FETCH_OP(xor, ^) 220 221 #define atomic64_and(i,v) (void)ia64_atomic64_fetch_and(i,v) 222 #define atomic64_or(i,v) (void)ia64_atomic64_fetch_or(i,v) 223 #define atomic64_xor(i,v) (void)ia64_atomic64_fetch_xor(i,v) 224 225 #define atomic64_fetch_and(i,v) ia64_atomic64_fetch_and(i,v) 226 #define atomic64_fetch_or(i,v) ia64_atomic64_fetch_or(i,v) 227 #define atomic64_fetch_xor(i,v) ia64_atomic64_fetch_xor(i,v) 228 229 #undef ATOMIC64_OPS 230 #undef ATOMIC64_FETCH_OP 231 #undef ATOMIC64_OP 232 233 #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) 234 #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 235 236 #define atomic64_cmpxchg(v, old, new) \ 237 (cmpxchg(&((v)->counter), old, new)) 238 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) 239 240 static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) 241 { 242 int c, old; 243 c = atomic_read(v); 244 for (;;) { 245 if (unlikely(c == (u))) 246 break; 247 old = atomic_cmpxchg((v), c, c + (a)); 248 if (likely(old == c)) 249 break; 250 c = old; 251 } 252 return c; 253 } 254 255 256 static __inline__ long atomic64_add_unless(atomic64_t *v, long a, long u) 257 { 258 long c, old; 259 c = atomic64_read(v); 260 for (;;) { 261 if (unlikely(c == (u))) 262 break; 263 old = atomic64_cmpxchg((v), c, c + (a)); 264 if (likely(old == c)) 265 break; 266 c = old; 267 } 268 return c != (u); 269 } 270 271 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) 272 273 static __inline__ long atomic64_dec_if_positive(atomic64_t *v) 274 { 275 long c, old, dec; 276 c = atomic64_read(v); 277 for (;;) { 278 dec = c - 1; 279 if (unlikely(dec < 0)) 280 break; 281 old = atomic64_cmpxchg((v), c, dec); 282 if (likely(old == c)) 283 break; 284 c = old; 285 } 286 return dec; 287 } 288 289 /* 290 * Atomically add I to V and return TRUE if the resulting value is 291 * negative. 292 */ 293 static __inline__ int 294 atomic_add_negative (int i, atomic_t *v) 295 { 296 return atomic_add_return(i, v) < 0; 297 } 298 299 static __inline__ long 300 atomic64_add_negative (__s64 i, atomic64_t *v) 301 { 302 return atomic64_add_return(i, v) < 0; 303 } 304 305 #define atomic_dec_return(v) atomic_sub_return(1, (v)) 306 #define atomic_inc_return(v) atomic_add_return(1, (v)) 307 #define atomic64_dec_return(v) atomic64_sub_return(1, (v)) 308 #define atomic64_inc_return(v) atomic64_add_return(1, (v)) 309 310 #define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0) 311 #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) 312 #define atomic_inc_and_test(v) (atomic_add_return(1, (v)) == 0) 313 #define atomic64_sub_and_test(i,v) (atomic64_sub_return((i), (v)) == 0) 314 #define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0) 315 #define atomic64_inc_and_test(v) (atomic64_add_return(1, (v)) == 0) 316 317 #define atomic_add(i,v) (void)atomic_add_return((i), (v)) 318 #define atomic_sub(i,v) (void)atomic_sub_return((i), (v)) 319 #define atomic_inc(v) atomic_add(1, (v)) 320 #define atomic_dec(v) atomic_sub(1, (v)) 321 322 #define atomic64_add(i,v) (void)atomic64_add_return((i), (v)) 323 #define atomic64_sub(i,v) (void)atomic64_sub_return((i), (v)) 324 #define atomic64_inc(v) atomic64_add(1, (v)) 325 #define atomic64_dec(v) atomic64_sub(1, (v)) 326 327 #endif /* _ASM_IA64_ATOMIC_H */ 328