1 /* 2 * Based on arch/arm/include/asm/atomic.h 3 * 4 * Copyright (C) 1996 Russell King. 5 * Copyright (C) 2002 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #ifndef __ASM_ATOMIC_LL_SC_H 22 #define __ASM_ATOMIC_LL_SC_H 23 24 #ifndef __ARM64_IN_ATOMIC_IMPL 25 #error "please don't include this file directly" 26 #endif 27 28 /* 29 * AArch64 UP and SMP safe atomic ops. We use load exclusive and 30 * store exclusive to ensure that these are atomic. We may loop 31 * to ensure that the update happens. 32 * 33 * NOTE: these functions do *not* follow the PCS and must explicitly 34 * save any clobbered registers other than x0 (regardless of return 35 * value). This is achieved through -fcall-saved-* compiler flags for 36 * this file, which unfortunately don't work on a per-function basis 37 * (the optimize attribute silently ignores these options). 38 */ 39 40 #define ATOMIC_OP(op, asm_op) \ 41 __LL_SC_INLINE void \ 42 __LL_SC_PREFIX(arch_atomic_##op(int i, atomic_t *v)) \ 43 { \ 44 unsigned long tmp; \ 45 int result; \ 46 \ 47 asm volatile("// atomic_" #op "\n" \ 48 " prfm pstl1strm, %2\n" \ 49 "1: ldxr %w0, %2\n" \ 50 " " #asm_op " %w0, %w0, %w3\n" \ 51 " stxr %w1, %w0, %2\n" \ 52 " cbnz %w1, 1b" \ 53 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 54 : "Ir" (i)); \ 55 } \ 56 __LL_SC_EXPORT(arch_atomic_##op); 57 58 #define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \ 59 __LL_SC_INLINE int \ 60 __LL_SC_PREFIX(arch_atomic_##op##_return##name(int i, atomic_t *v)) \ 61 { \ 62 unsigned long tmp; \ 63 int result; \ 64 \ 65 asm volatile("// atomic_" #op "_return" #name "\n" \ 66 " prfm pstl1strm, %2\n" \ 67 "1: ld" #acq "xr %w0, %2\n" \ 68 " " #asm_op " %w0, %w0, %w3\n" \ 69 " st" #rel "xr %w1, %w0, %2\n" \ 70 " cbnz %w1, 1b\n" \ 71 " " #mb \ 72 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 73 : "Ir" (i) \ 74 : cl); \ 75 \ 76 return result; \ 77 } \ 78 __LL_SC_EXPORT(arch_atomic_##op##_return##name); 79 80 #define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op) \ 81 __LL_SC_INLINE int \ 82 __LL_SC_PREFIX(arch_atomic_fetch_##op##name(int i, atomic_t *v)) \ 83 { \ 84 unsigned long tmp; \ 85 int val, result; \ 86 \ 87 asm volatile("// atomic_fetch_" #op #name "\n" \ 88 " prfm pstl1strm, %3\n" \ 89 "1: ld" #acq "xr %w0, %3\n" \ 90 " " #asm_op " %w1, %w0, %w4\n" \ 91 " st" #rel "xr %w2, %w1, %3\n" \ 92 " cbnz %w2, 1b\n" \ 93 " " #mb \ 94 : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ 95 : "Ir" (i) \ 96 : cl); \ 97 \ 98 return result; \ 99 } \ 100 __LL_SC_EXPORT(arch_atomic_fetch_##op##name); 101 102 #define ATOMIC_OPS(...) \ 103 ATOMIC_OP(__VA_ARGS__) \ 104 ATOMIC_OP_RETURN( , dmb ish, , l, "memory", __VA_ARGS__)\ 105 ATOMIC_OP_RETURN(_relaxed, , , , , __VA_ARGS__)\ 106 ATOMIC_OP_RETURN(_acquire, , a, , "memory", __VA_ARGS__)\ 107 ATOMIC_OP_RETURN(_release, , , l, "memory", __VA_ARGS__)\ 108 ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ 109 ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ 110 ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ 111 ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) 112 113 ATOMIC_OPS(add, add) 114 ATOMIC_OPS(sub, sub) 115 116 #undef ATOMIC_OPS 117 #define ATOMIC_OPS(...) \ 118 ATOMIC_OP(__VA_ARGS__) \ 119 ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ 120 ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ 121 ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ 122 ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) 123 124 ATOMIC_OPS(and, and) 125 ATOMIC_OPS(andnot, bic) 126 ATOMIC_OPS(or, orr) 127 ATOMIC_OPS(xor, eor) 128 129 #undef ATOMIC_OPS 130 #undef ATOMIC_FETCH_OP 131 #undef ATOMIC_OP_RETURN 132 #undef ATOMIC_OP 133 134 #define ATOMIC64_OP(op, asm_op) \ 135 __LL_SC_INLINE void \ 136 __LL_SC_PREFIX(arch_atomic64_##op(long i, atomic64_t *v)) \ 137 { \ 138 long result; \ 139 unsigned long tmp; \ 140 \ 141 asm volatile("// atomic64_" #op "\n" \ 142 " prfm pstl1strm, %2\n" \ 143 "1: ldxr %0, %2\n" \ 144 " " #asm_op " %0, %0, %3\n" \ 145 " stxr %w1, %0, %2\n" \ 146 " cbnz %w1, 1b" \ 147 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 148 : "Ir" (i)); \ 149 } \ 150 __LL_SC_EXPORT(arch_atomic64_##op); 151 152 #define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \ 153 __LL_SC_INLINE long \ 154 __LL_SC_PREFIX(arch_atomic64_##op##_return##name(long i, atomic64_t *v))\ 155 { \ 156 long result; \ 157 unsigned long tmp; \ 158 \ 159 asm volatile("// atomic64_" #op "_return" #name "\n" \ 160 " prfm pstl1strm, %2\n" \ 161 "1: ld" #acq "xr %0, %2\n" \ 162 " " #asm_op " %0, %0, %3\n" \ 163 " st" #rel "xr %w1, %0, %2\n" \ 164 " cbnz %w1, 1b\n" \ 165 " " #mb \ 166 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 167 : "Ir" (i) \ 168 : cl); \ 169 \ 170 return result; \ 171 } \ 172 __LL_SC_EXPORT(arch_atomic64_##op##_return##name); 173 174 #define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op) \ 175 __LL_SC_INLINE long \ 176 __LL_SC_PREFIX(arch_atomic64_fetch_##op##name(long i, atomic64_t *v)) \ 177 { \ 178 long result, val; \ 179 unsigned long tmp; \ 180 \ 181 asm volatile("// atomic64_fetch_" #op #name "\n" \ 182 " prfm pstl1strm, %3\n" \ 183 "1: ld" #acq "xr %0, %3\n" \ 184 " " #asm_op " %1, %0, %4\n" \ 185 " st" #rel "xr %w2, %1, %3\n" \ 186 " cbnz %w2, 1b\n" \ 187 " " #mb \ 188 : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ 189 : "Ir" (i) \ 190 : cl); \ 191 \ 192 return result; \ 193 } \ 194 __LL_SC_EXPORT(arch_atomic64_fetch_##op##name); 195 196 #define ATOMIC64_OPS(...) \ 197 ATOMIC64_OP(__VA_ARGS__) \ 198 ATOMIC64_OP_RETURN(, dmb ish, , l, "memory", __VA_ARGS__) \ 199 ATOMIC64_OP_RETURN(_relaxed,, , , , __VA_ARGS__) \ 200 ATOMIC64_OP_RETURN(_acquire,, a, , "memory", __VA_ARGS__) \ 201 ATOMIC64_OP_RETURN(_release,, , l, "memory", __VA_ARGS__) \ 202 ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ 203 ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ 204 ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ 205 ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) 206 207 ATOMIC64_OPS(add, add) 208 ATOMIC64_OPS(sub, sub) 209 210 #undef ATOMIC64_OPS 211 #define ATOMIC64_OPS(...) \ 212 ATOMIC64_OP(__VA_ARGS__) \ 213 ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ 214 ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ 215 ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ 216 ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) 217 218 ATOMIC64_OPS(and, and) 219 ATOMIC64_OPS(andnot, bic) 220 ATOMIC64_OPS(or, orr) 221 ATOMIC64_OPS(xor, eor) 222 223 #undef ATOMIC64_OPS 224 #undef ATOMIC64_FETCH_OP 225 #undef ATOMIC64_OP_RETURN 226 #undef ATOMIC64_OP 227 228 __LL_SC_INLINE long 229 __LL_SC_PREFIX(arch_atomic64_dec_if_positive(atomic64_t *v)) 230 { 231 long result; 232 unsigned long tmp; 233 234 asm volatile("// atomic64_dec_if_positive\n" 235 " prfm pstl1strm, %2\n" 236 "1: ldxr %0, %2\n" 237 " subs %0, %0, #1\n" 238 " b.lt 2f\n" 239 " stlxr %w1, %0, %2\n" 240 " cbnz %w1, 1b\n" 241 " dmb ish\n" 242 "2:" 243 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) 244 : 245 : "cc", "memory"); 246 247 return result; 248 } 249 __LL_SC_EXPORT(arch_atomic64_dec_if_positive); 250 251 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl) \ 252 __LL_SC_INLINE u##sz \ 253 __LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr, \ 254 unsigned long old, \ 255 u##sz new)) \ 256 { \ 257 unsigned long tmp; \ 258 u##sz oldval; \ 259 \ 260 /* \ 261 * Sub-word sizes require explicit casting so that the compare \ 262 * part of the cmpxchg doesn't end up interpreting non-zero \ 263 * upper bits of the register containing "old". \ 264 */ \ 265 if (sz < 32) \ 266 old = (u##sz)old; \ 267 \ 268 asm volatile( \ 269 " prfm pstl1strm, %[v]\n" \ 270 "1: ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n" \ 271 " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \ 272 " cbnz %" #w "[tmp], 2f\n" \ 273 " st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n" \ 274 " cbnz %w[tmp], 1b\n" \ 275 " " #mb "\n" \ 276 "2:" \ 277 : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \ 278 [v] "+Q" (*(u##sz *)ptr) \ 279 : [old] "Kr" (old), [new] "r" (new) \ 280 : cl); \ 281 \ 282 return oldval; \ 283 } \ 284 __LL_SC_EXPORT(__cmpxchg_case_##name##sz); 285 286 __CMPXCHG_CASE(w, b, , 8, , , , ) 287 __CMPXCHG_CASE(w, h, , 16, , , , ) 288 __CMPXCHG_CASE(w, , , 32, , , , ) 289 __CMPXCHG_CASE( , , , 64, , , , ) 290 __CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory") 291 __CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory") 292 __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory") 293 __CMPXCHG_CASE( , , acq_, 64, , a, , "memory") 294 __CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory") 295 __CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory") 296 __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory") 297 __CMPXCHG_CASE( , , rel_, 64, , , l, "memory") 298 __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory") 299 __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory") 300 __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory") 301 __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory") 302 303 #undef __CMPXCHG_CASE 304 305 #define __CMPXCHG_DBL(name, mb, rel, cl) \ 306 __LL_SC_INLINE long \ 307 __LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \ 308 unsigned long old2, \ 309 unsigned long new1, \ 310 unsigned long new2, \ 311 volatile void *ptr)) \ 312 { \ 313 unsigned long tmp, ret; \ 314 \ 315 asm volatile("// __cmpxchg_double" #name "\n" \ 316 " prfm pstl1strm, %2\n" \ 317 "1: ldxp %0, %1, %2\n" \ 318 " eor %0, %0, %3\n" \ 319 " eor %1, %1, %4\n" \ 320 " orr %1, %0, %1\n" \ 321 " cbnz %1, 2f\n" \ 322 " st" #rel "xp %w0, %5, %6, %2\n" \ 323 " cbnz %w0, 1b\n" \ 324 " " #mb "\n" \ 325 "2:" \ 326 : "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr) \ 327 : "r" (old1), "r" (old2), "r" (new1), "r" (new2) \ 328 : cl); \ 329 \ 330 return ret; \ 331 } \ 332 __LL_SC_EXPORT(__cmpxchg_double##name); 333 334 __CMPXCHG_DBL( , , , ) 335 __CMPXCHG_DBL(_mb, dmb ish, l, "memory") 336 337 #undef __CMPXCHG_DBL 338 339 #endif /* __ASM_ATOMIC_LL_SC_H */ 340