1 /* 2 * Based on arch/arm/include/asm/atomic.h 3 * 4 * Copyright (C) 1996 Russell King. 5 * Copyright (C) 2002 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #ifndef __ASM_ATOMIC_LL_SC_H 22 #define __ASM_ATOMIC_LL_SC_H 23 24 #ifndef __ARM64_IN_ATOMIC_IMPL 25 #error "please don't include this file directly" 26 #endif 27 28 /* 29 * AArch64 UP and SMP safe atomic ops. We use load exclusive and 30 * store exclusive to ensure that these are atomic. We may loop 31 * to ensure that the update happens. 32 * 33 * NOTE: these functions do *not* follow the PCS and must explicitly 34 * save any clobbered registers other than x0 (regardless of return 35 * value). This is achieved through -fcall-saved-* compiler flags for 36 * this file, which unfortunately don't work on a per-function basis 37 * (the optimize attribute silently ignores these options). 38 */ 39 40 #define ATOMIC_OP(op, asm_op) \ 41 __LL_SC_INLINE void \ 42 __LL_SC_PREFIX(atomic_##op(int i, atomic_t *v)) \ 43 { \ 44 unsigned long tmp; \ 45 int result; \ 46 \ 47 asm volatile("// atomic_" #op "\n" \ 48 " prfm pstl1strm, %2\n" \ 49 "1: ldxr %w0, %2\n" \ 50 " " #asm_op " %w0, %w0, %w3\n" \ 51 " stxr %w1, %w0, %2\n" \ 52 " cbnz %w1, 1b" \ 53 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 54 : "Ir" (i)); \ 55 } \ 56 __LL_SC_EXPORT(atomic_##op); 57 58 #define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \ 59 __LL_SC_INLINE int \ 60 __LL_SC_PREFIX(atomic_##op##_return##name(int i, atomic_t *v)) \ 61 { \ 62 unsigned long tmp; \ 63 int result; \ 64 \ 65 asm volatile("// atomic_" #op "_return" #name "\n" \ 66 " prfm pstl1strm, %2\n" \ 67 "1: ld" #acq "xr %w0, %2\n" \ 68 " " #asm_op " %w0, %w0, %w3\n" \ 69 " st" #rel "xr %w1, %w0, %2\n" \ 70 " cbnz %w1, 1b\n" \ 71 " " #mb \ 72 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 73 : "Ir" (i) \ 74 : cl); \ 75 \ 76 return result; \ 77 } \ 78 __LL_SC_EXPORT(atomic_##op##_return##name); 79 80 #define ATOMIC_OPS(...) \ 81 ATOMIC_OP(__VA_ARGS__) \ 82 ATOMIC_OP_RETURN( , dmb ish, , l, "memory", __VA_ARGS__) 83 84 #define ATOMIC_OPS_RLX(...) \ 85 ATOMIC_OPS(__VA_ARGS__) \ 86 ATOMIC_OP_RETURN(_relaxed, , , , , __VA_ARGS__)\ 87 ATOMIC_OP_RETURN(_acquire, , a, , "memory", __VA_ARGS__)\ 88 ATOMIC_OP_RETURN(_release, , , l, "memory", __VA_ARGS__) 89 90 ATOMIC_OPS_RLX(add, add) 91 ATOMIC_OPS_RLX(sub, sub) 92 93 ATOMIC_OP(and, and) 94 ATOMIC_OP(andnot, bic) 95 ATOMIC_OP(or, orr) 96 ATOMIC_OP(xor, eor) 97 98 #undef ATOMIC_OPS_RLX 99 #undef ATOMIC_OPS 100 #undef ATOMIC_OP_RETURN 101 #undef ATOMIC_OP 102 103 #define ATOMIC64_OP(op, asm_op) \ 104 __LL_SC_INLINE void \ 105 __LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v)) \ 106 { \ 107 long result; \ 108 unsigned long tmp; \ 109 \ 110 asm volatile("// atomic64_" #op "\n" \ 111 " prfm pstl1strm, %2\n" \ 112 "1: ldxr %0, %2\n" \ 113 " " #asm_op " %0, %0, %3\n" \ 114 " stxr %w1, %0, %2\n" \ 115 " cbnz %w1, 1b" \ 116 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 117 : "Ir" (i)); \ 118 } \ 119 __LL_SC_EXPORT(atomic64_##op); 120 121 #define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \ 122 __LL_SC_INLINE long \ 123 __LL_SC_PREFIX(atomic64_##op##_return##name(long i, atomic64_t *v)) \ 124 { \ 125 long result; \ 126 unsigned long tmp; \ 127 \ 128 asm volatile("// atomic64_" #op "_return" #name "\n" \ 129 " prfm pstl1strm, %2\n" \ 130 "1: ld" #acq "xr %0, %2\n" \ 131 " " #asm_op " %0, %0, %3\n" \ 132 " st" #rel "xr %w1, %0, %2\n" \ 133 " cbnz %w1, 1b\n" \ 134 " " #mb \ 135 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 136 : "Ir" (i) \ 137 : cl); \ 138 \ 139 return result; \ 140 } \ 141 __LL_SC_EXPORT(atomic64_##op##_return##name); 142 143 #define ATOMIC64_OPS(...) \ 144 ATOMIC64_OP(__VA_ARGS__) \ 145 ATOMIC64_OP_RETURN(, dmb ish, , l, "memory", __VA_ARGS__) 146 147 #define ATOMIC64_OPS_RLX(...) \ 148 ATOMIC64_OPS(__VA_ARGS__) \ 149 ATOMIC64_OP_RETURN(_relaxed,, , , , __VA_ARGS__) \ 150 ATOMIC64_OP_RETURN(_acquire,, a, , "memory", __VA_ARGS__) \ 151 ATOMIC64_OP_RETURN(_release,, , l, "memory", __VA_ARGS__) 152 153 ATOMIC64_OPS_RLX(add, add) 154 ATOMIC64_OPS_RLX(sub, sub) 155 156 ATOMIC64_OP(and, and) 157 ATOMIC64_OP(andnot, bic) 158 ATOMIC64_OP(or, orr) 159 ATOMIC64_OP(xor, eor) 160 161 #undef ATOMIC64_OPS_RLX 162 #undef ATOMIC64_OPS 163 #undef ATOMIC64_OP_RETURN 164 #undef ATOMIC64_OP 165 166 __LL_SC_INLINE long 167 __LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v)) 168 { 169 long result; 170 unsigned long tmp; 171 172 asm volatile("// atomic64_dec_if_positive\n" 173 " prfm pstl1strm, %2\n" 174 "1: ldxr %0, %2\n" 175 " subs %0, %0, #1\n" 176 " b.lt 2f\n" 177 " stlxr %w1, %0, %2\n" 178 " cbnz %w1, 1b\n" 179 " dmb ish\n" 180 "2:" 181 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) 182 : 183 : "cc", "memory"); 184 185 return result; 186 } 187 __LL_SC_EXPORT(atomic64_dec_if_positive); 188 189 #define __CMPXCHG_CASE(w, sz, name, mb, acq, rel, cl) \ 190 __LL_SC_INLINE unsigned long \ 191 __LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \ 192 unsigned long old, \ 193 unsigned long new)) \ 194 { \ 195 unsigned long tmp, oldval; \ 196 \ 197 asm volatile( \ 198 " prfm pstl1strm, %[v]\n" \ 199 "1: ld" #acq "xr" #sz "\t%" #w "[oldval], %[v]\n" \ 200 " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \ 201 " cbnz %" #w "[tmp], 2f\n" \ 202 " st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \ 203 " cbnz %w[tmp], 1b\n" \ 204 " " #mb "\n" \ 205 " mov %" #w "[oldval], %" #w "[old]\n" \ 206 "2:" \ 207 : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \ 208 [v] "+Q" (*(unsigned long *)ptr) \ 209 : [old] "Lr" (old), [new] "r" (new) \ 210 : cl); \ 211 \ 212 return oldval; \ 213 } \ 214 __LL_SC_EXPORT(__cmpxchg_case_##name); 215 216 __CMPXCHG_CASE(w, b, 1, , , , ) 217 __CMPXCHG_CASE(w, h, 2, , , , ) 218 __CMPXCHG_CASE(w, , 4, , , , ) 219 __CMPXCHG_CASE( , , 8, , , , ) 220 __CMPXCHG_CASE(w, b, acq_1, , a, , "memory") 221 __CMPXCHG_CASE(w, h, acq_2, , a, , "memory") 222 __CMPXCHG_CASE(w, , acq_4, , a, , "memory") 223 __CMPXCHG_CASE( , , acq_8, , a, , "memory") 224 __CMPXCHG_CASE(w, b, rel_1, , , l, "memory") 225 __CMPXCHG_CASE(w, h, rel_2, , , l, "memory") 226 __CMPXCHG_CASE(w, , rel_4, , , l, "memory") 227 __CMPXCHG_CASE( , , rel_8, , , l, "memory") 228 __CMPXCHG_CASE(w, b, mb_1, dmb ish, , l, "memory") 229 __CMPXCHG_CASE(w, h, mb_2, dmb ish, , l, "memory") 230 __CMPXCHG_CASE(w, , mb_4, dmb ish, , l, "memory") 231 __CMPXCHG_CASE( , , mb_8, dmb ish, , l, "memory") 232 233 #undef __CMPXCHG_CASE 234 235 #define __CMPXCHG_DBL(name, mb, rel, cl) \ 236 __LL_SC_INLINE long \ 237 __LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \ 238 unsigned long old2, \ 239 unsigned long new1, \ 240 unsigned long new2, \ 241 volatile void *ptr)) \ 242 { \ 243 unsigned long tmp, ret; \ 244 \ 245 asm volatile("// __cmpxchg_double" #name "\n" \ 246 " prfm pstl1strm, %2\n" \ 247 "1: ldxp %0, %1, %2\n" \ 248 " eor %0, %0, %3\n" \ 249 " eor %1, %1, %4\n" \ 250 " orr %1, %0, %1\n" \ 251 " cbnz %1, 2f\n" \ 252 " st" #rel "xp %w0, %5, %6, %2\n" \ 253 " cbnz %w0, 1b\n" \ 254 " " #mb "\n" \ 255 "2:" \ 256 : "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr) \ 257 : "r" (old1), "r" (old2), "r" (new1), "r" (new2) \ 258 : cl); \ 259 \ 260 return ret; \ 261 } \ 262 __LL_SC_EXPORT(__cmpxchg_double##name); 263 264 __CMPXCHG_DBL( , , , ) 265 __CMPXCHG_DBL(_mb, dmb ish, l, "memory") 266 267 #undef __CMPXCHG_DBL 268 269 #endif /* __ASM_ATOMIC_LL_SC_H */ 270