1 /* 2 * Based on arch/arm/include/asm/atomic.h 3 * 4 * Copyright (C) 1996 Russell King. 5 * Copyright (C) 2002 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #ifndef __ASM_ATOMIC_LSE_H 22 #define __ASM_ATOMIC_LSE_H 23 24 #ifndef __ARM64_IN_ATOMIC_IMPL 25 #error "please don't include this file directly" 26 #endif 27 28 #define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op) 29 30 static inline void atomic_andnot(int i, atomic_t *v) 31 { 32 register int w0 asm ("w0") = i; 33 register atomic_t *x1 asm ("x1") = v; 34 35 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(andnot), 36 " stclr %w[i], %[v]\n") 37 : [i] "+r" (w0), [v] "+Q" (v->counter) 38 : "r" (x1) 39 : "x30"); 40 } 41 42 static inline void atomic_or(int i, atomic_t *v) 43 { 44 register int w0 asm ("w0") = i; 45 register atomic_t *x1 asm ("x1") = v; 46 47 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(or), 48 " stset %w[i], %[v]\n") 49 : [i] "+r" (w0), [v] "+Q" (v->counter) 50 : "r" (x1) 51 : "x30"); 52 } 53 54 static inline void atomic_xor(int i, atomic_t *v) 55 { 56 register int w0 asm ("w0") = i; 57 register atomic_t *x1 asm ("x1") = v; 58 59 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(xor), 60 " steor %w[i], %[v]\n") 61 : [i] "+r" (w0), [v] "+Q" (v->counter) 62 : "r" (x1) 63 : "x30"); 64 } 65 66 static inline void atomic_add(int i, atomic_t *v) 67 { 68 register int w0 asm ("w0") = i; 69 register atomic_t *x1 asm ("x1") = v; 70 71 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(add), 72 " stadd %w[i], %[v]\n") 73 : [i] "+r" (w0), [v] "+Q" (v->counter) 74 : "r" (x1) 75 : "x30"); 76 } 77 78 #define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \ 79 static inline int atomic_add_return##name(int i, atomic_t *v) \ 80 { \ 81 register int w0 asm ("w0") = i; \ 82 register atomic_t *x1 asm ("x1") = v; \ 83 \ 84 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 85 /* LL/SC */ \ 86 " nop\n" \ 87 __LL_SC_ATOMIC(add_return##name), \ 88 /* LSE atomics */ \ 89 " ldadd" #mb " %w[i], w30, %[v]\n" \ 90 " add %w[i], %w[i], w30") \ 91 : [i] "+r" (w0), [v] "+Q" (v->counter) \ 92 : "r" (x1) \ 93 : "x30" , ##cl); \ 94 \ 95 return w0; \ 96 } 97 98 ATOMIC_OP_ADD_RETURN(_relaxed, ) 99 ATOMIC_OP_ADD_RETURN(_acquire, a, "memory") 100 ATOMIC_OP_ADD_RETURN(_release, l, "memory") 101 ATOMIC_OP_ADD_RETURN( , al, "memory") 102 103 #undef ATOMIC_OP_ADD_RETURN 104 105 static inline void atomic_and(int i, atomic_t *v) 106 { 107 register int w0 asm ("w0") = i; 108 register atomic_t *x1 asm ("x1") = v; 109 110 asm volatile(ARM64_LSE_ATOMIC_INSN( 111 /* LL/SC */ 112 " nop\n" 113 __LL_SC_ATOMIC(and), 114 /* LSE atomics */ 115 " mvn %w[i], %w[i]\n" 116 " stclr %w[i], %[v]") 117 : [i] "+r" (w0), [v] "+Q" (v->counter) 118 : "r" (x1) 119 : "x30"); 120 } 121 122 static inline void atomic_sub(int i, atomic_t *v) 123 { 124 register int w0 asm ("w0") = i; 125 register atomic_t *x1 asm ("x1") = v; 126 127 asm volatile(ARM64_LSE_ATOMIC_INSN( 128 /* LL/SC */ 129 " nop\n" 130 __LL_SC_ATOMIC(sub), 131 /* LSE atomics */ 132 " neg %w[i], %w[i]\n" 133 " stadd %w[i], %[v]") 134 : [i] "+r" (w0), [v] "+Q" (v->counter) 135 : "r" (x1) 136 : "x30"); 137 } 138 139 #define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \ 140 static inline int atomic_sub_return##name(int i, atomic_t *v) \ 141 { \ 142 register int w0 asm ("w0") = i; \ 143 register atomic_t *x1 asm ("x1") = v; \ 144 \ 145 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 146 /* LL/SC */ \ 147 " nop\n" \ 148 __LL_SC_ATOMIC(sub_return##name) \ 149 " nop", \ 150 /* LSE atomics */ \ 151 " neg %w[i], %w[i]\n" \ 152 " ldadd" #mb " %w[i], w30, %[v]\n" \ 153 " add %w[i], %w[i], w30") \ 154 : [i] "+r" (w0), [v] "+Q" (v->counter) \ 155 : "r" (x1) \ 156 : "x30" , ##cl); \ 157 \ 158 return w0; \ 159 } 160 161 ATOMIC_OP_SUB_RETURN(_relaxed, ) 162 ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") 163 ATOMIC_OP_SUB_RETURN(_release, l, "memory") 164 ATOMIC_OP_SUB_RETURN( , al, "memory") 165 166 #undef ATOMIC_OP_SUB_RETURN 167 #undef __LL_SC_ATOMIC 168 169 #define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op) 170 171 static inline void atomic64_andnot(long i, atomic64_t *v) 172 { 173 register long x0 asm ("x0") = i; 174 register atomic64_t *x1 asm ("x1") = v; 175 176 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(andnot), 177 " stclr %[i], %[v]\n") 178 : [i] "+r" (x0), [v] "+Q" (v->counter) 179 : "r" (x1) 180 : "x30"); 181 } 182 183 static inline void atomic64_or(long i, atomic64_t *v) 184 { 185 register long x0 asm ("x0") = i; 186 register atomic64_t *x1 asm ("x1") = v; 187 188 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(or), 189 " stset %[i], %[v]\n") 190 : [i] "+r" (x0), [v] "+Q" (v->counter) 191 : "r" (x1) 192 : "x30"); 193 } 194 195 static inline void atomic64_xor(long i, atomic64_t *v) 196 { 197 register long x0 asm ("x0") = i; 198 register atomic64_t *x1 asm ("x1") = v; 199 200 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(xor), 201 " steor %[i], %[v]\n") 202 : [i] "+r" (x0), [v] "+Q" (v->counter) 203 : "r" (x1) 204 : "x30"); 205 } 206 207 static inline void atomic64_add(long i, atomic64_t *v) 208 { 209 register long x0 asm ("x0") = i; 210 register atomic64_t *x1 asm ("x1") = v; 211 212 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(add), 213 " stadd %[i], %[v]\n") 214 : [i] "+r" (x0), [v] "+Q" (v->counter) 215 : "r" (x1) 216 : "x30"); 217 } 218 219 #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \ 220 static inline long atomic64_add_return##name(long i, atomic64_t *v) \ 221 { \ 222 register long x0 asm ("x0") = i; \ 223 register atomic64_t *x1 asm ("x1") = v; \ 224 \ 225 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 226 /* LL/SC */ \ 227 " nop\n" \ 228 __LL_SC_ATOMIC64(add_return##name), \ 229 /* LSE atomics */ \ 230 " ldadd" #mb " %[i], x30, %[v]\n" \ 231 " add %[i], %[i], x30") \ 232 : [i] "+r" (x0), [v] "+Q" (v->counter) \ 233 : "r" (x1) \ 234 : "x30" , ##cl); \ 235 \ 236 return x0; \ 237 } 238 239 ATOMIC64_OP_ADD_RETURN(_relaxed, ) 240 ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory") 241 ATOMIC64_OP_ADD_RETURN(_release, l, "memory") 242 ATOMIC64_OP_ADD_RETURN( , al, "memory") 243 244 #undef ATOMIC64_OP_ADD_RETURN 245 246 static inline void atomic64_and(long i, atomic64_t *v) 247 { 248 register long x0 asm ("x0") = i; 249 register atomic64_t *x1 asm ("x1") = v; 250 251 asm volatile(ARM64_LSE_ATOMIC_INSN( 252 /* LL/SC */ 253 " nop\n" 254 __LL_SC_ATOMIC64(and), 255 /* LSE atomics */ 256 " mvn %[i], %[i]\n" 257 " stclr %[i], %[v]") 258 : [i] "+r" (x0), [v] "+Q" (v->counter) 259 : "r" (x1) 260 : "x30"); 261 } 262 263 static inline void atomic64_sub(long i, atomic64_t *v) 264 { 265 register long x0 asm ("x0") = i; 266 register atomic64_t *x1 asm ("x1") = v; 267 268 asm volatile(ARM64_LSE_ATOMIC_INSN( 269 /* LL/SC */ 270 " nop\n" 271 __LL_SC_ATOMIC64(sub), 272 /* LSE atomics */ 273 " neg %[i], %[i]\n" 274 " stadd %[i], %[v]") 275 : [i] "+r" (x0), [v] "+Q" (v->counter) 276 : "r" (x1) 277 : "x30"); 278 } 279 280 #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \ 281 static inline long atomic64_sub_return##name(long i, atomic64_t *v) \ 282 { \ 283 register long x0 asm ("x0") = i; \ 284 register atomic64_t *x1 asm ("x1") = v; \ 285 \ 286 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 287 /* LL/SC */ \ 288 " nop\n" \ 289 __LL_SC_ATOMIC64(sub_return##name) \ 290 " nop", \ 291 /* LSE atomics */ \ 292 " neg %[i], %[i]\n" \ 293 " ldadd" #mb " %[i], x30, %[v]\n" \ 294 " add %[i], %[i], x30") \ 295 : [i] "+r" (x0), [v] "+Q" (v->counter) \ 296 : "r" (x1) \ 297 : "x30" , ##cl); \ 298 \ 299 return x0; \ 300 } 301 302 ATOMIC64_OP_SUB_RETURN(_relaxed, ) 303 ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") 304 ATOMIC64_OP_SUB_RETURN(_release, l, "memory") 305 ATOMIC64_OP_SUB_RETURN( , al, "memory") 306 307 #undef ATOMIC64_OP_SUB_RETURN 308 309 static inline long atomic64_dec_if_positive(atomic64_t *v) 310 { 311 register long x0 asm ("x0") = (long)v; 312 313 asm volatile(ARM64_LSE_ATOMIC_INSN( 314 /* LL/SC */ 315 " nop\n" 316 __LL_SC_ATOMIC64(dec_if_positive) 317 " nop\n" 318 " nop\n" 319 " nop\n" 320 " nop\n" 321 " nop", 322 /* LSE atomics */ 323 "1: ldr x30, %[v]\n" 324 " subs %[ret], x30, #1\n" 325 " b.lt 2f\n" 326 " casal x30, %[ret], %[v]\n" 327 " sub x30, x30, #1\n" 328 " sub x30, x30, %[ret]\n" 329 " cbnz x30, 1b\n" 330 "2:") 331 : [ret] "+&r" (x0), [v] "+Q" (v->counter) 332 : 333 : "x30", "cc", "memory"); 334 335 return x0; 336 } 337 338 #undef __LL_SC_ATOMIC64 339 340 #define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op) 341 342 #define __CMPXCHG_CASE(w, sz, name, mb, cl...) \ 343 static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \ 344 unsigned long old, \ 345 unsigned long new) \ 346 { \ 347 register unsigned long x0 asm ("x0") = (unsigned long)ptr; \ 348 register unsigned long x1 asm ("x1") = old; \ 349 register unsigned long x2 asm ("x2") = new; \ 350 \ 351 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 352 /* LL/SC */ \ 353 " nop\n" \ 354 __LL_SC_CMPXCHG(name) \ 355 " nop", \ 356 /* LSE atomics */ \ 357 " mov " #w "30, %" #w "[old]\n" \ 358 " cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \ 359 " mov %" #w "[ret], " #w "30") \ 360 : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \ 361 : [old] "r" (x1), [new] "r" (x2) \ 362 : "x30" , ##cl); \ 363 \ 364 return x0; \ 365 } 366 367 __CMPXCHG_CASE(w, b, 1, ) 368 __CMPXCHG_CASE(w, h, 2, ) 369 __CMPXCHG_CASE(w, , 4, ) 370 __CMPXCHG_CASE(x, , 8, ) 371 __CMPXCHG_CASE(w, b, acq_1, a, "memory") 372 __CMPXCHG_CASE(w, h, acq_2, a, "memory") 373 __CMPXCHG_CASE(w, , acq_4, a, "memory") 374 __CMPXCHG_CASE(x, , acq_8, a, "memory") 375 __CMPXCHG_CASE(w, b, rel_1, l, "memory") 376 __CMPXCHG_CASE(w, h, rel_2, l, "memory") 377 __CMPXCHG_CASE(w, , rel_4, l, "memory") 378 __CMPXCHG_CASE(x, , rel_8, l, "memory") 379 __CMPXCHG_CASE(w, b, mb_1, al, "memory") 380 __CMPXCHG_CASE(w, h, mb_2, al, "memory") 381 __CMPXCHG_CASE(w, , mb_4, al, "memory") 382 __CMPXCHG_CASE(x, , mb_8, al, "memory") 383 384 #undef __LL_SC_CMPXCHG 385 #undef __CMPXCHG_CASE 386 387 #define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op) 388 389 #define __CMPXCHG_DBL(name, mb, cl...) \ 390 static inline long __cmpxchg_double##name(unsigned long old1, \ 391 unsigned long old2, \ 392 unsigned long new1, \ 393 unsigned long new2, \ 394 volatile void *ptr) \ 395 { \ 396 unsigned long oldval1 = old1; \ 397 unsigned long oldval2 = old2; \ 398 register unsigned long x0 asm ("x0") = old1; \ 399 register unsigned long x1 asm ("x1") = old2; \ 400 register unsigned long x2 asm ("x2") = new1; \ 401 register unsigned long x3 asm ("x3") = new2; \ 402 register unsigned long x4 asm ("x4") = (unsigned long)ptr; \ 403 \ 404 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 405 /* LL/SC */ \ 406 " nop\n" \ 407 " nop\n" \ 408 " nop\n" \ 409 __LL_SC_CMPXCHG_DBL(name), \ 410 /* LSE atomics */ \ 411 " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\ 412 " eor %[old1], %[old1], %[oldval1]\n" \ 413 " eor %[old2], %[old2], %[oldval2]\n" \ 414 " orr %[old1], %[old1], %[old2]") \ 415 : [old1] "+r" (x0), [old2] "+r" (x1), \ 416 [v] "+Q" (*(unsigned long *)ptr) \ 417 : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \ 418 [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \ 419 : "x30" , ##cl); \ 420 \ 421 return x0; \ 422 } 423 424 __CMPXCHG_DBL( , ) 425 __CMPXCHG_DBL(_mb, al, "memory") 426 427 #undef __LL_SC_CMPXCHG_DBL 428 #undef __CMPXCHG_DBL 429 430 #endif /* __ASM_ATOMIC_LSE_H */ 431