1 /* 2 * Based on arch/arm/include/asm/atomic.h 3 * 4 * Copyright (C) 1996 Russell King. 5 * Copyright (C) 2002 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #ifndef __ASM_ATOMIC_LSE_H 22 #define __ASM_ATOMIC_LSE_H 23 24 #ifndef __ARM64_IN_ATOMIC_IMPL 25 #error "please don't include this file directly" 26 #endif 27 28 #define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op) 29 #define ATOMIC_OP(op, asm_op) \ 30 static inline void atomic_##op(int i, atomic_t *v) \ 31 { \ 32 register int w0 asm ("w0") = i; \ 33 register atomic_t *x1 asm ("x1") = v; \ 34 \ 35 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(op), \ 36 " " #asm_op " %w[i], %[v]\n") \ 37 : [i] "+r" (w0), [v] "+Q" (v->counter) \ 38 : "r" (x1) \ 39 : __LL_SC_CLOBBERS); \ 40 } 41 42 ATOMIC_OP(andnot, stclr) 43 ATOMIC_OP(or, stset) 44 ATOMIC_OP(xor, steor) 45 ATOMIC_OP(add, stadd) 46 47 #undef ATOMIC_OP 48 49 #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...) \ 50 static inline int atomic_fetch_##op##name(int i, atomic_t *v) \ 51 { \ 52 register int w0 asm ("w0") = i; \ 53 register atomic_t *x1 asm ("x1") = v; \ 54 \ 55 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 56 /* LL/SC */ \ 57 __LL_SC_ATOMIC(fetch_##op##name), \ 58 /* LSE atomics */ \ 59 " " #asm_op #mb " %w[i], %w[i], %[v]") \ 60 : [i] "+r" (w0), [v] "+Q" (v->counter) \ 61 : "r" (x1) \ 62 : __LL_SC_CLOBBERS, ##cl); \ 63 \ 64 return w0; \ 65 } 66 67 #define ATOMIC_FETCH_OPS(op, asm_op) \ 68 ATOMIC_FETCH_OP(_relaxed, , op, asm_op) \ 69 ATOMIC_FETCH_OP(_acquire, a, op, asm_op, "memory") \ 70 ATOMIC_FETCH_OP(_release, l, op, asm_op, "memory") \ 71 ATOMIC_FETCH_OP( , al, op, asm_op, "memory") 72 73 ATOMIC_FETCH_OPS(andnot, ldclr) 74 ATOMIC_FETCH_OPS(or, ldset) 75 ATOMIC_FETCH_OPS(xor, ldeor) 76 ATOMIC_FETCH_OPS(add, ldadd) 77 78 #undef ATOMIC_FETCH_OP 79 #undef ATOMIC_FETCH_OPS 80 81 #define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \ 82 static inline int atomic_add_return##name(int i, atomic_t *v) \ 83 { \ 84 register int w0 asm ("w0") = i; \ 85 register atomic_t *x1 asm ("x1") = v; \ 86 \ 87 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 88 /* LL/SC */ \ 89 " nop\n" \ 90 __LL_SC_ATOMIC(add_return##name), \ 91 /* LSE atomics */ \ 92 " ldadd" #mb " %w[i], w30, %[v]\n" \ 93 " add %w[i], %w[i], w30") \ 94 : [i] "+r" (w0), [v] "+Q" (v->counter) \ 95 : "r" (x1) \ 96 : __LL_SC_CLOBBERS, ##cl); \ 97 \ 98 return w0; \ 99 } 100 101 ATOMIC_OP_ADD_RETURN(_relaxed, ) 102 ATOMIC_OP_ADD_RETURN(_acquire, a, "memory") 103 ATOMIC_OP_ADD_RETURN(_release, l, "memory") 104 ATOMIC_OP_ADD_RETURN( , al, "memory") 105 106 #undef ATOMIC_OP_ADD_RETURN 107 108 static inline void atomic_and(int i, atomic_t *v) 109 { 110 register int w0 asm ("w0") = i; 111 register atomic_t *x1 asm ("x1") = v; 112 113 asm volatile(ARM64_LSE_ATOMIC_INSN( 114 /* LL/SC */ 115 " nop\n" 116 __LL_SC_ATOMIC(and), 117 /* LSE atomics */ 118 " mvn %w[i], %w[i]\n" 119 " stclr %w[i], %[v]") 120 : [i] "+r" (w0), [v] "+Q" (v->counter) 121 : "r" (x1) 122 : __LL_SC_CLOBBERS); 123 } 124 125 #define ATOMIC_FETCH_OP_AND(name, mb, cl...) \ 126 static inline int atomic_fetch_and##name(int i, atomic_t *v) \ 127 { \ 128 register int w0 asm ("w0") = i; \ 129 register atomic_t *x1 asm ("x1") = v; \ 130 \ 131 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 132 /* LL/SC */ \ 133 " nop\n" \ 134 __LL_SC_ATOMIC(fetch_and##name), \ 135 /* LSE atomics */ \ 136 " mvn %w[i], %w[i]\n" \ 137 " ldclr" #mb " %w[i], %w[i], %[v]") \ 138 : [i] "+r" (w0), [v] "+Q" (v->counter) \ 139 : "r" (x1) \ 140 : __LL_SC_CLOBBERS, ##cl); \ 141 \ 142 return w0; \ 143 } 144 145 ATOMIC_FETCH_OP_AND(_relaxed, ) 146 ATOMIC_FETCH_OP_AND(_acquire, a, "memory") 147 ATOMIC_FETCH_OP_AND(_release, l, "memory") 148 ATOMIC_FETCH_OP_AND( , al, "memory") 149 150 #undef ATOMIC_FETCH_OP_AND 151 152 static inline void atomic_sub(int i, atomic_t *v) 153 { 154 register int w0 asm ("w0") = i; 155 register atomic_t *x1 asm ("x1") = v; 156 157 asm volatile(ARM64_LSE_ATOMIC_INSN( 158 /* LL/SC */ 159 " nop\n" 160 __LL_SC_ATOMIC(sub), 161 /* LSE atomics */ 162 " neg %w[i], %w[i]\n" 163 " stadd %w[i], %[v]") 164 : [i] "+r" (w0), [v] "+Q" (v->counter) 165 : "r" (x1) 166 : __LL_SC_CLOBBERS); 167 } 168 169 #define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \ 170 static inline int atomic_sub_return##name(int i, atomic_t *v) \ 171 { \ 172 register int w0 asm ("w0") = i; \ 173 register atomic_t *x1 asm ("x1") = v; \ 174 \ 175 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 176 /* LL/SC */ \ 177 " nop\n" \ 178 __LL_SC_ATOMIC(sub_return##name) \ 179 " nop", \ 180 /* LSE atomics */ \ 181 " neg %w[i], %w[i]\n" \ 182 " ldadd" #mb " %w[i], w30, %[v]\n" \ 183 " add %w[i], %w[i], w30") \ 184 : [i] "+r" (w0), [v] "+Q" (v->counter) \ 185 : "r" (x1) \ 186 : __LL_SC_CLOBBERS , ##cl); \ 187 \ 188 return w0; \ 189 } 190 191 ATOMIC_OP_SUB_RETURN(_relaxed, ) 192 ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") 193 ATOMIC_OP_SUB_RETURN(_release, l, "memory") 194 ATOMIC_OP_SUB_RETURN( , al, "memory") 195 196 #undef ATOMIC_OP_SUB_RETURN 197 198 #define ATOMIC_FETCH_OP_SUB(name, mb, cl...) \ 199 static inline int atomic_fetch_sub##name(int i, atomic_t *v) \ 200 { \ 201 register int w0 asm ("w0") = i; \ 202 register atomic_t *x1 asm ("x1") = v; \ 203 \ 204 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 205 /* LL/SC */ \ 206 " nop\n" \ 207 __LL_SC_ATOMIC(fetch_sub##name), \ 208 /* LSE atomics */ \ 209 " neg %w[i], %w[i]\n" \ 210 " ldadd" #mb " %w[i], %w[i], %[v]") \ 211 : [i] "+r" (w0), [v] "+Q" (v->counter) \ 212 : "r" (x1) \ 213 : __LL_SC_CLOBBERS, ##cl); \ 214 \ 215 return w0; \ 216 } 217 218 ATOMIC_FETCH_OP_SUB(_relaxed, ) 219 ATOMIC_FETCH_OP_SUB(_acquire, a, "memory") 220 ATOMIC_FETCH_OP_SUB(_release, l, "memory") 221 ATOMIC_FETCH_OP_SUB( , al, "memory") 222 223 #undef ATOMIC_FETCH_OP_SUB 224 #undef __LL_SC_ATOMIC 225 226 #define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op) 227 #define ATOMIC64_OP(op, asm_op) \ 228 static inline void atomic64_##op(long i, atomic64_t *v) \ 229 { \ 230 register long x0 asm ("x0") = i; \ 231 register atomic64_t *x1 asm ("x1") = v; \ 232 \ 233 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(op), \ 234 " " #asm_op " %[i], %[v]\n") \ 235 : [i] "+r" (x0), [v] "+Q" (v->counter) \ 236 : "r" (x1) \ 237 : __LL_SC_CLOBBERS); \ 238 } 239 240 ATOMIC64_OP(andnot, stclr) 241 ATOMIC64_OP(or, stset) 242 ATOMIC64_OP(xor, steor) 243 ATOMIC64_OP(add, stadd) 244 245 #undef ATOMIC64_OP 246 247 #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...) \ 248 static inline long atomic64_fetch_##op##name(long i, atomic64_t *v) \ 249 { \ 250 register long x0 asm ("x0") = i; \ 251 register atomic64_t *x1 asm ("x1") = v; \ 252 \ 253 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 254 /* LL/SC */ \ 255 __LL_SC_ATOMIC64(fetch_##op##name), \ 256 /* LSE atomics */ \ 257 " " #asm_op #mb " %[i], %[i], %[v]") \ 258 : [i] "+r" (x0), [v] "+Q" (v->counter) \ 259 : "r" (x1) \ 260 : __LL_SC_CLOBBERS, ##cl); \ 261 \ 262 return x0; \ 263 } 264 265 #define ATOMIC64_FETCH_OPS(op, asm_op) \ 266 ATOMIC64_FETCH_OP(_relaxed, , op, asm_op) \ 267 ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ 268 ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ 269 ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") 270 271 ATOMIC64_FETCH_OPS(andnot, ldclr) 272 ATOMIC64_FETCH_OPS(or, ldset) 273 ATOMIC64_FETCH_OPS(xor, ldeor) 274 ATOMIC64_FETCH_OPS(add, ldadd) 275 276 #undef ATOMIC64_FETCH_OP 277 #undef ATOMIC64_FETCH_OPS 278 279 #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \ 280 static inline long atomic64_add_return##name(long i, atomic64_t *v) \ 281 { \ 282 register long x0 asm ("x0") = i; \ 283 register atomic64_t *x1 asm ("x1") = v; \ 284 \ 285 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 286 /* LL/SC */ \ 287 " nop\n" \ 288 __LL_SC_ATOMIC64(add_return##name), \ 289 /* LSE atomics */ \ 290 " ldadd" #mb " %[i], x30, %[v]\n" \ 291 " add %[i], %[i], x30") \ 292 : [i] "+r" (x0), [v] "+Q" (v->counter) \ 293 : "r" (x1) \ 294 : __LL_SC_CLOBBERS, ##cl); \ 295 \ 296 return x0; \ 297 } 298 299 ATOMIC64_OP_ADD_RETURN(_relaxed, ) 300 ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory") 301 ATOMIC64_OP_ADD_RETURN(_release, l, "memory") 302 ATOMIC64_OP_ADD_RETURN( , al, "memory") 303 304 #undef ATOMIC64_OP_ADD_RETURN 305 306 static inline void atomic64_and(long i, atomic64_t *v) 307 { 308 register long x0 asm ("x0") = i; 309 register atomic64_t *x1 asm ("x1") = v; 310 311 asm volatile(ARM64_LSE_ATOMIC_INSN( 312 /* LL/SC */ 313 " nop\n" 314 __LL_SC_ATOMIC64(and), 315 /* LSE atomics */ 316 " mvn %[i], %[i]\n" 317 " stclr %[i], %[v]") 318 : [i] "+r" (x0), [v] "+Q" (v->counter) 319 : "r" (x1) 320 : __LL_SC_CLOBBERS); 321 } 322 323 #define ATOMIC64_FETCH_OP_AND(name, mb, cl...) \ 324 static inline long atomic64_fetch_and##name(long i, atomic64_t *v) \ 325 { \ 326 register long x0 asm ("w0") = i; \ 327 register atomic64_t *x1 asm ("x1") = v; \ 328 \ 329 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 330 /* LL/SC */ \ 331 " nop\n" \ 332 __LL_SC_ATOMIC64(fetch_and##name), \ 333 /* LSE atomics */ \ 334 " mvn %[i], %[i]\n" \ 335 " ldclr" #mb " %[i], %[i], %[v]") \ 336 : [i] "+r" (x0), [v] "+Q" (v->counter) \ 337 : "r" (x1) \ 338 : __LL_SC_CLOBBERS, ##cl); \ 339 \ 340 return x0; \ 341 } 342 343 ATOMIC64_FETCH_OP_AND(_relaxed, ) 344 ATOMIC64_FETCH_OP_AND(_acquire, a, "memory") 345 ATOMIC64_FETCH_OP_AND(_release, l, "memory") 346 ATOMIC64_FETCH_OP_AND( , al, "memory") 347 348 #undef ATOMIC64_FETCH_OP_AND 349 350 static inline void atomic64_sub(long i, atomic64_t *v) 351 { 352 register long x0 asm ("x0") = i; 353 register atomic64_t *x1 asm ("x1") = v; 354 355 asm volatile(ARM64_LSE_ATOMIC_INSN( 356 /* LL/SC */ 357 " nop\n" 358 __LL_SC_ATOMIC64(sub), 359 /* LSE atomics */ 360 " neg %[i], %[i]\n" 361 " stadd %[i], %[v]") 362 : [i] "+r" (x0), [v] "+Q" (v->counter) 363 : "r" (x1) 364 : __LL_SC_CLOBBERS); 365 } 366 367 #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \ 368 static inline long atomic64_sub_return##name(long i, atomic64_t *v) \ 369 { \ 370 register long x0 asm ("x0") = i; \ 371 register atomic64_t *x1 asm ("x1") = v; \ 372 \ 373 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 374 /* LL/SC */ \ 375 " nop\n" \ 376 __LL_SC_ATOMIC64(sub_return##name) \ 377 " nop", \ 378 /* LSE atomics */ \ 379 " neg %[i], %[i]\n" \ 380 " ldadd" #mb " %[i], x30, %[v]\n" \ 381 " add %[i], %[i], x30") \ 382 : [i] "+r" (x0), [v] "+Q" (v->counter) \ 383 : "r" (x1) \ 384 : __LL_SC_CLOBBERS, ##cl); \ 385 \ 386 return x0; \ 387 } 388 389 ATOMIC64_OP_SUB_RETURN(_relaxed, ) 390 ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") 391 ATOMIC64_OP_SUB_RETURN(_release, l, "memory") 392 ATOMIC64_OP_SUB_RETURN( , al, "memory") 393 394 #undef ATOMIC64_OP_SUB_RETURN 395 396 #define ATOMIC64_FETCH_OP_SUB(name, mb, cl...) \ 397 static inline long atomic64_fetch_sub##name(long i, atomic64_t *v) \ 398 { \ 399 register long x0 asm ("w0") = i; \ 400 register atomic64_t *x1 asm ("x1") = v; \ 401 \ 402 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 403 /* LL/SC */ \ 404 " nop\n" \ 405 __LL_SC_ATOMIC64(fetch_sub##name), \ 406 /* LSE atomics */ \ 407 " neg %[i], %[i]\n" \ 408 " ldadd" #mb " %[i], %[i], %[v]") \ 409 : [i] "+r" (x0), [v] "+Q" (v->counter) \ 410 : "r" (x1) \ 411 : __LL_SC_CLOBBERS, ##cl); \ 412 \ 413 return x0; \ 414 } 415 416 ATOMIC64_FETCH_OP_SUB(_relaxed, ) 417 ATOMIC64_FETCH_OP_SUB(_acquire, a, "memory") 418 ATOMIC64_FETCH_OP_SUB(_release, l, "memory") 419 ATOMIC64_FETCH_OP_SUB( , al, "memory") 420 421 #undef ATOMIC64_FETCH_OP_SUB 422 423 static inline long atomic64_dec_if_positive(atomic64_t *v) 424 { 425 register long x0 asm ("x0") = (long)v; 426 427 asm volatile(ARM64_LSE_ATOMIC_INSN( 428 /* LL/SC */ 429 " nop\n" 430 __LL_SC_ATOMIC64(dec_if_positive) 431 " nop\n" 432 " nop\n" 433 " nop\n" 434 " nop\n" 435 " nop", 436 /* LSE atomics */ 437 "1: ldr x30, %[v]\n" 438 " subs %[ret], x30, #1\n" 439 " b.lt 2f\n" 440 " casal x30, %[ret], %[v]\n" 441 " sub x30, x30, #1\n" 442 " sub x30, x30, %[ret]\n" 443 " cbnz x30, 1b\n" 444 "2:") 445 : [ret] "+&r" (x0), [v] "+Q" (v->counter) 446 : 447 : __LL_SC_CLOBBERS, "cc", "memory"); 448 449 return x0; 450 } 451 452 #undef __LL_SC_ATOMIC64 453 454 #define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op) 455 456 #define __CMPXCHG_CASE(w, sz, name, mb, cl...) \ 457 static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \ 458 unsigned long old, \ 459 unsigned long new) \ 460 { \ 461 register unsigned long x0 asm ("x0") = (unsigned long)ptr; \ 462 register unsigned long x1 asm ("x1") = old; \ 463 register unsigned long x2 asm ("x2") = new; \ 464 \ 465 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 466 /* LL/SC */ \ 467 " nop\n" \ 468 __LL_SC_CMPXCHG(name) \ 469 " nop", \ 470 /* LSE atomics */ \ 471 " mov " #w "30, %" #w "[old]\n" \ 472 " cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \ 473 " mov %" #w "[ret], " #w "30") \ 474 : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \ 475 : [old] "r" (x1), [new] "r" (x2) \ 476 : __LL_SC_CLOBBERS, ##cl); \ 477 \ 478 return x0; \ 479 } 480 481 __CMPXCHG_CASE(w, b, 1, ) 482 __CMPXCHG_CASE(w, h, 2, ) 483 __CMPXCHG_CASE(w, , 4, ) 484 __CMPXCHG_CASE(x, , 8, ) 485 __CMPXCHG_CASE(w, b, acq_1, a, "memory") 486 __CMPXCHG_CASE(w, h, acq_2, a, "memory") 487 __CMPXCHG_CASE(w, , acq_4, a, "memory") 488 __CMPXCHG_CASE(x, , acq_8, a, "memory") 489 __CMPXCHG_CASE(w, b, rel_1, l, "memory") 490 __CMPXCHG_CASE(w, h, rel_2, l, "memory") 491 __CMPXCHG_CASE(w, , rel_4, l, "memory") 492 __CMPXCHG_CASE(x, , rel_8, l, "memory") 493 __CMPXCHG_CASE(w, b, mb_1, al, "memory") 494 __CMPXCHG_CASE(w, h, mb_2, al, "memory") 495 __CMPXCHG_CASE(w, , mb_4, al, "memory") 496 __CMPXCHG_CASE(x, , mb_8, al, "memory") 497 498 #undef __LL_SC_CMPXCHG 499 #undef __CMPXCHG_CASE 500 501 #define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op) 502 503 #define __CMPXCHG_DBL(name, mb, cl...) \ 504 static inline long __cmpxchg_double##name(unsigned long old1, \ 505 unsigned long old2, \ 506 unsigned long new1, \ 507 unsigned long new2, \ 508 volatile void *ptr) \ 509 { \ 510 unsigned long oldval1 = old1; \ 511 unsigned long oldval2 = old2; \ 512 register unsigned long x0 asm ("x0") = old1; \ 513 register unsigned long x1 asm ("x1") = old2; \ 514 register unsigned long x2 asm ("x2") = new1; \ 515 register unsigned long x3 asm ("x3") = new2; \ 516 register unsigned long x4 asm ("x4") = (unsigned long)ptr; \ 517 \ 518 asm volatile(ARM64_LSE_ATOMIC_INSN( \ 519 /* LL/SC */ \ 520 " nop\n" \ 521 " nop\n" \ 522 " nop\n" \ 523 __LL_SC_CMPXCHG_DBL(name), \ 524 /* LSE atomics */ \ 525 " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\ 526 " eor %[old1], %[old1], %[oldval1]\n" \ 527 " eor %[old2], %[old2], %[oldval2]\n" \ 528 " orr %[old1], %[old1], %[old2]") \ 529 : [old1] "+r" (x0), [old2] "+r" (x1), \ 530 [v] "+Q" (*(unsigned long *)ptr) \ 531 : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \ 532 [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \ 533 : __LL_SC_CLOBBERS, ##cl); \ 534 \ 535 return x0; \ 536 } 537 538 __CMPXCHG_DBL( , ) 539 __CMPXCHG_DBL(_mb, al, "memory") 540 541 #undef __LL_SC_CMPXCHG_DBL 542 #undef __CMPXCHG_DBL 543 544 #endif /* __ASM_ATOMIC_LSE_H */ 545