1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_POWERPC_ATOMIC_H_ 3 #define _ASM_POWERPC_ATOMIC_H_ 4 5 /* 6 * PowerPC atomic operations 7 */ 8 9 #ifdef __KERNEL__ 10 #include <linux/types.h> 11 #include <asm/cmpxchg.h> 12 #include <asm/barrier.h> 13 14 /* 15 * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with 16 * a "bne-" instruction at the end, so an isync is enough as a acquire barrier 17 * on the platform without lwsync. 18 */ 19 #define __atomic_acquire_fence() \ 20 __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory") 21 22 #define __atomic_release_fence() \ 23 __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory") 24 25 static __inline__ int atomic_read(const atomic_t *v) 26 { 27 int t; 28 29 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 30 31 return t; 32 } 33 34 static __inline__ void atomic_set(atomic_t *v, int i) 35 { 36 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 37 } 38 39 #define ATOMIC_OP(op, asm_op) \ 40 static __inline__ void atomic_##op(int a, atomic_t *v) \ 41 { \ 42 int t; \ 43 \ 44 __asm__ __volatile__( \ 45 "1: lwarx %0,0,%3 # atomic_" #op "\n" \ 46 #asm_op " %0,%2,%0\n" \ 47 " stwcx. %0,0,%3 \n" \ 48 " bne- 1b\n" \ 49 : "=&r" (t), "+m" (v->counter) \ 50 : "r" (a), "r" (&v->counter) \ 51 : "cc"); \ 52 } \ 53 54 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ 55 static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \ 56 { \ 57 int t; \ 58 \ 59 __asm__ __volatile__( \ 60 "1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \ 61 #asm_op " %0,%2,%0\n" \ 62 " stwcx. %0,0,%3\n" \ 63 " bne- 1b\n" \ 64 : "=&r" (t), "+m" (v->counter) \ 65 : "r" (a), "r" (&v->counter) \ 66 : "cc"); \ 67 \ 68 return t; \ 69 } 70 71 #define ATOMIC_FETCH_OP_RELAXED(op, asm_op) \ 72 static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ 73 { \ 74 int res, t; \ 75 \ 76 __asm__ __volatile__( \ 77 "1: lwarx %0,0,%4 # atomic_fetch_" #op "_relaxed\n" \ 78 #asm_op " %1,%3,%0\n" \ 79 " stwcx. %1,0,%4\n" \ 80 " bne- 1b\n" \ 81 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 82 : "r" (a), "r" (&v->counter) \ 83 : "cc"); \ 84 \ 85 return res; \ 86 } 87 88 #define ATOMIC_OPS(op, asm_op) \ 89 ATOMIC_OP(op, asm_op) \ 90 ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ 91 ATOMIC_FETCH_OP_RELAXED(op, asm_op) 92 93 ATOMIC_OPS(add, add) 94 ATOMIC_OPS(sub, subf) 95 96 #define atomic_add_return_relaxed atomic_add_return_relaxed 97 #define atomic_sub_return_relaxed atomic_sub_return_relaxed 98 99 #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed 100 #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed 101 102 #undef ATOMIC_OPS 103 #define ATOMIC_OPS(op, asm_op) \ 104 ATOMIC_OP(op, asm_op) \ 105 ATOMIC_FETCH_OP_RELAXED(op, asm_op) 106 107 ATOMIC_OPS(and, and) 108 ATOMIC_OPS(or, or) 109 ATOMIC_OPS(xor, xor) 110 111 #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed 112 #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed 113 #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed 114 115 #undef ATOMIC_OPS 116 #undef ATOMIC_FETCH_OP_RELAXED 117 #undef ATOMIC_OP_RETURN_RELAXED 118 #undef ATOMIC_OP 119 120 static __inline__ void atomic_inc(atomic_t *v) 121 { 122 int t; 123 124 __asm__ __volatile__( 125 "1: lwarx %0,0,%2 # atomic_inc\n\ 126 addic %0,%0,1\n" 127 " stwcx. %0,0,%2 \n\ 128 bne- 1b" 129 : "=&r" (t), "+m" (v->counter) 130 : "r" (&v->counter) 131 : "cc", "xer"); 132 } 133 #define atomic_inc atomic_inc 134 135 static __inline__ int atomic_inc_return_relaxed(atomic_t *v) 136 { 137 int t; 138 139 __asm__ __volatile__( 140 "1: lwarx %0,0,%2 # atomic_inc_return_relaxed\n" 141 " addic %0,%0,1\n" 142 " stwcx. %0,0,%2\n" 143 " bne- 1b" 144 : "=&r" (t), "+m" (v->counter) 145 : "r" (&v->counter) 146 : "cc", "xer"); 147 148 return t; 149 } 150 151 static __inline__ void atomic_dec(atomic_t *v) 152 { 153 int t; 154 155 __asm__ __volatile__( 156 "1: lwarx %0,0,%2 # atomic_dec\n\ 157 addic %0,%0,-1\n" 158 " stwcx. %0,0,%2\n\ 159 bne- 1b" 160 : "=&r" (t), "+m" (v->counter) 161 : "r" (&v->counter) 162 : "cc", "xer"); 163 } 164 #define atomic_dec atomic_dec 165 166 static __inline__ int atomic_dec_return_relaxed(atomic_t *v) 167 { 168 int t; 169 170 __asm__ __volatile__( 171 "1: lwarx %0,0,%2 # atomic_dec_return_relaxed\n" 172 " addic %0,%0,-1\n" 173 " stwcx. %0,0,%2\n" 174 " bne- 1b" 175 : "=&r" (t), "+m" (v->counter) 176 : "r" (&v->counter) 177 : "cc", "xer"); 178 179 return t; 180 } 181 182 #define atomic_inc_return_relaxed atomic_inc_return_relaxed 183 #define atomic_dec_return_relaxed atomic_dec_return_relaxed 184 185 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 186 #define atomic_cmpxchg_relaxed(v, o, n) \ 187 cmpxchg_relaxed(&((v)->counter), (o), (n)) 188 #define atomic_cmpxchg_acquire(v, o, n) \ 189 cmpxchg_acquire(&((v)->counter), (o), (n)) 190 191 #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 192 #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) 193 194 /* 195 * Don't want to override the generic atomic_try_cmpxchg_acquire, because 196 * we add a lock hint to the lwarx, which may not be wanted for the 197 * _acquire case (and is not used by the other _acquire variants so it 198 * would be a surprise). 199 */ 200 static __always_inline bool 201 atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new) 202 { 203 int r, o = *old; 204 205 __asm__ __volatile__ ( 206 "1:\t" PPC_LWARX(%0,0,%2,1) " # atomic_try_cmpxchg_acquire \n" 207 " cmpw 0,%0,%3 \n" 208 " bne- 2f \n" 209 " stwcx. %4,0,%2 \n" 210 " bne- 1b \n" 211 "\t" PPC_ACQUIRE_BARRIER " \n" 212 "2: \n" 213 : "=&r" (r), "+m" (v->counter) 214 : "r" (&v->counter), "r" (o), "r" (new) 215 : "cr0", "memory"); 216 217 if (unlikely(r != o)) 218 *old = r; 219 return likely(r == o); 220 } 221 222 /** 223 * atomic_fetch_add_unless - add unless the number is a given value 224 * @v: pointer of type atomic_t 225 * @a: the amount to add to v... 226 * @u: ...unless v is equal to u. 227 * 228 * Atomically adds @a to @v, so long as it was not @u. 229 * Returns the old value of @v. 230 */ 231 static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) 232 { 233 int t; 234 235 __asm__ __volatile__ ( 236 PPC_ATOMIC_ENTRY_BARRIER 237 "1: lwarx %0,0,%1 # atomic_fetch_add_unless\n\ 238 cmpw 0,%0,%3 \n\ 239 beq 2f \n\ 240 add %0,%2,%0 \n" 241 " stwcx. %0,0,%1 \n\ 242 bne- 1b \n" 243 PPC_ATOMIC_EXIT_BARRIER 244 " subf %0,%2,%0 \n\ 245 2:" 246 : "=&r" (t) 247 : "r" (&v->counter), "r" (a), "r" (u) 248 : "cc", "memory"); 249 250 return t; 251 } 252 #define atomic_fetch_add_unless atomic_fetch_add_unless 253 254 /** 255 * atomic_inc_not_zero - increment unless the number is zero 256 * @v: pointer of type atomic_t 257 * 258 * Atomically increments @v by 1, so long as @v is non-zero. 259 * Returns non-zero if @v was non-zero, and zero otherwise. 260 */ 261 static __inline__ int atomic_inc_not_zero(atomic_t *v) 262 { 263 int t1, t2; 264 265 __asm__ __volatile__ ( 266 PPC_ATOMIC_ENTRY_BARRIER 267 "1: lwarx %0,0,%2 # atomic_inc_not_zero\n\ 268 cmpwi 0,%0,0\n\ 269 beq- 2f\n\ 270 addic %1,%0,1\n" 271 " stwcx. %1,0,%2\n\ 272 bne- 1b\n" 273 PPC_ATOMIC_EXIT_BARRIER 274 "\n\ 275 2:" 276 : "=&r" (t1), "=&r" (t2) 277 : "r" (&v->counter) 278 : "cc", "xer", "memory"); 279 280 return t1; 281 } 282 #define atomic_inc_not_zero(v) atomic_inc_not_zero((v)) 283 284 /* 285 * Atomically test *v and decrement if it is greater than 0. 286 * The function returns the old value of *v minus 1, even if 287 * the atomic variable, v, was not decremented. 288 */ 289 static __inline__ int atomic_dec_if_positive(atomic_t *v) 290 { 291 int t; 292 293 __asm__ __volatile__( 294 PPC_ATOMIC_ENTRY_BARRIER 295 "1: lwarx %0,0,%1 # atomic_dec_if_positive\n\ 296 cmpwi %0,1\n\ 297 addi %0,%0,-1\n\ 298 blt- 2f\n" 299 " stwcx. %0,0,%1\n\ 300 bne- 1b" 301 PPC_ATOMIC_EXIT_BARRIER 302 "\n\ 303 2:" : "=&b" (t) 304 : "r" (&v->counter) 305 : "cc", "memory"); 306 307 return t; 308 } 309 #define atomic_dec_if_positive atomic_dec_if_positive 310 311 #ifdef __powerpc64__ 312 313 #define ATOMIC64_INIT(i) { (i) } 314 315 static __inline__ s64 atomic64_read(const atomic64_t *v) 316 { 317 s64 t; 318 319 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 320 321 return t; 322 } 323 324 static __inline__ void atomic64_set(atomic64_t *v, s64 i) 325 { 326 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 327 } 328 329 #define ATOMIC64_OP(op, asm_op) \ 330 static __inline__ void atomic64_##op(s64 a, atomic64_t *v) \ 331 { \ 332 s64 t; \ 333 \ 334 __asm__ __volatile__( \ 335 "1: ldarx %0,0,%3 # atomic64_" #op "\n" \ 336 #asm_op " %0,%2,%0\n" \ 337 " stdcx. %0,0,%3 \n" \ 338 " bne- 1b\n" \ 339 : "=&r" (t), "+m" (v->counter) \ 340 : "r" (a), "r" (&v->counter) \ 341 : "cc"); \ 342 } 343 344 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 345 static inline s64 \ 346 atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ 347 { \ 348 s64 t; \ 349 \ 350 __asm__ __volatile__( \ 351 "1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \ 352 #asm_op " %0,%2,%0\n" \ 353 " stdcx. %0,0,%3\n" \ 354 " bne- 1b\n" \ 355 : "=&r" (t), "+m" (v->counter) \ 356 : "r" (a), "r" (&v->counter) \ 357 : "cc"); \ 358 \ 359 return t; \ 360 } 361 362 #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \ 363 static inline s64 \ 364 atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ 365 { \ 366 s64 res, t; \ 367 \ 368 __asm__ __volatile__( \ 369 "1: ldarx %0,0,%4 # atomic64_fetch_" #op "_relaxed\n" \ 370 #asm_op " %1,%3,%0\n" \ 371 " stdcx. %1,0,%4\n" \ 372 " bne- 1b\n" \ 373 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 374 : "r" (a), "r" (&v->counter) \ 375 : "cc"); \ 376 \ 377 return res; \ 378 } 379 380 #define ATOMIC64_OPS(op, asm_op) \ 381 ATOMIC64_OP(op, asm_op) \ 382 ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 383 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 384 385 ATOMIC64_OPS(add, add) 386 ATOMIC64_OPS(sub, subf) 387 388 #define atomic64_add_return_relaxed atomic64_add_return_relaxed 389 #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed 390 391 #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed 392 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed 393 394 #undef ATOMIC64_OPS 395 #define ATOMIC64_OPS(op, asm_op) \ 396 ATOMIC64_OP(op, asm_op) \ 397 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 398 399 ATOMIC64_OPS(and, and) 400 ATOMIC64_OPS(or, or) 401 ATOMIC64_OPS(xor, xor) 402 403 #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed 404 #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed 405 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed 406 407 #undef ATOPIC64_OPS 408 #undef ATOMIC64_FETCH_OP_RELAXED 409 #undef ATOMIC64_OP_RETURN_RELAXED 410 #undef ATOMIC64_OP 411 412 static __inline__ void atomic64_inc(atomic64_t *v) 413 { 414 s64 t; 415 416 __asm__ __volatile__( 417 "1: ldarx %0,0,%2 # atomic64_inc\n\ 418 addic %0,%0,1\n\ 419 stdcx. %0,0,%2 \n\ 420 bne- 1b" 421 : "=&r" (t), "+m" (v->counter) 422 : "r" (&v->counter) 423 : "cc", "xer"); 424 } 425 #define atomic64_inc atomic64_inc 426 427 static __inline__ s64 atomic64_inc_return_relaxed(atomic64_t *v) 428 { 429 s64 t; 430 431 __asm__ __volatile__( 432 "1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n" 433 " addic %0,%0,1\n" 434 " stdcx. %0,0,%2\n" 435 " bne- 1b" 436 : "=&r" (t), "+m" (v->counter) 437 : "r" (&v->counter) 438 : "cc", "xer"); 439 440 return t; 441 } 442 443 static __inline__ void atomic64_dec(atomic64_t *v) 444 { 445 s64 t; 446 447 __asm__ __volatile__( 448 "1: ldarx %0,0,%2 # atomic64_dec\n\ 449 addic %0,%0,-1\n\ 450 stdcx. %0,0,%2\n\ 451 bne- 1b" 452 : "=&r" (t), "+m" (v->counter) 453 : "r" (&v->counter) 454 : "cc", "xer"); 455 } 456 #define atomic64_dec atomic64_dec 457 458 static __inline__ s64 atomic64_dec_return_relaxed(atomic64_t *v) 459 { 460 s64 t; 461 462 __asm__ __volatile__( 463 "1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n" 464 " addic %0,%0,-1\n" 465 " stdcx. %0,0,%2\n" 466 " bne- 1b" 467 : "=&r" (t), "+m" (v->counter) 468 : "r" (&v->counter) 469 : "cc", "xer"); 470 471 return t; 472 } 473 474 #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed 475 #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed 476 477 /* 478 * Atomically test *v and decrement if it is greater than 0. 479 * The function returns the old value of *v minus 1. 480 */ 481 static __inline__ s64 atomic64_dec_if_positive(atomic64_t *v) 482 { 483 s64 t; 484 485 __asm__ __volatile__( 486 PPC_ATOMIC_ENTRY_BARRIER 487 "1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\ 488 addic. %0,%0,-1\n\ 489 blt- 2f\n\ 490 stdcx. %0,0,%1\n\ 491 bne- 1b" 492 PPC_ATOMIC_EXIT_BARRIER 493 "\n\ 494 2:" : "=&r" (t) 495 : "r" (&v->counter) 496 : "cc", "xer", "memory"); 497 498 return t; 499 } 500 #define atomic64_dec_if_positive atomic64_dec_if_positive 501 502 #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 503 #define atomic64_cmpxchg_relaxed(v, o, n) \ 504 cmpxchg_relaxed(&((v)->counter), (o), (n)) 505 #define atomic64_cmpxchg_acquire(v, o, n) \ 506 cmpxchg_acquire(&((v)->counter), (o), (n)) 507 508 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) 509 #define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) 510 511 /** 512 * atomic64_fetch_add_unless - add unless the number is a given value 513 * @v: pointer of type atomic64_t 514 * @a: the amount to add to v... 515 * @u: ...unless v is equal to u. 516 * 517 * Atomically adds @a to @v, so long as it was not @u. 518 * Returns the old value of @v. 519 */ 520 static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 521 { 522 s64 t; 523 524 __asm__ __volatile__ ( 525 PPC_ATOMIC_ENTRY_BARRIER 526 "1: ldarx %0,0,%1 # atomic64_fetch_add_unless\n\ 527 cmpd 0,%0,%3 \n\ 528 beq 2f \n\ 529 add %0,%2,%0 \n" 530 " stdcx. %0,0,%1 \n\ 531 bne- 1b \n" 532 PPC_ATOMIC_EXIT_BARRIER 533 " subf %0,%2,%0 \n\ 534 2:" 535 : "=&r" (t) 536 : "r" (&v->counter), "r" (a), "r" (u) 537 : "cc", "memory"); 538 539 return t; 540 } 541 #define atomic64_fetch_add_unless atomic64_fetch_add_unless 542 543 /** 544 * atomic_inc64_not_zero - increment unless the number is zero 545 * @v: pointer of type atomic64_t 546 * 547 * Atomically increments @v by 1, so long as @v is non-zero. 548 * Returns non-zero if @v was non-zero, and zero otherwise. 549 */ 550 static __inline__ int atomic64_inc_not_zero(atomic64_t *v) 551 { 552 s64 t1, t2; 553 554 __asm__ __volatile__ ( 555 PPC_ATOMIC_ENTRY_BARRIER 556 "1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\ 557 cmpdi 0,%0,0\n\ 558 beq- 2f\n\ 559 addic %1,%0,1\n\ 560 stdcx. %1,0,%2\n\ 561 bne- 1b\n" 562 PPC_ATOMIC_EXIT_BARRIER 563 "\n\ 564 2:" 565 : "=&r" (t1), "=&r" (t2) 566 : "r" (&v->counter) 567 : "cc", "xer", "memory"); 568 569 return t1 != 0; 570 } 571 #define atomic64_inc_not_zero(v) atomic64_inc_not_zero((v)) 572 573 #endif /* __powerpc64__ */ 574 575 #endif /* __KERNEL__ */ 576 #endif /* _ASM_POWERPC_ATOMIC_H_ */ 577