1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_POWERPC_ATOMIC_H_ 3 #define _ASM_POWERPC_ATOMIC_H_ 4 5 /* 6 * PowerPC atomic operations 7 */ 8 9 #ifdef __KERNEL__ 10 #include <linux/types.h> 11 #include <asm/cmpxchg.h> 12 #include <asm/barrier.h> 13 #include <asm/asm-405.h> 14 15 #define ATOMIC_INIT(i) { (i) } 16 17 /* 18 * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with 19 * a "bne-" instruction at the end, so an isync is enough as a acquire barrier 20 * on the platform without lwsync. 21 */ 22 #define __atomic_acquire_fence() \ 23 __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory") 24 25 #define __atomic_release_fence() \ 26 __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory") 27 28 static __inline__ int atomic_read(const atomic_t *v) 29 { 30 int t; 31 32 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 33 34 return t; 35 } 36 37 static __inline__ void atomic_set(atomic_t *v, int i) 38 { 39 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 40 } 41 42 #define ATOMIC_OP(op, asm_op) \ 43 static __inline__ void atomic_##op(int a, atomic_t *v) \ 44 { \ 45 int t; \ 46 \ 47 __asm__ __volatile__( \ 48 "1: lwarx %0,0,%3 # atomic_" #op "\n" \ 49 #asm_op " %0,%2,%0\n" \ 50 PPC405_ERR77(0,%3) \ 51 " stwcx. %0,0,%3 \n" \ 52 " bne- 1b\n" \ 53 : "=&r" (t), "+m" (v->counter) \ 54 : "r" (a), "r" (&v->counter) \ 55 : "cc"); \ 56 } \ 57 58 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ 59 static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \ 60 { \ 61 int t; \ 62 \ 63 __asm__ __volatile__( \ 64 "1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \ 65 #asm_op " %0,%2,%0\n" \ 66 PPC405_ERR77(0, %3) \ 67 " stwcx. %0,0,%3\n" \ 68 " bne- 1b\n" \ 69 : "=&r" (t), "+m" (v->counter) \ 70 : "r" (a), "r" (&v->counter) \ 71 : "cc"); \ 72 \ 73 return t; \ 74 } 75 76 #define ATOMIC_FETCH_OP_RELAXED(op, asm_op) \ 77 static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ 78 { \ 79 int res, t; \ 80 \ 81 __asm__ __volatile__( \ 82 "1: lwarx %0,0,%4 # atomic_fetch_" #op "_relaxed\n" \ 83 #asm_op " %1,%3,%0\n" \ 84 PPC405_ERR77(0, %4) \ 85 " stwcx. %1,0,%4\n" \ 86 " bne- 1b\n" \ 87 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 88 : "r" (a), "r" (&v->counter) \ 89 : "cc"); \ 90 \ 91 return res; \ 92 } 93 94 #define ATOMIC_OPS(op, asm_op) \ 95 ATOMIC_OP(op, asm_op) \ 96 ATOMIC_OP_RETURN_RELAXED(op, asm_op) \ 97 ATOMIC_FETCH_OP_RELAXED(op, asm_op) 98 99 ATOMIC_OPS(add, add) 100 ATOMIC_OPS(sub, subf) 101 102 #define atomic_add_return_relaxed atomic_add_return_relaxed 103 #define atomic_sub_return_relaxed atomic_sub_return_relaxed 104 105 #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed 106 #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed 107 108 #undef ATOMIC_OPS 109 #define ATOMIC_OPS(op, asm_op) \ 110 ATOMIC_OP(op, asm_op) \ 111 ATOMIC_FETCH_OP_RELAXED(op, asm_op) 112 113 ATOMIC_OPS(and, and) 114 ATOMIC_OPS(or, or) 115 ATOMIC_OPS(xor, xor) 116 117 #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed 118 #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed 119 #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed 120 121 #undef ATOMIC_OPS 122 #undef ATOMIC_FETCH_OP_RELAXED 123 #undef ATOMIC_OP_RETURN_RELAXED 124 #undef ATOMIC_OP 125 126 static __inline__ void atomic_inc(atomic_t *v) 127 { 128 int t; 129 130 __asm__ __volatile__( 131 "1: lwarx %0,0,%2 # atomic_inc\n\ 132 addic %0,%0,1\n" 133 PPC405_ERR77(0,%2) 134 " stwcx. %0,0,%2 \n\ 135 bne- 1b" 136 : "=&r" (t), "+m" (v->counter) 137 : "r" (&v->counter) 138 : "cc", "xer"); 139 } 140 #define atomic_inc atomic_inc 141 142 static __inline__ int atomic_inc_return_relaxed(atomic_t *v) 143 { 144 int t; 145 146 __asm__ __volatile__( 147 "1: lwarx %0,0,%2 # atomic_inc_return_relaxed\n" 148 " addic %0,%0,1\n" 149 PPC405_ERR77(0, %2) 150 " stwcx. %0,0,%2\n" 151 " bne- 1b" 152 : "=&r" (t), "+m" (v->counter) 153 : "r" (&v->counter) 154 : "cc", "xer"); 155 156 return t; 157 } 158 159 static __inline__ void atomic_dec(atomic_t *v) 160 { 161 int t; 162 163 __asm__ __volatile__( 164 "1: lwarx %0,0,%2 # atomic_dec\n\ 165 addic %0,%0,-1\n" 166 PPC405_ERR77(0,%2)\ 167 " stwcx. %0,0,%2\n\ 168 bne- 1b" 169 : "=&r" (t), "+m" (v->counter) 170 : "r" (&v->counter) 171 : "cc", "xer"); 172 } 173 #define atomic_dec atomic_dec 174 175 static __inline__ int atomic_dec_return_relaxed(atomic_t *v) 176 { 177 int t; 178 179 __asm__ __volatile__( 180 "1: lwarx %0,0,%2 # atomic_dec_return_relaxed\n" 181 " addic %0,%0,-1\n" 182 PPC405_ERR77(0, %2) 183 " stwcx. %0,0,%2\n" 184 " bne- 1b" 185 : "=&r" (t), "+m" (v->counter) 186 : "r" (&v->counter) 187 : "cc", "xer"); 188 189 return t; 190 } 191 192 #define atomic_inc_return_relaxed atomic_inc_return_relaxed 193 #define atomic_dec_return_relaxed atomic_dec_return_relaxed 194 195 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 196 #define atomic_cmpxchg_relaxed(v, o, n) \ 197 cmpxchg_relaxed(&((v)->counter), (o), (n)) 198 #define atomic_cmpxchg_acquire(v, o, n) \ 199 cmpxchg_acquire(&((v)->counter), (o), (n)) 200 201 #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 202 #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) 203 204 /** 205 * atomic_fetch_add_unless - add unless the number is a given value 206 * @v: pointer of type atomic_t 207 * @a: the amount to add to v... 208 * @u: ...unless v is equal to u. 209 * 210 * Atomically adds @a to @v, so long as it was not @u. 211 * Returns the old value of @v. 212 */ 213 static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u) 214 { 215 int t; 216 217 __asm__ __volatile__ ( 218 PPC_ATOMIC_ENTRY_BARRIER 219 "1: lwarx %0,0,%1 # atomic_fetch_add_unless\n\ 220 cmpw 0,%0,%3 \n\ 221 beq 2f \n\ 222 add %0,%2,%0 \n" 223 PPC405_ERR77(0,%2) 224 " stwcx. %0,0,%1 \n\ 225 bne- 1b \n" 226 PPC_ATOMIC_EXIT_BARRIER 227 " subf %0,%2,%0 \n\ 228 2:" 229 : "=&r" (t) 230 : "r" (&v->counter), "r" (a), "r" (u) 231 : "cc", "memory"); 232 233 return t; 234 } 235 #define atomic_fetch_add_unless atomic_fetch_add_unless 236 237 /** 238 * atomic_inc_not_zero - increment unless the number is zero 239 * @v: pointer of type atomic_t 240 * 241 * Atomically increments @v by 1, so long as @v is non-zero. 242 * Returns non-zero if @v was non-zero, and zero otherwise. 243 */ 244 static __inline__ int atomic_inc_not_zero(atomic_t *v) 245 { 246 int t1, t2; 247 248 __asm__ __volatile__ ( 249 PPC_ATOMIC_ENTRY_BARRIER 250 "1: lwarx %0,0,%2 # atomic_inc_not_zero\n\ 251 cmpwi 0,%0,0\n\ 252 beq- 2f\n\ 253 addic %1,%0,1\n" 254 PPC405_ERR77(0,%2) 255 " stwcx. %1,0,%2\n\ 256 bne- 1b\n" 257 PPC_ATOMIC_EXIT_BARRIER 258 "\n\ 259 2:" 260 : "=&r" (t1), "=&r" (t2) 261 : "r" (&v->counter) 262 : "cc", "xer", "memory"); 263 264 return t1; 265 } 266 #define atomic_inc_not_zero(v) atomic_inc_not_zero((v)) 267 268 /* 269 * Atomically test *v and decrement if it is greater than 0. 270 * The function returns the old value of *v minus 1, even if 271 * the atomic variable, v, was not decremented. 272 */ 273 static __inline__ int atomic_dec_if_positive(atomic_t *v) 274 { 275 int t; 276 277 __asm__ __volatile__( 278 PPC_ATOMIC_ENTRY_BARRIER 279 "1: lwarx %0,0,%1 # atomic_dec_if_positive\n\ 280 cmpwi %0,1\n\ 281 addi %0,%0,-1\n\ 282 blt- 2f\n" 283 PPC405_ERR77(0,%1) 284 " stwcx. %0,0,%1\n\ 285 bne- 1b" 286 PPC_ATOMIC_EXIT_BARRIER 287 "\n\ 288 2:" : "=&b" (t) 289 : "r" (&v->counter) 290 : "cc", "memory"); 291 292 return t; 293 } 294 #define atomic_dec_if_positive atomic_dec_if_positive 295 296 #ifdef __powerpc64__ 297 298 #define ATOMIC64_INIT(i) { (i) } 299 300 static __inline__ s64 atomic64_read(const atomic64_t *v) 301 { 302 s64 t; 303 304 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter)); 305 306 return t; 307 } 308 309 static __inline__ void atomic64_set(atomic64_t *v, s64 i) 310 { 311 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i)); 312 } 313 314 #define ATOMIC64_OP(op, asm_op) \ 315 static __inline__ void atomic64_##op(s64 a, atomic64_t *v) \ 316 { \ 317 s64 t; \ 318 \ 319 __asm__ __volatile__( \ 320 "1: ldarx %0,0,%3 # atomic64_" #op "\n" \ 321 #asm_op " %0,%2,%0\n" \ 322 " stdcx. %0,0,%3 \n" \ 323 " bne- 1b\n" \ 324 : "=&r" (t), "+m" (v->counter) \ 325 : "r" (a), "r" (&v->counter) \ 326 : "cc"); \ 327 } 328 329 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 330 static inline s64 \ 331 atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ 332 { \ 333 s64 t; \ 334 \ 335 __asm__ __volatile__( \ 336 "1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \ 337 #asm_op " %0,%2,%0\n" \ 338 " stdcx. %0,0,%3\n" \ 339 " bne- 1b\n" \ 340 : "=&r" (t), "+m" (v->counter) \ 341 : "r" (a), "r" (&v->counter) \ 342 : "cc"); \ 343 \ 344 return t; \ 345 } 346 347 #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \ 348 static inline s64 \ 349 atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ 350 { \ 351 s64 res, t; \ 352 \ 353 __asm__ __volatile__( \ 354 "1: ldarx %0,0,%4 # atomic64_fetch_" #op "_relaxed\n" \ 355 #asm_op " %1,%3,%0\n" \ 356 " stdcx. %1,0,%4\n" \ 357 " bne- 1b\n" \ 358 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 359 : "r" (a), "r" (&v->counter) \ 360 : "cc"); \ 361 \ 362 return res; \ 363 } 364 365 #define ATOMIC64_OPS(op, asm_op) \ 366 ATOMIC64_OP(op, asm_op) \ 367 ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 368 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 369 370 ATOMIC64_OPS(add, add) 371 ATOMIC64_OPS(sub, subf) 372 373 #define atomic64_add_return_relaxed atomic64_add_return_relaxed 374 #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed 375 376 #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed 377 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed 378 379 #undef ATOMIC64_OPS 380 #define ATOMIC64_OPS(op, asm_op) \ 381 ATOMIC64_OP(op, asm_op) \ 382 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 383 384 ATOMIC64_OPS(and, and) 385 ATOMIC64_OPS(or, or) 386 ATOMIC64_OPS(xor, xor) 387 388 #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed 389 #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed 390 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed 391 392 #undef ATOPIC64_OPS 393 #undef ATOMIC64_FETCH_OP_RELAXED 394 #undef ATOMIC64_OP_RETURN_RELAXED 395 #undef ATOMIC64_OP 396 397 static __inline__ void atomic64_inc(atomic64_t *v) 398 { 399 s64 t; 400 401 __asm__ __volatile__( 402 "1: ldarx %0,0,%2 # atomic64_inc\n\ 403 addic %0,%0,1\n\ 404 stdcx. %0,0,%2 \n\ 405 bne- 1b" 406 : "=&r" (t), "+m" (v->counter) 407 : "r" (&v->counter) 408 : "cc", "xer"); 409 } 410 #define atomic64_inc atomic64_inc 411 412 static __inline__ s64 atomic64_inc_return_relaxed(atomic64_t *v) 413 { 414 s64 t; 415 416 __asm__ __volatile__( 417 "1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n" 418 " addic %0,%0,1\n" 419 " stdcx. %0,0,%2\n" 420 " bne- 1b" 421 : "=&r" (t), "+m" (v->counter) 422 : "r" (&v->counter) 423 : "cc", "xer"); 424 425 return t; 426 } 427 428 static __inline__ void atomic64_dec(atomic64_t *v) 429 { 430 s64 t; 431 432 __asm__ __volatile__( 433 "1: ldarx %0,0,%2 # atomic64_dec\n\ 434 addic %0,%0,-1\n\ 435 stdcx. %0,0,%2\n\ 436 bne- 1b" 437 : "=&r" (t), "+m" (v->counter) 438 : "r" (&v->counter) 439 : "cc", "xer"); 440 } 441 #define atomic64_dec atomic64_dec 442 443 static __inline__ s64 atomic64_dec_return_relaxed(atomic64_t *v) 444 { 445 s64 t; 446 447 __asm__ __volatile__( 448 "1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n" 449 " addic %0,%0,-1\n" 450 " stdcx. %0,0,%2\n" 451 " bne- 1b" 452 : "=&r" (t), "+m" (v->counter) 453 : "r" (&v->counter) 454 : "cc", "xer"); 455 456 return t; 457 } 458 459 #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed 460 #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed 461 462 /* 463 * Atomically test *v and decrement if it is greater than 0. 464 * The function returns the old value of *v minus 1. 465 */ 466 static __inline__ s64 atomic64_dec_if_positive(atomic64_t *v) 467 { 468 s64 t; 469 470 __asm__ __volatile__( 471 PPC_ATOMIC_ENTRY_BARRIER 472 "1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\ 473 addic. %0,%0,-1\n\ 474 blt- 2f\n\ 475 stdcx. %0,0,%1\n\ 476 bne- 1b" 477 PPC_ATOMIC_EXIT_BARRIER 478 "\n\ 479 2:" : "=&r" (t) 480 : "r" (&v->counter) 481 : "cc", "xer", "memory"); 482 483 return t; 484 } 485 #define atomic64_dec_if_positive atomic64_dec_if_positive 486 487 #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) 488 #define atomic64_cmpxchg_relaxed(v, o, n) \ 489 cmpxchg_relaxed(&((v)->counter), (o), (n)) 490 #define atomic64_cmpxchg_acquire(v, o, n) \ 491 cmpxchg_acquire(&((v)->counter), (o), (n)) 492 493 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) 494 #define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new)) 495 496 /** 497 * atomic64_fetch_add_unless - add unless the number is a given value 498 * @v: pointer of type atomic64_t 499 * @a: the amount to add to v... 500 * @u: ...unless v is equal to u. 501 * 502 * Atomically adds @a to @v, so long as it was not @u. 503 * Returns the old value of @v. 504 */ 505 static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 506 { 507 s64 t; 508 509 __asm__ __volatile__ ( 510 PPC_ATOMIC_ENTRY_BARRIER 511 "1: ldarx %0,0,%1 # atomic64_fetch_add_unless\n\ 512 cmpd 0,%0,%3 \n\ 513 beq 2f \n\ 514 add %0,%2,%0 \n" 515 " stdcx. %0,0,%1 \n\ 516 bne- 1b \n" 517 PPC_ATOMIC_EXIT_BARRIER 518 " subf %0,%2,%0 \n\ 519 2:" 520 : "=&r" (t) 521 : "r" (&v->counter), "r" (a), "r" (u) 522 : "cc", "memory"); 523 524 return t; 525 } 526 #define atomic64_fetch_add_unless atomic64_fetch_add_unless 527 528 /** 529 * atomic_inc64_not_zero - increment unless the number is zero 530 * @v: pointer of type atomic64_t 531 * 532 * Atomically increments @v by 1, so long as @v is non-zero. 533 * Returns non-zero if @v was non-zero, and zero otherwise. 534 */ 535 static __inline__ int atomic64_inc_not_zero(atomic64_t *v) 536 { 537 s64 t1, t2; 538 539 __asm__ __volatile__ ( 540 PPC_ATOMIC_ENTRY_BARRIER 541 "1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\ 542 cmpdi 0,%0,0\n\ 543 beq- 2f\n\ 544 addic %1,%0,1\n\ 545 stdcx. %1,0,%2\n\ 546 bne- 1b\n" 547 PPC_ATOMIC_EXIT_BARRIER 548 "\n\ 549 2:" 550 : "=&r" (t1), "=&r" (t2) 551 : "r" (&v->counter) 552 : "cc", "xer", "memory"); 553 554 return t1 != 0; 555 } 556 #define atomic64_inc_not_zero(v) atomic64_inc_not_zero((v)) 557 558 #endif /* __powerpc64__ */ 559 560 #endif /* __KERNEL__ */ 561 #endif /* _ASM_POWERPC_ATOMIC_H_ */ 562