1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_POWERPC_ATOMIC_H_ 3 #define _ASM_POWERPC_ATOMIC_H_ 4 5 /* 6 * PowerPC atomic operations 7 */ 8 9 #ifdef __KERNEL__ 10 #include <linux/types.h> 11 #include <asm/cmpxchg.h> 12 #include <asm/barrier.h> 13 #include <asm/asm-const.h> 14 15 /* 16 * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with 17 * a "bne-" instruction at the end, so an isync is enough as a acquire barrier 18 * on the platform without lwsync. 19 */ 20 #define __atomic_acquire_fence() \ 21 __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory") 22 23 #define __atomic_release_fence() \ 24 __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory") 25 26 static __inline__ int arch_atomic_read(const atomic_t *v) 27 { 28 int t; 29 30 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m<>"(v->counter)); 31 32 return t; 33 } 34 35 static __inline__ void arch_atomic_set(atomic_t *v, int i) 36 { 37 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m<>"(v->counter) : "r"(i)); 38 } 39 40 #define ATOMIC_OP(op, asm_op, suffix, sign, ...) \ 41 static __inline__ void arch_atomic_##op(int a, atomic_t *v) \ 42 { \ 43 int t; \ 44 \ 45 __asm__ __volatile__( \ 46 "1: lwarx %0,0,%3 # atomic_" #op "\n" \ 47 #asm_op "%I2" suffix " %0,%0,%2\n" \ 48 " stwcx. %0,0,%3 \n" \ 49 " bne- 1b\n" \ 50 : "=&r" (t), "+m" (v->counter) \ 51 : "r"#sign (a), "r" (&v->counter) \ 52 : "cc", ##__VA_ARGS__); \ 53 } \ 54 55 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op, suffix, sign, ...) \ 56 static inline int arch_atomic_##op##_return_relaxed(int a, atomic_t *v) \ 57 { \ 58 int t; \ 59 \ 60 __asm__ __volatile__( \ 61 "1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \ 62 #asm_op "%I2" suffix " %0,%0,%2\n" \ 63 " stwcx. %0,0,%3\n" \ 64 " bne- 1b\n" \ 65 : "=&r" (t), "+m" (v->counter) \ 66 : "r"#sign (a), "r" (&v->counter) \ 67 : "cc", ##__VA_ARGS__); \ 68 \ 69 return t; \ 70 } 71 72 #define ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign, ...) \ 73 static inline int arch_atomic_fetch_##op##_relaxed(int a, atomic_t *v) \ 74 { \ 75 int res, t; \ 76 \ 77 __asm__ __volatile__( \ 78 "1: lwarx %0,0,%4 # atomic_fetch_" #op "_relaxed\n" \ 79 #asm_op "%I3" suffix " %1,%0,%3\n" \ 80 " stwcx. %1,0,%4\n" \ 81 " bne- 1b\n" \ 82 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 83 : "r"#sign (a), "r" (&v->counter) \ 84 : "cc", ##__VA_ARGS__); \ 85 \ 86 return res; \ 87 } 88 89 #define ATOMIC_OPS(op, asm_op, suffix, sign, ...) \ 90 ATOMIC_OP(op, asm_op, suffix, sign, ##__VA_ARGS__) \ 91 ATOMIC_OP_RETURN_RELAXED(op, asm_op, suffix, sign, ##__VA_ARGS__)\ 92 ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign, ##__VA_ARGS__) 93 94 ATOMIC_OPS(add, add, "c", I, "xer") 95 ATOMIC_OPS(sub, sub, "c", I, "xer") 96 97 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed 98 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed 99 100 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed 101 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed 102 103 #undef ATOMIC_OPS 104 #define ATOMIC_OPS(op, asm_op, suffix, sign) \ 105 ATOMIC_OP(op, asm_op, suffix, sign) \ 106 ATOMIC_FETCH_OP_RELAXED(op, asm_op, suffix, sign) 107 108 ATOMIC_OPS(and, and, ".", K) 109 ATOMIC_OPS(or, or, "", K) 110 ATOMIC_OPS(xor, xor, "", K) 111 112 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed 113 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed 114 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed 115 116 #undef ATOMIC_OPS 117 #undef ATOMIC_FETCH_OP_RELAXED 118 #undef ATOMIC_OP_RETURN_RELAXED 119 #undef ATOMIC_OP 120 121 #define arch_atomic_cmpxchg(v, o, n) \ 122 (arch_cmpxchg(&((v)->counter), (o), (n))) 123 #define arch_atomic_cmpxchg_relaxed(v, o, n) \ 124 arch_cmpxchg_relaxed(&((v)->counter), (o), (n)) 125 #define arch_atomic_cmpxchg_acquire(v, o, n) \ 126 arch_cmpxchg_acquire(&((v)->counter), (o), (n)) 127 128 #define arch_atomic_xchg(v, new) \ 129 (arch_xchg(&((v)->counter), new)) 130 #define arch_atomic_xchg_relaxed(v, new) \ 131 arch_xchg_relaxed(&((v)->counter), (new)) 132 133 /* 134 * Don't want to override the generic atomic_try_cmpxchg_acquire, because 135 * we add a lock hint to the lwarx, which may not be wanted for the 136 * _acquire case (and is not used by the other _acquire variants so it 137 * would be a surprise). 138 */ 139 static __always_inline bool 140 arch_atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new) 141 { 142 int r, o = *old; 143 unsigned int eh = IS_ENABLED(CONFIG_PPC64); 144 145 __asm__ __volatile__ ( 146 "1: lwarx %0,0,%2,%[eh] # atomic_try_cmpxchg_acquire \n" 147 " cmpw 0,%0,%3 \n" 148 " bne- 2f \n" 149 " stwcx. %4,0,%2 \n" 150 " bne- 1b \n" 151 "\t" PPC_ACQUIRE_BARRIER " \n" 152 "2: \n" 153 : "=&r" (r), "+m" (v->counter) 154 : "r" (&v->counter), "r" (o), "r" (new), [eh] "n" (eh) 155 : "cr0", "memory"); 156 157 if (unlikely(r != o)) 158 *old = r; 159 return likely(r == o); 160 } 161 162 /** 163 * atomic_fetch_add_unless - add unless the number is a given value 164 * @v: pointer of type atomic_t 165 * @a: the amount to add to v... 166 * @u: ...unless v is equal to u. 167 * 168 * Atomically adds @a to @v, so long as it was not @u. 169 * Returns the old value of @v. 170 */ 171 static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 172 { 173 int t; 174 175 __asm__ __volatile__ ( 176 PPC_ATOMIC_ENTRY_BARRIER 177 "1: lwarx %0,0,%1 # atomic_fetch_add_unless\n\ 178 cmpw 0,%0,%3 \n\ 179 beq 2f \n\ 180 add%I2c %0,%0,%2 \n" 181 " stwcx. %0,0,%1 \n\ 182 bne- 1b \n" 183 PPC_ATOMIC_EXIT_BARRIER 184 " sub%I2c %0,%0,%2 \n\ 185 2:" 186 : "=&r" (t) 187 : "r" (&v->counter), "rI" (a), "r" (u) 188 : "cc", "memory", "xer"); 189 190 return t; 191 } 192 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 193 194 /* 195 * Atomically test *v and decrement if it is greater than 0. 196 * The function returns the old value of *v minus 1, even if 197 * the atomic variable, v, was not decremented. 198 */ 199 static __inline__ int arch_atomic_dec_if_positive(atomic_t *v) 200 { 201 int t; 202 203 __asm__ __volatile__( 204 PPC_ATOMIC_ENTRY_BARRIER 205 "1: lwarx %0,0,%1 # atomic_dec_if_positive\n\ 206 cmpwi %0,1\n\ 207 addi %0,%0,-1\n\ 208 blt- 2f\n" 209 " stwcx. %0,0,%1\n\ 210 bne- 1b" 211 PPC_ATOMIC_EXIT_BARRIER 212 "\n\ 213 2:" : "=&b" (t) 214 : "r" (&v->counter) 215 : "cc", "memory"); 216 217 return t; 218 } 219 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive 220 221 #ifdef __powerpc64__ 222 223 #define ATOMIC64_INIT(i) { (i) } 224 225 static __inline__ s64 arch_atomic64_read(const atomic64_t *v) 226 { 227 s64 t; 228 229 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m<>"(v->counter)); 230 231 return t; 232 } 233 234 static __inline__ void arch_atomic64_set(atomic64_t *v, s64 i) 235 { 236 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m<>"(v->counter) : "r"(i)); 237 } 238 239 #define ATOMIC64_OP(op, asm_op) \ 240 static __inline__ void arch_atomic64_##op(s64 a, atomic64_t *v) \ 241 { \ 242 s64 t; \ 243 \ 244 __asm__ __volatile__( \ 245 "1: ldarx %0,0,%3 # atomic64_" #op "\n" \ 246 #asm_op " %0,%2,%0\n" \ 247 " stdcx. %0,0,%3 \n" \ 248 " bne- 1b\n" \ 249 : "=&r" (t), "+m" (v->counter) \ 250 : "r" (a), "r" (&v->counter) \ 251 : "cc"); \ 252 } 253 254 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 255 static inline s64 \ 256 arch_atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ 257 { \ 258 s64 t; \ 259 \ 260 __asm__ __volatile__( \ 261 "1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \ 262 #asm_op " %0,%2,%0\n" \ 263 " stdcx. %0,0,%3\n" \ 264 " bne- 1b\n" \ 265 : "=&r" (t), "+m" (v->counter) \ 266 : "r" (a), "r" (&v->counter) \ 267 : "cc"); \ 268 \ 269 return t; \ 270 } 271 272 #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \ 273 static inline s64 \ 274 arch_atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ 275 { \ 276 s64 res, t; \ 277 \ 278 __asm__ __volatile__( \ 279 "1: ldarx %0,0,%4 # atomic64_fetch_" #op "_relaxed\n" \ 280 #asm_op " %1,%3,%0\n" \ 281 " stdcx. %1,0,%4\n" \ 282 " bne- 1b\n" \ 283 : "=&r" (res), "=&r" (t), "+m" (v->counter) \ 284 : "r" (a), "r" (&v->counter) \ 285 : "cc"); \ 286 \ 287 return res; \ 288 } 289 290 #define ATOMIC64_OPS(op, asm_op) \ 291 ATOMIC64_OP(op, asm_op) \ 292 ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \ 293 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 294 295 ATOMIC64_OPS(add, add) 296 ATOMIC64_OPS(sub, subf) 297 298 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 299 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 300 301 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 302 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 303 304 #undef ATOMIC64_OPS 305 #define ATOMIC64_OPS(op, asm_op) \ 306 ATOMIC64_OP(op, asm_op) \ 307 ATOMIC64_FETCH_OP_RELAXED(op, asm_op) 308 309 ATOMIC64_OPS(and, and) 310 ATOMIC64_OPS(or, or) 311 ATOMIC64_OPS(xor, xor) 312 313 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 314 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 315 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 316 317 #undef ATOPIC64_OPS 318 #undef ATOMIC64_FETCH_OP_RELAXED 319 #undef ATOMIC64_OP_RETURN_RELAXED 320 #undef ATOMIC64_OP 321 322 static __inline__ void arch_atomic64_inc(atomic64_t *v) 323 { 324 s64 t; 325 326 __asm__ __volatile__( 327 "1: ldarx %0,0,%2 # atomic64_inc\n\ 328 addic %0,%0,1\n\ 329 stdcx. %0,0,%2 \n\ 330 bne- 1b" 331 : "=&r" (t), "+m" (v->counter) 332 : "r" (&v->counter) 333 : "cc", "xer"); 334 } 335 #define arch_atomic64_inc arch_atomic64_inc 336 337 static __inline__ s64 arch_atomic64_inc_return_relaxed(atomic64_t *v) 338 { 339 s64 t; 340 341 __asm__ __volatile__( 342 "1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n" 343 " addic %0,%0,1\n" 344 " stdcx. %0,0,%2\n" 345 " bne- 1b" 346 : "=&r" (t), "+m" (v->counter) 347 : "r" (&v->counter) 348 : "cc", "xer"); 349 350 return t; 351 } 352 353 static __inline__ void arch_atomic64_dec(atomic64_t *v) 354 { 355 s64 t; 356 357 __asm__ __volatile__( 358 "1: ldarx %0,0,%2 # atomic64_dec\n\ 359 addic %0,%0,-1\n\ 360 stdcx. %0,0,%2\n\ 361 bne- 1b" 362 : "=&r" (t), "+m" (v->counter) 363 : "r" (&v->counter) 364 : "cc", "xer"); 365 } 366 #define arch_atomic64_dec arch_atomic64_dec 367 368 static __inline__ s64 arch_atomic64_dec_return_relaxed(atomic64_t *v) 369 { 370 s64 t; 371 372 __asm__ __volatile__( 373 "1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n" 374 " addic %0,%0,-1\n" 375 " stdcx. %0,0,%2\n" 376 " bne- 1b" 377 : "=&r" (t), "+m" (v->counter) 378 : "r" (&v->counter) 379 : "cc", "xer"); 380 381 return t; 382 } 383 384 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed 385 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed 386 387 /* 388 * Atomically test *v and decrement if it is greater than 0. 389 * The function returns the old value of *v minus 1. 390 */ 391 static __inline__ s64 arch_atomic64_dec_if_positive(atomic64_t *v) 392 { 393 s64 t; 394 395 __asm__ __volatile__( 396 PPC_ATOMIC_ENTRY_BARRIER 397 "1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\ 398 addic. %0,%0,-1\n\ 399 blt- 2f\n\ 400 stdcx. %0,0,%1\n\ 401 bne- 1b" 402 PPC_ATOMIC_EXIT_BARRIER 403 "\n\ 404 2:" : "=&r" (t) 405 : "r" (&v->counter) 406 : "cc", "xer", "memory"); 407 408 return t; 409 } 410 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 411 412 #define arch_atomic64_cmpxchg(v, o, n) \ 413 (arch_cmpxchg(&((v)->counter), (o), (n))) 414 #define arch_atomic64_cmpxchg_relaxed(v, o, n) \ 415 arch_cmpxchg_relaxed(&((v)->counter), (o), (n)) 416 #define arch_atomic64_cmpxchg_acquire(v, o, n) \ 417 arch_cmpxchg_acquire(&((v)->counter), (o), (n)) 418 419 #define arch_atomic64_xchg(v, new) \ 420 (arch_xchg(&((v)->counter), new)) 421 #define arch_atomic64_xchg_relaxed(v, new) \ 422 arch_xchg_relaxed(&((v)->counter), (new)) 423 424 /** 425 * atomic64_fetch_add_unless - add unless the number is a given value 426 * @v: pointer of type atomic64_t 427 * @a: the amount to add to v... 428 * @u: ...unless v is equal to u. 429 * 430 * Atomically adds @a to @v, so long as it was not @u. 431 * Returns the old value of @v. 432 */ 433 static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 434 { 435 s64 t; 436 437 __asm__ __volatile__ ( 438 PPC_ATOMIC_ENTRY_BARRIER 439 "1: ldarx %0,0,%1 # atomic64_fetch_add_unless\n\ 440 cmpd 0,%0,%3 \n\ 441 beq 2f \n\ 442 add %0,%2,%0 \n" 443 " stdcx. %0,0,%1 \n\ 444 bne- 1b \n" 445 PPC_ATOMIC_EXIT_BARRIER 446 " subf %0,%2,%0 \n\ 447 2:" 448 : "=&r" (t) 449 : "r" (&v->counter), "r" (a), "r" (u) 450 : "cc", "memory"); 451 452 return t; 453 } 454 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 455 456 /** 457 * atomic_inc64_not_zero - increment unless the number is zero 458 * @v: pointer of type atomic64_t 459 * 460 * Atomically increments @v by 1, so long as @v is non-zero. 461 * Returns non-zero if @v was non-zero, and zero otherwise. 462 */ 463 static __inline__ int arch_atomic64_inc_not_zero(atomic64_t *v) 464 { 465 s64 t1, t2; 466 467 __asm__ __volatile__ ( 468 PPC_ATOMIC_ENTRY_BARRIER 469 "1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\ 470 cmpdi 0,%0,0\n\ 471 beq- 2f\n\ 472 addic %1,%0,1\n\ 473 stdcx. %1,0,%2\n\ 474 bne- 1b\n" 475 PPC_ATOMIC_EXIT_BARRIER 476 "\n\ 477 2:" 478 : "=&r" (t1), "=&r" (t2) 479 : "r" (&v->counter) 480 : "cc", "xer", "memory"); 481 482 return t1 != 0; 483 } 484 #define arch_atomic64_inc_not_zero(v) arch_atomic64_inc_not_zero((v)) 485 486 #endif /* __powerpc64__ */ 487 488 #endif /* __KERNEL__ */ 489 #endif /* _ASM_POWERPC_ATOMIC_H_ */ 490