1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. 4 * Copyright (C) 2012 Regents of the University of California 5 * Copyright (C) 2017 SiFive 6 */ 7 8 #ifndef _ASM_RISCV_ATOMIC_H 9 #define _ASM_RISCV_ATOMIC_H 10 11 #ifdef CONFIG_GENERIC_ATOMIC64 12 # include <asm-generic/atomic64.h> 13 #else 14 # if (__riscv_xlen < 64) 15 # error "64-bit atomics require XLEN to be at least 64" 16 # endif 17 #endif 18 19 #include <asm/cmpxchg.h> 20 #include <asm/barrier.h> 21 22 #define __atomic_acquire_fence() \ 23 __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory") 24 25 #define __atomic_release_fence() \ 26 __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory"); 27 28 static __always_inline int arch_atomic_read(const atomic_t *v) 29 { 30 return READ_ONCE(v->counter); 31 } 32 static __always_inline void arch_atomic_set(atomic_t *v, int i) 33 { 34 WRITE_ONCE(v->counter, i); 35 } 36 37 #ifndef CONFIG_GENERIC_ATOMIC64 38 #define ATOMIC64_INIT(i) { (i) } 39 static __always_inline s64 arch_atomic64_read(const atomic64_t *v) 40 { 41 return READ_ONCE(v->counter); 42 } 43 static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) 44 { 45 WRITE_ONCE(v->counter, i); 46 } 47 #endif 48 49 /* 50 * First, the atomic ops that have no ordering constraints and therefor don't 51 * have the AQ or RL bits set. These don't return anything, so there's only 52 * one version to worry about. 53 */ 54 #define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \ 55 static __always_inline \ 56 void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \ 57 { \ 58 __asm__ __volatile__ ( \ 59 " amo" #asm_op "." #asm_type " zero, %1, %0" \ 60 : "+A" (v->counter) \ 61 : "r" (I) \ 62 : "memory"); \ 63 } \ 64 65 #ifdef CONFIG_GENERIC_ATOMIC64 66 #define ATOMIC_OPS(op, asm_op, I) \ 67 ATOMIC_OP (op, asm_op, I, w, int, ) 68 #else 69 #define ATOMIC_OPS(op, asm_op, I) \ 70 ATOMIC_OP (op, asm_op, I, w, int, ) \ 71 ATOMIC_OP (op, asm_op, I, d, s64, 64) 72 #endif 73 74 ATOMIC_OPS(add, add, i) 75 ATOMIC_OPS(sub, add, -i) 76 ATOMIC_OPS(and, and, i) 77 ATOMIC_OPS( or, or, i) 78 ATOMIC_OPS(xor, xor, i) 79 80 #undef ATOMIC_OP 81 #undef ATOMIC_OPS 82 83 /* 84 * Atomic ops that have ordered, relaxed, acquire, and release variants. 85 * There's two flavors of these: the arithmatic ops have both fetch and return 86 * versions, while the logical ops only have fetch versions. 87 */ 88 #define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \ 89 static __always_inline \ 90 c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \ 91 atomic##prefix##_t *v) \ 92 { \ 93 register c_type ret; \ 94 __asm__ __volatile__ ( \ 95 " amo" #asm_op "." #asm_type " %1, %2, %0" \ 96 : "+A" (v->counter), "=r" (ret) \ 97 : "r" (I) \ 98 : "memory"); \ 99 return ret; \ 100 } \ 101 static __always_inline \ 102 c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \ 103 { \ 104 register c_type ret; \ 105 __asm__ __volatile__ ( \ 106 " amo" #asm_op "." #asm_type ".aqrl %1, %2, %0" \ 107 : "+A" (v->counter), "=r" (ret) \ 108 : "r" (I) \ 109 : "memory"); \ 110 return ret; \ 111 } 112 113 #define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \ 114 static __always_inline \ 115 c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i, \ 116 atomic##prefix##_t *v) \ 117 { \ 118 return arch_atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \ 119 } \ 120 static __always_inline \ 121 c_type arch_atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \ 122 { \ 123 return arch_atomic##prefix##_fetch_##op(i, v) c_op I; \ 124 } 125 126 #ifdef CONFIG_GENERIC_ATOMIC64 127 #define ATOMIC_OPS(op, asm_op, c_op, I) \ 128 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \ 129 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) 130 #else 131 #define ATOMIC_OPS(op, asm_op, c_op, I) \ 132 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \ 133 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) \ 134 ATOMIC_FETCH_OP( op, asm_op, I, d, s64, 64) \ 135 ATOMIC_OP_RETURN(op, asm_op, c_op, I, d, s64, 64) 136 #endif 137 138 ATOMIC_OPS(add, add, +, i) 139 ATOMIC_OPS(sub, add, +, -i) 140 141 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed 142 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed 143 #define arch_atomic_add_return arch_atomic_add_return 144 #define arch_atomic_sub_return arch_atomic_sub_return 145 146 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed 147 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed 148 #define arch_atomic_fetch_add arch_atomic_fetch_add 149 #define arch_atomic_fetch_sub arch_atomic_fetch_sub 150 151 #ifndef CONFIG_GENERIC_ATOMIC64 152 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 153 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 154 #define arch_atomic64_add_return arch_atomic64_add_return 155 #define arch_atomic64_sub_return arch_atomic64_sub_return 156 157 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 158 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 159 #define arch_atomic64_fetch_add arch_atomic64_fetch_add 160 #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub 161 #endif 162 163 #undef ATOMIC_OPS 164 165 #ifdef CONFIG_GENERIC_ATOMIC64 166 #define ATOMIC_OPS(op, asm_op, I) \ 167 ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) 168 #else 169 #define ATOMIC_OPS(op, asm_op, I) \ 170 ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) \ 171 ATOMIC_FETCH_OP(op, asm_op, I, d, s64, 64) 172 #endif 173 174 ATOMIC_OPS(and, and, i) 175 ATOMIC_OPS( or, or, i) 176 ATOMIC_OPS(xor, xor, i) 177 178 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed 179 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed 180 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed 181 #define arch_atomic_fetch_and arch_atomic_fetch_and 182 #define arch_atomic_fetch_or arch_atomic_fetch_or 183 #define arch_atomic_fetch_xor arch_atomic_fetch_xor 184 185 #ifndef CONFIG_GENERIC_ATOMIC64 186 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 187 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 188 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 189 #define arch_atomic64_fetch_and arch_atomic64_fetch_and 190 #define arch_atomic64_fetch_or arch_atomic64_fetch_or 191 #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor 192 #endif 193 194 #undef ATOMIC_OPS 195 196 #undef ATOMIC_FETCH_OP 197 #undef ATOMIC_OP_RETURN 198 199 /* This is required to provide a full barrier on success. */ 200 static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 201 { 202 int prev, rc; 203 204 __asm__ __volatile__ ( 205 "0: lr.w %[p], %[c]\n" 206 " beq %[p], %[u], 1f\n" 207 " add %[rc], %[p], %[a]\n" 208 " sc.w.rl %[rc], %[rc], %[c]\n" 209 " bnez %[rc], 0b\n" 210 " fence rw, rw\n" 211 "1:\n" 212 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 213 : [a]"r" (a), [u]"r" (u) 214 : "memory"); 215 return prev; 216 } 217 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 218 219 #ifndef CONFIG_GENERIC_ATOMIC64 220 static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 221 { 222 s64 prev; 223 long rc; 224 225 __asm__ __volatile__ ( 226 "0: lr.d %[p], %[c]\n" 227 " beq %[p], %[u], 1f\n" 228 " add %[rc], %[p], %[a]\n" 229 " sc.d.rl %[rc], %[rc], %[c]\n" 230 " bnez %[rc], 0b\n" 231 " fence rw, rw\n" 232 "1:\n" 233 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 234 : [a]"r" (a), [u]"r" (u) 235 : "memory"); 236 return prev; 237 } 238 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 239 #endif 240 241 /* 242 * atomic_{cmp,}xchg is required to have exactly the same ordering semantics as 243 * {cmp,}xchg and the operations that return, so they need a full barrier. 244 */ 245 #define ATOMIC_OP(c_t, prefix, size) \ 246 static __always_inline \ 247 c_t arch_atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n) \ 248 { \ 249 return __xchg_relaxed(&(v->counter), n, size); \ 250 } \ 251 static __always_inline \ 252 c_t arch_atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n) \ 253 { \ 254 return __xchg_acquire(&(v->counter), n, size); \ 255 } \ 256 static __always_inline \ 257 c_t arch_atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n) \ 258 { \ 259 return __xchg_release(&(v->counter), n, size); \ 260 } \ 261 static __always_inline \ 262 c_t arch_atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \ 263 { \ 264 return __arch_xchg(&(v->counter), n, size); \ 265 } \ 266 static __always_inline \ 267 c_t arch_atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \ 268 c_t o, c_t n) \ 269 { \ 270 return __cmpxchg_relaxed(&(v->counter), o, n, size); \ 271 } \ 272 static __always_inline \ 273 c_t arch_atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v, \ 274 c_t o, c_t n) \ 275 { \ 276 return __cmpxchg_acquire(&(v->counter), o, n, size); \ 277 } \ 278 static __always_inline \ 279 c_t arch_atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v, \ 280 c_t o, c_t n) \ 281 { \ 282 return __cmpxchg_release(&(v->counter), o, n, size); \ 283 } \ 284 static __always_inline \ 285 c_t arch_atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \ 286 { \ 287 return __cmpxchg(&(v->counter), o, n, size); \ 288 } 289 290 #ifdef CONFIG_GENERIC_ATOMIC64 291 #define ATOMIC_OPS() \ 292 ATOMIC_OP(int, , 4) 293 #else 294 #define ATOMIC_OPS() \ 295 ATOMIC_OP(int, , 4) \ 296 ATOMIC_OP(s64, 64, 8) 297 #endif 298 299 ATOMIC_OPS() 300 301 #define arch_atomic_xchg_relaxed arch_atomic_xchg_relaxed 302 #define arch_atomic_xchg_acquire arch_atomic_xchg_acquire 303 #define arch_atomic_xchg_release arch_atomic_xchg_release 304 #define arch_atomic_xchg arch_atomic_xchg 305 #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed 306 #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire 307 #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release 308 #define arch_atomic_cmpxchg arch_atomic_cmpxchg 309 310 #undef ATOMIC_OPS 311 #undef ATOMIC_OP 312 313 static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v) 314 { 315 int prev, rc; 316 317 __asm__ __volatile__ ( 318 "0: lr.w %[p], %[c]\n" 319 " bltz %[p], 1f\n" 320 " addi %[rc], %[p], 1\n" 321 " sc.w.rl %[rc], %[rc], %[c]\n" 322 " bnez %[rc], 0b\n" 323 " fence rw, rw\n" 324 "1:\n" 325 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 326 : 327 : "memory"); 328 return !(prev < 0); 329 } 330 331 #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative 332 333 static __always_inline bool arch_atomic_dec_unless_positive(atomic_t *v) 334 { 335 int prev, rc; 336 337 __asm__ __volatile__ ( 338 "0: lr.w %[p], %[c]\n" 339 " bgtz %[p], 1f\n" 340 " addi %[rc], %[p], -1\n" 341 " sc.w.rl %[rc], %[rc], %[c]\n" 342 " bnez %[rc], 0b\n" 343 " fence rw, rw\n" 344 "1:\n" 345 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 346 : 347 : "memory"); 348 return !(prev > 0); 349 } 350 351 #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive 352 353 static __always_inline int arch_atomic_dec_if_positive(atomic_t *v) 354 { 355 int prev, rc; 356 357 __asm__ __volatile__ ( 358 "0: lr.w %[p], %[c]\n" 359 " addi %[rc], %[p], -1\n" 360 " bltz %[rc], 1f\n" 361 " sc.w.rl %[rc], %[rc], %[c]\n" 362 " bnez %[rc], 0b\n" 363 " fence rw, rw\n" 364 "1:\n" 365 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 366 : 367 : "memory"); 368 return prev - 1; 369 } 370 371 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive 372 373 #ifndef CONFIG_GENERIC_ATOMIC64 374 static __always_inline bool arch_atomic64_inc_unless_negative(atomic64_t *v) 375 { 376 s64 prev; 377 long rc; 378 379 __asm__ __volatile__ ( 380 "0: lr.d %[p], %[c]\n" 381 " bltz %[p], 1f\n" 382 " addi %[rc], %[p], 1\n" 383 " sc.d.rl %[rc], %[rc], %[c]\n" 384 " bnez %[rc], 0b\n" 385 " fence rw, rw\n" 386 "1:\n" 387 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 388 : 389 : "memory"); 390 return !(prev < 0); 391 } 392 393 #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative 394 395 static __always_inline bool arch_atomic64_dec_unless_positive(atomic64_t *v) 396 { 397 s64 prev; 398 long rc; 399 400 __asm__ __volatile__ ( 401 "0: lr.d %[p], %[c]\n" 402 " bgtz %[p], 1f\n" 403 " addi %[rc], %[p], -1\n" 404 " sc.d.rl %[rc], %[rc], %[c]\n" 405 " bnez %[rc], 0b\n" 406 " fence rw, rw\n" 407 "1:\n" 408 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 409 : 410 : "memory"); 411 return !(prev > 0); 412 } 413 414 #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive 415 416 static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 417 { 418 s64 prev; 419 long rc; 420 421 __asm__ __volatile__ ( 422 "0: lr.d %[p], %[c]\n" 423 " addi %[rc], %[p], -1\n" 424 " bltz %[rc], 1f\n" 425 " sc.d.rl %[rc], %[rc], %[c]\n" 426 " bnez %[rc], 0b\n" 427 " fence rw, rw\n" 428 "1:\n" 429 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter) 430 : 431 : "memory"); 432 return prev - 1; 433 } 434 435 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 436 #endif 437 438 #endif /* _ASM_RISCV_ATOMIC_H */ 439