1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2014 Regents of the University of California 4 */ 5 6 #ifndef _ASM_RISCV_CMPXCHG_H 7 #define _ASM_RISCV_CMPXCHG_H 8 9 #include <linux/bug.h> 10 11 #include <asm/barrier.h> 12 #include <asm/fence.h> 13 14 #define __xchg_relaxed(ptr, new, size) \ 15 ({ \ 16 __typeof__(ptr) __ptr = (ptr); \ 17 __typeof__(new) __new = (new); \ 18 __typeof__(*(ptr)) __ret; \ 19 switch (size) { \ 20 case 4: \ 21 __asm__ __volatile__ ( \ 22 " amoswap.w %0, %2, %1\n" \ 23 : "=r" (__ret), "+A" (*__ptr) \ 24 : "r" (__new) \ 25 : "memory"); \ 26 break; \ 27 case 8: \ 28 __asm__ __volatile__ ( \ 29 " amoswap.d %0, %2, %1\n" \ 30 : "=r" (__ret), "+A" (*__ptr) \ 31 : "r" (__new) \ 32 : "memory"); \ 33 break; \ 34 default: \ 35 BUILD_BUG(); \ 36 } \ 37 __ret; \ 38 }) 39 40 #define xchg_relaxed(ptr, x) \ 41 ({ \ 42 __typeof__(*(ptr)) _x_ = (x); \ 43 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \ 44 _x_, sizeof(*(ptr))); \ 45 }) 46 47 #define __xchg_acquire(ptr, new, size) \ 48 ({ \ 49 __typeof__(ptr) __ptr = (ptr); \ 50 __typeof__(new) __new = (new); \ 51 __typeof__(*(ptr)) __ret; \ 52 switch (size) { \ 53 case 4: \ 54 __asm__ __volatile__ ( \ 55 " amoswap.w %0, %2, %1\n" \ 56 RISCV_ACQUIRE_BARRIER \ 57 : "=r" (__ret), "+A" (*__ptr) \ 58 : "r" (__new) \ 59 : "memory"); \ 60 break; \ 61 case 8: \ 62 __asm__ __volatile__ ( \ 63 " amoswap.d %0, %2, %1\n" \ 64 RISCV_ACQUIRE_BARRIER \ 65 : "=r" (__ret), "+A" (*__ptr) \ 66 : "r" (__new) \ 67 : "memory"); \ 68 break; \ 69 default: \ 70 BUILD_BUG(); \ 71 } \ 72 __ret; \ 73 }) 74 75 #define xchg_acquire(ptr, x) \ 76 ({ \ 77 __typeof__(*(ptr)) _x_ = (x); \ 78 (__typeof__(*(ptr))) __xchg_acquire((ptr), \ 79 _x_, sizeof(*(ptr))); \ 80 }) 81 82 #define __xchg_release(ptr, new, size) \ 83 ({ \ 84 __typeof__(ptr) __ptr = (ptr); \ 85 __typeof__(new) __new = (new); \ 86 __typeof__(*(ptr)) __ret; \ 87 switch (size) { \ 88 case 4: \ 89 __asm__ __volatile__ ( \ 90 RISCV_RELEASE_BARRIER \ 91 " amoswap.w %0, %2, %1\n" \ 92 : "=r" (__ret), "+A" (*__ptr) \ 93 : "r" (__new) \ 94 : "memory"); \ 95 break; \ 96 case 8: \ 97 __asm__ __volatile__ ( \ 98 RISCV_RELEASE_BARRIER \ 99 " amoswap.d %0, %2, %1\n" \ 100 : "=r" (__ret), "+A" (*__ptr) \ 101 : "r" (__new) \ 102 : "memory"); \ 103 break; \ 104 default: \ 105 BUILD_BUG(); \ 106 } \ 107 __ret; \ 108 }) 109 110 #define xchg_release(ptr, x) \ 111 ({ \ 112 __typeof__(*(ptr)) _x_ = (x); \ 113 (__typeof__(*(ptr))) __xchg_release((ptr), \ 114 _x_, sizeof(*(ptr))); \ 115 }) 116 117 #define __xchg(ptr, new, size) \ 118 ({ \ 119 __typeof__(ptr) __ptr = (ptr); \ 120 __typeof__(new) __new = (new); \ 121 __typeof__(*(ptr)) __ret; \ 122 switch (size) { \ 123 case 4: \ 124 __asm__ __volatile__ ( \ 125 " amoswap.w.aqrl %0, %2, %1\n" \ 126 : "=r" (__ret), "+A" (*__ptr) \ 127 : "r" (__new) \ 128 : "memory"); \ 129 break; \ 130 case 8: \ 131 __asm__ __volatile__ ( \ 132 " amoswap.d.aqrl %0, %2, %1\n" \ 133 : "=r" (__ret), "+A" (*__ptr) \ 134 : "r" (__new) \ 135 : "memory"); \ 136 break; \ 137 default: \ 138 BUILD_BUG(); \ 139 } \ 140 __ret; \ 141 }) 142 143 #define xchg(ptr, x) \ 144 ({ \ 145 __typeof__(*(ptr)) _x_ = (x); \ 146 (__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \ 147 }) 148 149 #define xchg32(ptr, x) \ 150 ({ \ 151 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ 152 xchg((ptr), (x)); \ 153 }) 154 155 #define xchg64(ptr, x) \ 156 ({ \ 157 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 158 xchg((ptr), (x)); \ 159 }) 160 161 /* 162 * Atomic compare and exchange. Compare OLD with MEM, if identical, 163 * store NEW in MEM. Return the initial value in MEM. Success is 164 * indicated by comparing RETURN with OLD. 165 */ 166 #define __cmpxchg_relaxed(ptr, old, new, size) \ 167 ({ \ 168 __typeof__(ptr) __ptr = (ptr); \ 169 __typeof__(*(ptr)) __old = (old); \ 170 __typeof__(*(ptr)) __new = (new); \ 171 __typeof__(*(ptr)) __ret; \ 172 register unsigned int __rc; \ 173 switch (size) { \ 174 case 4: \ 175 __asm__ __volatile__ ( \ 176 "0: lr.w %0, %2\n" \ 177 " bne %0, %z3, 1f\n" \ 178 " sc.w %1, %z4, %2\n" \ 179 " bnez %1, 0b\n" \ 180 "1:\n" \ 181 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 182 : "rJ" (__old), "rJ" (__new) \ 183 : "memory"); \ 184 break; \ 185 case 8: \ 186 __asm__ __volatile__ ( \ 187 "0: lr.d %0, %2\n" \ 188 " bne %0, %z3, 1f\n" \ 189 " sc.d %1, %z4, %2\n" \ 190 " bnez %1, 0b\n" \ 191 "1:\n" \ 192 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 193 : "rJ" (__old), "rJ" (__new) \ 194 : "memory"); \ 195 break; \ 196 default: \ 197 BUILD_BUG(); \ 198 } \ 199 __ret; \ 200 }) 201 202 #define cmpxchg_relaxed(ptr, o, n) \ 203 ({ \ 204 __typeof__(*(ptr)) _o_ = (o); \ 205 __typeof__(*(ptr)) _n_ = (n); \ 206 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \ 207 _o_, _n_, sizeof(*(ptr))); \ 208 }) 209 210 #define __cmpxchg_acquire(ptr, old, new, size) \ 211 ({ \ 212 __typeof__(ptr) __ptr = (ptr); \ 213 __typeof__(*(ptr)) __old = (old); \ 214 __typeof__(*(ptr)) __new = (new); \ 215 __typeof__(*(ptr)) __ret; \ 216 register unsigned int __rc; \ 217 switch (size) { \ 218 case 4: \ 219 __asm__ __volatile__ ( \ 220 "0: lr.w %0, %2\n" \ 221 " bne %0, %z3, 1f\n" \ 222 " sc.w %1, %z4, %2\n" \ 223 " bnez %1, 0b\n" \ 224 RISCV_ACQUIRE_BARRIER \ 225 "1:\n" \ 226 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 227 : "rJ" (__old), "rJ" (__new) \ 228 : "memory"); \ 229 break; \ 230 case 8: \ 231 __asm__ __volatile__ ( \ 232 "0: lr.d %0, %2\n" \ 233 " bne %0, %z3, 1f\n" \ 234 " sc.d %1, %z4, %2\n" \ 235 " bnez %1, 0b\n" \ 236 RISCV_ACQUIRE_BARRIER \ 237 "1:\n" \ 238 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 239 : "rJ" (__old), "rJ" (__new) \ 240 : "memory"); \ 241 break; \ 242 default: \ 243 BUILD_BUG(); \ 244 } \ 245 __ret; \ 246 }) 247 248 #define cmpxchg_acquire(ptr, o, n) \ 249 ({ \ 250 __typeof__(*(ptr)) _o_ = (o); \ 251 __typeof__(*(ptr)) _n_ = (n); \ 252 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \ 253 _o_, _n_, sizeof(*(ptr))); \ 254 }) 255 256 #define __cmpxchg_release(ptr, old, new, size) \ 257 ({ \ 258 __typeof__(ptr) __ptr = (ptr); \ 259 __typeof__(*(ptr)) __old = (old); \ 260 __typeof__(*(ptr)) __new = (new); \ 261 __typeof__(*(ptr)) __ret; \ 262 register unsigned int __rc; \ 263 switch (size) { \ 264 case 4: \ 265 __asm__ __volatile__ ( \ 266 RISCV_RELEASE_BARRIER \ 267 "0: lr.w %0, %2\n" \ 268 " bne %0, %z3, 1f\n" \ 269 " sc.w %1, %z4, %2\n" \ 270 " bnez %1, 0b\n" \ 271 "1:\n" \ 272 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 273 : "rJ" (__old), "rJ" (__new) \ 274 : "memory"); \ 275 break; \ 276 case 8: \ 277 __asm__ __volatile__ ( \ 278 RISCV_RELEASE_BARRIER \ 279 "0: lr.d %0, %2\n" \ 280 " bne %0, %z3, 1f\n" \ 281 " sc.d %1, %z4, %2\n" \ 282 " bnez %1, 0b\n" \ 283 "1:\n" \ 284 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 285 : "rJ" (__old), "rJ" (__new) \ 286 : "memory"); \ 287 break; \ 288 default: \ 289 BUILD_BUG(); \ 290 } \ 291 __ret; \ 292 }) 293 294 #define cmpxchg_release(ptr, o, n) \ 295 ({ \ 296 __typeof__(*(ptr)) _o_ = (o); \ 297 __typeof__(*(ptr)) _n_ = (n); \ 298 (__typeof__(*(ptr))) __cmpxchg_release((ptr), \ 299 _o_, _n_, sizeof(*(ptr))); \ 300 }) 301 302 #define __cmpxchg(ptr, old, new, size) \ 303 ({ \ 304 __typeof__(ptr) __ptr = (ptr); \ 305 __typeof__(*(ptr)) __old = (old); \ 306 __typeof__(*(ptr)) __new = (new); \ 307 __typeof__(*(ptr)) __ret; \ 308 register unsigned int __rc; \ 309 switch (size) { \ 310 case 4: \ 311 __asm__ __volatile__ ( \ 312 "0: lr.w %0, %2\n" \ 313 " bne %0, %z3, 1f\n" \ 314 " sc.w.rl %1, %z4, %2\n" \ 315 " bnez %1, 0b\n" \ 316 " fence rw, rw\n" \ 317 "1:\n" \ 318 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 319 : "rJ" (__old), "rJ" (__new) \ 320 : "memory"); \ 321 break; \ 322 case 8: \ 323 __asm__ __volatile__ ( \ 324 "0: lr.d %0, %2\n" \ 325 " bne %0, %z3, 1f\n" \ 326 " sc.d.rl %1, %z4, %2\n" \ 327 " bnez %1, 0b\n" \ 328 " fence rw, rw\n" \ 329 "1:\n" \ 330 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 331 : "rJ" (__old), "rJ" (__new) \ 332 : "memory"); \ 333 break; \ 334 default: \ 335 BUILD_BUG(); \ 336 } \ 337 __ret; \ 338 }) 339 340 #define cmpxchg(ptr, o, n) \ 341 ({ \ 342 __typeof__(*(ptr)) _o_ = (o); \ 343 __typeof__(*(ptr)) _n_ = (n); \ 344 (__typeof__(*(ptr))) __cmpxchg((ptr), \ 345 _o_, _n_, sizeof(*(ptr))); \ 346 }) 347 348 #define cmpxchg_local(ptr, o, n) \ 349 (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) 350 351 #define cmpxchg32(ptr, o, n) \ 352 ({ \ 353 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ 354 cmpxchg((ptr), (o), (n)); \ 355 }) 356 357 #define cmpxchg32_local(ptr, o, n) \ 358 ({ \ 359 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ 360 cmpxchg_relaxed((ptr), (o), (n)) \ 361 }) 362 363 #define cmpxchg64(ptr, o, n) \ 364 ({ \ 365 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 366 cmpxchg((ptr), (o), (n)); \ 367 }) 368 369 #define cmpxchg64_local(ptr, o, n) \ 370 ({ \ 371 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 372 cmpxchg_relaxed((ptr), (o), (n)); \ 373 }) 374 375 #endif /* _ASM_RISCV_CMPXCHG_H */ 376