1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/atomic.h 4 * 5 * Copyright (C) 1996 Russell King. 6 * Copyright (C) 2002 Deep Blue Solutions Ltd. 7 * Copyright (C) 2012 ARM Ltd. 8 */ 9 10 #ifndef __ASM_ATOMIC_LSE_H 11 #define __ASM_ATOMIC_LSE_H 12 13 #define ATOMIC_OP(op, asm_op) \ 14 static inline void __lse_atomic_##op(int i, atomic_t *v) \ 15 { \ 16 asm volatile( \ 17 " " #asm_op " %w[i], %[v]\n" \ 18 : [i] "+r" (i), [v] "+Q" (v->counter) \ 19 : "r" (v)); \ 20 } 21 22 ATOMIC_OP(andnot, stclr) 23 ATOMIC_OP(or, stset) 24 ATOMIC_OP(xor, steor) 25 ATOMIC_OP(add, stadd) 26 27 #undef ATOMIC_OP 28 29 #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...) \ 30 static inline int __lse_atomic_fetch_##op##name(int i, atomic_t *v) \ 31 { \ 32 asm volatile( \ 33 " " #asm_op #mb " %w[i], %w[i], %[v]" \ 34 : [i] "+r" (i), [v] "+Q" (v->counter) \ 35 : "r" (v) \ 36 : cl); \ 37 \ 38 return i; \ 39 } 40 41 #define ATOMIC_FETCH_OPS(op, asm_op) \ 42 ATOMIC_FETCH_OP(_relaxed, , op, asm_op) \ 43 ATOMIC_FETCH_OP(_acquire, a, op, asm_op, "memory") \ 44 ATOMIC_FETCH_OP(_release, l, op, asm_op, "memory") \ 45 ATOMIC_FETCH_OP( , al, op, asm_op, "memory") 46 47 ATOMIC_FETCH_OPS(andnot, ldclr) 48 ATOMIC_FETCH_OPS(or, ldset) 49 ATOMIC_FETCH_OPS(xor, ldeor) 50 ATOMIC_FETCH_OPS(add, ldadd) 51 52 #undef ATOMIC_FETCH_OP 53 #undef ATOMIC_FETCH_OPS 54 55 #define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \ 56 static inline int __lse_atomic_add_return##name(int i, atomic_t *v) \ 57 { \ 58 u32 tmp; \ 59 \ 60 asm volatile( \ 61 " ldadd" #mb " %w[i], %w[tmp], %[v]\n" \ 62 " add %w[i], %w[i], %w[tmp]" \ 63 : [i] "+r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \ 64 : "r" (v) \ 65 : cl); \ 66 \ 67 return i; \ 68 } 69 70 ATOMIC_OP_ADD_RETURN(_relaxed, ) 71 ATOMIC_OP_ADD_RETURN(_acquire, a, "memory") 72 ATOMIC_OP_ADD_RETURN(_release, l, "memory") 73 ATOMIC_OP_ADD_RETURN( , al, "memory") 74 75 #undef ATOMIC_OP_ADD_RETURN 76 77 static inline void __lse_atomic_and(int i, atomic_t *v) 78 { 79 asm volatile( 80 " mvn %w[i], %w[i]\n" 81 " stclr %w[i], %[v]" 82 : [i] "+&r" (i), [v] "+Q" (v->counter) 83 : "r" (v)); 84 } 85 86 #define ATOMIC_FETCH_OP_AND(name, mb, cl...) \ 87 static inline int __lse_atomic_fetch_and##name(int i, atomic_t *v) \ 88 { \ 89 asm volatile( \ 90 " mvn %w[i], %w[i]\n" \ 91 " ldclr" #mb " %w[i], %w[i], %[v]" \ 92 : [i] "+&r" (i), [v] "+Q" (v->counter) \ 93 : "r" (v) \ 94 : cl); \ 95 \ 96 return i; \ 97 } 98 99 ATOMIC_FETCH_OP_AND(_relaxed, ) 100 ATOMIC_FETCH_OP_AND(_acquire, a, "memory") 101 ATOMIC_FETCH_OP_AND(_release, l, "memory") 102 ATOMIC_FETCH_OP_AND( , al, "memory") 103 104 #undef ATOMIC_FETCH_OP_AND 105 106 static inline void __lse_atomic_sub(int i, atomic_t *v) 107 { 108 asm volatile( 109 " neg %w[i], %w[i]\n" 110 " stadd %w[i], %[v]" 111 : [i] "+&r" (i), [v] "+Q" (v->counter) 112 : "r" (v)); 113 } 114 115 #define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \ 116 static inline int __lse_atomic_sub_return##name(int i, atomic_t *v) \ 117 { \ 118 u32 tmp; \ 119 \ 120 asm volatile( \ 121 " neg %w[i], %w[i]\n" \ 122 " ldadd" #mb " %w[i], %w[tmp], %[v]\n" \ 123 " add %w[i], %w[i], %w[tmp]" \ 124 : [i] "+&r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \ 125 : "r" (v) \ 126 : cl); \ 127 \ 128 return i; \ 129 } 130 131 ATOMIC_OP_SUB_RETURN(_relaxed, ) 132 ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") 133 ATOMIC_OP_SUB_RETURN(_release, l, "memory") 134 ATOMIC_OP_SUB_RETURN( , al, "memory") 135 136 #undef ATOMIC_OP_SUB_RETURN 137 138 #define ATOMIC_FETCH_OP_SUB(name, mb, cl...) \ 139 static inline int __lse_atomic_fetch_sub##name(int i, atomic_t *v) \ 140 { \ 141 asm volatile( \ 142 " neg %w[i], %w[i]\n" \ 143 " ldadd" #mb " %w[i], %w[i], %[v]" \ 144 : [i] "+&r" (i), [v] "+Q" (v->counter) \ 145 : "r" (v) \ 146 : cl); \ 147 \ 148 return i; \ 149 } 150 151 ATOMIC_FETCH_OP_SUB(_relaxed, ) 152 ATOMIC_FETCH_OP_SUB(_acquire, a, "memory") 153 ATOMIC_FETCH_OP_SUB(_release, l, "memory") 154 ATOMIC_FETCH_OP_SUB( , al, "memory") 155 156 #undef ATOMIC_FETCH_OP_SUB 157 158 #define ATOMIC64_OP(op, asm_op) \ 159 static inline void __lse_atomic64_##op(s64 i, atomic64_t *v) \ 160 { \ 161 asm volatile( \ 162 " " #asm_op " %[i], %[v]\n" \ 163 : [i] "+r" (i), [v] "+Q" (v->counter) \ 164 : "r" (v)); \ 165 } 166 167 ATOMIC64_OP(andnot, stclr) 168 ATOMIC64_OP(or, stset) 169 ATOMIC64_OP(xor, steor) 170 ATOMIC64_OP(add, stadd) 171 172 #undef ATOMIC64_OP 173 174 #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...) \ 175 static inline long __lse_atomic64_fetch_##op##name(s64 i, atomic64_t *v)\ 176 { \ 177 asm volatile( \ 178 " " #asm_op #mb " %[i], %[i], %[v]" \ 179 : [i] "+r" (i), [v] "+Q" (v->counter) \ 180 : "r" (v) \ 181 : cl); \ 182 \ 183 return i; \ 184 } 185 186 #define ATOMIC64_FETCH_OPS(op, asm_op) \ 187 ATOMIC64_FETCH_OP(_relaxed, , op, asm_op) \ 188 ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \ 189 ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \ 190 ATOMIC64_FETCH_OP( , al, op, asm_op, "memory") 191 192 ATOMIC64_FETCH_OPS(andnot, ldclr) 193 ATOMIC64_FETCH_OPS(or, ldset) 194 ATOMIC64_FETCH_OPS(xor, ldeor) 195 ATOMIC64_FETCH_OPS(add, ldadd) 196 197 #undef ATOMIC64_FETCH_OP 198 #undef ATOMIC64_FETCH_OPS 199 200 #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \ 201 static inline long __lse_atomic64_add_return##name(s64 i, atomic64_t *v)\ 202 { \ 203 unsigned long tmp; \ 204 \ 205 asm volatile( \ 206 " ldadd" #mb " %[i], %x[tmp], %[v]\n" \ 207 " add %[i], %[i], %x[tmp]" \ 208 : [i] "+r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \ 209 : "r" (v) \ 210 : cl); \ 211 \ 212 return i; \ 213 } 214 215 ATOMIC64_OP_ADD_RETURN(_relaxed, ) 216 ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory") 217 ATOMIC64_OP_ADD_RETURN(_release, l, "memory") 218 ATOMIC64_OP_ADD_RETURN( , al, "memory") 219 220 #undef ATOMIC64_OP_ADD_RETURN 221 222 static inline void __lse_atomic64_and(s64 i, atomic64_t *v) 223 { 224 asm volatile( 225 " mvn %[i], %[i]\n" 226 " stclr %[i], %[v]" 227 : [i] "+&r" (i), [v] "+Q" (v->counter) 228 : "r" (v)); 229 } 230 231 #define ATOMIC64_FETCH_OP_AND(name, mb, cl...) \ 232 static inline long __lse_atomic64_fetch_and##name(s64 i, atomic64_t *v) \ 233 { \ 234 asm volatile( \ 235 " mvn %[i], %[i]\n" \ 236 " ldclr" #mb " %[i], %[i], %[v]" \ 237 : [i] "+&r" (i), [v] "+Q" (v->counter) \ 238 : "r" (v) \ 239 : cl); \ 240 \ 241 return i; \ 242 } 243 244 ATOMIC64_FETCH_OP_AND(_relaxed, ) 245 ATOMIC64_FETCH_OP_AND(_acquire, a, "memory") 246 ATOMIC64_FETCH_OP_AND(_release, l, "memory") 247 ATOMIC64_FETCH_OP_AND( , al, "memory") 248 249 #undef ATOMIC64_FETCH_OP_AND 250 251 static inline void __lse_atomic64_sub(s64 i, atomic64_t *v) 252 { 253 asm volatile( 254 " neg %[i], %[i]\n" 255 " stadd %[i], %[v]" 256 : [i] "+&r" (i), [v] "+Q" (v->counter) 257 : "r" (v)); 258 } 259 260 #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \ 261 static inline long __lse_atomic64_sub_return##name(s64 i, atomic64_t *v) \ 262 { \ 263 unsigned long tmp; \ 264 \ 265 asm volatile( \ 266 " neg %[i], %[i]\n" \ 267 " ldadd" #mb " %[i], %x[tmp], %[v]\n" \ 268 " add %[i], %[i], %x[tmp]" \ 269 : [i] "+&r" (i), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) \ 270 : "r" (v) \ 271 : cl); \ 272 \ 273 return i; \ 274 } 275 276 ATOMIC64_OP_SUB_RETURN(_relaxed, ) 277 ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") 278 ATOMIC64_OP_SUB_RETURN(_release, l, "memory") 279 ATOMIC64_OP_SUB_RETURN( , al, "memory") 280 281 #undef ATOMIC64_OP_SUB_RETURN 282 283 #define ATOMIC64_FETCH_OP_SUB(name, mb, cl...) \ 284 static inline long __lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v) \ 285 { \ 286 asm volatile( \ 287 " neg %[i], %[i]\n" \ 288 " ldadd" #mb " %[i], %[i], %[v]" \ 289 : [i] "+&r" (i), [v] "+Q" (v->counter) \ 290 : "r" (v) \ 291 : cl); \ 292 \ 293 return i; \ 294 } 295 296 ATOMIC64_FETCH_OP_SUB(_relaxed, ) 297 ATOMIC64_FETCH_OP_SUB(_acquire, a, "memory") 298 ATOMIC64_FETCH_OP_SUB(_release, l, "memory") 299 ATOMIC64_FETCH_OP_SUB( , al, "memory") 300 301 #undef ATOMIC64_FETCH_OP_SUB 302 303 static inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v) 304 { 305 unsigned long tmp; 306 307 asm volatile( 308 "1: ldr %x[tmp], %[v]\n" 309 " subs %[ret], %x[tmp], #1\n" 310 " b.lt 2f\n" 311 " casal %x[tmp], %[ret], %[v]\n" 312 " sub %x[tmp], %x[tmp], #1\n" 313 " sub %x[tmp], %x[tmp], %[ret]\n" 314 " cbnz %x[tmp], 1b\n" 315 "2:" 316 : [ret] "+&r" (v), [v] "+Q" (v->counter), [tmp] "=&r" (tmp) 317 : 318 : "cc", "memory"); 319 320 return (long)v; 321 } 322 323 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...) \ 324 static inline u##sz __lse__cmpxchg_case_##name##sz(volatile void *ptr, \ 325 u##sz old, \ 326 u##sz new) \ 327 { \ 328 register unsigned long x0 asm ("x0") = (unsigned long)ptr; \ 329 register u##sz x1 asm ("x1") = old; \ 330 register u##sz x2 asm ("x2") = new; \ 331 unsigned long tmp; \ 332 \ 333 asm volatile( \ 334 " mov %" #w "[tmp], %" #w "[old]\n" \ 335 " cas" #mb #sfx "\t%" #w "[tmp], %" #w "[new], %[v]\n" \ 336 " mov %" #w "[ret], %" #w "[tmp]" \ 337 : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr), \ 338 [tmp] "=&r" (tmp) \ 339 : [old] "r" (x1), [new] "r" (x2) \ 340 : cl); \ 341 \ 342 return x0; \ 343 } 344 345 __CMPXCHG_CASE(w, b, , 8, ) 346 __CMPXCHG_CASE(w, h, , 16, ) 347 __CMPXCHG_CASE(w, , , 32, ) 348 __CMPXCHG_CASE(x, , , 64, ) 349 __CMPXCHG_CASE(w, b, acq_, 8, a, "memory") 350 __CMPXCHG_CASE(w, h, acq_, 16, a, "memory") 351 __CMPXCHG_CASE(w, , acq_, 32, a, "memory") 352 __CMPXCHG_CASE(x, , acq_, 64, a, "memory") 353 __CMPXCHG_CASE(w, b, rel_, 8, l, "memory") 354 __CMPXCHG_CASE(w, h, rel_, 16, l, "memory") 355 __CMPXCHG_CASE(w, , rel_, 32, l, "memory") 356 __CMPXCHG_CASE(x, , rel_, 64, l, "memory") 357 __CMPXCHG_CASE(w, b, mb_, 8, al, "memory") 358 __CMPXCHG_CASE(w, h, mb_, 16, al, "memory") 359 __CMPXCHG_CASE(w, , mb_, 32, al, "memory") 360 __CMPXCHG_CASE(x, , mb_, 64, al, "memory") 361 362 #undef __CMPXCHG_CASE 363 364 #define __CMPXCHG_DBL(name, mb, cl...) \ 365 static inline long __lse__cmpxchg_double##name(unsigned long old1, \ 366 unsigned long old2, \ 367 unsigned long new1, \ 368 unsigned long new2, \ 369 volatile void *ptr) \ 370 { \ 371 unsigned long oldval1 = old1; \ 372 unsigned long oldval2 = old2; \ 373 register unsigned long x0 asm ("x0") = old1; \ 374 register unsigned long x1 asm ("x1") = old2; \ 375 register unsigned long x2 asm ("x2") = new1; \ 376 register unsigned long x3 asm ("x3") = new2; \ 377 register unsigned long x4 asm ("x4") = (unsigned long)ptr; \ 378 \ 379 asm volatile( \ 380 " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\ 381 " eor %[old1], %[old1], %[oldval1]\n" \ 382 " eor %[old2], %[old2], %[oldval2]\n" \ 383 " orr %[old1], %[old1], %[old2]" \ 384 : [old1] "+&r" (x0), [old2] "+&r" (x1), \ 385 [v] "+Q" (*(unsigned long *)ptr) \ 386 : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \ 387 [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \ 388 : cl); \ 389 \ 390 return x0; \ 391 } 392 393 __CMPXCHG_DBL( , ) 394 __CMPXCHG_DBL(_mb, al, "memory") 395 396 #undef __CMPXCHG_DBL 397 398 #endif /* __ASM_ATOMIC_LSE_H */ 399