1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 3 /* 4 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD) 5 * - The address HAS to be 64-bit aligned 6 */ 7 8 #ifndef _ASM_ARC_ATOMIC64_ARCV2_H 9 #define _ASM_ARC_ATOMIC64_ARCV2_H 10 11 typedef struct { 12 s64 __aligned(8) counter; 13 } atomic64_t; 14 15 #define ATOMIC64_INIT(a) { (a) } 16 17 static inline s64 arch_atomic64_read(const atomic64_t *v) 18 { 19 s64 val; 20 21 __asm__ __volatile__( 22 " ldd %0, [%1] \n" 23 : "=r"(val) 24 : "r"(&v->counter)); 25 26 return val; 27 } 28 29 static inline void arch_atomic64_set(atomic64_t *v, s64 a) 30 { 31 /* 32 * This could have been a simple assignment in "C" but would need 33 * explicit volatile. Otherwise gcc optimizers could elide the store 34 * which borked atomic64 self-test 35 * In the inline asm version, memory clobber needed for exact same 36 * reason, to tell gcc about the store. 37 * 38 * This however is not needed for sibling atomic64_add() etc since both 39 * load/store are explicitly done in inline asm. As long as API is used 40 * for each access, gcc has no way to optimize away any load/store 41 */ 42 __asm__ __volatile__( 43 " std %0, [%1] \n" 44 : 45 : "r"(a), "r"(&v->counter) 46 : "memory"); 47 } 48 49 #define ATOMIC64_OP(op, op1, op2) \ 50 static inline void arch_atomic64_##op(s64 a, atomic64_t *v) \ 51 { \ 52 s64 val; \ 53 \ 54 __asm__ __volatile__( \ 55 "1: \n" \ 56 " llockd %0, [%1] \n" \ 57 " " #op1 " %L0, %L0, %L2 \n" \ 58 " " #op2 " %H0, %H0, %H2 \n" \ 59 " scondd %0, [%1] \n" \ 60 " bnz 1b \n" \ 61 : "=&r"(val) \ 62 : "r"(&v->counter), "ir"(a) \ 63 : "cc"); \ 64 } \ 65 66 #define ATOMIC64_OP_RETURN(op, op1, op2) \ 67 static inline s64 arch_atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ 68 { \ 69 s64 val; \ 70 \ 71 __asm__ __volatile__( \ 72 "1: \n" \ 73 " llockd %0, [%1] \n" \ 74 " " #op1 " %L0, %L0, %L2 \n" \ 75 " " #op2 " %H0, %H0, %H2 \n" \ 76 " scondd %0, [%1] \n" \ 77 " bnz 1b \n" \ 78 : [val] "=&r"(val) \ 79 : "r"(&v->counter), "ir"(a) \ 80 : "cc"); /* memory clobber comes from smp_mb() */ \ 81 \ 82 return val; \ 83 } 84 85 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 86 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 87 88 #define ATOMIC64_FETCH_OP(op, op1, op2) \ 89 static inline s64 arch_atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ 90 { \ 91 s64 val, orig; \ 92 \ 93 __asm__ __volatile__( \ 94 "1: \n" \ 95 " llockd %0, [%2] \n" \ 96 " " #op1 " %L1, %L0, %L3 \n" \ 97 " " #op2 " %H1, %H0, %H3 \n" \ 98 " scondd %1, [%2] \n" \ 99 " bnz 1b \n" \ 100 : "=&r"(orig), "=&r"(val) \ 101 : "r"(&v->counter), "ir"(a) \ 102 : "cc"); /* memory clobber comes from smp_mb() */ \ 103 \ 104 return orig; \ 105 } 106 107 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 108 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 109 110 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 111 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed 112 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 113 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 114 115 #define ATOMIC64_OPS(op, op1, op2) \ 116 ATOMIC64_OP(op, op1, op2) \ 117 ATOMIC64_OP_RETURN(op, op1, op2) \ 118 ATOMIC64_FETCH_OP(op, op1, op2) 119 120 ATOMIC64_OPS(add, add.f, adc) 121 ATOMIC64_OPS(sub, sub.f, sbc) 122 123 #undef ATOMIC64_OPS 124 #define ATOMIC64_OPS(op, op1, op2) \ 125 ATOMIC64_OP(op, op1, op2) \ 126 ATOMIC64_FETCH_OP(op, op1, op2) 127 128 ATOMIC64_OPS(and, and, and) 129 ATOMIC64_OPS(andnot, bic, bic) 130 ATOMIC64_OPS(or, or, or) 131 ATOMIC64_OPS(xor, xor, xor) 132 133 #define arch_atomic64_andnot arch_atomic64_andnot 134 135 #undef ATOMIC64_OPS 136 #undef ATOMIC64_FETCH_OP 137 #undef ATOMIC64_OP_RETURN 138 #undef ATOMIC64_OP 139 140 static inline s64 141 arch_atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new) 142 { 143 s64 prev; 144 145 smp_mb(); 146 147 __asm__ __volatile__( 148 "1: llockd %0, [%1] \n" 149 " brne %L0, %L2, 2f \n" 150 " brne %H0, %H2, 2f \n" 151 " scondd %3, [%1] \n" 152 " bnz 1b \n" 153 "2: \n" 154 : "=&r"(prev) 155 : "r"(ptr), "ir"(expected), "r"(new) 156 : "cc"); /* memory clobber comes from smp_mb() */ 157 158 smp_mb(); 159 160 return prev; 161 } 162 #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg 163 164 static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new) 165 { 166 s64 prev; 167 168 smp_mb(); 169 170 __asm__ __volatile__( 171 "1: llockd %0, [%1] \n" 172 " scondd %2, [%1] \n" 173 " bnz 1b \n" 174 "2: \n" 175 : "=&r"(prev) 176 : "r"(ptr), "r"(new) 177 : "cc"); /* memory clobber comes from smp_mb() */ 178 179 smp_mb(); 180 181 return prev; 182 } 183 #define arch_atomic64_xchg arch_atomic64_xchg 184 185 static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 186 { 187 s64 val; 188 189 smp_mb(); 190 191 __asm__ __volatile__( 192 "1: llockd %0, [%1] \n" 193 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n" 194 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n" 195 " brlt %H0, 0, 2f \n" 196 " scondd %0, [%1] \n" 197 " bnz 1b \n" 198 "2: \n" 199 : "=&r"(val) 200 : "r"(&v->counter) 201 : "cc"); /* memory clobber comes from smp_mb() */ 202 203 smp_mb(); 204 205 return val; 206 } 207 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 208 209 static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 210 { 211 s64 old, temp; 212 213 smp_mb(); 214 215 __asm__ __volatile__( 216 "1: llockd %0, [%2] \n" 217 " brne %L0, %L4, 2f # continue to add since v != u \n" 218 " breq.d %H0, %H4, 3f # return since v == u \n" 219 "2: \n" 220 " add.f %L1, %L0, %L3 \n" 221 " adc %H1, %H0, %H3 \n" 222 " scondd %1, [%2] \n" 223 " bnz 1b \n" 224 "3: \n" 225 : "=&r"(old), "=&r" (temp) 226 : "r"(&v->counter), "r"(a), "r"(u) 227 : "cc"); /* memory clobber comes from smp_mb() */ 228 229 smp_mb(); 230 231 return old; 232 } 233 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 234 235 #endif 236