1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ALPHA_CMPXCHG_H 3 #error Do not include xchg.h directly! 4 #else 5 /* 6 * xchg/xchg_local and cmpxchg/cmpxchg_local share the same code 7 * except that local version do not have the expensive memory barrier. 8 * So this file is included twice from asm/cmpxchg.h. 9 */ 10 11 /* 12 * Atomic exchange. 13 * Since it can be used to implement critical sections 14 * it must clobber "memory" (also for interrupts in UP). 15 */ 16 17 static inline unsigned long 18 ____xchg(_u8, volatile char *m, unsigned long val) 19 { 20 unsigned long ret, tmp, addr64; 21 22 __asm__ __volatile__( 23 " andnot %4,7,%3\n" 24 " insbl %1,%4,%1\n" 25 "1: ldq_l %2,0(%3)\n" 26 " extbl %2,%4,%0\n" 27 " mskbl %2,%4,%2\n" 28 " or %1,%2,%2\n" 29 " stq_c %2,0(%3)\n" 30 " beq %2,2f\n" 31 ".subsection 2\n" 32 "2: br 1b\n" 33 ".previous" 34 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) 35 : "r" ((long)m), "1" (val) : "memory"); 36 37 return ret; 38 } 39 40 static inline unsigned long 41 ____xchg(_u16, volatile short *m, unsigned long val) 42 { 43 unsigned long ret, tmp, addr64; 44 45 __asm__ __volatile__( 46 " andnot %4,7,%3\n" 47 " inswl %1,%4,%1\n" 48 "1: ldq_l %2,0(%3)\n" 49 " extwl %2,%4,%0\n" 50 " mskwl %2,%4,%2\n" 51 " or %1,%2,%2\n" 52 " stq_c %2,0(%3)\n" 53 " beq %2,2f\n" 54 ".subsection 2\n" 55 "2: br 1b\n" 56 ".previous" 57 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) 58 : "r" ((long)m), "1" (val) : "memory"); 59 60 return ret; 61 } 62 63 static inline unsigned long 64 ____xchg(_u32, volatile int *m, unsigned long val) 65 { 66 unsigned long dummy; 67 68 __asm__ __volatile__( 69 "1: ldl_l %0,%4\n" 70 " bis $31,%3,%1\n" 71 " stl_c %1,%2\n" 72 " beq %1,2f\n" 73 ".subsection 2\n" 74 "2: br 1b\n" 75 ".previous" 76 : "=&r" (val), "=&r" (dummy), "=m" (*m) 77 : "rI" (val), "m" (*m) : "memory"); 78 79 return val; 80 } 81 82 static inline unsigned long 83 ____xchg(_u64, volatile long *m, unsigned long val) 84 { 85 unsigned long dummy; 86 87 __asm__ __volatile__( 88 "1: ldq_l %0,%4\n" 89 " bis $31,%3,%1\n" 90 " stq_c %1,%2\n" 91 " beq %1,2f\n" 92 ".subsection 2\n" 93 "2: br 1b\n" 94 ".previous" 95 : "=&r" (val), "=&r" (dummy), "=m" (*m) 96 : "rI" (val), "m" (*m) : "memory"); 97 98 return val; 99 } 100 101 /* This function doesn't exist, so you'll get a linker error 102 if something tries to do an invalid xchg(). */ 103 extern void __xchg_called_with_bad_pointer(void); 104 105 static __always_inline unsigned long 106 ____xchg(, volatile void *ptr, unsigned long x, int size) 107 { 108 switch (size) { 109 case 1: 110 return ____xchg(_u8, ptr, x); 111 case 2: 112 return ____xchg(_u16, ptr, x); 113 case 4: 114 return ____xchg(_u32, ptr, x); 115 case 8: 116 return ____xchg(_u64, ptr, x); 117 } 118 __xchg_called_with_bad_pointer(); 119 return x; 120 } 121 122 /* 123 * Atomic compare and exchange. Compare OLD with MEM, if identical, 124 * store NEW in MEM. Return the initial value in MEM. Success is 125 * indicated by comparing RETURN with OLD. 126 */ 127 128 static inline unsigned long 129 ____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new) 130 { 131 unsigned long prev, tmp, cmp, addr64; 132 133 __asm__ __volatile__( 134 " andnot %5,7,%4\n" 135 " insbl %1,%5,%1\n" 136 "1: ldq_l %2,0(%4)\n" 137 " extbl %2,%5,%0\n" 138 " cmpeq %0,%6,%3\n" 139 " beq %3,2f\n" 140 " mskbl %2,%5,%2\n" 141 " or %1,%2,%2\n" 142 " stq_c %2,0(%4)\n" 143 " beq %2,3f\n" 144 "2:\n" 145 ".subsection 2\n" 146 "3: br 1b\n" 147 ".previous" 148 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) 149 : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); 150 151 return prev; 152 } 153 154 static inline unsigned long 155 ____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new) 156 { 157 unsigned long prev, tmp, cmp, addr64; 158 159 __asm__ __volatile__( 160 " andnot %5,7,%4\n" 161 " inswl %1,%5,%1\n" 162 "1: ldq_l %2,0(%4)\n" 163 " extwl %2,%5,%0\n" 164 " cmpeq %0,%6,%3\n" 165 " beq %3,2f\n" 166 " mskwl %2,%5,%2\n" 167 " or %1,%2,%2\n" 168 " stq_c %2,0(%4)\n" 169 " beq %2,3f\n" 170 "2:\n" 171 ".subsection 2\n" 172 "3: br 1b\n" 173 ".previous" 174 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) 175 : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); 176 177 return prev; 178 } 179 180 static inline unsigned long 181 ____cmpxchg(_u32, volatile int *m, int old, int new) 182 { 183 unsigned long prev, cmp; 184 185 __asm__ __volatile__( 186 "1: ldl_l %0,%5\n" 187 " cmpeq %0,%3,%1\n" 188 " beq %1,2f\n" 189 " mov %4,%1\n" 190 " stl_c %1,%2\n" 191 " beq %1,3f\n" 192 "2:\n" 193 ".subsection 2\n" 194 "3: br 1b\n" 195 ".previous" 196 : "=&r"(prev), "=&r"(cmp), "=m"(*m) 197 : "r"((long) old), "r"(new), "m"(*m) : "memory"); 198 199 return prev; 200 } 201 202 static inline unsigned long 203 ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) 204 { 205 unsigned long prev, cmp; 206 207 __asm__ __volatile__( 208 "1: ldq_l %0,%5\n" 209 " cmpeq %0,%3,%1\n" 210 " beq %1,2f\n" 211 " mov %4,%1\n" 212 " stq_c %1,%2\n" 213 " beq %1,3f\n" 214 "2:\n" 215 ".subsection 2\n" 216 "3: br 1b\n" 217 ".previous" 218 : "=&r"(prev), "=&r"(cmp), "=m"(*m) 219 : "r"((long) old), "r"(new), "m"(*m) : "memory"); 220 221 return prev; 222 } 223 224 /* This function doesn't exist, so you'll get a linker error 225 if something tries to do an invalid cmpxchg(). */ 226 extern void __cmpxchg_called_with_bad_pointer(void); 227 228 static __always_inline unsigned long 229 ____cmpxchg(, volatile void *ptr, unsigned long old, unsigned long new, 230 int size) 231 { 232 switch (size) { 233 case 1: 234 return ____cmpxchg(_u8, ptr, old, new); 235 case 2: 236 return ____cmpxchg(_u16, ptr, old, new); 237 case 4: 238 return ____cmpxchg(_u32, ptr, old, new); 239 case 8: 240 return ____cmpxchg(_u64, ptr, old, new); 241 } 242 __cmpxchg_called_with_bad_pointer(); 243 return old; 244 } 245 246 #endif 247