1 #ifndef __ASM_ARM_CMPXCHG_H 2 #define __ASM_ARM_CMPXCHG_H 3 4 #include <linux/irqflags.h> 5 #include <linux/prefetch.h> 6 #include <asm/barrier.h> 7 8 #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110) 9 /* 10 * On the StrongARM, "swp" is terminally broken since it bypasses the 11 * cache totally. This means that the cache becomes inconsistent, and, 12 * since we use normal loads/stores as well, this is really bad. 13 * Typically, this causes oopsen in filp_close, but could have other, 14 * more disastrous effects. There are two work-arounds: 15 * 1. Disable interrupts and emulate the atomic swap 16 * 2. Clean the cache, perform atomic swap, flush the cache 17 * 18 * We choose (1) since its the "easiest" to achieve here and is not 19 * dependent on the processor type. 20 * 21 * NOTE that this solution won't work on an SMP system, so explcitly 22 * forbid it here. 23 */ 24 #define swp_is_buggy 25 #endif 26 27 static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) 28 { 29 extern void __bad_xchg(volatile void *, int); 30 unsigned long ret; 31 #ifdef swp_is_buggy 32 unsigned long flags; 33 #endif 34 #if __LINUX_ARM_ARCH__ >= 6 35 unsigned int tmp; 36 #endif 37 38 smp_mb(); 39 prefetchw((const void *)ptr); 40 41 switch (size) { 42 #if __LINUX_ARM_ARCH__ >= 6 43 case 1: 44 asm volatile("@ __xchg1\n" 45 "1: ldrexb %0, [%3]\n" 46 " strexb %1, %2, [%3]\n" 47 " teq %1, #0\n" 48 " bne 1b" 49 : "=&r" (ret), "=&r" (tmp) 50 : "r" (x), "r" (ptr) 51 : "memory", "cc"); 52 break; 53 case 4: 54 asm volatile("@ __xchg4\n" 55 "1: ldrex %0, [%3]\n" 56 " strex %1, %2, [%3]\n" 57 " teq %1, #0\n" 58 " bne 1b" 59 : "=&r" (ret), "=&r" (tmp) 60 : "r" (x), "r" (ptr) 61 : "memory", "cc"); 62 break; 63 #elif defined(swp_is_buggy) 64 #ifdef CONFIG_SMP 65 #error SMP is not supported on this platform 66 #endif 67 case 1: 68 raw_local_irq_save(flags); 69 ret = *(volatile unsigned char *)ptr; 70 *(volatile unsigned char *)ptr = x; 71 raw_local_irq_restore(flags); 72 break; 73 74 case 4: 75 raw_local_irq_save(flags); 76 ret = *(volatile unsigned long *)ptr; 77 *(volatile unsigned long *)ptr = x; 78 raw_local_irq_restore(flags); 79 break; 80 #else 81 case 1: 82 asm volatile("@ __xchg1\n" 83 " swpb %0, %1, [%2]" 84 : "=&r" (ret) 85 : "r" (x), "r" (ptr) 86 : "memory", "cc"); 87 break; 88 case 4: 89 asm volatile("@ __xchg4\n" 90 " swp %0, %1, [%2]" 91 : "=&r" (ret) 92 : "r" (x), "r" (ptr) 93 : "memory", "cc"); 94 break; 95 #endif 96 default: 97 __bad_xchg(ptr, size), ret = 0; 98 break; 99 } 100 smp_mb(); 101 102 return ret; 103 } 104 105 #define xchg(ptr,x) \ 106 ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) 107 108 #include <asm-generic/cmpxchg-local.h> 109 110 #if __LINUX_ARM_ARCH__ < 6 111 /* min ARCH < ARMv6 */ 112 113 #ifdef CONFIG_SMP 114 #error "SMP is not supported on this platform" 115 #endif 116 117 /* 118 * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make 119 * them available. 120 */ 121 #define cmpxchg_local(ptr, o, n) \ 122 ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\ 123 (unsigned long)(n), sizeof(*(ptr)))) 124 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) 125 126 #ifndef CONFIG_SMP 127 #include <asm-generic/cmpxchg.h> 128 #endif 129 130 #else /* min ARCH >= ARMv6 */ 131 132 extern void __bad_cmpxchg(volatile void *ptr, int size); 133 134 /* 135 * cmpxchg only support 32-bits operands on ARMv6. 136 */ 137 138 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, 139 unsigned long new, int size) 140 { 141 unsigned long oldval, res; 142 143 prefetchw((const void *)ptr); 144 145 switch (size) { 146 #ifndef CONFIG_CPU_V6 /* min ARCH >= ARMv6K */ 147 case 1: 148 do { 149 asm volatile("@ __cmpxchg1\n" 150 " ldrexb %1, [%2]\n" 151 " mov %0, #0\n" 152 " teq %1, %3\n" 153 " strexbeq %0, %4, [%2]\n" 154 : "=&r" (res), "=&r" (oldval) 155 : "r" (ptr), "Ir" (old), "r" (new) 156 : "memory", "cc"); 157 } while (res); 158 break; 159 case 2: 160 do { 161 asm volatile("@ __cmpxchg1\n" 162 " ldrexh %1, [%2]\n" 163 " mov %0, #0\n" 164 " teq %1, %3\n" 165 " strexheq %0, %4, [%2]\n" 166 : "=&r" (res), "=&r" (oldval) 167 : "r" (ptr), "Ir" (old), "r" (new) 168 : "memory", "cc"); 169 } while (res); 170 break; 171 #endif 172 case 4: 173 do { 174 asm volatile("@ __cmpxchg4\n" 175 " ldrex %1, [%2]\n" 176 " mov %0, #0\n" 177 " teq %1, %3\n" 178 " strexeq %0, %4, [%2]\n" 179 : "=&r" (res), "=&r" (oldval) 180 : "r" (ptr), "Ir" (old), "r" (new) 181 : "memory", "cc"); 182 } while (res); 183 break; 184 default: 185 __bad_cmpxchg(ptr, size); 186 oldval = 0; 187 } 188 189 return oldval; 190 } 191 192 static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, 193 unsigned long new, int size) 194 { 195 unsigned long ret; 196 197 smp_mb(); 198 ret = __cmpxchg(ptr, old, new, size); 199 smp_mb(); 200 201 return ret; 202 } 203 204 #define cmpxchg(ptr,o,n) \ 205 ((__typeof__(*(ptr)))__cmpxchg_mb((ptr), \ 206 (unsigned long)(o), \ 207 (unsigned long)(n), \ 208 sizeof(*(ptr)))) 209 210 static inline unsigned long __cmpxchg_local(volatile void *ptr, 211 unsigned long old, 212 unsigned long new, int size) 213 { 214 unsigned long ret; 215 216 switch (size) { 217 #ifdef CONFIG_CPU_V6 /* min ARCH == ARMv6 */ 218 case 1: 219 case 2: 220 ret = __cmpxchg_local_generic(ptr, old, new, size); 221 break; 222 #endif 223 default: 224 ret = __cmpxchg(ptr, old, new, size); 225 } 226 227 return ret; 228 } 229 230 static inline unsigned long long __cmpxchg64(unsigned long long *ptr, 231 unsigned long long old, 232 unsigned long long new) 233 { 234 unsigned long long oldval; 235 unsigned long res; 236 237 prefetchw(ptr); 238 239 __asm__ __volatile__( 240 "1: ldrexd %1, %H1, [%3]\n" 241 " teq %1, %4\n" 242 " teqeq %H1, %H4\n" 243 " bne 2f\n" 244 " strexd %0, %5, %H5, [%3]\n" 245 " teq %0, #0\n" 246 " bne 1b\n" 247 "2:" 248 : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) 249 : "r" (ptr), "r" (old), "r" (new) 250 : "cc"); 251 252 return oldval; 253 } 254 255 static inline unsigned long long __cmpxchg64_mb(unsigned long long *ptr, 256 unsigned long long old, 257 unsigned long long new) 258 { 259 unsigned long long ret; 260 261 smp_mb(); 262 ret = __cmpxchg64(ptr, old, new); 263 smp_mb(); 264 265 return ret; 266 } 267 268 #define cmpxchg_local(ptr,o,n) \ 269 ((__typeof__(*(ptr)))__cmpxchg_local((ptr), \ 270 (unsigned long)(o), \ 271 (unsigned long)(n), \ 272 sizeof(*(ptr)))) 273 274 #define cmpxchg64(ptr, o, n) \ 275 ((__typeof__(*(ptr)))__cmpxchg64_mb((ptr), \ 276 (unsigned long long)(o), \ 277 (unsigned long long)(n))) 278 279 #define cmpxchg64_relaxed(ptr, o, n) \ 280 ((__typeof__(*(ptr)))__cmpxchg64((ptr), \ 281 (unsigned long long)(o), \ 282 (unsigned long long)(n))) 283 284 #define cmpxchg64_local(ptr, o, n) cmpxchg64_relaxed((ptr), (o), (n)) 285 286 #endif /* __LINUX_ARM_ARCH__ >= 6 */ 287 288 #endif /* __ASM_ARM_CMPXCHG_H */ 289