1 #ifndef __ASM_ARM_CMPXCHG_H 2 #define __ASM_ARM_CMPXCHG_H 3 4 #include <linux/irqflags.h> 5 #include <asm/barrier.h> 6 7 #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110) 8 /* 9 * On the StrongARM, "swp" is terminally broken since it bypasses the 10 * cache totally. This means that the cache becomes inconsistent, and, 11 * since we use normal loads/stores as well, this is really bad. 12 * Typically, this causes oopsen in filp_close, but could have other, 13 * more disastrous effects. There are two work-arounds: 14 * 1. Disable interrupts and emulate the atomic swap 15 * 2. Clean the cache, perform atomic swap, flush the cache 16 * 17 * We choose (1) since its the "easiest" to achieve here and is not 18 * dependent on the processor type. 19 * 20 * NOTE that this solution won't work on an SMP system, so explcitly 21 * forbid it here. 22 */ 23 #define swp_is_buggy 24 #endif 25 26 static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) 27 { 28 extern void __bad_xchg(volatile void *, int); 29 unsigned long ret; 30 #ifdef swp_is_buggy 31 unsigned long flags; 32 #endif 33 #if __LINUX_ARM_ARCH__ >= 6 34 unsigned int tmp; 35 #endif 36 37 smp_mb(); 38 39 switch (size) { 40 #if __LINUX_ARM_ARCH__ >= 6 41 case 1: 42 asm volatile("@ __xchg1\n" 43 "1: ldrexb %0, [%3]\n" 44 " strexb %1, %2, [%3]\n" 45 " teq %1, #0\n" 46 " bne 1b" 47 : "=&r" (ret), "=&r" (tmp) 48 : "r" (x), "r" (ptr) 49 : "memory", "cc"); 50 break; 51 case 4: 52 asm volatile("@ __xchg4\n" 53 "1: ldrex %0, [%3]\n" 54 " strex %1, %2, [%3]\n" 55 " teq %1, #0\n" 56 " bne 1b" 57 : "=&r" (ret), "=&r" (tmp) 58 : "r" (x), "r" (ptr) 59 : "memory", "cc"); 60 break; 61 #elif defined(swp_is_buggy) 62 #ifdef CONFIG_SMP 63 #error SMP is not supported on this platform 64 #endif 65 case 1: 66 raw_local_irq_save(flags); 67 ret = *(volatile unsigned char *)ptr; 68 *(volatile unsigned char *)ptr = x; 69 raw_local_irq_restore(flags); 70 break; 71 72 case 4: 73 raw_local_irq_save(flags); 74 ret = *(volatile unsigned long *)ptr; 75 *(volatile unsigned long *)ptr = x; 76 raw_local_irq_restore(flags); 77 break; 78 #else 79 case 1: 80 asm volatile("@ __xchg1\n" 81 " swpb %0, %1, [%2]" 82 : "=&r" (ret) 83 : "r" (x), "r" (ptr) 84 : "memory", "cc"); 85 break; 86 case 4: 87 asm volatile("@ __xchg4\n" 88 " swp %0, %1, [%2]" 89 : "=&r" (ret) 90 : "r" (x), "r" (ptr) 91 : "memory", "cc"); 92 break; 93 #endif 94 default: 95 __bad_xchg(ptr, size), ret = 0; 96 break; 97 } 98 smp_mb(); 99 100 return ret; 101 } 102 103 #define xchg(ptr,x) \ 104 ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) 105 106 #include <asm-generic/cmpxchg-local.h> 107 108 #if __LINUX_ARM_ARCH__ < 6 109 /* min ARCH < ARMv6 */ 110 111 #ifdef CONFIG_SMP 112 #error "SMP is not supported on this platform" 113 #endif 114 115 /* 116 * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make 117 * them available. 118 */ 119 #define cmpxchg_local(ptr, o, n) \ 120 ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\ 121 (unsigned long)(n), sizeof(*(ptr)))) 122 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) 123 124 #ifndef CONFIG_SMP 125 #include <asm-generic/cmpxchg.h> 126 #endif 127 128 #else /* min ARCH >= ARMv6 */ 129 130 extern void __bad_cmpxchg(volatile void *ptr, int size); 131 132 /* 133 * cmpxchg only support 32-bits operands on ARMv6. 134 */ 135 136 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, 137 unsigned long new, int size) 138 { 139 unsigned long oldval, res; 140 141 switch (size) { 142 #ifndef CONFIG_CPU_V6 /* min ARCH >= ARMv6K */ 143 case 1: 144 do { 145 asm volatile("@ __cmpxchg1\n" 146 " ldrexb %1, [%2]\n" 147 " mov %0, #0\n" 148 " teq %1, %3\n" 149 " strexbeq %0, %4, [%2]\n" 150 : "=&r" (res), "=&r" (oldval) 151 : "r" (ptr), "Ir" (old), "r" (new) 152 : "memory", "cc"); 153 } while (res); 154 break; 155 case 2: 156 do { 157 asm volatile("@ __cmpxchg1\n" 158 " ldrexh %1, [%2]\n" 159 " mov %0, #0\n" 160 " teq %1, %3\n" 161 " strexheq %0, %4, [%2]\n" 162 : "=&r" (res), "=&r" (oldval) 163 : "r" (ptr), "Ir" (old), "r" (new) 164 : "memory", "cc"); 165 } while (res); 166 break; 167 #endif 168 case 4: 169 do { 170 asm volatile("@ __cmpxchg4\n" 171 " ldrex %1, [%2]\n" 172 " mov %0, #0\n" 173 " teq %1, %3\n" 174 " strexeq %0, %4, [%2]\n" 175 : "=&r" (res), "=&r" (oldval) 176 : "r" (ptr), "Ir" (old), "r" (new) 177 : "memory", "cc"); 178 } while (res); 179 break; 180 default: 181 __bad_cmpxchg(ptr, size); 182 oldval = 0; 183 } 184 185 return oldval; 186 } 187 188 static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, 189 unsigned long new, int size) 190 { 191 unsigned long ret; 192 193 smp_mb(); 194 ret = __cmpxchg(ptr, old, new, size); 195 smp_mb(); 196 197 return ret; 198 } 199 200 #define cmpxchg(ptr,o,n) \ 201 ((__typeof__(*(ptr)))__cmpxchg_mb((ptr), \ 202 (unsigned long)(o), \ 203 (unsigned long)(n), \ 204 sizeof(*(ptr)))) 205 206 static inline unsigned long __cmpxchg_local(volatile void *ptr, 207 unsigned long old, 208 unsigned long new, int size) 209 { 210 unsigned long ret; 211 212 switch (size) { 213 #ifdef CONFIG_CPU_V6 /* min ARCH == ARMv6 */ 214 case 1: 215 case 2: 216 ret = __cmpxchg_local_generic(ptr, old, new, size); 217 break; 218 #endif 219 default: 220 ret = __cmpxchg(ptr, old, new, size); 221 } 222 223 return ret; 224 } 225 226 #define cmpxchg_local(ptr,o,n) \ 227 ((__typeof__(*(ptr)))__cmpxchg_local((ptr), \ 228 (unsigned long)(o), \ 229 (unsigned long)(n), \ 230 sizeof(*(ptr)))) 231 232 #ifndef CONFIG_CPU_V6 /* min ARCH >= ARMv6K */ 233 234 /* 235 * Note : ARMv7-M (currently unsupported by Linux) does not support 236 * ldrexd/strexd. If ARMv7-M is ever supported by the Linux kernel, it should 237 * not be allowed to use __cmpxchg64. 238 */ 239 static inline unsigned long long __cmpxchg64(volatile void *ptr, 240 unsigned long long old, 241 unsigned long long new) 242 { 243 register unsigned long long oldval asm("r0"); 244 register unsigned long long __old asm("r2") = old; 245 register unsigned long long __new asm("r4") = new; 246 unsigned long res; 247 248 do { 249 asm volatile( 250 " @ __cmpxchg8\n" 251 " ldrexd %1, %H1, [%2]\n" 252 " mov %0, #0\n" 253 " teq %1, %3\n" 254 " teqeq %H1, %H3\n" 255 " strexdeq %0, %4, %H4, [%2]\n" 256 : "=&r" (res), "=&r" (oldval) 257 : "r" (ptr), "Ir" (__old), "r" (__new) 258 : "memory", "cc"); 259 } while (res); 260 261 return oldval; 262 } 263 264 static inline unsigned long long __cmpxchg64_mb(volatile void *ptr, 265 unsigned long long old, 266 unsigned long long new) 267 { 268 unsigned long long ret; 269 270 smp_mb(); 271 ret = __cmpxchg64(ptr, old, new); 272 smp_mb(); 273 274 return ret; 275 } 276 277 #define cmpxchg64(ptr,o,n) \ 278 ((__typeof__(*(ptr)))__cmpxchg64_mb((ptr), \ 279 (unsigned long long)(o), \ 280 (unsigned long long)(n))) 281 282 #define cmpxchg64_local(ptr,o,n) \ 283 ((__typeof__(*(ptr)))__cmpxchg64((ptr), \ 284 (unsigned long long)(o), \ 285 (unsigned long long)(n))) 286 287 #else /* min ARCH = ARMv6 */ 288 289 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) 290 291 #endif 292 293 #endif /* __LINUX_ARM_ARCH__ >= 6 */ 294 295 #endif /* __ASM_ARM_CMPXCHG_H */ 296