1 #ifndef _ASM_X86_CMPXCHG_32_H 2 #define _ASM_X86_CMPXCHG_32_H 3 4 #include <linux/bitops.h> /* for LOCK_PREFIX */ 5 6 /* 7 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you 8 * you need to test for the feature in boot_cpu_data. 9 */ 10 11 #define xchg(ptr, v) \ 12 ((__typeof__(*(ptr)))__xchg((unsigned long)(v), (ptr), sizeof(*(ptr)))) 13 14 struct __xchg_dummy { 15 unsigned long a[100]; 16 }; 17 #define __xg(x) ((struct __xchg_dummy *)(x)) 18 19 /* 20 * The semantics of XCHGCMP8B are a bit strange, this is why 21 * there is a loop and the loading of %%eax and %%edx has to 22 * be inside. This inlines well in most cases, the cached 23 * cost is around ~38 cycles. (in the future we might want 24 * to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that 25 * might have an implicit FPU-save as a cost, so it's not 26 * clear which path to go.) 27 * 28 * cmpxchg8b must be used with the lock prefix here to allow 29 * the instruction to be executed atomically, see page 3-102 30 * of the instruction set reference 24319102.pdf. We need 31 * the reader side to see the coherent 64bit value. 32 */ 33 static inline void __set_64bit(unsigned long long *ptr, 34 unsigned int low, unsigned int high) 35 { 36 asm volatile("\n1:\t" 37 "movl (%0), %%eax\n\t" 38 "movl 4(%0), %%edx\n\t" 39 LOCK_PREFIX "cmpxchg8b (%0)\n\t" 40 "jnz 1b" 41 : /* no outputs */ 42 : "D"(ptr), 43 "b"(low), 44 "c"(high) 45 : "ax", "dx", "memory"); 46 } 47 48 static inline void __set_64bit_constant(unsigned long long *ptr, 49 unsigned long long value) 50 { 51 __set_64bit(ptr, (unsigned int)value, (unsigned int)(value >> 32)); 52 } 53 54 #define ll_low(x) *(((unsigned int *)&(x)) + 0) 55 #define ll_high(x) *(((unsigned int *)&(x)) + 1) 56 57 static inline void __set_64bit_var(unsigned long long *ptr, 58 unsigned long long value) 59 { 60 __set_64bit(ptr, ll_low(value), ll_high(value)); 61 } 62 63 #define set_64bit(ptr, value) \ 64 (__builtin_constant_p((value)) \ 65 ? __set_64bit_constant((ptr), (value)) \ 66 : __set_64bit_var((ptr), (value))) 67 68 #define _set_64bit(ptr, value) \ 69 (__builtin_constant_p(value) \ 70 ? __set_64bit(ptr, (unsigned int)(value), \ 71 (unsigned int)((value) >> 32)) \ 72 : __set_64bit(ptr, ll_low((value)), ll_high((value)))) 73 74 /* 75 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway 76 * Note 2: xchg has side effect, so that attribute volatile is necessary, 77 * but generally the primitive is invalid, *ptr is output argument. --ANK 78 */ 79 static inline unsigned long __xchg(unsigned long x, volatile void *ptr, 80 int size) 81 { 82 switch (size) { 83 case 1: 84 asm volatile("xchgb %b0,%1" 85 : "=q" (x) 86 : "m" (*__xg(ptr)), "0" (x) 87 : "memory"); 88 break; 89 case 2: 90 asm volatile("xchgw %w0,%1" 91 : "=r" (x) 92 : "m" (*__xg(ptr)), "0" (x) 93 : "memory"); 94 break; 95 case 4: 96 asm volatile("xchgl %0,%1" 97 : "=r" (x) 98 : "m" (*__xg(ptr)), "0" (x) 99 : "memory"); 100 break; 101 } 102 return x; 103 } 104 105 /* 106 * Atomic compare and exchange. Compare OLD with MEM, if identical, 107 * store NEW in MEM. Return the initial value in MEM. Success is 108 * indicated by comparing RETURN with OLD. 109 */ 110 111 #ifdef CONFIG_X86_CMPXCHG 112 #define __HAVE_ARCH_CMPXCHG 1 113 #define cmpxchg(ptr, o, n) \ 114 ((__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ 115 (unsigned long)(n), \ 116 sizeof(*(ptr)))) 117 #define sync_cmpxchg(ptr, o, n) \ 118 ((__typeof__(*(ptr)))__sync_cmpxchg((ptr), (unsigned long)(o), \ 119 (unsigned long)(n), \ 120 sizeof(*(ptr)))) 121 #define cmpxchg_local(ptr, o, n) \ 122 ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ 123 (unsigned long)(n), \ 124 sizeof(*(ptr)))) 125 #endif 126 127 #ifdef CONFIG_X86_CMPXCHG64 128 #define cmpxchg64(ptr, o, n) \ 129 ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ 130 (unsigned long long)(n))) 131 #define cmpxchg64_local(ptr, o, n) \ 132 ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ 133 (unsigned long long)(n))) 134 #endif 135 136 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, 137 unsigned long new, int size) 138 { 139 unsigned long prev; 140 switch (size) { 141 case 1: 142 asm volatile(LOCK_PREFIX "cmpxchgb %b1,%2" 143 : "=a"(prev) 144 : "q"(new), "m"(*__xg(ptr)), "0"(old) 145 : "memory"); 146 return prev; 147 case 2: 148 asm volatile(LOCK_PREFIX "cmpxchgw %w1,%2" 149 : "=a"(prev) 150 : "r"(new), "m"(*__xg(ptr)), "0"(old) 151 : "memory"); 152 return prev; 153 case 4: 154 asm volatile(LOCK_PREFIX "cmpxchgl %1,%2" 155 : "=a"(prev) 156 : "r"(new), "m"(*__xg(ptr)), "0"(old) 157 : "memory"); 158 return prev; 159 } 160 return old; 161 } 162 163 /* 164 * Always use locked operations when touching memory shared with a 165 * hypervisor, since the system may be SMP even if the guest kernel 166 * isn't. 167 */ 168 static inline unsigned long __sync_cmpxchg(volatile void *ptr, 169 unsigned long old, 170 unsigned long new, int size) 171 { 172 unsigned long prev; 173 switch (size) { 174 case 1: 175 asm volatile("lock; cmpxchgb %b1,%2" 176 : "=a"(prev) 177 : "q"(new), "m"(*__xg(ptr)), "0"(old) 178 : "memory"); 179 return prev; 180 case 2: 181 asm volatile("lock; cmpxchgw %w1,%2" 182 : "=a"(prev) 183 : "r"(new), "m"(*__xg(ptr)), "0"(old) 184 : "memory"); 185 return prev; 186 case 4: 187 asm volatile("lock; cmpxchgl %1,%2" 188 : "=a"(prev) 189 : "r"(new), "m"(*__xg(ptr)), "0"(old) 190 : "memory"); 191 return prev; 192 } 193 return old; 194 } 195 196 static inline unsigned long __cmpxchg_local(volatile void *ptr, 197 unsigned long old, 198 unsigned long new, int size) 199 { 200 unsigned long prev; 201 switch (size) { 202 case 1: 203 asm volatile("cmpxchgb %b1,%2" 204 : "=a"(prev) 205 : "q"(new), "m"(*__xg(ptr)), "0"(old) 206 : "memory"); 207 return prev; 208 case 2: 209 asm volatile("cmpxchgw %w1,%2" 210 : "=a"(prev) 211 : "r"(new), "m"(*__xg(ptr)), "0"(old) 212 : "memory"); 213 return prev; 214 case 4: 215 asm volatile("cmpxchgl %1,%2" 216 : "=a"(prev) 217 : "r"(new), "m"(*__xg(ptr)), "0"(old) 218 : "memory"); 219 return prev; 220 } 221 return old; 222 } 223 224 static inline unsigned long long __cmpxchg64(volatile void *ptr, 225 unsigned long long old, 226 unsigned long long new) 227 { 228 unsigned long long prev; 229 asm volatile(LOCK_PREFIX "cmpxchg8b %3" 230 : "=A"(prev) 231 : "b"((unsigned long)new), 232 "c"((unsigned long)(new >> 32)), 233 "m"(*__xg(ptr)), 234 "0"(old) 235 : "memory"); 236 return prev; 237 } 238 239 static inline unsigned long long __cmpxchg64_local(volatile void *ptr, 240 unsigned long long old, 241 unsigned long long new) 242 { 243 unsigned long long prev; 244 asm volatile("cmpxchg8b %3" 245 : "=A"(prev) 246 : "b"((unsigned long)new), 247 "c"((unsigned long)(new >> 32)), 248 "m"(*__xg(ptr)), 249 "0"(old) 250 : "memory"); 251 return prev; 252 } 253 254 #ifndef CONFIG_X86_CMPXCHG 255 /* 256 * Building a kernel capable running on 80386. It may be necessary to 257 * simulate the cmpxchg on the 80386 CPU. For that purpose we define 258 * a function for each of the sizes we support. 259 */ 260 261 extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8); 262 extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16); 263 extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32); 264 265 static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old, 266 unsigned long new, int size) 267 { 268 switch (size) { 269 case 1: 270 return cmpxchg_386_u8(ptr, old, new); 271 case 2: 272 return cmpxchg_386_u16(ptr, old, new); 273 case 4: 274 return cmpxchg_386_u32(ptr, old, new); 275 } 276 return old; 277 } 278 279 #define cmpxchg(ptr, o, n) \ 280 ({ \ 281 __typeof__(*(ptr)) __ret; \ 282 if (likely(boot_cpu_data.x86 > 3)) \ 283 __ret = (__typeof__(*(ptr)))__cmpxchg((ptr), \ 284 (unsigned long)(o), (unsigned long)(n), \ 285 sizeof(*(ptr))); \ 286 else \ 287 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \ 288 (unsigned long)(o), (unsigned long)(n), \ 289 sizeof(*(ptr))); \ 290 __ret; \ 291 }) 292 #define cmpxchg_local(ptr, o, n) \ 293 ({ \ 294 __typeof__(*(ptr)) __ret; \ 295 if (likely(boot_cpu_data.x86 > 3)) \ 296 __ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr), \ 297 (unsigned long)(o), (unsigned long)(n), \ 298 sizeof(*(ptr))); \ 299 else \ 300 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \ 301 (unsigned long)(o), (unsigned long)(n), \ 302 sizeof(*(ptr))); \ 303 __ret; \ 304 }) 305 #endif 306 307 #ifndef CONFIG_X86_CMPXCHG64 308 /* 309 * Building a kernel capable running on 80386 and 80486. It may be necessary 310 * to simulate the cmpxchg8b on the 80386 and 80486 CPU. 311 */ 312 313 extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64); 314 315 #define cmpxchg64(ptr, o, n) \ 316 ({ \ 317 __typeof__(*(ptr)) __ret; \ 318 __typeof__(*(ptr)) __old = (o); \ 319 __typeof__(*(ptr)) __new = (n); \ 320 alternative_io("call cmpxchg8b_emu", \ 321 "lock; cmpxchg8b (%%esi)" , \ 322 X86_FEATURE_CX8, \ 323 "=A" (__ret), \ 324 "S" ((ptr)), "0" (__old), \ 325 "b" ((unsigned int)__new), \ 326 "c" ((unsigned int)(__new>>32)) \ 327 : "memory"); \ 328 __ret; }) 329 330 331 332 #define cmpxchg64_local(ptr, o, n) \ 333 ({ \ 334 __typeof__(*(ptr)) __ret; \ 335 if (likely(boot_cpu_data.x86 > 4)) \ 336 __ret = (__typeof__(*(ptr)))__cmpxchg64_local((ptr), \ 337 (unsigned long long)(o), \ 338 (unsigned long long)(n)); \ 339 else \ 340 __ret = (__typeof__(*(ptr)))cmpxchg_486_u64((ptr), \ 341 (unsigned long long)(o), \ 342 (unsigned long long)(n)); \ 343 __ret; \ 344 }) 345 346 #endif 347 348 #endif /* _ASM_X86_CMPXCHG_32_H */ 349