1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_CMPXCHG_32_H 3 #define _ASM_X86_CMPXCHG_32_H 4 5 /* 6 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, 7 * you need to test for the feature in boot_cpu_data. 8 */ 9 10 #ifdef CONFIG_X86_CMPXCHG64 11 #define arch_cmpxchg64(ptr, o, n) \ 12 ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ 13 (unsigned long long)(n))) 14 #define arch_cmpxchg64_local(ptr, o, n) \ 15 ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ 16 (unsigned long long)(n))) 17 #define arch_try_cmpxchg64(ptr, po, n) \ 18 __try_cmpxchg64((ptr), (unsigned long long *)(po), \ 19 (unsigned long long)(n)) 20 #endif 21 22 static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new) 23 { 24 u64 prev; 25 asm volatile(LOCK_PREFIX "cmpxchg8b %1" 26 : "=A" (prev), 27 "+m" (*ptr) 28 : "b" ((u32)new), 29 "c" ((u32)(new >> 32)), 30 "0" (old) 31 : "memory"); 32 return prev; 33 } 34 35 static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) 36 { 37 u64 prev; 38 asm volatile("cmpxchg8b %1" 39 : "=A" (prev), 40 "+m" (*ptr) 41 : "b" ((u32)new), 42 "c" ((u32)(new >> 32)), 43 "0" (old) 44 : "memory"); 45 return prev; 46 } 47 48 static inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *pold, u64 new) 49 { 50 bool success; 51 u64 old = *pold; 52 asm volatile(LOCK_PREFIX "cmpxchg8b %[ptr]" 53 CC_SET(z) 54 : CC_OUT(z) (success), 55 [ptr] "+m" (*ptr), 56 "+A" (old) 57 : "b" ((u32)new), 58 "c" ((u32)(new >> 32)) 59 : "memory"); 60 61 if (unlikely(!success)) 62 *pold = old; 63 return success; 64 } 65 66 #ifndef CONFIG_X86_CMPXCHG64 67 /* 68 * Building a kernel capable running on 80386 and 80486. It may be necessary 69 * to simulate the cmpxchg8b on the 80386 and 80486 CPU. 70 */ 71 72 #define arch_cmpxchg64(ptr, o, n) \ 73 ({ \ 74 __typeof__(*(ptr)) __ret; \ 75 __typeof__(*(ptr)) __old = (o); \ 76 __typeof__(*(ptr)) __new = (n); \ 77 alternative_io(LOCK_PREFIX_HERE \ 78 "call cmpxchg8b_emu", \ 79 "lock; cmpxchg8b (%%esi)" , \ 80 X86_FEATURE_CX8, \ 81 "=A" (__ret), \ 82 "S" ((ptr)), "0" (__old), \ 83 "b" ((unsigned int)__new), \ 84 "c" ((unsigned int)(__new>>32)) \ 85 : "memory"); \ 86 __ret; }) 87 88 89 #define arch_cmpxchg64_local(ptr, o, n) \ 90 ({ \ 91 __typeof__(*(ptr)) __ret; \ 92 __typeof__(*(ptr)) __old = (o); \ 93 __typeof__(*(ptr)) __new = (n); \ 94 alternative_io("call cmpxchg8b_emu", \ 95 "cmpxchg8b (%%esi)" , \ 96 X86_FEATURE_CX8, \ 97 "=A" (__ret), \ 98 "S" ((ptr)), "0" (__old), \ 99 "b" ((unsigned int)__new), \ 100 "c" ((unsigned int)(__new>>32)) \ 101 : "memory"); \ 102 __ret; }) 103 104 #endif 105 106 #define system_has_cmpxchg64() boot_cpu_has(X86_FEATURE_CX8) 107 108 #endif /* _ASM_X86_CMPXCHG_32_H */ 109