1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) 7 */ 8 #ifndef __ASM_CMPXCHG_H 9 #define __ASM_CMPXCHG_H 10 11 #include <linux/bug.h> 12 #include <linux/irqflags.h> 13 #include <asm/compiler.h> 14 #include <asm/war.h> 15 16 /* 17 * Using a branch-likely instruction to check the result of an sc instruction 18 * works around a bug present in R10000 CPUs prior to revision 3.0 that could 19 * cause ll-sc sequences to execute non-atomically. 20 */ 21 #if R10000_LLSC_WAR 22 # define __scbeqz "beqzl" 23 #else 24 # define __scbeqz "beqz" 25 #endif 26 27 /* 28 * These functions doesn't exist, so if they are called you'll either: 29 * 30 * - Get an error at compile-time due to __compiletime_error, if supported by 31 * your compiler. 32 * 33 * or: 34 * 35 * - Get an error at link-time due to the call to the missing function. 36 */ 37 extern unsigned long __cmpxchg_called_with_bad_pointer(void) 38 __compiletime_error("Bad argument size for cmpxchg"); 39 extern unsigned long __xchg_called_with_bad_pointer(void) 40 __compiletime_error("Bad argument size for xchg"); 41 42 #define __xchg_asm(ld, st, m, val) \ 43 ({ \ 44 __typeof(*(m)) __ret; \ 45 \ 46 if (kernel_uses_llsc) { \ 47 __asm__ __volatile__( \ 48 " .set push \n" \ 49 " .set noat \n" \ 50 " .set push \n" \ 51 " .set " MIPS_ISA_ARCH_LEVEL " \n" \ 52 "1: " ld " %0, %2 # __xchg_asm \n" \ 53 " .set pop \n" \ 54 " move $1, %z3 \n" \ 55 " .set " MIPS_ISA_ARCH_LEVEL " \n" \ 56 " " st " $1, %1 \n" \ 57 "\t" __scbeqz " $1, 1b \n" \ 58 " .set pop \n" \ 59 : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ 60 : GCC_OFF_SMALL_ASM() (*m), "Jr" (val) \ 61 : "memory"); \ 62 } else { \ 63 unsigned long __flags; \ 64 \ 65 raw_local_irq_save(__flags); \ 66 __ret = *m; \ 67 *m = val; \ 68 raw_local_irq_restore(__flags); \ 69 } \ 70 \ 71 __ret; \ 72 }) 73 74 extern unsigned long __xchg_small(volatile void *ptr, unsigned long val, 75 unsigned int size); 76 77 static inline unsigned long __xchg(volatile void *ptr, unsigned long x, 78 int size) 79 { 80 switch (size) { 81 case 1: 82 case 2: 83 return __xchg_small(ptr, x, size); 84 85 case 4: 86 return __xchg_asm("ll", "sc", (volatile u32 *)ptr, x); 87 88 case 8: 89 if (!IS_ENABLED(CONFIG_64BIT)) 90 return __xchg_called_with_bad_pointer(); 91 92 return __xchg_asm("lld", "scd", (volatile u64 *)ptr, x); 93 94 default: 95 return __xchg_called_with_bad_pointer(); 96 } 97 } 98 99 #define xchg(ptr, x) \ 100 ({ \ 101 __typeof__(*(ptr)) __res; \ 102 \ 103 smp_mb__before_llsc(); \ 104 \ 105 __res = (__typeof__(*(ptr))) \ 106 __xchg((ptr), (unsigned long)(x), sizeof(*(ptr))); \ 107 \ 108 smp_llsc_mb(); \ 109 \ 110 __res; \ 111 }) 112 113 #define __cmpxchg_asm(ld, st, m, old, new) \ 114 ({ \ 115 __typeof(*(m)) __ret; \ 116 \ 117 if (kernel_uses_llsc) { \ 118 __asm__ __volatile__( \ 119 " .set push \n" \ 120 " .set noat \n" \ 121 " .set push \n" \ 122 " .set "MIPS_ISA_ARCH_LEVEL" \n" \ 123 "1: " ld " %0, %2 # __cmpxchg_asm \n" \ 124 " bne %0, %z3, 2f \n" \ 125 " .set pop \n" \ 126 " move $1, %z4 \n" \ 127 " .set "MIPS_ISA_ARCH_LEVEL" \n" \ 128 " " st " $1, %1 \n" \ 129 "\t" __scbeqz " $1, 1b \n" \ 130 " .set pop \n" \ 131 "2: \n" \ 132 : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ 133 : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \ 134 : "memory"); \ 135 } else { \ 136 unsigned long __flags; \ 137 \ 138 raw_local_irq_save(__flags); \ 139 __ret = *m; \ 140 if (__ret == old) \ 141 *m = new; \ 142 raw_local_irq_restore(__flags); \ 143 } \ 144 \ 145 __ret; \ 146 }) 147 148 extern unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old, 149 unsigned long new, unsigned int size); 150 151 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, 152 unsigned long new, unsigned int size) 153 { 154 switch (size) { 155 case 1: 156 case 2: 157 return __cmpxchg_small(ptr, old, new, size); 158 159 case 4: 160 return __cmpxchg_asm("ll", "sc", (volatile u32 *)ptr, 161 (u32)old, new); 162 163 case 8: 164 /* lld/scd are only available for MIPS64 */ 165 if (!IS_ENABLED(CONFIG_64BIT)) 166 return __cmpxchg_called_with_bad_pointer(); 167 168 return __cmpxchg_asm("lld", "scd", (volatile u64 *)ptr, 169 (u64)old, new); 170 171 default: 172 return __cmpxchg_called_with_bad_pointer(); 173 } 174 } 175 176 #define cmpxchg_local(ptr, old, new) \ 177 ((__typeof__(*(ptr))) \ 178 __cmpxchg((ptr), \ 179 (unsigned long)(__typeof__(*(ptr)))(old), \ 180 (unsigned long)(__typeof__(*(ptr)))(new), \ 181 sizeof(*(ptr)))) 182 183 #define cmpxchg(ptr, old, new) \ 184 ({ \ 185 __typeof__(*(ptr)) __res; \ 186 \ 187 smp_mb__before_llsc(); \ 188 __res = cmpxchg_local((ptr), (old), (new)); \ 189 smp_llsc_mb(); \ 190 \ 191 __res; \ 192 }) 193 194 #ifdef CONFIG_64BIT 195 #define cmpxchg64_local(ptr, o, n) \ 196 ({ \ 197 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 198 cmpxchg_local((ptr), (o), (n)); \ 199 }) 200 201 #define cmpxchg64(ptr, o, n) \ 202 ({ \ 203 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 204 cmpxchg((ptr), (o), (n)); \ 205 }) 206 #else 207 #include <asm-generic/cmpxchg-local.h> 208 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) 209 #ifndef CONFIG_SMP 210 #define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) 211 #endif 212 #endif 213 214 #undef __scbeqz 215 216 #endif /* __ASM_CMPXCHG_H */ 217