1 /* 2 * Copyright IBM Corp. 1999, 2011 3 * 4 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, 5 */ 6 7 #ifndef __ASM_CMPXCHG_H 8 #define __ASM_CMPXCHG_H 9 10 #include <linux/mmdebug.h> 11 #include <linux/types.h> 12 #include <linux/bug.h> 13 14 extern void __xchg_called_with_bad_pointer(void); 15 16 static inline unsigned long __xchg(unsigned long x, void *ptr, int size) 17 { 18 unsigned long addr, old; 19 int shift; 20 21 switch (size) { 22 case 1: 23 addr = (unsigned long) ptr; 24 shift = (3 ^ (addr & 3)) << 3; 25 addr ^= addr & 3; 26 asm volatile( 27 " l %0,%4\n" 28 "0: lr 0,%0\n" 29 " nr 0,%3\n" 30 " or 0,%2\n" 31 " cs %0,0,%4\n" 32 " jl 0b\n" 33 : "=&d" (old), "=Q" (*(int *) addr) 34 : "d" ((x & 0xff) << shift), "d" (~(0xff << shift)), 35 "Q" (*(int *) addr) : "memory", "cc", "0"); 36 return old >> shift; 37 case 2: 38 addr = (unsigned long) ptr; 39 shift = (2 ^ (addr & 2)) << 3; 40 addr ^= addr & 2; 41 asm volatile( 42 " l %0,%4\n" 43 "0: lr 0,%0\n" 44 " nr 0,%3\n" 45 " or 0,%2\n" 46 " cs %0,0,%4\n" 47 " jl 0b\n" 48 : "=&d" (old), "=Q" (*(int *) addr) 49 : "d" ((x & 0xffff) << shift), "d" (~(0xffff << shift)), 50 "Q" (*(int *) addr) : "memory", "cc", "0"); 51 return old >> shift; 52 case 4: 53 asm volatile( 54 " l %0,%3\n" 55 "0: cs %0,%2,%3\n" 56 " jl 0b\n" 57 : "=&d" (old), "=Q" (*(int *) ptr) 58 : "d" (x), "Q" (*(int *) ptr) 59 : "memory", "cc"); 60 return old; 61 #ifdef CONFIG_64BIT 62 case 8: 63 asm volatile( 64 " lg %0,%3\n" 65 "0: csg %0,%2,%3\n" 66 " jl 0b\n" 67 : "=&d" (old), "=m" (*(long *) ptr) 68 : "d" (x), "Q" (*(long *) ptr) 69 : "memory", "cc"); 70 return old; 71 #endif /* CONFIG_64BIT */ 72 } 73 __xchg_called_with_bad_pointer(); 74 return x; 75 } 76 77 #define xchg(ptr, x) \ 78 ({ \ 79 __typeof__(*(ptr)) __ret; \ 80 __ret = (__typeof__(*(ptr))) \ 81 __xchg((unsigned long)(x), (void *)(ptr), sizeof(*(ptr)));\ 82 __ret; \ 83 }) 84 85 /* 86 * Atomic compare and exchange. Compare OLD with MEM, if identical, 87 * store NEW in MEM. Return the initial value in MEM. Success is 88 * indicated by comparing RETURN with OLD. 89 */ 90 91 #define __HAVE_ARCH_CMPXCHG 92 93 extern void __cmpxchg_called_with_bad_pointer(void); 94 95 static inline unsigned long __cmpxchg(void *ptr, unsigned long old, 96 unsigned long new, int size) 97 { 98 unsigned long addr, prev, tmp; 99 int shift; 100 101 switch (size) { 102 case 1: 103 addr = (unsigned long) ptr; 104 shift = (3 ^ (addr & 3)) << 3; 105 addr ^= addr & 3; 106 asm volatile( 107 " l %0,%2\n" 108 "0: nr %0,%5\n" 109 " lr %1,%0\n" 110 " or %0,%3\n" 111 " or %1,%4\n" 112 " cs %0,%1,%2\n" 113 " jnl 1f\n" 114 " xr %1,%0\n" 115 " nr %1,%5\n" 116 " jnz 0b\n" 117 "1:" 118 : "=&d" (prev), "=&d" (tmp), "+Q" (*(int *) addr) 119 : "d" ((old & 0xff) << shift), 120 "d" ((new & 0xff) << shift), 121 "d" (~(0xff << shift)) 122 : "memory", "cc"); 123 return prev >> shift; 124 case 2: 125 addr = (unsigned long) ptr; 126 shift = (2 ^ (addr & 2)) << 3; 127 addr ^= addr & 2; 128 asm volatile( 129 " l %0,%2\n" 130 "0: nr %0,%5\n" 131 " lr %1,%0\n" 132 " or %0,%3\n" 133 " or %1,%4\n" 134 " cs %0,%1,%2\n" 135 " jnl 1f\n" 136 " xr %1,%0\n" 137 " nr %1,%5\n" 138 " jnz 0b\n" 139 "1:" 140 : "=&d" (prev), "=&d" (tmp), "+Q" (*(int *) addr) 141 : "d" ((old & 0xffff) << shift), 142 "d" ((new & 0xffff) << shift), 143 "d" (~(0xffff << shift)) 144 : "memory", "cc"); 145 return prev >> shift; 146 case 4: 147 asm volatile( 148 " cs %0,%3,%1\n" 149 : "=&d" (prev), "=Q" (*(int *) ptr) 150 : "0" (old), "d" (new), "Q" (*(int *) ptr) 151 : "memory", "cc"); 152 return prev; 153 #ifdef CONFIG_64BIT 154 case 8: 155 asm volatile( 156 " csg %0,%3,%1\n" 157 : "=&d" (prev), "=Q" (*(long *) ptr) 158 : "0" (old), "d" (new), "Q" (*(long *) ptr) 159 : "memory", "cc"); 160 return prev; 161 #endif /* CONFIG_64BIT */ 162 } 163 __cmpxchg_called_with_bad_pointer(); 164 return old; 165 } 166 167 #define cmpxchg(ptr, o, n) \ 168 ({ \ 169 __typeof__(*(ptr)) __ret; \ 170 __ret = (__typeof__(*(ptr))) \ 171 __cmpxchg((ptr), (unsigned long)(o), (unsigned long)(n), \ 172 sizeof(*(ptr))); \ 173 __ret; \ 174 }) 175 176 #ifdef CONFIG_64BIT 177 #define cmpxchg64(ptr, o, n) \ 178 ({ \ 179 cmpxchg((ptr), (o), (n)); \ 180 }) 181 #else /* CONFIG_64BIT */ 182 static inline unsigned long long __cmpxchg64(void *ptr, 183 unsigned long long old, 184 unsigned long long new) 185 { 186 register_pair rp_old = {.pair = old}; 187 register_pair rp_new = {.pair = new}; 188 189 asm volatile( 190 " cds %0,%2,%1" 191 : "+&d" (rp_old), "=Q" (ptr) 192 : "d" (rp_new), "Q" (ptr) 193 : "memory", "cc"); 194 return rp_old.pair; 195 } 196 197 #define cmpxchg64(ptr, o, n) \ 198 ({ \ 199 __typeof__(*(ptr)) __ret; \ 200 __ret = (__typeof__(*(ptr))) \ 201 __cmpxchg64((ptr), \ 202 (unsigned long long)(o), \ 203 (unsigned long long)(n)); \ 204 __ret; \ 205 }) 206 #endif /* CONFIG_64BIT */ 207 208 #define __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, insn) \ 209 ({ \ 210 register __typeof__(*(p1)) __old1 asm("2") = (o1); \ 211 register __typeof__(*(p2)) __old2 asm("3") = (o2); \ 212 register __typeof__(*(p1)) __new1 asm("4") = (n1); \ 213 register __typeof__(*(p2)) __new2 asm("5") = (n2); \ 214 int cc; \ 215 asm volatile( \ 216 insn " %[old],%[new],%[ptr]\n" \ 217 " ipm %[cc]\n" \ 218 " srl %[cc],28" \ 219 : [cc] "=d" (cc), [old] "+d" (__old1), "+d" (__old2) \ 220 : [new] "d" (__new1), "d" (__new2), \ 221 [ptr] "Q" (*(p1)), "Q" (*(p2)) \ 222 : "memory", "cc"); \ 223 !cc; \ 224 }) 225 226 #define __cmpxchg_double_4(p1, p2, o1, o2, n1, n2) \ 227 __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, "cds") 228 229 #define __cmpxchg_double_8(p1, p2, o1, o2, n1, n2) \ 230 __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, "cdsg") 231 232 extern void __cmpxchg_double_called_with_bad_pointer(void); 233 234 #define __cmpxchg_double(p1, p2, o1, o2, n1, n2) \ 235 ({ \ 236 int __ret; \ 237 switch (sizeof(*(p1))) { \ 238 case 4: \ 239 __ret = __cmpxchg_double_4(p1, p2, o1, o2, n1, n2); \ 240 break; \ 241 case 8: \ 242 __ret = __cmpxchg_double_8(p1, p2, o1, o2, n1, n2); \ 243 break; \ 244 default: \ 245 __cmpxchg_double_called_with_bad_pointer(); \ 246 } \ 247 __ret; \ 248 }) 249 250 #define cmpxchg_double(p1, p2, o1, o2, n1, n2) \ 251 ({ \ 252 __typeof__(p1) __p1 = (p1); \ 253 __typeof__(p2) __p2 = (p2); \ 254 int __ret; \ 255 BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \ 256 BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \ 257 VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\ 258 if (sizeof(long) == 4) \ 259 __ret = __cmpxchg_double_4(__p1, __p2, o1, o2, n1, n2); \ 260 else \ 261 __ret = __cmpxchg_double_8(__p1, __p2, o1, o2, n1, n2); \ 262 __ret; \ 263 }) 264 265 #define system_has_cmpxchg_double() 1 266 267 #include <asm-generic/cmpxchg-local.h> 268 269 static inline unsigned long __cmpxchg_local(void *ptr, 270 unsigned long old, 271 unsigned long new, int size) 272 { 273 switch (size) { 274 case 1: 275 case 2: 276 case 4: 277 #ifdef CONFIG_64BIT 278 case 8: 279 #endif 280 return __cmpxchg(ptr, old, new, size); 281 default: 282 return __cmpxchg_local_generic(ptr, old, new, size); 283 } 284 285 return old; 286 } 287 288 /* 289 * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make 290 * them available. 291 */ 292 #define cmpxchg_local(ptr, o, n) \ 293 ({ \ 294 __typeof__(*(ptr)) __ret; \ 295 __ret = (__typeof__(*(ptr))) \ 296 __cmpxchg_local((ptr), (unsigned long)(o), \ 297 (unsigned long)(n), sizeof(*(ptr))); \ 298 __ret; \ 299 }) 300 301 #define cmpxchg64_local(ptr, o, n) cmpxchg64((ptr), (o), (n)) 302 303 #endif /* __ASM_CMPXCHG_H */ 304