1 /* 2 * Copyright IBM Corp. 1999, 2011 3 * 4 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, 5 */ 6 7 #ifndef __ASM_CMPXCHG_H 8 #define __ASM_CMPXCHG_H 9 10 #include <linux/types.h> 11 12 extern void __xchg_called_with_bad_pointer(void); 13 14 static inline unsigned long __xchg(unsigned long x, void *ptr, int size) 15 { 16 unsigned long addr, old; 17 int shift; 18 19 switch (size) { 20 case 1: 21 addr = (unsigned long) ptr; 22 shift = (3 ^ (addr & 3)) << 3; 23 addr ^= addr & 3; 24 asm volatile( 25 " l %0,%4\n" 26 "0: lr 0,%0\n" 27 " nr 0,%3\n" 28 " or 0,%2\n" 29 " cs %0,0,%4\n" 30 " jl 0b\n" 31 : "=&d" (old), "=Q" (*(int *) addr) 32 : "d" (x << shift), "d" (~(255 << shift)), 33 "Q" (*(int *) addr) : "memory", "cc", "0"); 34 return old >> shift; 35 case 2: 36 addr = (unsigned long) ptr; 37 shift = (2 ^ (addr & 2)) << 3; 38 addr ^= addr & 2; 39 asm volatile( 40 " l %0,%4\n" 41 "0: lr 0,%0\n" 42 " nr 0,%3\n" 43 " or 0,%2\n" 44 " cs %0,0,%4\n" 45 " jl 0b\n" 46 : "=&d" (old), "=Q" (*(int *) addr) 47 : "d" (x << shift), "d" (~(65535 << shift)), 48 "Q" (*(int *) addr) : "memory", "cc", "0"); 49 return old >> shift; 50 case 4: 51 asm volatile( 52 " l %0,%3\n" 53 "0: cs %0,%2,%3\n" 54 " jl 0b\n" 55 : "=&d" (old), "=Q" (*(int *) ptr) 56 : "d" (x), "Q" (*(int *) ptr) 57 : "memory", "cc"); 58 return old; 59 #ifdef CONFIG_64BIT 60 case 8: 61 asm volatile( 62 " lg %0,%3\n" 63 "0: csg %0,%2,%3\n" 64 " jl 0b\n" 65 : "=&d" (old), "=m" (*(long *) ptr) 66 : "d" (x), "Q" (*(long *) ptr) 67 : "memory", "cc"); 68 return old; 69 #endif /* CONFIG_64BIT */ 70 } 71 __xchg_called_with_bad_pointer(); 72 return x; 73 } 74 75 #define xchg(ptr, x) \ 76 ({ \ 77 __typeof__(*(ptr)) __ret; \ 78 __ret = (__typeof__(*(ptr))) \ 79 __xchg((unsigned long)(x), (void *)(ptr), sizeof(*(ptr)));\ 80 __ret; \ 81 }) 82 83 /* 84 * Atomic compare and exchange. Compare OLD with MEM, if identical, 85 * store NEW in MEM. Return the initial value in MEM. Success is 86 * indicated by comparing RETURN with OLD. 87 */ 88 89 #define __HAVE_ARCH_CMPXCHG 90 91 extern void __cmpxchg_called_with_bad_pointer(void); 92 93 static inline unsigned long __cmpxchg(void *ptr, unsigned long old, 94 unsigned long new, int size) 95 { 96 unsigned long addr, prev, tmp; 97 int shift; 98 99 switch (size) { 100 case 1: 101 addr = (unsigned long) ptr; 102 shift = (3 ^ (addr & 3)) << 3; 103 addr ^= addr & 3; 104 asm volatile( 105 " l %0,%2\n" 106 "0: nr %0,%5\n" 107 " lr %1,%0\n" 108 " or %0,%3\n" 109 " or %1,%4\n" 110 " cs %0,%1,%2\n" 111 " jnl 1f\n" 112 " xr %1,%0\n" 113 " nr %1,%5\n" 114 " jnz 0b\n" 115 "1:" 116 : "=&d" (prev), "=&d" (tmp), "=Q" (*(int *) ptr) 117 : "d" (old << shift), "d" (new << shift), 118 "d" (~(255 << shift)), "Q" (*(int *) ptr) 119 : "memory", "cc"); 120 return prev >> shift; 121 case 2: 122 addr = (unsigned long) ptr; 123 shift = (2 ^ (addr & 2)) << 3; 124 addr ^= addr & 2; 125 asm volatile( 126 " l %0,%2\n" 127 "0: nr %0,%5\n" 128 " lr %1,%0\n" 129 " or %0,%3\n" 130 " or %1,%4\n" 131 " cs %0,%1,%2\n" 132 " jnl 1f\n" 133 " xr %1,%0\n" 134 " nr %1,%5\n" 135 " jnz 0b\n" 136 "1:" 137 : "=&d" (prev), "=&d" (tmp), "=Q" (*(int *) ptr) 138 : "d" (old << shift), "d" (new << shift), 139 "d" (~(65535 << shift)), "Q" (*(int *) ptr) 140 : "memory", "cc"); 141 return prev >> shift; 142 case 4: 143 asm volatile( 144 " cs %0,%3,%1\n" 145 : "=&d" (prev), "=Q" (*(int *) ptr) 146 : "0" (old), "d" (new), "Q" (*(int *) ptr) 147 : "memory", "cc"); 148 return prev; 149 #ifdef CONFIG_64BIT 150 case 8: 151 asm volatile( 152 " csg %0,%3,%1\n" 153 : "=&d" (prev), "=Q" (*(long *) ptr) 154 : "0" (old), "d" (new), "Q" (*(long *) ptr) 155 : "memory", "cc"); 156 return prev; 157 #endif /* CONFIG_64BIT */ 158 } 159 __cmpxchg_called_with_bad_pointer(); 160 return old; 161 } 162 163 #define cmpxchg(ptr, o, n) \ 164 ((__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ 165 (unsigned long)(n), sizeof(*(ptr)))) 166 167 #ifdef CONFIG_64BIT 168 #define cmpxchg64(ptr, o, n) \ 169 ({ \ 170 cmpxchg((ptr), (o), (n)); \ 171 }) 172 #else /* CONFIG_64BIT */ 173 static inline unsigned long long __cmpxchg64(void *ptr, 174 unsigned long long old, 175 unsigned long long new) 176 { 177 register_pair rp_old = {.pair = old}; 178 register_pair rp_new = {.pair = new}; 179 180 asm volatile( 181 " cds %0,%2,%1" 182 : "+&d" (rp_old), "=Q" (ptr) 183 : "d" (rp_new), "Q" (ptr) 184 : "cc"); 185 return rp_old.pair; 186 } 187 #define cmpxchg64(ptr, o, n) \ 188 ((__typeof__(*(ptr)))__cmpxchg64((ptr), \ 189 (unsigned long long)(o), \ 190 (unsigned long long)(n))) 191 #endif /* CONFIG_64BIT */ 192 193 #include <asm-generic/cmpxchg-local.h> 194 195 static inline unsigned long __cmpxchg_local(void *ptr, 196 unsigned long old, 197 unsigned long new, int size) 198 { 199 switch (size) { 200 case 1: 201 case 2: 202 case 4: 203 #ifdef CONFIG_64BIT 204 case 8: 205 #endif 206 return __cmpxchg(ptr, old, new, size); 207 default: 208 return __cmpxchg_local_generic(ptr, old, new, size); 209 } 210 211 return old; 212 } 213 214 /* 215 * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make 216 * them available. 217 */ 218 #define cmpxchg_local(ptr, o, n) \ 219 ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ 220 (unsigned long)(n), sizeof(*(ptr)))) 221 222 #define cmpxchg64_local(ptr, o, n) cmpxchg64((ptr), (o), (n)) 223 224 #endif /* __ASM_CMPXCHG_H */ 225