1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright IBM Corp. 1999, 2011 4 * 5 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, 6 */ 7 8 #ifndef __ASM_CMPXCHG_H 9 #define __ASM_CMPXCHG_H 10 11 #include <linux/mmdebug.h> 12 #include <linux/types.h> 13 #include <linux/bug.h> 14 15 void __xchg_called_with_bad_pointer(void); 16 17 static __always_inline unsigned long __xchg(unsigned long x, 18 unsigned long address, int size) 19 { 20 unsigned long old; 21 int shift; 22 23 switch (size) { 24 case 1: 25 shift = (3 ^ (address & 3)) << 3; 26 address ^= address & 3; 27 asm volatile( 28 " l %0,%1\n" 29 "0: lr 0,%0\n" 30 " nr 0,%3\n" 31 " or 0,%2\n" 32 " cs %0,0,%1\n" 33 " jl 0b\n" 34 : "=&d" (old), "+Q" (*(int *) address) 35 : "d" ((x & 0xff) << shift), "d" (~(0xff << shift)) 36 : "memory", "cc", "0"); 37 return old >> shift; 38 case 2: 39 shift = (2 ^ (address & 2)) << 3; 40 address ^= address & 2; 41 asm volatile( 42 " l %0,%1\n" 43 "0: lr 0,%0\n" 44 " nr 0,%3\n" 45 " or 0,%2\n" 46 " cs %0,0,%1\n" 47 " jl 0b\n" 48 : "=&d" (old), "+Q" (*(int *) address) 49 : "d" ((x & 0xffff) << shift), "d" (~(0xffff << shift)) 50 : "memory", "cc", "0"); 51 return old >> shift; 52 case 4: 53 asm volatile( 54 " l %0,%1\n" 55 "0: cs %0,%2,%1\n" 56 " jl 0b\n" 57 : "=&d" (old), "+Q" (*(int *) address) 58 : "d" (x) 59 : "memory", "cc"); 60 return old; 61 case 8: 62 asm volatile( 63 " lg %0,%1\n" 64 "0: csg %0,%2,%1\n" 65 " jl 0b\n" 66 : "=&d" (old), "+QS" (*(long *) address) 67 : "d" (x) 68 : "memory", "cc"); 69 return old; 70 } 71 __xchg_called_with_bad_pointer(); 72 return x; 73 } 74 75 #define arch_xchg(ptr, x) \ 76 ({ \ 77 __typeof__(*(ptr)) __ret; \ 78 \ 79 __ret = (__typeof__(*(ptr))) \ 80 __xchg((unsigned long)(x), (unsigned long)(ptr), \ 81 sizeof(*(ptr))); \ 82 __ret; \ 83 }) 84 85 void __cmpxchg_called_with_bad_pointer(void); 86 87 static __always_inline unsigned long __cmpxchg(unsigned long address, 88 unsigned long old, 89 unsigned long new, int size) 90 { 91 unsigned long prev, tmp; 92 int shift; 93 94 switch (size) { 95 case 1: 96 shift = (3 ^ (address & 3)) << 3; 97 address ^= address & 3; 98 asm volatile( 99 " l %0,%2\n" 100 "0: nr %0,%5\n" 101 " lr %1,%0\n" 102 " or %0,%3\n" 103 " or %1,%4\n" 104 " cs %0,%1,%2\n" 105 " jnl 1f\n" 106 " xr %1,%0\n" 107 " nr %1,%5\n" 108 " jnz 0b\n" 109 "1:" 110 : "=&d" (prev), "=&d" (tmp), "+Q" (*(int *) address) 111 : "d" ((old & 0xff) << shift), 112 "d" ((new & 0xff) << shift), 113 "d" (~(0xff << shift)) 114 : "memory", "cc"); 115 return prev >> shift; 116 case 2: 117 shift = (2 ^ (address & 2)) << 3; 118 address ^= address & 2; 119 asm volatile( 120 " l %0,%2\n" 121 "0: nr %0,%5\n" 122 " lr %1,%0\n" 123 " or %0,%3\n" 124 " or %1,%4\n" 125 " cs %0,%1,%2\n" 126 " jnl 1f\n" 127 " xr %1,%0\n" 128 " nr %1,%5\n" 129 " jnz 0b\n" 130 "1:" 131 : "=&d" (prev), "=&d" (tmp), "+Q" (*(int *) address) 132 : "d" ((old & 0xffff) << shift), 133 "d" ((new & 0xffff) << shift), 134 "d" (~(0xffff << shift)) 135 : "memory", "cc"); 136 return prev >> shift; 137 case 4: 138 asm volatile( 139 " cs %0,%3,%1\n" 140 : "=&d" (prev), "+Q" (*(int *) address) 141 : "0" (old), "d" (new) 142 : "memory", "cc"); 143 return prev; 144 case 8: 145 asm volatile( 146 " csg %0,%3,%1\n" 147 : "=&d" (prev), "+QS" (*(long *) address) 148 : "0" (old), "d" (new) 149 : "memory", "cc"); 150 return prev; 151 } 152 __cmpxchg_called_with_bad_pointer(); 153 return old; 154 } 155 156 #define arch_cmpxchg(ptr, o, n) \ 157 ({ \ 158 __typeof__(*(ptr)) __ret; \ 159 \ 160 __ret = (__typeof__(*(ptr))) \ 161 __cmpxchg((unsigned long)(ptr), (unsigned long)(o), \ 162 (unsigned long)(n), sizeof(*(ptr))); \ 163 __ret; \ 164 }) 165 166 #define arch_cmpxchg64 arch_cmpxchg 167 #define arch_cmpxchg_local arch_cmpxchg 168 #define arch_cmpxchg64_local arch_cmpxchg 169 170 #define system_has_cmpxchg_double() 1 171 172 #define __cmpxchg_double(p1, p2, o1, o2, n1, n2) \ 173 ({ \ 174 register __typeof__(*(p1)) __old1 asm("2") = (o1); \ 175 register __typeof__(*(p2)) __old2 asm("3") = (o2); \ 176 register __typeof__(*(p1)) __new1 asm("4") = (n1); \ 177 register __typeof__(*(p2)) __new2 asm("5") = (n2); \ 178 int cc; \ 179 asm volatile( \ 180 " cdsg %[old],%[new],%[ptr]\n" \ 181 " ipm %[cc]\n" \ 182 " srl %[cc],28" \ 183 : [cc] "=d" (cc), [old] "+d" (__old1), "+d" (__old2) \ 184 : [new] "d" (__new1), "d" (__new2), \ 185 [ptr] "Q" (*(p1)), "Q" (*(p2)) \ 186 : "memory", "cc"); \ 187 !cc; \ 188 }) 189 190 #define arch_cmpxchg_double(p1, p2, o1, o2, n1, n2) \ 191 ({ \ 192 __typeof__(p1) __p1 = (p1); \ 193 __typeof__(p2) __p2 = (p2); \ 194 BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \ 195 BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \ 196 VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\ 197 __cmpxchg_double(__p1, __p2, o1, o2, n1, n2); \ 198 }) 199 200 #endif /* __ASM_CMPXCHG_H */ 201