1 /* 2 * Based on arch/arm/include/asm/cmpxchg.h 3 * 4 * Copyright (C) 2012 ARM Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 #ifndef __ASM_CMPXCHG_H 19 #define __ASM_CMPXCHG_H 20 21 #include <linux/bug.h> 22 23 #include <asm/barrier.h> 24 25 static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) 26 { 27 unsigned long ret, tmp; 28 29 switch (size) { 30 case 1: 31 asm volatile("// __xchg1\n" 32 "1: ldxrb %w0, %2\n" 33 " stlxrb %w1, %w3, %2\n" 34 " cbnz %w1, 1b\n" 35 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr) 36 : "r" (x) 37 : "memory"); 38 break; 39 case 2: 40 asm volatile("// __xchg2\n" 41 "1: ldxrh %w0, %2\n" 42 " stlxrh %w1, %w3, %2\n" 43 " cbnz %w1, 1b\n" 44 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr) 45 : "r" (x) 46 : "memory"); 47 break; 48 case 4: 49 asm volatile("// __xchg4\n" 50 "1: ldxr %w0, %2\n" 51 " stlxr %w1, %w3, %2\n" 52 " cbnz %w1, 1b\n" 53 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr) 54 : "r" (x) 55 : "memory"); 56 break; 57 case 8: 58 asm volatile("// __xchg8\n" 59 "1: ldxr %0, %2\n" 60 " stlxr %w1, %3, %2\n" 61 " cbnz %w1, 1b\n" 62 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr) 63 : "r" (x) 64 : "memory"); 65 break; 66 default: 67 BUILD_BUG(); 68 } 69 70 smp_mb(); 71 return ret; 72 } 73 74 #define xchg(ptr,x) \ 75 ({ \ 76 __typeof__(*(ptr)) __ret; \ 77 __ret = (__typeof__(*(ptr))) \ 78 __xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \ 79 __ret; \ 80 }) 81 82 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, 83 unsigned long new, int size) 84 { 85 unsigned long oldval = 0, res; 86 87 switch (size) { 88 case 1: 89 do { 90 asm volatile("// __cmpxchg1\n" 91 " ldxrb %w1, %2\n" 92 " mov %w0, #0\n" 93 " cmp %w1, %w3\n" 94 " b.ne 1f\n" 95 " stxrb %w0, %w4, %2\n" 96 "1:\n" 97 : "=&r" (res), "=&r" (oldval), "+Q" (*(u8 *)ptr) 98 : "Ir" (old), "r" (new) 99 : "cc"); 100 } while (res); 101 break; 102 103 case 2: 104 do { 105 asm volatile("// __cmpxchg2\n" 106 " ldxrh %w1, %2\n" 107 " mov %w0, #0\n" 108 " cmp %w1, %w3\n" 109 " b.ne 1f\n" 110 " stxrh %w0, %w4, %2\n" 111 "1:\n" 112 : "=&r" (res), "=&r" (oldval), "+Q" (*(u16 *)ptr) 113 : "Ir" (old), "r" (new) 114 : "cc"); 115 } while (res); 116 break; 117 118 case 4: 119 do { 120 asm volatile("// __cmpxchg4\n" 121 " ldxr %w1, %2\n" 122 " mov %w0, #0\n" 123 " cmp %w1, %w3\n" 124 " b.ne 1f\n" 125 " stxr %w0, %w4, %2\n" 126 "1:\n" 127 : "=&r" (res), "=&r" (oldval), "+Q" (*(u32 *)ptr) 128 : "Ir" (old), "r" (new) 129 : "cc"); 130 } while (res); 131 break; 132 133 case 8: 134 do { 135 asm volatile("// __cmpxchg8\n" 136 " ldxr %1, %2\n" 137 " mov %w0, #0\n" 138 " cmp %1, %3\n" 139 " b.ne 1f\n" 140 " stxr %w0, %4, %2\n" 141 "1:\n" 142 : "=&r" (res), "=&r" (oldval), "+Q" (*(u64 *)ptr) 143 : "Ir" (old), "r" (new) 144 : "cc"); 145 } while (res); 146 break; 147 148 default: 149 BUILD_BUG(); 150 } 151 152 return oldval; 153 } 154 155 static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, 156 unsigned long new, int size) 157 { 158 unsigned long ret; 159 160 smp_mb(); 161 ret = __cmpxchg(ptr, old, new, size); 162 smp_mb(); 163 164 return ret; 165 } 166 167 #define cmpxchg(ptr, o, n) \ 168 ({ \ 169 __typeof__(*(ptr)) __ret; \ 170 __ret = (__typeof__(*(ptr))) \ 171 __cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \ 172 sizeof(*(ptr))); \ 173 __ret; \ 174 }) 175 176 #define cmpxchg_local(ptr, o, n) \ 177 ({ \ 178 __typeof__(*(ptr)) __ret; \ 179 __ret = (__typeof__(*(ptr))) \ 180 __cmpxchg((ptr), (unsigned long)(o), \ 181 (unsigned long)(n), sizeof(*(ptr))); \ 182 __ret; \ 183 }) 184 185 #define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n)) 186 #define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n)) 187 188 #define cmpxchg64_relaxed(ptr,o,n) cmpxchg_local((ptr),(o),(n)) 189 190 #endif /* __ASM_CMPXCHG_H */ 191