1 /* 2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #ifndef __ASM_ARC_CMPXCHG_H 10 #define __ASM_ARC_CMPXCHG_H 11 12 #include <linux/types.h> 13 14 #include <asm/barrier.h> 15 #include <asm/smp.h> 16 17 #ifdef CONFIG_ARC_HAS_LLSC 18 19 static inline unsigned long 20 __cmpxchg(volatile void *ptr, unsigned long expected, unsigned long new) 21 { 22 unsigned long prev; 23 24 /* 25 * Explicit full memory barrier needed before/after as 26 * LLOCK/SCOND thmeselves don't provide any such semantics 27 */ 28 smp_mb(); 29 30 __asm__ __volatile__( 31 "1: llock %0, [%1] \n" 32 " brne %0, %2, 2f \n" 33 " scond %3, [%1] \n" 34 " bnz 1b \n" 35 "2: \n" 36 : "=&r"(prev) /* Early clobber, to prevent reg reuse */ 37 : "r"(ptr), /* Not "m": llock only supports reg direct addr mode */ 38 "ir"(expected), 39 "r"(new) /* can't be "ir". scond can't take LIMM for "b" */ 40 : "cc", "memory"); /* so that gcc knows memory is being written here */ 41 42 smp_mb(); 43 44 return prev; 45 } 46 47 #elif !defined(CONFIG_ARC_PLAT_EZNPS) 48 49 static inline unsigned long 50 __cmpxchg(volatile void *ptr, unsigned long expected, unsigned long new) 51 { 52 unsigned long flags; 53 int prev; 54 volatile unsigned long *p = ptr; 55 56 /* 57 * spin lock/unlock provide the needed smp_mb() before/after 58 */ 59 atomic_ops_lock(flags); 60 prev = *p; 61 if (prev == expected) 62 *p = new; 63 atomic_ops_unlock(flags); 64 return prev; 65 } 66 67 #else /* CONFIG_ARC_PLAT_EZNPS */ 68 69 static inline unsigned long 70 __cmpxchg(volatile void *ptr, unsigned long expected, unsigned long new) 71 { 72 /* 73 * Explicit full memory barrier needed before/after 74 */ 75 smp_mb(); 76 77 write_aux_reg(CTOP_AUX_GPA1, expected); 78 79 __asm__ __volatile__( 80 " mov r2, %0\n" 81 " mov r3, %1\n" 82 " .word %2\n" 83 " mov %0, r2" 84 : "+r"(new) 85 : "r"(ptr), "i"(CTOP_INST_EXC_DI_R2_R2_R3) 86 : "r2", "r3", "memory"); 87 88 smp_mb(); 89 90 return new; 91 } 92 93 #endif /* CONFIG_ARC_HAS_LLSC */ 94 95 #define cmpxchg(ptr, o, n) ({ \ 96 (typeof(*(ptr)))__cmpxchg((ptr), \ 97 (unsigned long)(o), \ 98 (unsigned long)(n)); \ 99 }) 100 101 /* 102 * atomic_cmpxchg is same as cmpxchg 103 * LLSC: only different in data-type, semantics are exactly same 104 * !LLSC: cmpxchg() has to use an external lock atomic_ops_lock to guarantee 105 * semantics, and this lock also happens to be used by atomic_*() 106 */ 107 #define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n))) 108 109 110 #ifndef CONFIG_ARC_PLAT_EZNPS 111 112 /* 113 * xchg (reg with memory) based on "Native atomic" EX insn 114 */ 115 static inline unsigned long __xchg(unsigned long val, volatile void *ptr, 116 int size) 117 { 118 extern unsigned long __xchg_bad_pointer(void); 119 120 switch (size) { 121 case 4: 122 smp_mb(); 123 124 __asm__ __volatile__( 125 " ex %0, [%1] \n" 126 : "+r"(val) 127 : "r"(ptr) 128 : "memory"); 129 130 smp_mb(); 131 132 return val; 133 } 134 return __xchg_bad_pointer(); 135 } 136 137 #define _xchg(ptr, with) ((typeof(*(ptr)))__xchg((unsigned long)(with), (ptr), \ 138 sizeof(*(ptr)))) 139 140 /* 141 * xchg() maps directly to ARC EX instruction which guarantees atomicity. 142 * However in !LLSC config, it also needs to be use @atomic_ops_lock spinlock 143 * due to a subtle reason: 144 * - For !LLSC, cmpxchg() needs to use that lock (see above) and there is lot 145 * of kernel code which calls xchg()/cmpxchg() on same data (see llist.h) 146 * Hence xchg() needs to follow same locking rules. 147 * 148 * Technically the lock is also needed for UP (boils down to irq save/restore) 149 * but we can cheat a bit since cmpxchg() atomic_ops_lock() would cause irqs to 150 * be disabled thus can't possibly be interrpted/preempted/clobbered by xchg() 151 * Other way around, xchg is one instruction anyways, so can't be interrupted 152 * as such 153 */ 154 155 #if !defined(CONFIG_ARC_HAS_LLSC) && defined(CONFIG_SMP) 156 157 #define xchg(ptr, with) \ 158 ({ \ 159 unsigned long flags; \ 160 typeof(*(ptr)) old_val; \ 161 \ 162 atomic_ops_lock(flags); \ 163 old_val = _xchg(ptr, with); \ 164 atomic_ops_unlock(flags); \ 165 old_val; \ 166 }) 167 168 #else 169 170 #define xchg(ptr, with) _xchg(ptr, with) 171 172 #endif 173 174 #else /* CONFIG_ARC_PLAT_EZNPS */ 175 176 static inline unsigned long __xchg(unsigned long val, volatile void *ptr, 177 int size) 178 { 179 extern unsigned long __xchg_bad_pointer(void); 180 181 switch (size) { 182 case 4: 183 /* 184 * Explicit full memory barrier needed before/after 185 */ 186 smp_mb(); 187 188 __asm__ __volatile__( 189 " mov r2, %0\n" 190 " mov r3, %1\n" 191 " .word %2\n" 192 " mov %0, r2\n" 193 : "+r"(val) 194 : "r"(ptr), "i"(CTOP_INST_XEX_DI_R2_R2_R3) 195 : "r2", "r3", "memory"); 196 197 smp_mb(); 198 199 return val; 200 } 201 return __xchg_bad_pointer(); 202 } 203 204 #define xchg(ptr, with) ({ \ 205 (typeof(*(ptr)))__xchg((unsigned long)(with), \ 206 (ptr), \ 207 sizeof(*(ptr))); \ 208 }) 209 210 #endif /* CONFIG_ARC_PLAT_EZNPS */ 211 212 /* 213 * "atomic" variant of xchg() 214 * REQ: It needs to follow the same serialization rules as other atomic_xxx() 215 * Since xchg() doesn't always do that, it would seem that following defintion 216 * is incorrect. But here's the rationale: 217 * SMP : Even xchg() takes the atomic_ops_lock, so OK. 218 * LLSC: atomic_ops_lock are not relevant at all (even if SMP, since LLSC 219 * is natively "SMP safe", no serialization required). 220 * UP : other atomics disable IRQ, so no way a difft ctxt atomic_xchg() 221 * could clobber them. atomic_xchg() itself would be 1 insn, so it 222 * can't be clobbered by others. Thus no serialization required when 223 * atomic_xchg is involved. 224 */ 225 #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) 226 227 #endif 228