1 /* 2 * Out of line spinlock code. 3 * 4 * Copyright IBM Corp. 2004, 2006 5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) 6 */ 7 8 #include <linux/types.h> 9 #include <linux/module.h> 10 #include <linux/spinlock.h> 11 #include <linux/init.h> 12 #include <linux/smp.h> 13 #include <asm/io.h> 14 15 int spin_retry = -1; 16 17 static int __init spin_retry_init(void) 18 { 19 if (spin_retry < 0) 20 spin_retry = MACHINE_HAS_CAD ? 10 : 1000; 21 return 0; 22 } 23 early_initcall(spin_retry_init); 24 25 /** 26 * spin_retry= parameter 27 */ 28 static int __init spin_retry_setup(char *str) 29 { 30 spin_retry = simple_strtoul(str, &str, 0); 31 return 1; 32 } 33 __setup("spin_retry=", spin_retry_setup); 34 35 static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old) 36 { 37 asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock)); 38 } 39 40 static inline int cpu_is_preempted(int cpu) 41 { 42 if (test_cpu_flag_of(CIF_ENABLED_WAIT, cpu)) 43 return 0; 44 if (smp_vcpu_scheduled(cpu)) 45 return 0; 46 return 1; 47 } 48 49 void arch_spin_lock_wait(arch_spinlock_t *lp) 50 { 51 unsigned int cpu = SPINLOCK_LOCKVAL; 52 unsigned int owner; 53 int count, first_diag; 54 55 first_diag = 1; 56 while (1) { 57 owner = ACCESS_ONCE(lp->lock); 58 /* Try to get the lock if it is free. */ 59 if (!owner) { 60 if (_raw_compare_and_swap(&lp->lock, 0, cpu)) 61 return; 62 continue; 63 } 64 /* First iteration: check if the lock owner is running. */ 65 if (first_diag && cpu_is_preempted(~owner)) { 66 smp_yield_cpu(~owner); 67 first_diag = 0; 68 continue; 69 } 70 /* Loop for a while on the lock value. */ 71 count = spin_retry; 72 do { 73 if (MACHINE_HAS_CAD) 74 _raw_compare_and_delay(&lp->lock, owner); 75 owner = ACCESS_ONCE(lp->lock); 76 } while (owner && count-- > 0); 77 if (!owner) 78 continue; 79 /* 80 * For multiple layers of hypervisors, e.g. z/VM + LPAR 81 * yield the CPU unconditionally. For LPAR rely on the 82 * sense running status. 83 */ 84 if (!MACHINE_IS_LPAR || cpu_is_preempted(~owner)) { 85 smp_yield_cpu(~owner); 86 first_diag = 0; 87 } 88 } 89 } 90 EXPORT_SYMBOL(arch_spin_lock_wait); 91 92 void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) 93 { 94 unsigned int cpu = SPINLOCK_LOCKVAL; 95 unsigned int owner; 96 int count, first_diag; 97 98 local_irq_restore(flags); 99 first_diag = 1; 100 while (1) { 101 owner = ACCESS_ONCE(lp->lock); 102 /* Try to get the lock if it is free. */ 103 if (!owner) { 104 local_irq_disable(); 105 if (_raw_compare_and_swap(&lp->lock, 0, cpu)) 106 return; 107 local_irq_restore(flags); 108 continue; 109 } 110 /* Check if the lock owner is running. */ 111 if (first_diag && cpu_is_preempted(~owner)) { 112 smp_yield_cpu(~owner); 113 first_diag = 0; 114 continue; 115 } 116 /* Loop for a while on the lock value. */ 117 count = spin_retry; 118 do { 119 if (MACHINE_HAS_CAD) 120 _raw_compare_and_delay(&lp->lock, owner); 121 owner = ACCESS_ONCE(lp->lock); 122 } while (owner && count-- > 0); 123 if (!owner) 124 continue; 125 /* 126 * For multiple layers of hypervisors, e.g. z/VM + LPAR 127 * yield the CPU unconditionally. For LPAR rely on the 128 * sense running status. 129 */ 130 if (!MACHINE_IS_LPAR || cpu_is_preempted(~owner)) { 131 smp_yield_cpu(~owner); 132 first_diag = 0; 133 } 134 } 135 } 136 EXPORT_SYMBOL(arch_spin_lock_wait_flags); 137 138 int arch_spin_trylock_retry(arch_spinlock_t *lp) 139 { 140 unsigned int cpu = SPINLOCK_LOCKVAL; 141 unsigned int owner; 142 int count; 143 144 for (count = spin_retry; count > 0; count--) { 145 owner = ACCESS_ONCE(lp->lock); 146 /* Try to get the lock if it is free. */ 147 if (!owner) { 148 if (_raw_compare_and_swap(&lp->lock, 0, cpu)) 149 return 1; 150 } else if (MACHINE_HAS_CAD) 151 _raw_compare_and_delay(&lp->lock, owner); 152 } 153 return 0; 154 } 155 EXPORT_SYMBOL(arch_spin_trylock_retry); 156 157 void _raw_read_lock_wait(arch_rwlock_t *rw) 158 { 159 unsigned int owner, old; 160 int count = spin_retry; 161 162 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES 163 __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD); 164 #endif 165 owner = 0; 166 while (1) { 167 if (count-- <= 0) { 168 if (owner && cpu_is_preempted(~owner)) 169 smp_yield_cpu(~owner); 170 count = spin_retry; 171 } 172 old = ACCESS_ONCE(rw->lock); 173 owner = ACCESS_ONCE(rw->owner); 174 if ((int) old < 0) { 175 if (MACHINE_HAS_CAD) 176 _raw_compare_and_delay(&rw->lock, old); 177 continue; 178 } 179 if (_raw_compare_and_swap(&rw->lock, old, old + 1)) 180 return; 181 } 182 } 183 EXPORT_SYMBOL(_raw_read_lock_wait); 184 185 int _raw_read_trylock_retry(arch_rwlock_t *rw) 186 { 187 unsigned int old; 188 int count = spin_retry; 189 190 while (count-- > 0) { 191 old = ACCESS_ONCE(rw->lock); 192 if ((int) old < 0) { 193 if (MACHINE_HAS_CAD) 194 _raw_compare_and_delay(&rw->lock, old); 195 continue; 196 } 197 if (_raw_compare_and_swap(&rw->lock, old, old + 1)) 198 return 1; 199 } 200 return 0; 201 } 202 EXPORT_SYMBOL(_raw_read_trylock_retry); 203 204 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES 205 206 void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev) 207 { 208 unsigned int owner, old; 209 int count = spin_retry; 210 211 owner = 0; 212 while (1) { 213 if (count-- <= 0) { 214 if (owner && cpu_is_preempted(~owner)) 215 smp_yield_cpu(~owner); 216 count = spin_retry; 217 } 218 old = ACCESS_ONCE(rw->lock); 219 owner = ACCESS_ONCE(rw->owner); 220 smp_mb(); 221 if ((int) old >= 0) { 222 prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR); 223 old = prev; 224 } 225 if ((old & 0x7fffffff) == 0 && (int) prev >= 0) 226 break; 227 if (MACHINE_HAS_CAD) 228 _raw_compare_and_delay(&rw->lock, old); 229 } 230 } 231 EXPORT_SYMBOL(_raw_write_lock_wait); 232 233 #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 234 235 void _raw_write_lock_wait(arch_rwlock_t *rw) 236 { 237 unsigned int owner, old, prev; 238 int count = spin_retry; 239 240 prev = 0x80000000; 241 owner = 0; 242 while (1) { 243 if (count-- <= 0) { 244 if (owner && cpu_is_preempted(~owner)) 245 smp_yield_cpu(~owner); 246 count = spin_retry; 247 } 248 old = ACCESS_ONCE(rw->lock); 249 owner = ACCESS_ONCE(rw->owner); 250 if ((int) old >= 0 && 251 _raw_compare_and_swap(&rw->lock, old, old | 0x80000000)) 252 prev = old; 253 else 254 smp_mb(); 255 if ((old & 0x7fffffff) == 0 && (int) prev >= 0) 256 break; 257 if (MACHINE_HAS_CAD) 258 _raw_compare_and_delay(&rw->lock, old); 259 } 260 } 261 EXPORT_SYMBOL(_raw_write_lock_wait); 262 263 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 264 265 int _raw_write_trylock_retry(arch_rwlock_t *rw) 266 { 267 unsigned int old; 268 int count = spin_retry; 269 270 while (count-- > 0) { 271 old = ACCESS_ONCE(rw->lock); 272 if (old) { 273 if (MACHINE_HAS_CAD) 274 _raw_compare_and_delay(&rw->lock, old); 275 continue; 276 } 277 if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000)) 278 return 1; 279 } 280 return 0; 281 } 282 EXPORT_SYMBOL(_raw_write_trylock_retry); 283 284 void arch_lock_relax(unsigned int cpu) 285 { 286 if (!cpu) 287 return; 288 if (MACHINE_IS_LPAR && !cpu_is_preempted(~cpu)) 289 return; 290 smp_yield_cpu(~cpu); 291 } 292 EXPORT_SYMBOL(arch_lock_relax); 293