1 /* 2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #ifndef __ASM_SPINLOCK_H 10 #define __ASM_SPINLOCK_H 11 12 #include <asm/spinlock_types.h> 13 #include <asm/processor.h> 14 #include <asm/barrier.h> 15 16 #define arch_spin_is_locked(x) ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__) 17 #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 18 19 #ifdef CONFIG_ARC_HAS_LLSC 20 21 static inline void arch_spin_lock(arch_spinlock_t *lock) 22 { 23 unsigned int val; 24 25 smp_mb(); 26 27 __asm__ __volatile__( 28 "1: llock %[val], [%[slock]] \n" 29 " breq %[val], %[LOCKED], 1b \n" /* spin while LOCKED */ 30 " scond %[LOCKED], [%[slock]] \n" /* acquire */ 31 " bnz 1b \n" 32 " \n" 33 : [val] "=&r" (val) 34 : [slock] "r" (&(lock->slock)), 35 [LOCKED] "r" (__ARCH_SPIN_LOCK_LOCKED__) 36 : "memory", "cc"); 37 38 smp_mb(); 39 } 40 41 /* 1 - lock taken successfully */ 42 static inline int arch_spin_trylock(arch_spinlock_t *lock) 43 { 44 unsigned int val, got_it = 0; 45 46 smp_mb(); 47 48 __asm__ __volatile__( 49 "1: llock %[val], [%[slock]] \n" 50 " breq %[val], %[LOCKED], 4f \n" /* already LOCKED, just bail */ 51 " scond %[LOCKED], [%[slock]] \n" /* acquire */ 52 " bnz 1b \n" 53 " mov %[got_it], 1 \n" 54 "4: \n" 55 " \n" 56 : [val] "=&r" (val), 57 [got_it] "+&r" (got_it) 58 : [slock] "r" (&(lock->slock)), 59 [LOCKED] "r" (__ARCH_SPIN_LOCK_LOCKED__) 60 : "memory", "cc"); 61 62 smp_mb(); 63 64 return got_it; 65 } 66 67 static inline void arch_spin_unlock(arch_spinlock_t *lock) 68 { 69 smp_mb(); 70 71 lock->slock = __ARCH_SPIN_LOCK_UNLOCKED__; 72 73 smp_mb(); 74 } 75 76 /* 77 * Read-write spinlocks, allowing multiple readers but only one writer. 78 * Unfair locking as Writers could be starved indefinitely by Reader(s) 79 */ 80 81 static inline void arch_read_lock(arch_rwlock_t *rw) 82 { 83 unsigned int val; 84 85 smp_mb(); 86 87 /* 88 * zero means writer holds the lock exclusively, deny Reader. 89 * Otherwise grant lock to first/subseq reader 90 * 91 * if (rw->counter > 0) { 92 * rw->counter--; 93 * ret = 1; 94 * } 95 */ 96 97 __asm__ __volatile__( 98 "1: llock %[val], [%[rwlock]] \n" 99 " brls %[val], %[WR_LOCKED], 1b\n" /* <= 0: spin while write locked */ 100 " sub %[val], %[val], 1 \n" /* reader lock */ 101 " scond %[val], [%[rwlock]] \n" 102 " bnz 1b \n" 103 " \n" 104 : [val] "=&r" (val) 105 : [rwlock] "r" (&(rw->counter)), 106 [WR_LOCKED] "ir" (0) 107 : "memory", "cc"); 108 109 smp_mb(); 110 } 111 112 /* 1 - lock taken successfully */ 113 static inline int arch_read_trylock(arch_rwlock_t *rw) 114 { 115 unsigned int val, got_it = 0; 116 117 smp_mb(); 118 119 __asm__ __volatile__( 120 "1: llock %[val], [%[rwlock]] \n" 121 " brls %[val], %[WR_LOCKED], 4f\n" /* <= 0: already write locked, bail */ 122 " sub %[val], %[val], 1 \n" /* counter-- */ 123 " scond %[val], [%[rwlock]] \n" 124 " bnz 1b \n" /* retry if collided with someone */ 125 " mov %[got_it], 1 \n" 126 " \n" 127 "4: ; --- done --- \n" 128 129 : [val] "=&r" (val), 130 [got_it] "+&r" (got_it) 131 : [rwlock] "r" (&(rw->counter)), 132 [WR_LOCKED] "ir" (0) 133 : "memory", "cc"); 134 135 smp_mb(); 136 137 return got_it; 138 } 139 140 static inline void arch_write_lock(arch_rwlock_t *rw) 141 { 142 unsigned int val; 143 144 smp_mb(); 145 146 /* 147 * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__), 148 * deny writer. Otherwise if unlocked grant to writer 149 * Hence the claim that Linux rwlocks are unfair to writers. 150 * (can be starved for an indefinite time by readers). 151 * 152 * if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) { 153 * rw->counter = 0; 154 * ret = 1; 155 * } 156 */ 157 158 __asm__ __volatile__( 159 "1: llock %[val], [%[rwlock]] \n" 160 " brne %[val], %[UNLOCKED], 1b \n" /* while !UNLOCKED spin */ 161 " mov %[val], %[WR_LOCKED] \n" 162 " scond %[val], [%[rwlock]] \n" 163 " bnz 1b \n" 164 " \n" 165 : [val] "=&r" (val) 166 : [rwlock] "r" (&(rw->counter)), 167 [UNLOCKED] "ir" (__ARCH_RW_LOCK_UNLOCKED__), 168 [WR_LOCKED] "ir" (0) 169 : "memory", "cc"); 170 171 smp_mb(); 172 } 173 174 /* 1 - lock taken successfully */ 175 static inline int arch_write_trylock(arch_rwlock_t *rw) 176 { 177 unsigned int val, got_it = 0; 178 179 smp_mb(); 180 181 __asm__ __volatile__( 182 "1: llock %[val], [%[rwlock]] \n" 183 " brne %[val], %[UNLOCKED], 4f \n" /* !UNLOCKED, bail */ 184 " mov %[val], %[WR_LOCKED] \n" 185 " scond %[val], [%[rwlock]] \n" 186 " bnz 1b \n" /* retry if collided with someone */ 187 " mov %[got_it], 1 \n" 188 " \n" 189 "4: ; --- done --- \n" 190 191 : [val] "=&r" (val), 192 [got_it] "+&r" (got_it) 193 : [rwlock] "r" (&(rw->counter)), 194 [UNLOCKED] "ir" (__ARCH_RW_LOCK_UNLOCKED__), 195 [WR_LOCKED] "ir" (0) 196 : "memory", "cc"); 197 198 smp_mb(); 199 200 return got_it; 201 } 202 203 static inline void arch_read_unlock(arch_rwlock_t *rw) 204 { 205 unsigned int val; 206 207 smp_mb(); 208 209 /* 210 * rw->counter++; 211 */ 212 __asm__ __volatile__( 213 "1: llock %[val], [%[rwlock]] \n" 214 " add %[val], %[val], 1 \n" 215 " scond %[val], [%[rwlock]] \n" 216 " bnz 1b \n" 217 " \n" 218 : [val] "=&r" (val) 219 : [rwlock] "r" (&(rw->counter)) 220 : "memory", "cc"); 221 222 smp_mb(); 223 } 224 225 static inline void arch_write_unlock(arch_rwlock_t *rw) 226 { 227 smp_mb(); 228 229 rw->counter = __ARCH_RW_LOCK_UNLOCKED__; 230 231 smp_mb(); 232 } 233 234 #else /* !CONFIG_ARC_HAS_LLSC */ 235 236 static inline void arch_spin_lock(arch_spinlock_t *lock) 237 { 238 unsigned int val = __ARCH_SPIN_LOCK_LOCKED__; 239 240 /* 241 * This smp_mb() is technically superfluous, we only need the one 242 * after the lock for providing the ACQUIRE semantics. 243 * However doing the "right" thing was regressing hackbench 244 * so keeping this, pending further investigation 245 */ 246 smp_mb(); 247 248 __asm__ __volatile__( 249 "1: ex %0, [%1] \n" 250 #ifdef CONFIG_EZNPS_MTM_EXT 251 " .word %3 \n" 252 #endif 253 " breq %0, %2, 1b \n" 254 : "+&r" (val) 255 : "r"(&(lock->slock)), "ir"(__ARCH_SPIN_LOCK_LOCKED__) 256 #ifdef CONFIG_EZNPS_MTM_EXT 257 , "i"(CTOP_INST_SCHD_RW) 258 #endif 259 : "memory"); 260 261 /* 262 * ACQUIRE barrier to ensure load/store after taking the lock 263 * don't "bleed-up" out of the critical section (leak-in is allowed) 264 * http://www.spinics.net/lists/kernel/msg2010409.html 265 * 266 * ARCv2 only has load-load, store-store and all-all barrier 267 * thus need the full all-all barrier 268 */ 269 smp_mb(); 270 } 271 272 /* 1 - lock taken successfully */ 273 static inline int arch_spin_trylock(arch_spinlock_t *lock) 274 { 275 unsigned int val = __ARCH_SPIN_LOCK_LOCKED__; 276 277 smp_mb(); 278 279 __asm__ __volatile__( 280 "1: ex %0, [%1] \n" 281 : "+r" (val) 282 : "r"(&(lock->slock)) 283 : "memory"); 284 285 smp_mb(); 286 287 return (val == __ARCH_SPIN_LOCK_UNLOCKED__); 288 } 289 290 static inline void arch_spin_unlock(arch_spinlock_t *lock) 291 { 292 unsigned int val = __ARCH_SPIN_LOCK_UNLOCKED__; 293 294 /* 295 * RELEASE barrier: given the instructions avail on ARCv2, full barrier 296 * is the only option 297 */ 298 smp_mb(); 299 300 /* 301 * EX is not really required here, a simple STore of 0 suffices. 302 * However this causes tasklist livelocks in SystemC based SMP virtual 303 * platforms where the systemc core scheduler uses EX as a cue for 304 * moving to next core. Do a git log of this file for details 305 */ 306 __asm__ __volatile__( 307 " ex %0, [%1] \n" 308 : "+r" (val) 309 : "r"(&(lock->slock)) 310 : "memory"); 311 312 /* 313 * superfluous, but keeping for now - see pairing version in 314 * arch_spin_lock above 315 */ 316 smp_mb(); 317 } 318 319 /* 320 * Read-write spinlocks, allowing multiple readers but only one writer. 321 * Unfair locking as Writers could be starved indefinitely by Reader(s) 322 * 323 * The spinlock itself is contained in @counter and access to it is 324 * serialized with @lock_mutex. 325 */ 326 327 /* 1 - lock taken successfully */ 328 static inline int arch_read_trylock(arch_rwlock_t *rw) 329 { 330 int ret = 0; 331 unsigned long flags; 332 333 local_irq_save(flags); 334 arch_spin_lock(&(rw->lock_mutex)); 335 336 /* 337 * zero means writer holds the lock exclusively, deny Reader. 338 * Otherwise grant lock to first/subseq reader 339 */ 340 if (rw->counter > 0) { 341 rw->counter--; 342 ret = 1; 343 } 344 345 arch_spin_unlock(&(rw->lock_mutex)); 346 local_irq_restore(flags); 347 348 smp_mb(); 349 return ret; 350 } 351 352 /* 1 - lock taken successfully */ 353 static inline int arch_write_trylock(arch_rwlock_t *rw) 354 { 355 int ret = 0; 356 unsigned long flags; 357 358 local_irq_save(flags); 359 arch_spin_lock(&(rw->lock_mutex)); 360 361 /* 362 * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__), 363 * deny writer. Otherwise if unlocked grant to writer 364 * Hence the claim that Linux rwlocks are unfair to writers. 365 * (can be starved for an indefinite time by readers). 366 */ 367 if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) { 368 rw->counter = 0; 369 ret = 1; 370 } 371 arch_spin_unlock(&(rw->lock_mutex)); 372 local_irq_restore(flags); 373 374 return ret; 375 } 376 377 static inline void arch_read_lock(arch_rwlock_t *rw) 378 { 379 while (!arch_read_trylock(rw)) 380 cpu_relax(); 381 } 382 383 static inline void arch_write_lock(arch_rwlock_t *rw) 384 { 385 while (!arch_write_trylock(rw)) 386 cpu_relax(); 387 } 388 389 static inline void arch_read_unlock(arch_rwlock_t *rw) 390 { 391 unsigned long flags; 392 393 local_irq_save(flags); 394 arch_spin_lock(&(rw->lock_mutex)); 395 rw->counter++; 396 arch_spin_unlock(&(rw->lock_mutex)); 397 local_irq_restore(flags); 398 } 399 400 static inline void arch_write_unlock(arch_rwlock_t *rw) 401 { 402 unsigned long flags; 403 404 local_irq_save(flags); 405 arch_spin_lock(&(rw->lock_mutex)); 406 rw->counter = __ARCH_RW_LOCK_UNLOCKED__; 407 arch_spin_unlock(&(rw->lock_mutex)); 408 local_irq_restore(flags); 409 } 410 411 #endif 412 413 #define arch_read_can_lock(x) ((x)->counter > 0) 414 #define arch_write_can_lock(x) ((x)->counter == __ARCH_RW_LOCK_UNLOCKED__) 415 416 #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 417 #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 418 419 #define arch_spin_relax(lock) cpu_relax() 420 #define arch_read_relax(lock) cpu_relax() 421 #define arch_write_relax(lock) cpu_relax() 422 423 #endif /* __ASM_SPINLOCK_H */ 424