1 /* 2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #ifndef __ASM_SPINLOCK_H 10 #define __ASM_SPINLOCK_H 11 12 #include <asm/spinlock_types.h> 13 #include <asm/processor.h> 14 #include <asm/barrier.h> 15 16 #define arch_spin_is_locked(x) ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__) 17 #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 18 #define arch_spin_unlock_wait(x) \ 19 do { while (arch_spin_is_locked(x)) cpu_relax(); } while (0) 20 21 #ifdef CONFIG_ARC_HAS_LLSC 22 23 static inline void arch_spin_lock(arch_spinlock_t *lock) 24 { 25 unsigned int val; 26 27 smp_mb(); 28 29 __asm__ __volatile__( 30 "1: llock %[val], [%[slock]] \n" 31 " breq %[val], %[LOCKED], 1b \n" /* spin while LOCKED */ 32 " scond %[LOCKED], [%[slock]] \n" /* acquire */ 33 " bnz 1b \n" 34 " \n" 35 : [val] "=&r" (val) 36 : [slock] "r" (&(lock->slock)), 37 [LOCKED] "r" (__ARCH_SPIN_LOCK_LOCKED__) 38 : "memory", "cc"); 39 40 smp_mb(); 41 } 42 43 /* 1 - lock taken successfully */ 44 static inline int arch_spin_trylock(arch_spinlock_t *lock) 45 { 46 unsigned int val, got_it = 0; 47 48 smp_mb(); 49 50 __asm__ __volatile__( 51 "1: llock %[val], [%[slock]] \n" 52 " breq %[val], %[LOCKED], 4f \n" /* already LOCKED, just bail */ 53 " scond %[LOCKED], [%[slock]] \n" /* acquire */ 54 " bnz 1b \n" 55 " mov %[got_it], 1 \n" 56 "4: \n" 57 " \n" 58 : [val] "=&r" (val), 59 [got_it] "+&r" (got_it) 60 : [slock] "r" (&(lock->slock)), 61 [LOCKED] "r" (__ARCH_SPIN_LOCK_LOCKED__) 62 : "memory", "cc"); 63 64 smp_mb(); 65 66 return got_it; 67 } 68 69 static inline void arch_spin_unlock(arch_spinlock_t *lock) 70 { 71 smp_mb(); 72 73 lock->slock = __ARCH_SPIN_LOCK_UNLOCKED__; 74 75 smp_mb(); 76 } 77 78 /* 79 * Read-write spinlocks, allowing multiple readers but only one writer. 80 * Unfair locking as Writers could be starved indefinitely by Reader(s) 81 */ 82 83 static inline void arch_read_lock(arch_rwlock_t *rw) 84 { 85 unsigned int val; 86 87 smp_mb(); 88 89 /* 90 * zero means writer holds the lock exclusively, deny Reader. 91 * Otherwise grant lock to first/subseq reader 92 * 93 * if (rw->counter > 0) { 94 * rw->counter--; 95 * ret = 1; 96 * } 97 */ 98 99 __asm__ __volatile__( 100 "1: llock %[val], [%[rwlock]] \n" 101 " brls %[val], %[WR_LOCKED], 1b\n" /* <= 0: spin while write locked */ 102 " sub %[val], %[val], 1 \n" /* reader lock */ 103 " scond %[val], [%[rwlock]] \n" 104 " bnz 1b \n" 105 " \n" 106 : [val] "=&r" (val) 107 : [rwlock] "r" (&(rw->counter)), 108 [WR_LOCKED] "ir" (0) 109 : "memory", "cc"); 110 111 smp_mb(); 112 } 113 114 /* 1 - lock taken successfully */ 115 static inline int arch_read_trylock(arch_rwlock_t *rw) 116 { 117 unsigned int val, got_it = 0; 118 119 smp_mb(); 120 121 __asm__ __volatile__( 122 "1: llock %[val], [%[rwlock]] \n" 123 " brls %[val], %[WR_LOCKED], 4f\n" /* <= 0: already write locked, bail */ 124 " sub %[val], %[val], 1 \n" /* counter-- */ 125 " scond %[val], [%[rwlock]] \n" 126 " bnz 1b \n" /* retry if collided with someone */ 127 " mov %[got_it], 1 \n" 128 " \n" 129 "4: ; --- done --- \n" 130 131 : [val] "=&r" (val), 132 [got_it] "+&r" (got_it) 133 : [rwlock] "r" (&(rw->counter)), 134 [WR_LOCKED] "ir" (0) 135 : "memory", "cc"); 136 137 smp_mb(); 138 139 return got_it; 140 } 141 142 static inline void arch_write_lock(arch_rwlock_t *rw) 143 { 144 unsigned int val; 145 146 smp_mb(); 147 148 /* 149 * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__), 150 * deny writer. Otherwise if unlocked grant to writer 151 * Hence the claim that Linux rwlocks are unfair to writers. 152 * (can be starved for an indefinite time by readers). 153 * 154 * if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) { 155 * rw->counter = 0; 156 * ret = 1; 157 * } 158 */ 159 160 __asm__ __volatile__( 161 "1: llock %[val], [%[rwlock]] \n" 162 " brne %[val], %[UNLOCKED], 1b \n" /* while !UNLOCKED spin */ 163 " mov %[val], %[WR_LOCKED] \n" 164 " scond %[val], [%[rwlock]] \n" 165 " bnz 1b \n" 166 " \n" 167 : [val] "=&r" (val) 168 : [rwlock] "r" (&(rw->counter)), 169 [UNLOCKED] "ir" (__ARCH_RW_LOCK_UNLOCKED__), 170 [WR_LOCKED] "ir" (0) 171 : "memory", "cc"); 172 173 smp_mb(); 174 } 175 176 /* 1 - lock taken successfully */ 177 static inline int arch_write_trylock(arch_rwlock_t *rw) 178 { 179 unsigned int val, got_it = 0; 180 181 smp_mb(); 182 183 __asm__ __volatile__( 184 "1: llock %[val], [%[rwlock]] \n" 185 " brne %[val], %[UNLOCKED], 4f \n" /* !UNLOCKED, bail */ 186 " mov %[val], %[WR_LOCKED] \n" 187 " scond %[val], [%[rwlock]] \n" 188 " bnz 1b \n" /* retry if collided with someone */ 189 " mov %[got_it], 1 \n" 190 " \n" 191 "4: ; --- done --- \n" 192 193 : [val] "=&r" (val), 194 [got_it] "+&r" (got_it) 195 : [rwlock] "r" (&(rw->counter)), 196 [UNLOCKED] "ir" (__ARCH_RW_LOCK_UNLOCKED__), 197 [WR_LOCKED] "ir" (0) 198 : "memory", "cc"); 199 200 smp_mb(); 201 202 return got_it; 203 } 204 205 static inline void arch_read_unlock(arch_rwlock_t *rw) 206 { 207 unsigned int val; 208 209 smp_mb(); 210 211 /* 212 * rw->counter++; 213 */ 214 __asm__ __volatile__( 215 "1: llock %[val], [%[rwlock]] \n" 216 " add %[val], %[val], 1 \n" 217 " scond %[val], [%[rwlock]] \n" 218 " bnz 1b \n" 219 " \n" 220 : [val] "=&r" (val) 221 : [rwlock] "r" (&(rw->counter)) 222 : "memory", "cc"); 223 224 smp_mb(); 225 } 226 227 static inline void arch_write_unlock(arch_rwlock_t *rw) 228 { 229 smp_mb(); 230 231 rw->counter = __ARCH_RW_LOCK_UNLOCKED__; 232 233 smp_mb(); 234 } 235 236 #else /* !CONFIG_ARC_HAS_LLSC */ 237 238 static inline void arch_spin_lock(arch_spinlock_t *lock) 239 { 240 unsigned int val = __ARCH_SPIN_LOCK_LOCKED__; 241 242 /* 243 * This smp_mb() is technically superfluous, we only need the one 244 * after the lock for providing the ACQUIRE semantics. 245 * However doing the "right" thing was regressing hackbench 246 * so keeping this, pending further investigation 247 */ 248 smp_mb(); 249 250 __asm__ __volatile__( 251 "1: ex %0, [%1] \n" 252 " breq %0, %2, 1b \n" 253 : "+&r" (val) 254 : "r"(&(lock->slock)), "ir"(__ARCH_SPIN_LOCK_LOCKED__) 255 : "memory"); 256 257 /* 258 * ACQUIRE barrier to ensure load/store after taking the lock 259 * don't "bleed-up" out of the critical section (leak-in is allowed) 260 * http://www.spinics.net/lists/kernel/msg2010409.html 261 * 262 * ARCv2 only has load-load, store-store and all-all barrier 263 * thus need the full all-all barrier 264 */ 265 smp_mb(); 266 } 267 268 /* 1 - lock taken successfully */ 269 static inline int arch_spin_trylock(arch_spinlock_t *lock) 270 { 271 unsigned int val = __ARCH_SPIN_LOCK_LOCKED__; 272 273 smp_mb(); 274 275 __asm__ __volatile__( 276 "1: ex %0, [%1] \n" 277 : "+r" (val) 278 : "r"(&(lock->slock)) 279 : "memory"); 280 281 smp_mb(); 282 283 return (val == __ARCH_SPIN_LOCK_UNLOCKED__); 284 } 285 286 static inline void arch_spin_unlock(arch_spinlock_t *lock) 287 { 288 unsigned int val = __ARCH_SPIN_LOCK_UNLOCKED__; 289 290 /* 291 * RELEASE barrier: given the instructions avail on ARCv2, full barrier 292 * is the only option 293 */ 294 smp_mb(); 295 296 __asm__ __volatile__( 297 " ex %0, [%1] \n" 298 : "+r" (val) 299 : "r"(&(lock->slock)) 300 : "memory"); 301 302 /* 303 * superfluous, but keeping for now - see pairing version in 304 * arch_spin_lock above 305 */ 306 smp_mb(); 307 } 308 309 /* 310 * Read-write spinlocks, allowing multiple readers but only one writer. 311 * Unfair locking as Writers could be starved indefinitely by Reader(s) 312 * 313 * The spinlock itself is contained in @counter and access to it is 314 * serialized with @lock_mutex. 315 */ 316 317 /* 1 - lock taken successfully */ 318 static inline int arch_read_trylock(arch_rwlock_t *rw) 319 { 320 int ret = 0; 321 unsigned long flags; 322 323 local_irq_save(flags); 324 arch_spin_lock(&(rw->lock_mutex)); 325 326 /* 327 * zero means writer holds the lock exclusively, deny Reader. 328 * Otherwise grant lock to first/subseq reader 329 */ 330 if (rw->counter > 0) { 331 rw->counter--; 332 ret = 1; 333 } 334 335 arch_spin_unlock(&(rw->lock_mutex)); 336 local_irq_restore(flags); 337 338 smp_mb(); 339 return ret; 340 } 341 342 /* 1 - lock taken successfully */ 343 static inline int arch_write_trylock(arch_rwlock_t *rw) 344 { 345 int ret = 0; 346 unsigned long flags; 347 348 local_irq_save(flags); 349 arch_spin_lock(&(rw->lock_mutex)); 350 351 /* 352 * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__), 353 * deny writer. Otherwise if unlocked grant to writer 354 * Hence the claim that Linux rwlocks are unfair to writers. 355 * (can be starved for an indefinite time by readers). 356 */ 357 if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) { 358 rw->counter = 0; 359 ret = 1; 360 } 361 arch_spin_unlock(&(rw->lock_mutex)); 362 local_irq_restore(flags); 363 364 return ret; 365 } 366 367 static inline void arch_read_lock(arch_rwlock_t *rw) 368 { 369 while (!arch_read_trylock(rw)) 370 cpu_relax(); 371 } 372 373 static inline void arch_write_lock(arch_rwlock_t *rw) 374 { 375 while (!arch_write_trylock(rw)) 376 cpu_relax(); 377 } 378 379 static inline void arch_read_unlock(arch_rwlock_t *rw) 380 { 381 unsigned long flags; 382 383 local_irq_save(flags); 384 arch_spin_lock(&(rw->lock_mutex)); 385 rw->counter++; 386 arch_spin_unlock(&(rw->lock_mutex)); 387 local_irq_restore(flags); 388 } 389 390 static inline void arch_write_unlock(arch_rwlock_t *rw) 391 { 392 unsigned long flags; 393 394 local_irq_save(flags); 395 arch_spin_lock(&(rw->lock_mutex)); 396 rw->counter = __ARCH_RW_LOCK_UNLOCKED__; 397 arch_spin_unlock(&(rw->lock_mutex)); 398 local_irq_restore(flags); 399 } 400 401 #endif 402 403 #define arch_read_can_lock(x) ((x)->counter > 0) 404 #define arch_write_can_lock(x) ((x)->counter == __ARCH_RW_LOCK_UNLOCKED__) 405 406 #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 407 #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 408 409 #define arch_spin_relax(lock) cpu_relax() 410 #define arch_read_relax(lock) cpu_relax() 411 #define arch_write_relax(lock) cpu_relax() 412 413 #endif /* __ASM_SPINLOCK_H */ 414