1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 4 */ 5 6 #ifndef __ASM_SPINLOCK_H 7 #define __ASM_SPINLOCK_H 8 9 #include <asm/spinlock_types.h> 10 #include <asm/processor.h> 11 #include <asm/barrier.h> 12 13 #define arch_spin_is_locked(x) ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__) 14 15 #ifdef CONFIG_ARC_HAS_LLSC 16 17 static inline void arch_spin_lock(arch_spinlock_t *lock) 18 { 19 unsigned int val; 20 21 __asm__ __volatile__( 22 "1: llock %[val], [%[slock]] \n" 23 " breq %[val], %[LOCKED], 1b \n" /* spin while LOCKED */ 24 " scond %[LOCKED], [%[slock]] \n" /* acquire */ 25 " bnz 1b \n" 26 " \n" 27 : [val] "=&r" (val) 28 : [slock] "r" (&(lock->slock)), 29 [LOCKED] "r" (__ARCH_SPIN_LOCK_LOCKED__) 30 : "memory", "cc"); 31 32 /* 33 * ACQUIRE barrier to ensure load/store after taking the lock 34 * don't "bleed-up" out of the critical section (leak-in is allowed) 35 * http://www.spinics.net/lists/kernel/msg2010409.html 36 * 37 * ARCv2 only has load-load, store-store and all-all barrier 38 * thus need the full all-all barrier 39 */ 40 smp_mb(); 41 } 42 43 /* 1 - lock taken successfully */ 44 static inline int arch_spin_trylock(arch_spinlock_t *lock) 45 { 46 unsigned int val, got_it = 0; 47 48 __asm__ __volatile__( 49 "1: llock %[val], [%[slock]] \n" 50 " breq %[val], %[LOCKED], 4f \n" /* already LOCKED, just bail */ 51 " scond %[LOCKED], [%[slock]] \n" /* acquire */ 52 " bnz 1b \n" 53 " mov %[got_it], 1 \n" 54 "4: \n" 55 " \n" 56 : [val] "=&r" (val), 57 [got_it] "+&r" (got_it) 58 : [slock] "r" (&(lock->slock)), 59 [LOCKED] "r" (__ARCH_SPIN_LOCK_LOCKED__) 60 : "memory", "cc"); 61 62 smp_mb(); 63 64 return got_it; 65 } 66 67 static inline void arch_spin_unlock(arch_spinlock_t *lock) 68 { 69 smp_mb(); 70 71 WRITE_ONCE(lock->slock, __ARCH_SPIN_LOCK_UNLOCKED__); 72 } 73 74 /* 75 * Read-write spinlocks, allowing multiple readers but only one writer. 76 * Unfair locking as Writers could be starved indefinitely by Reader(s) 77 */ 78 79 static inline void arch_read_lock(arch_rwlock_t *rw) 80 { 81 unsigned int val; 82 83 /* 84 * zero means writer holds the lock exclusively, deny Reader. 85 * Otherwise grant lock to first/subseq reader 86 * 87 * if (rw->counter > 0) { 88 * rw->counter--; 89 * ret = 1; 90 * } 91 */ 92 93 __asm__ __volatile__( 94 "1: llock %[val], [%[rwlock]] \n" 95 " brls %[val], %[WR_LOCKED], 1b\n" /* <= 0: spin while write locked */ 96 " sub %[val], %[val], 1 \n" /* reader lock */ 97 " scond %[val], [%[rwlock]] \n" 98 " bnz 1b \n" 99 " \n" 100 : [val] "=&r" (val) 101 : [rwlock] "r" (&(rw->counter)), 102 [WR_LOCKED] "ir" (0) 103 : "memory", "cc"); 104 105 smp_mb(); 106 } 107 108 /* 1 - lock taken successfully */ 109 static inline int arch_read_trylock(arch_rwlock_t *rw) 110 { 111 unsigned int val, got_it = 0; 112 113 __asm__ __volatile__( 114 "1: llock %[val], [%[rwlock]] \n" 115 " brls %[val], %[WR_LOCKED], 4f\n" /* <= 0: already write locked, bail */ 116 " sub %[val], %[val], 1 \n" /* counter-- */ 117 " scond %[val], [%[rwlock]] \n" 118 " bnz 1b \n" /* retry if collided with someone */ 119 " mov %[got_it], 1 \n" 120 " \n" 121 "4: ; --- done --- \n" 122 123 : [val] "=&r" (val), 124 [got_it] "+&r" (got_it) 125 : [rwlock] "r" (&(rw->counter)), 126 [WR_LOCKED] "ir" (0) 127 : "memory", "cc"); 128 129 smp_mb(); 130 131 return got_it; 132 } 133 134 static inline void arch_write_lock(arch_rwlock_t *rw) 135 { 136 unsigned int val; 137 138 /* 139 * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__), 140 * deny writer. Otherwise if unlocked grant to writer 141 * Hence the claim that Linux rwlocks are unfair to writers. 142 * (can be starved for an indefinite time by readers). 143 * 144 * if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) { 145 * rw->counter = 0; 146 * ret = 1; 147 * } 148 */ 149 150 __asm__ __volatile__( 151 "1: llock %[val], [%[rwlock]] \n" 152 " brne %[val], %[UNLOCKED], 1b \n" /* while !UNLOCKED spin */ 153 " mov %[val], %[WR_LOCKED] \n" 154 " scond %[val], [%[rwlock]] \n" 155 " bnz 1b \n" 156 " \n" 157 : [val] "=&r" (val) 158 : [rwlock] "r" (&(rw->counter)), 159 [UNLOCKED] "ir" (__ARCH_RW_LOCK_UNLOCKED__), 160 [WR_LOCKED] "ir" (0) 161 : "memory", "cc"); 162 163 smp_mb(); 164 } 165 166 /* 1 - lock taken successfully */ 167 static inline int arch_write_trylock(arch_rwlock_t *rw) 168 { 169 unsigned int val, got_it = 0; 170 171 __asm__ __volatile__( 172 "1: llock %[val], [%[rwlock]] \n" 173 " brne %[val], %[UNLOCKED], 4f \n" /* !UNLOCKED, bail */ 174 " mov %[val], %[WR_LOCKED] \n" 175 " scond %[val], [%[rwlock]] \n" 176 " bnz 1b \n" /* retry if collided with someone */ 177 " mov %[got_it], 1 \n" 178 " \n" 179 "4: ; --- done --- \n" 180 181 : [val] "=&r" (val), 182 [got_it] "+&r" (got_it) 183 : [rwlock] "r" (&(rw->counter)), 184 [UNLOCKED] "ir" (__ARCH_RW_LOCK_UNLOCKED__), 185 [WR_LOCKED] "ir" (0) 186 : "memory", "cc"); 187 188 smp_mb(); 189 190 return got_it; 191 } 192 193 static inline void arch_read_unlock(arch_rwlock_t *rw) 194 { 195 unsigned int val; 196 197 smp_mb(); 198 199 /* 200 * rw->counter++; 201 */ 202 __asm__ __volatile__( 203 "1: llock %[val], [%[rwlock]] \n" 204 " add %[val], %[val], 1 \n" 205 " scond %[val], [%[rwlock]] \n" 206 " bnz 1b \n" 207 " \n" 208 : [val] "=&r" (val) 209 : [rwlock] "r" (&(rw->counter)) 210 : "memory", "cc"); 211 } 212 213 static inline void arch_write_unlock(arch_rwlock_t *rw) 214 { 215 smp_mb(); 216 217 WRITE_ONCE(rw->counter, __ARCH_RW_LOCK_UNLOCKED__); 218 } 219 220 #else /* !CONFIG_ARC_HAS_LLSC */ 221 222 static inline void arch_spin_lock(arch_spinlock_t *lock) 223 { 224 unsigned int val = __ARCH_SPIN_LOCK_LOCKED__; 225 226 /* 227 * Per lkmm, smp_mb() is only required after _lock (and before_unlock) 228 * for ACQ and REL semantics respectively. However EX based spinlocks 229 * need the extra smp_mb to workaround a hardware quirk. 230 */ 231 smp_mb(); 232 233 __asm__ __volatile__( 234 "1: ex %0, [%1] \n" 235 " breq %0, %2, 1b \n" 236 : "+&r" (val) 237 : "r"(&(lock->slock)), "ir"(__ARCH_SPIN_LOCK_LOCKED__) 238 : "memory"); 239 240 smp_mb(); 241 } 242 243 /* 1 - lock taken successfully */ 244 static inline int arch_spin_trylock(arch_spinlock_t *lock) 245 { 246 unsigned int val = __ARCH_SPIN_LOCK_LOCKED__; 247 248 smp_mb(); 249 250 __asm__ __volatile__( 251 "1: ex %0, [%1] \n" 252 : "+r" (val) 253 : "r"(&(lock->slock)) 254 : "memory"); 255 256 smp_mb(); 257 258 return (val == __ARCH_SPIN_LOCK_UNLOCKED__); 259 } 260 261 static inline void arch_spin_unlock(arch_spinlock_t *lock) 262 { 263 unsigned int val = __ARCH_SPIN_LOCK_UNLOCKED__; 264 265 /* 266 * RELEASE barrier: given the instructions avail on ARCv2, full barrier 267 * is the only option 268 */ 269 smp_mb(); 270 271 /* 272 * EX is not really required here, a simple STore of 0 suffices. 273 * However this causes tasklist livelocks in SystemC based SMP virtual 274 * platforms where the systemc core scheduler uses EX as a cue for 275 * moving to next core. Do a git log of this file for details 276 */ 277 __asm__ __volatile__( 278 " ex %0, [%1] \n" 279 : "+r" (val) 280 : "r"(&(lock->slock)) 281 : "memory"); 282 283 /* 284 * see pairing version/comment in arch_spin_lock above 285 */ 286 smp_mb(); 287 } 288 289 /* 290 * Read-write spinlocks, allowing multiple readers but only one writer. 291 * Unfair locking as Writers could be starved indefinitely by Reader(s) 292 * 293 * The spinlock itself is contained in @counter and access to it is 294 * serialized with @lock_mutex. 295 */ 296 297 /* 1 - lock taken successfully */ 298 static inline int arch_read_trylock(arch_rwlock_t *rw) 299 { 300 int ret = 0; 301 unsigned long flags; 302 303 local_irq_save(flags); 304 arch_spin_lock(&(rw->lock_mutex)); 305 306 /* 307 * zero means writer holds the lock exclusively, deny Reader. 308 * Otherwise grant lock to first/subseq reader 309 */ 310 if (rw->counter > 0) { 311 rw->counter--; 312 ret = 1; 313 } 314 315 arch_spin_unlock(&(rw->lock_mutex)); 316 local_irq_restore(flags); 317 318 return ret; 319 } 320 321 /* 1 - lock taken successfully */ 322 static inline int arch_write_trylock(arch_rwlock_t *rw) 323 { 324 int ret = 0; 325 unsigned long flags; 326 327 local_irq_save(flags); 328 arch_spin_lock(&(rw->lock_mutex)); 329 330 /* 331 * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__), 332 * deny writer. Otherwise if unlocked grant to writer 333 * Hence the claim that Linux rwlocks are unfair to writers. 334 * (can be starved for an indefinite time by readers). 335 */ 336 if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) { 337 rw->counter = 0; 338 ret = 1; 339 } 340 arch_spin_unlock(&(rw->lock_mutex)); 341 local_irq_restore(flags); 342 343 return ret; 344 } 345 346 static inline void arch_read_lock(arch_rwlock_t *rw) 347 { 348 while (!arch_read_trylock(rw)) 349 cpu_relax(); 350 } 351 352 static inline void arch_write_lock(arch_rwlock_t *rw) 353 { 354 while (!arch_write_trylock(rw)) 355 cpu_relax(); 356 } 357 358 static inline void arch_read_unlock(arch_rwlock_t *rw) 359 { 360 unsigned long flags; 361 362 local_irq_save(flags); 363 arch_spin_lock(&(rw->lock_mutex)); 364 rw->counter++; 365 arch_spin_unlock(&(rw->lock_mutex)); 366 local_irq_restore(flags); 367 } 368 369 static inline void arch_write_unlock(arch_rwlock_t *rw) 370 { 371 unsigned long flags; 372 373 local_irq_save(flags); 374 arch_spin_lock(&(rw->lock_mutex)); 375 rw->counter = __ARCH_RW_LOCK_UNLOCKED__; 376 arch_spin_unlock(&(rw->lock_mutex)); 377 local_irq_restore(flags); 378 } 379 380 #endif 381 382 #endif /* __ASM_SPINLOCK_H */ 383