1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 4 */ 5 6 #ifndef __ASM_SPINLOCK_H 7 #define __ASM_SPINLOCK_H 8 9 #include <asm/spinlock_types.h> 10 #include <asm/processor.h> 11 #include <asm/barrier.h> 12 13 #define arch_spin_is_locked(x) ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__) 14 15 #ifdef CONFIG_ARC_HAS_LLSC 16 17 static inline void arch_spin_lock(arch_spinlock_t *lock) 18 { 19 unsigned int val; 20 21 __asm__ __volatile__( 22 "1: llock %[val], [%[slock]] \n" 23 " breq %[val], %[LOCKED], 1b \n" /* spin while LOCKED */ 24 " scond %[LOCKED], [%[slock]] \n" /* acquire */ 25 " bnz 1b \n" 26 " \n" 27 : [val] "=&r" (val) 28 : [slock] "r" (&(lock->slock)), 29 [LOCKED] "r" (__ARCH_SPIN_LOCK_LOCKED__) 30 : "memory", "cc"); 31 32 /* 33 * ACQUIRE barrier to ensure load/store after taking the lock 34 * don't "bleed-up" out of the critical section (leak-in is allowed) 35 * http://www.spinics.net/lists/kernel/msg2010409.html 36 * 37 * ARCv2 only has load-load, store-store and all-all barrier 38 * thus need the full all-all barrier 39 */ 40 smp_mb(); 41 } 42 43 /* 1 - lock taken successfully */ 44 static inline int arch_spin_trylock(arch_spinlock_t *lock) 45 { 46 unsigned int val, got_it = 0; 47 48 __asm__ __volatile__( 49 "1: llock %[val], [%[slock]] \n" 50 " breq %[val], %[LOCKED], 4f \n" /* already LOCKED, just bail */ 51 " scond %[LOCKED], [%[slock]] \n" /* acquire */ 52 " bnz 1b \n" 53 " mov %[got_it], 1 \n" 54 "4: \n" 55 " \n" 56 : [val] "=&r" (val), 57 [got_it] "+&r" (got_it) 58 : [slock] "r" (&(lock->slock)), 59 [LOCKED] "r" (__ARCH_SPIN_LOCK_LOCKED__) 60 : "memory", "cc"); 61 62 smp_mb(); 63 64 return got_it; 65 } 66 67 static inline void arch_spin_unlock(arch_spinlock_t *lock) 68 { 69 smp_mb(); 70 71 WRITE_ONCE(lock->slock, __ARCH_SPIN_LOCK_UNLOCKED__); 72 } 73 74 /* 75 * Read-write spinlocks, allowing multiple readers but only one writer. 76 * Unfair locking as Writers could be starved indefinitely by Reader(s) 77 */ 78 79 static inline void arch_read_lock(arch_rwlock_t *rw) 80 { 81 unsigned int val; 82 83 /* 84 * zero means writer holds the lock exclusively, deny Reader. 85 * Otherwise grant lock to first/subseq reader 86 * 87 * if (rw->counter > 0) { 88 * rw->counter--; 89 * ret = 1; 90 * } 91 */ 92 93 __asm__ __volatile__( 94 "1: llock %[val], [%[rwlock]] \n" 95 " brls %[val], %[WR_LOCKED], 1b\n" /* <= 0: spin while write locked */ 96 " sub %[val], %[val], 1 \n" /* reader lock */ 97 " scond %[val], [%[rwlock]] \n" 98 " bnz 1b \n" 99 " \n" 100 : [val] "=&r" (val) 101 : [rwlock] "r" (&(rw->counter)), 102 [WR_LOCKED] "ir" (0) 103 : "memory", "cc"); 104 105 smp_mb(); 106 } 107 108 /* 1 - lock taken successfully */ 109 static inline int arch_read_trylock(arch_rwlock_t *rw) 110 { 111 unsigned int val, got_it = 0; 112 113 __asm__ __volatile__( 114 "1: llock %[val], [%[rwlock]] \n" 115 " brls %[val], %[WR_LOCKED], 4f\n" /* <= 0: already write locked, bail */ 116 " sub %[val], %[val], 1 \n" /* counter-- */ 117 " scond %[val], [%[rwlock]] \n" 118 " bnz 1b \n" /* retry if collided with someone */ 119 " mov %[got_it], 1 \n" 120 " \n" 121 "4: ; --- done --- \n" 122 123 : [val] "=&r" (val), 124 [got_it] "+&r" (got_it) 125 : [rwlock] "r" (&(rw->counter)), 126 [WR_LOCKED] "ir" (0) 127 : "memory", "cc"); 128 129 smp_mb(); 130 131 return got_it; 132 } 133 134 static inline void arch_write_lock(arch_rwlock_t *rw) 135 { 136 unsigned int val; 137 138 /* 139 * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__), 140 * deny writer. Otherwise if unlocked grant to writer 141 * Hence the claim that Linux rwlocks are unfair to writers. 142 * (can be starved for an indefinite time by readers). 143 * 144 * if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) { 145 * rw->counter = 0; 146 * ret = 1; 147 * } 148 */ 149 150 __asm__ __volatile__( 151 "1: llock %[val], [%[rwlock]] \n" 152 " brne %[val], %[UNLOCKED], 1b \n" /* while !UNLOCKED spin */ 153 " mov %[val], %[WR_LOCKED] \n" 154 " scond %[val], [%[rwlock]] \n" 155 " bnz 1b \n" 156 " \n" 157 : [val] "=&r" (val) 158 : [rwlock] "r" (&(rw->counter)), 159 [UNLOCKED] "ir" (__ARCH_RW_LOCK_UNLOCKED__), 160 [WR_LOCKED] "ir" (0) 161 : "memory", "cc"); 162 163 smp_mb(); 164 } 165 166 /* 1 - lock taken successfully */ 167 static inline int arch_write_trylock(arch_rwlock_t *rw) 168 { 169 unsigned int val, got_it = 0; 170 171 __asm__ __volatile__( 172 "1: llock %[val], [%[rwlock]] \n" 173 " brne %[val], %[UNLOCKED], 4f \n" /* !UNLOCKED, bail */ 174 " mov %[val], %[WR_LOCKED] \n" 175 " scond %[val], [%[rwlock]] \n" 176 " bnz 1b \n" /* retry if collided with someone */ 177 " mov %[got_it], 1 \n" 178 " \n" 179 "4: ; --- done --- \n" 180 181 : [val] "=&r" (val), 182 [got_it] "+&r" (got_it) 183 : [rwlock] "r" (&(rw->counter)), 184 [UNLOCKED] "ir" (__ARCH_RW_LOCK_UNLOCKED__), 185 [WR_LOCKED] "ir" (0) 186 : "memory", "cc"); 187 188 smp_mb(); 189 190 return got_it; 191 } 192 193 static inline void arch_read_unlock(arch_rwlock_t *rw) 194 { 195 unsigned int val; 196 197 smp_mb(); 198 199 /* 200 * rw->counter++; 201 */ 202 __asm__ __volatile__( 203 "1: llock %[val], [%[rwlock]] \n" 204 " add %[val], %[val], 1 \n" 205 " scond %[val], [%[rwlock]] \n" 206 " bnz 1b \n" 207 " \n" 208 : [val] "=&r" (val) 209 : [rwlock] "r" (&(rw->counter)) 210 : "memory", "cc"); 211 } 212 213 static inline void arch_write_unlock(arch_rwlock_t *rw) 214 { 215 smp_mb(); 216 217 WRITE_ONCE(rw->counter, __ARCH_RW_LOCK_UNLOCKED__); 218 } 219 220 #else /* !CONFIG_ARC_HAS_LLSC */ 221 222 static inline void arch_spin_lock(arch_spinlock_t *lock) 223 { 224 unsigned int val = __ARCH_SPIN_LOCK_LOCKED__; 225 226 /* 227 * Per lkmm, smp_mb() is only required after _lock (and before_unlock) 228 * for ACQ and REL semantics respectively. However EX based spinlocks 229 * need the extra smp_mb to workaround a hardware quirk. 230 */ 231 smp_mb(); 232 233 __asm__ __volatile__( 234 "1: ex %0, [%1] \n" 235 #ifdef CONFIG_EZNPS_MTM_EXT 236 " .word %3 \n" 237 #endif 238 " breq %0, %2, 1b \n" 239 : "+&r" (val) 240 : "r"(&(lock->slock)), "ir"(__ARCH_SPIN_LOCK_LOCKED__) 241 #ifdef CONFIG_EZNPS_MTM_EXT 242 , "i"(CTOP_INST_SCHD_RW) 243 #endif 244 : "memory"); 245 246 smp_mb(); 247 } 248 249 /* 1 - lock taken successfully */ 250 static inline int arch_spin_trylock(arch_spinlock_t *lock) 251 { 252 unsigned int val = __ARCH_SPIN_LOCK_LOCKED__; 253 254 smp_mb(); 255 256 __asm__ __volatile__( 257 "1: ex %0, [%1] \n" 258 : "+r" (val) 259 : "r"(&(lock->slock)) 260 : "memory"); 261 262 smp_mb(); 263 264 return (val == __ARCH_SPIN_LOCK_UNLOCKED__); 265 } 266 267 static inline void arch_spin_unlock(arch_spinlock_t *lock) 268 { 269 unsigned int val = __ARCH_SPIN_LOCK_UNLOCKED__; 270 271 /* 272 * RELEASE barrier: given the instructions avail on ARCv2, full barrier 273 * is the only option 274 */ 275 smp_mb(); 276 277 /* 278 * EX is not really required here, a simple STore of 0 suffices. 279 * However this causes tasklist livelocks in SystemC based SMP virtual 280 * platforms where the systemc core scheduler uses EX as a cue for 281 * moving to next core. Do a git log of this file for details 282 */ 283 __asm__ __volatile__( 284 " ex %0, [%1] \n" 285 : "+r" (val) 286 : "r"(&(lock->slock)) 287 : "memory"); 288 289 /* 290 * see pairing version/comment in arch_spin_lock above 291 */ 292 smp_mb(); 293 } 294 295 /* 296 * Read-write spinlocks, allowing multiple readers but only one writer. 297 * Unfair locking as Writers could be starved indefinitely by Reader(s) 298 * 299 * The spinlock itself is contained in @counter and access to it is 300 * serialized with @lock_mutex. 301 */ 302 303 /* 1 - lock taken successfully */ 304 static inline int arch_read_trylock(arch_rwlock_t *rw) 305 { 306 int ret = 0; 307 unsigned long flags; 308 309 local_irq_save(flags); 310 arch_spin_lock(&(rw->lock_mutex)); 311 312 /* 313 * zero means writer holds the lock exclusively, deny Reader. 314 * Otherwise grant lock to first/subseq reader 315 */ 316 if (rw->counter > 0) { 317 rw->counter--; 318 ret = 1; 319 } 320 321 arch_spin_unlock(&(rw->lock_mutex)); 322 local_irq_restore(flags); 323 324 return ret; 325 } 326 327 /* 1 - lock taken successfully */ 328 static inline int arch_write_trylock(arch_rwlock_t *rw) 329 { 330 int ret = 0; 331 unsigned long flags; 332 333 local_irq_save(flags); 334 arch_spin_lock(&(rw->lock_mutex)); 335 336 /* 337 * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__), 338 * deny writer. Otherwise if unlocked grant to writer 339 * Hence the claim that Linux rwlocks are unfair to writers. 340 * (can be starved for an indefinite time by readers). 341 */ 342 if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) { 343 rw->counter = 0; 344 ret = 1; 345 } 346 arch_spin_unlock(&(rw->lock_mutex)); 347 local_irq_restore(flags); 348 349 return ret; 350 } 351 352 static inline void arch_read_lock(arch_rwlock_t *rw) 353 { 354 while (!arch_read_trylock(rw)) 355 cpu_relax(); 356 } 357 358 static inline void arch_write_lock(arch_rwlock_t *rw) 359 { 360 while (!arch_write_trylock(rw)) 361 cpu_relax(); 362 } 363 364 static inline void arch_read_unlock(arch_rwlock_t *rw) 365 { 366 unsigned long flags; 367 368 local_irq_save(flags); 369 arch_spin_lock(&(rw->lock_mutex)); 370 rw->counter++; 371 arch_spin_unlock(&(rw->lock_mutex)); 372 local_irq_restore(flags); 373 } 374 375 static inline void arch_write_unlock(arch_rwlock_t *rw) 376 { 377 unsigned long flags; 378 379 local_irq_save(flags); 380 arch_spin_lock(&(rw->lock_mutex)); 381 rw->counter = __ARCH_RW_LOCK_UNLOCKED__; 382 arch_spin_unlock(&(rw->lock_mutex)); 383 local_irq_restore(flags); 384 } 385 386 #endif 387 388 #endif /* __ASM_SPINLOCK_H */ 389