1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org) 7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 8 */ 9 #ifndef _ASM_SPINLOCK_H 10 #define _ASM_SPINLOCK_H 11 12 #include <linux/compiler.h> 13 14 #include <asm/barrier.h> 15 #include <asm/compiler.h> 16 #include <asm/war.h> 17 18 /* 19 * Your basic SMP spinlocks, allowing only a single CPU anywhere 20 * 21 * Simple spin lock operations. There are two variants, one clears IRQ's 22 * on the local processor, one does not. 23 * 24 * These are fair FIFO ticket locks 25 * 26 * (the type definitions are in asm/spinlock_types.h) 27 */ 28 29 30 /* 31 * Ticket locks are conceptually two parts, one indicating the current head of 32 * the queue, and the other indicating the current tail. The lock is acquired 33 * by atomically noting the tail and incrementing it by one (thus adding 34 * ourself to the queue and noting our position), then waiting until the head 35 * becomes equal to the the initial value of the tail. 36 */ 37 38 static inline int arch_spin_is_locked(arch_spinlock_t *lock) 39 { 40 u32 counters = ACCESS_ONCE(lock->lock); 41 42 return ((counters >> 16) ^ counters) & 0xffff; 43 } 44 45 #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 46 #define arch_spin_unlock_wait(x) \ 47 while (arch_spin_is_locked(x)) { cpu_relax(); } 48 49 static inline int arch_spin_is_contended(arch_spinlock_t *lock) 50 { 51 u32 counters = ACCESS_ONCE(lock->lock); 52 53 return (((counters >> 16) - counters) & 0xffff) > 1; 54 } 55 #define arch_spin_is_contended arch_spin_is_contended 56 57 static inline void arch_spin_lock(arch_spinlock_t *lock) 58 { 59 int my_ticket; 60 int tmp; 61 int inc = 0x10000; 62 63 if (R10000_LLSC_WAR) { 64 __asm__ __volatile__ ( 65 " .set push # arch_spin_lock \n" 66 " .set noreorder \n" 67 " \n" 68 "1: ll %[ticket], %[ticket_ptr] \n" 69 " addu %[my_ticket], %[ticket], %[inc] \n" 70 " sc %[my_ticket], %[ticket_ptr] \n" 71 " beqzl %[my_ticket], 1b \n" 72 " nop \n" 73 " srl %[my_ticket], %[ticket], 16 \n" 74 " andi %[ticket], %[ticket], 0xffff \n" 75 " bne %[ticket], %[my_ticket], 4f \n" 76 " subu %[ticket], %[my_ticket], %[ticket] \n" 77 "2: \n" 78 " .subsection 2 \n" 79 "4: andi %[ticket], %[ticket], 0xffff \n" 80 " sll %[ticket], 5 \n" 81 " \n" 82 "6: bnez %[ticket], 6b \n" 83 " subu %[ticket], 1 \n" 84 " \n" 85 " lhu %[ticket], %[serving_now_ptr] \n" 86 " beq %[ticket], %[my_ticket], 2b \n" 87 " subu %[ticket], %[my_ticket], %[ticket] \n" 88 " b 4b \n" 89 " subu %[ticket], %[ticket], 1 \n" 90 " .previous \n" 91 " .set pop \n" 92 : [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock), 93 [serving_now_ptr] "+m" (lock->h.serving_now), 94 [ticket] "=&r" (tmp), 95 [my_ticket] "=&r" (my_ticket) 96 : [inc] "r" (inc)); 97 } else { 98 __asm__ __volatile__ ( 99 " .set push # arch_spin_lock \n" 100 " .set noreorder \n" 101 " \n" 102 "1: ll %[ticket], %[ticket_ptr] \n" 103 " addu %[my_ticket], %[ticket], %[inc] \n" 104 " sc %[my_ticket], %[ticket_ptr] \n" 105 " beqz %[my_ticket], 1b \n" 106 " srl %[my_ticket], %[ticket], 16 \n" 107 " andi %[ticket], %[ticket], 0xffff \n" 108 " bne %[ticket], %[my_ticket], 4f \n" 109 " subu %[ticket], %[my_ticket], %[ticket] \n" 110 "2: \n" 111 " .subsection 2 \n" 112 "4: andi %[ticket], %[ticket], 0x1fff \n" 113 " sll %[ticket], 5 \n" 114 " \n" 115 "6: bnez %[ticket], 6b \n" 116 " subu %[ticket], 1 \n" 117 " \n" 118 " lhu %[ticket], %[serving_now_ptr] \n" 119 " beq %[ticket], %[my_ticket], 2b \n" 120 " subu %[ticket], %[my_ticket], %[ticket] \n" 121 " b 4b \n" 122 " subu %[ticket], %[ticket], 1 \n" 123 " .previous \n" 124 " .set pop \n" 125 : [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock), 126 [serving_now_ptr] "+m" (lock->h.serving_now), 127 [ticket] "=&r" (tmp), 128 [my_ticket] "=&r" (my_ticket) 129 : [inc] "r" (inc)); 130 } 131 132 smp_llsc_mb(); 133 } 134 135 static inline void arch_spin_unlock(arch_spinlock_t *lock) 136 { 137 unsigned int serving_now = lock->h.serving_now + 1; 138 wmb(); 139 lock->h.serving_now = (u16)serving_now; 140 nudge_writes(); 141 } 142 143 static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock) 144 { 145 int tmp, tmp2, tmp3; 146 int inc = 0x10000; 147 148 if (R10000_LLSC_WAR) { 149 __asm__ __volatile__ ( 150 " .set push # arch_spin_trylock \n" 151 " .set noreorder \n" 152 " \n" 153 "1: ll %[ticket], %[ticket_ptr] \n" 154 " srl %[my_ticket], %[ticket], 16 \n" 155 " andi %[now_serving], %[ticket], 0xffff \n" 156 " bne %[my_ticket], %[now_serving], 3f \n" 157 " addu %[ticket], %[ticket], %[inc] \n" 158 " sc %[ticket], %[ticket_ptr] \n" 159 " beqzl %[ticket], 1b \n" 160 " li %[ticket], 1 \n" 161 "2: \n" 162 " .subsection 2 \n" 163 "3: b 2b \n" 164 " li %[ticket], 0 \n" 165 " .previous \n" 166 " .set pop \n" 167 : [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock), 168 [ticket] "=&r" (tmp), 169 [my_ticket] "=&r" (tmp2), 170 [now_serving] "=&r" (tmp3) 171 : [inc] "r" (inc)); 172 } else { 173 __asm__ __volatile__ ( 174 " .set push # arch_spin_trylock \n" 175 " .set noreorder \n" 176 " \n" 177 "1: ll %[ticket], %[ticket_ptr] \n" 178 " srl %[my_ticket], %[ticket], 16 \n" 179 " andi %[now_serving], %[ticket], 0xffff \n" 180 " bne %[my_ticket], %[now_serving], 3f \n" 181 " addu %[ticket], %[ticket], %[inc] \n" 182 " sc %[ticket], %[ticket_ptr] \n" 183 " beqz %[ticket], 1b \n" 184 " li %[ticket], 1 \n" 185 "2: \n" 186 " .subsection 2 \n" 187 "3: b 2b \n" 188 " li %[ticket], 0 \n" 189 " .previous \n" 190 " .set pop \n" 191 : [ticket_ptr] "+" GCC_OFF12_ASM() (lock->lock), 192 [ticket] "=&r" (tmp), 193 [my_ticket] "=&r" (tmp2), 194 [now_serving] "=&r" (tmp3) 195 : [inc] "r" (inc)); 196 } 197 198 smp_llsc_mb(); 199 200 return tmp; 201 } 202 203 /* 204 * Read-write spinlocks, allowing multiple readers but only one writer. 205 * 206 * NOTE! it is quite common to have readers in interrupts but no interrupt 207 * writers. For those circumstances we can "mix" irq-safe locks - any writer 208 * needs to get a irq-safe write-lock, but readers can get non-irqsafe 209 * read-locks. 210 */ 211 212 /* 213 * read_can_lock - would read_trylock() succeed? 214 * @lock: the rwlock in question. 215 */ 216 #define arch_read_can_lock(rw) ((rw)->lock >= 0) 217 218 /* 219 * write_can_lock - would write_trylock() succeed? 220 * @lock: the rwlock in question. 221 */ 222 #define arch_write_can_lock(rw) (!(rw)->lock) 223 224 static inline void arch_read_lock(arch_rwlock_t *rw) 225 { 226 unsigned int tmp; 227 228 if (R10000_LLSC_WAR) { 229 __asm__ __volatile__( 230 " .set noreorder # arch_read_lock \n" 231 "1: ll %1, %2 \n" 232 " bltz %1, 1b \n" 233 " addu %1, 1 \n" 234 " sc %1, %0 \n" 235 " beqzl %1, 1b \n" 236 " nop \n" 237 " .set reorder \n" 238 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp) 239 : GCC_OFF12_ASM() (rw->lock) 240 : "memory"); 241 } else { 242 do { 243 __asm__ __volatile__( 244 "1: ll %1, %2 # arch_read_lock \n" 245 " bltz %1, 1b \n" 246 " addu %1, 1 \n" 247 "2: sc %1, %0 \n" 248 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp) 249 : GCC_OFF12_ASM() (rw->lock) 250 : "memory"); 251 } while (unlikely(!tmp)); 252 } 253 254 smp_llsc_mb(); 255 } 256 257 /* Note the use of sub, not subu which will make the kernel die with an 258 overflow exception if we ever try to unlock an rwlock that is already 259 unlocked or is being held by a writer. */ 260 static inline void arch_read_unlock(arch_rwlock_t *rw) 261 { 262 unsigned int tmp; 263 264 smp_mb__before_llsc(); 265 266 if (R10000_LLSC_WAR) { 267 __asm__ __volatile__( 268 "1: ll %1, %2 # arch_read_unlock \n" 269 " sub %1, 1 \n" 270 " sc %1, %0 \n" 271 " beqzl %1, 1b \n" 272 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp) 273 : GCC_OFF12_ASM() (rw->lock) 274 : "memory"); 275 } else { 276 do { 277 __asm__ __volatile__( 278 "1: ll %1, %2 # arch_read_unlock \n" 279 " sub %1, 1 \n" 280 " sc %1, %0 \n" 281 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp) 282 : GCC_OFF12_ASM() (rw->lock) 283 : "memory"); 284 } while (unlikely(!tmp)); 285 } 286 } 287 288 static inline void arch_write_lock(arch_rwlock_t *rw) 289 { 290 unsigned int tmp; 291 292 if (R10000_LLSC_WAR) { 293 __asm__ __volatile__( 294 " .set noreorder # arch_write_lock \n" 295 "1: ll %1, %2 \n" 296 " bnez %1, 1b \n" 297 " lui %1, 0x8000 \n" 298 " sc %1, %0 \n" 299 " beqzl %1, 1b \n" 300 " nop \n" 301 " .set reorder \n" 302 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp) 303 : GCC_OFF12_ASM() (rw->lock) 304 : "memory"); 305 } else { 306 do { 307 __asm__ __volatile__( 308 "1: ll %1, %2 # arch_write_lock \n" 309 " bnez %1, 1b \n" 310 " lui %1, 0x8000 \n" 311 "2: sc %1, %0 \n" 312 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp) 313 : GCC_OFF12_ASM() (rw->lock) 314 : "memory"); 315 } while (unlikely(!tmp)); 316 } 317 318 smp_llsc_mb(); 319 } 320 321 static inline void arch_write_unlock(arch_rwlock_t *rw) 322 { 323 smp_mb(); 324 325 __asm__ __volatile__( 326 " # arch_write_unlock \n" 327 " sw $0, %0 \n" 328 : "=m" (rw->lock) 329 : "m" (rw->lock) 330 : "memory"); 331 } 332 333 static inline int arch_read_trylock(arch_rwlock_t *rw) 334 { 335 unsigned int tmp; 336 int ret; 337 338 if (R10000_LLSC_WAR) { 339 __asm__ __volatile__( 340 " .set noreorder # arch_read_trylock \n" 341 " li %2, 0 \n" 342 "1: ll %1, %3 \n" 343 " bltz %1, 2f \n" 344 " addu %1, 1 \n" 345 " sc %1, %0 \n" 346 " .set reorder \n" 347 " beqzl %1, 1b \n" 348 " nop \n" 349 __WEAK_LLSC_MB 350 " li %2, 1 \n" 351 "2: \n" 352 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret) 353 : GCC_OFF12_ASM() (rw->lock) 354 : "memory"); 355 } else { 356 __asm__ __volatile__( 357 " .set noreorder # arch_read_trylock \n" 358 " li %2, 0 \n" 359 "1: ll %1, %3 \n" 360 " bltz %1, 2f \n" 361 " addu %1, 1 \n" 362 " sc %1, %0 \n" 363 " beqz %1, 1b \n" 364 " nop \n" 365 " .set reorder \n" 366 __WEAK_LLSC_MB 367 " li %2, 1 \n" 368 "2: \n" 369 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret) 370 : GCC_OFF12_ASM() (rw->lock) 371 : "memory"); 372 } 373 374 return ret; 375 } 376 377 static inline int arch_write_trylock(arch_rwlock_t *rw) 378 { 379 unsigned int tmp; 380 int ret; 381 382 if (R10000_LLSC_WAR) { 383 __asm__ __volatile__( 384 " .set noreorder # arch_write_trylock \n" 385 " li %2, 0 \n" 386 "1: ll %1, %3 \n" 387 " bnez %1, 2f \n" 388 " lui %1, 0x8000 \n" 389 " sc %1, %0 \n" 390 " beqzl %1, 1b \n" 391 " nop \n" 392 __WEAK_LLSC_MB 393 " li %2, 1 \n" 394 " .set reorder \n" 395 "2: \n" 396 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret) 397 : GCC_OFF12_ASM() (rw->lock) 398 : "memory"); 399 } else { 400 do { 401 __asm__ __volatile__( 402 " ll %1, %3 # arch_write_trylock \n" 403 " li %2, 0 \n" 404 " bnez %1, 2f \n" 405 " lui %1, 0x8000 \n" 406 " sc %1, %0 \n" 407 " li %2, 1 \n" 408 "2: \n" 409 : "=" GCC_OFF12_ASM() (rw->lock), "=&r" (tmp), 410 "=&r" (ret) 411 : GCC_OFF12_ASM() (rw->lock) 412 : "memory"); 413 } while (unlikely(!tmp)); 414 415 smp_llsc_mb(); 416 } 417 418 return ret; 419 } 420 421 #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 422 #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 423 424 #define arch_spin_relax(lock) cpu_relax() 425 #define arch_read_relax(lock) cpu_relax() 426 #define arch_write_relax(lock) cpu_relax() 427 428 #endif /* _ASM_SPINLOCK_H */ 429