11965aae3SH. Peter Anvin #ifndef _ASM_X86_SPINLOCK_H 21965aae3SH. Peter Anvin #define _ASM_X86_SPINLOCK_H 3bb898558SAl Viro 460063497SArun Sharma #include <linux/atomic.h> 5bb898558SAl Viro #include <asm/page.h> 6bb898558SAl Viro #include <asm/processor.h> 7bb898558SAl Viro #include <linux/compiler.h> 8bb898558SAl Viro #include <asm/paravirt.h> 9bb898558SAl Viro /* 10bb898558SAl Viro * Your basic SMP spinlocks, allowing only a single CPU anywhere 11bb898558SAl Viro * 12bb898558SAl Viro * Simple spin lock operations. There are two variants, one clears IRQ's 13bb898558SAl Viro * on the local processor, one does not. 14bb898558SAl Viro * 15bb898558SAl Viro * These are fair FIFO ticket locks, which are currently limited to 256 16bb898558SAl Viro * CPUs. 17bb898558SAl Viro * 18bb898558SAl Viro * (the type definitions are in asm/spinlock_types.h) 19bb898558SAl Viro */ 20bb898558SAl Viro 21bb898558SAl Viro #ifdef CONFIG_X86_32 22bb898558SAl Viro # define LOCK_PTR_REG "a" 23bb898558SAl Viro # define REG_PTR_MODE "k" 24bb898558SAl Viro #else 25bb898558SAl Viro # define LOCK_PTR_REG "D" 26bb898558SAl Viro # define REG_PTR_MODE "q" 27bb898558SAl Viro #endif 28bb898558SAl Viro 29bb898558SAl Viro #if defined(CONFIG_X86_32) && \ 30bb898558SAl Viro (defined(CONFIG_X86_OOSTORE) || defined(CONFIG_X86_PPRO_FENCE)) 31bb898558SAl Viro /* 32bb898558SAl Viro * On PPro SMP or if we are using OOSTORE, we use a locked operation to unlock 33bb898558SAl Viro * (PPro errata 66, 92) 34bb898558SAl Viro */ 35bb898558SAl Viro # define UNLOCK_LOCK_PREFIX LOCK_PREFIX 36bb898558SAl Viro #else 37bb898558SAl Viro # define UNLOCK_LOCK_PREFIX 38bb898558SAl Viro #endif 39bb898558SAl Viro 40bb898558SAl Viro /* 41bb898558SAl Viro * Ticket locks are conceptually two parts, one indicating the current head of 42bb898558SAl Viro * the queue, and the other indicating the current tail. The lock is acquired 43bb898558SAl Viro * by atomically noting the tail and incrementing it by one (thus adding 44bb898558SAl Viro * ourself to the queue and noting our position), then waiting until the head 45bb898558SAl Viro * becomes equal to the the initial value of the tail. 46bb898558SAl Viro * 47bb898558SAl Viro * We use an xadd covering *both* parts of the lock, to increment the tail and 48bb898558SAl Viro * also load the position of the head, which takes care of memory ordering 49bb898558SAl Viro * issues and should be optimal for the uncontended case. Note the tail must be 50bb898558SAl Viro * in the high part, because a wide xadd increment of the low part would carry 51bb898558SAl Viro * up and contaminate the high part. 52bb898558SAl Viro * 53bb898558SAl Viro * With fewer than 2^8 possible CPUs, we can use x86's partial registers to 54bb898558SAl Viro * save some instructions and make the code more elegant. There really isn't 55bb898558SAl Viro * much between them in performance though, especially as locks are out of line. 56bb898558SAl Viro */ 57445c8951SThomas Gleixner static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock) 58bb898558SAl Viro { 592994488fSJeremy Fitzhardinge register struct __raw_tickets inc = { .tail = 1 }; 60bb898558SAl Viro 612994488fSJeremy Fitzhardinge inc = xadd(&lock->tickets, inc); 62c576a3eaSJeremy Fitzhardinge 63c576a3eaSJeremy Fitzhardinge for (;;) { 642994488fSJeremy Fitzhardinge if (inc.head == inc.tail) 65c576a3eaSJeremy Fitzhardinge break; 66c576a3eaSJeremy Fitzhardinge cpu_relax(); 672994488fSJeremy Fitzhardinge inc.head = ACCESS_ONCE(lock->tickets.head); 68c576a3eaSJeremy Fitzhardinge } 69c576a3eaSJeremy Fitzhardinge barrier(); /* make sure nothing creeps before the lock is taken */ 70bb898558SAl Viro } 71bb898558SAl Viro 72445c8951SThomas Gleixner static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock) 73bb898558SAl Viro { 74229855d6SJeremy Fitzhardinge arch_spinlock_t old, new; 75bb898558SAl Viro 76229855d6SJeremy Fitzhardinge old.tickets = ACCESS_ONCE(lock->tickets); 77229855d6SJeremy Fitzhardinge if (old.tickets.head != old.tickets.tail) 78229855d6SJeremy Fitzhardinge return 0; 79bb898558SAl Viro 80229855d6SJeremy Fitzhardinge new.head_tail = old.head_tail + (1 << TICKET_SHIFT); 81229855d6SJeremy Fitzhardinge 82229855d6SJeremy Fitzhardinge /* cmpxchg is a full barrier, so nothing can move before it */ 83229855d6SJeremy Fitzhardinge return cmpxchg(&lock->head_tail, old.head_tail, new.head_tail) == old.head_tail; 84bb898558SAl Viro } 85bb898558SAl Viro 86229855d6SJeremy Fitzhardinge #if (NR_CPUS < 256) 87445c8951SThomas Gleixner static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock) 88bb898558SAl Viro { 89bb898558SAl Viro asm volatile(UNLOCK_LOCK_PREFIX "incb %0" 90229855d6SJeremy Fitzhardinge : "+m" (lock->head_tail) 91bb898558SAl Viro : 92bb898558SAl Viro : "memory", "cc"); 93bb898558SAl Viro } 94bb898558SAl Viro #else 95445c8951SThomas Gleixner static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock) 96bb898558SAl Viro { 97bb898558SAl Viro asm volatile(UNLOCK_LOCK_PREFIX "incw %0" 98229855d6SJeremy Fitzhardinge : "+m" (lock->head_tail) 99bb898558SAl Viro : 100bb898558SAl Viro : "memory", "cc"); 101bb898558SAl Viro } 102bb898558SAl Viro #endif 103bb898558SAl Viro 104445c8951SThomas Gleixner static inline int __ticket_spin_is_locked(arch_spinlock_t *lock) 105bb898558SAl Viro { 10684eb950dSJeremy Fitzhardinge struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets); 107bb898558SAl Viro 10884eb950dSJeremy Fitzhardinge return !!(tmp.tail ^ tmp.head); 109bb898558SAl Viro } 110bb898558SAl Viro 111445c8951SThomas Gleixner static inline int __ticket_spin_is_contended(arch_spinlock_t *lock) 112bb898558SAl Viro { 11384eb950dSJeremy Fitzhardinge struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets); 114bb898558SAl Viro 11584eb950dSJeremy Fitzhardinge return ((tmp.tail - tmp.head) & TICKET_MASK) > 1; 116bb898558SAl Viro } 117bb898558SAl Viro 118b4ecc126SJeremy Fitzhardinge #ifndef CONFIG_PARAVIRT_SPINLOCKS 119bb898558SAl Viro 1200199c4e6SThomas Gleixner static inline int arch_spin_is_locked(arch_spinlock_t *lock) 121bb898558SAl Viro { 122bb898558SAl Viro return __ticket_spin_is_locked(lock); 123bb898558SAl Viro } 124bb898558SAl Viro 1250199c4e6SThomas Gleixner static inline int arch_spin_is_contended(arch_spinlock_t *lock) 126bb898558SAl Viro { 127bb898558SAl Viro return __ticket_spin_is_contended(lock); 128bb898558SAl Viro } 1290199c4e6SThomas Gleixner #define arch_spin_is_contended arch_spin_is_contended 130bb898558SAl Viro 1310199c4e6SThomas Gleixner static __always_inline void arch_spin_lock(arch_spinlock_t *lock) 132bb898558SAl Viro { 133bb898558SAl Viro __ticket_spin_lock(lock); 134bb898558SAl Viro } 135bb898558SAl Viro 1360199c4e6SThomas Gleixner static __always_inline int arch_spin_trylock(arch_spinlock_t *lock) 137bb898558SAl Viro { 138bb898558SAl Viro return __ticket_spin_trylock(lock); 139bb898558SAl Viro } 140bb898558SAl Viro 1410199c4e6SThomas Gleixner static __always_inline void arch_spin_unlock(arch_spinlock_t *lock) 142bb898558SAl Viro { 143bb898558SAl Viro __ticket_spin_unlock(lock); 144bb898558SAl Viro } 145bb898558SAl Viro 1460199c4e6SThomas Gleixner static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock, 147bb898558SAl Viro unsigned long flags) 148bb898558SAl Viro { 1490199c4e6SThomas Gleixner arch_spin_lock(lock); 150bb898558SAl Viro } 151bb898558SAl Viro 152b4ecc126SJeremy Fitzhardinge #endif /* CONFIG_PARAVIRT_SPINLOCKS */ 153bb898558SAl Viro 1540199c4e6SThomas Gleixner static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) 155bb898558SAl Viro { 1560199c4e6SThomas Gleixner while (arch_spin_is_locked(lock)) 157bb898558SAl Viro cpu_relax(); 158bb898558SAl Viro } 159bb898558SAl Viro 160bb898558SAl Viro /* 161bb898558SAl Viro * Read-write spinlocks, allowing multiple readers 162bb898558SAl Viro * but only one writer. 163bb898558SAl Viro * 164bb898558SAl Viro * NOTE! it is quite common to have readers in interrupts 165bb898558SAl Viro * but no interrupt writers. For those circumstances we 166bb898558SAl Viro * can "mix" irq-safe locks - any writer needs to get a 167bb898558SAl Viro * irq-safe write-lock, but readers can get non-irqsafe 168bb898558SAl Viro * read-locks. 169bb898558SAl Viro * 170bb898558SAl Viro * On x86, we implement read-write locks as a 32-bit counter 171bb898558SAl Viro * with the high bit (sign) being the "contended" bit. 172bb898558SAl Viro */ 173bb898558SAl Viro 174bb898558SAl Viro /** 175bb898558SAl Viro * read_can_lock - would read_trylock() succeed? 176bb898558SAl Viro * @lock: the rwlock in question. 177bb898558SAl Viro */ 178e5931943SThomas Gleixner static inline int arch_read_can_lock(arch_rwlock_t *lock) 179bb898558SAl Viro { 180a750036fSJan Beulich return lock->lock > 0; 181bb898558SAl Viro } 182bb898558SAl Viro 183bb898558SAl Viro /** 184bb898558SAl Viro * write_can_lock - would write_trylock() succeed? 185bb898558SAl Viro * @lock: the rwlock in question. 186bb898558SAl Viro */ 187e5931943SThomas Gleixner static inline int arch_write_can_lock(arch_rwlock_t *lock) 188bb898558SAl Viro { 189a750036fSJan Beulich return lock->write == WRITE_LOCK_CMP; 190bb898558SAl Viro } 191bb898558SAl Viro 192e5931943SThomas Gleixner static inline void arch_read_lock(arch_rwlock_t *rw) 193bb898558SAl Viro { 194a750036fSJan Beulich asm volatile(LOCK_PREFIX READ_LOCK_SIZE(dec) " (%0)\n\t" 195bb898558SAl Viro "jns 1f\n" 196bb898558SAl Viro "call __read_lock_failed\n\t" 197bb898558SAl Viro "1:\n" 198bb898558SAl Viro ::LOCK_PTR_REG (rw) : "memory"); 199bb898558SAl Viro } 200bb898558SAl Viro 201e5931943SThomas Gleixner static inline void arch_write_lock(arch_rwlock_t *rw) 202bb898558SAl Viro { 203a750036fSJan Beulich asm volatile(LOCK_PREFIX WRITE_LOCK_SUB(%1) "(%0)\n\t" 204bb898558SAl Viro "jz 1f\n" 205bb898558SAl Viro "call __write_lock_failed\n\t" 206bb898558SAl Viro "1:\n" 207a750036fSJan Beulich ::LOCK_PTR_REG (&rw->write), "i" (RW_LOCK_BIAS) 208a750036fSJan Beulich : "memory"); 209bb898558SAl Viro } 210bb898558SAl Viro 211e5931943SThomas Gleixner static inline int arch_read_trylock(arch_rwlock_t *lock) 212bb898558SAl Viro { 213a750036fSJan Beulich READ_LOCK_ATOMIC(t) *count = (READ_LOCK_ATOMIC(t) *)lock; 214bb898558SAl Viro 215a750036fSJan Beulich if (READ_LOCK_ATOMIC(dec_return)(count) >= 0) 216bb898558SAl Viro return 1; 217a750036fSJan Beulich READ_LOCK_ATOMIC(inc)(count); 218bb898558SAl Viro return 0; 219bb898558SAl Viro } 220bb898558SAl Viro 221e5931943SThomas Gleixner static inline int arch_write_trylock(arch_rwlock_t *lock) 222bb898558SAl Viro { 223a750036fSJan Beulich atomic_t *count = (atomic_t *)&lock->write; 224bb898558SAl Viro 225a750036fSJan Beulich if (atomic_sub_and_test(WRITE_LOCK_CMP, count)) 226bb898558SAl Viro return 1; 227a750036fSJan Beulich atomic_add(WRITE_LOCK_CMP, count); 228bb898558SAl Viro return 0; 229bb898558SAl Viro } 230bb898558SAl Viro 231e5931943SThomas Gleixner static inline void arch_read_unlock(arch_rwlock_t *rw) 232bb898558SAl Viro { 233a750036fSJan Beulich asm volatile(LOCK_PREFIX READ_LOCK_SIZE(inc) " %0" 234a750036fSJan Beulich :"+m" (rw->lock) : : "memory"); 235bb898558SAl Viro } 236bb898558SAl Viro 237e5931943SThomas Gleixner static inline void arch_write_unlock(arch_rwlock_t *rw) 238bb898558SAl Viro { 239a750036fSJan Beulich asm volatile(LOCK_PREFIX WRITE_LOCK_ADD(%1) "%0" 240a750036fSJan Beulich : "+m" (rw->write) : "i" (RW_LOCK_BIAS) : "memory"); 241bb898558SAl Viro } 242bb898558SAl Viro 243e5931943SThomas Gleixner #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 244e5931943SThomas Gleixner #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 245f5f7eac4SRobin Holt 246a750036fSJan Beulich #undef READ_LOCK_SIZE 247a750036fSJan Beulich #undef READ_LOCK_ATOMIC 248a750036fSJan Beulich #undef WRITE_LOCK_ADD 249a750036fSJan Beulich #undef WRITE_LOCK_SUB 250a750036fSJan Beulich #undef WRITE_LOCK_CMP 251a750036fSJan Beulich 2520199c4e6SThomas Gleixner #define arch_spin_relax(lock) cpu_relax() 2530199c4e6SThomas Gleixner #define arch_read_relax(lock) cpu_relax() 2540199c4e6SThomas Gleixner #define arch_write_relax(lock) cpu_relax() 255bb898558SAl Viro 256ad462769SJiri Olsa /* The {read|write|spin}_lock() on x86 are full memory barriers. */ 257ad462769SJiri Olsa static inline void smp_mb__after_lock(void) { } 258ad462769SJiri Olsa #define ARCH_HAS_SMP_MB_AFTER_LOCK 259ad462769SJiri Olsa 2601965aae3SH. Peter Anvin #endif /* _ASM_X86_SPINLOCK_H */ 261