1c6557e7fSMartin Schwidefsky /* 2c6557e7fSMartin Schwidefsky * S390 version 3a53c8fabSHeiko Carstens * Copyright IBM Corp. 1999 4c6557e7fSMartin Schwidefsky * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) 5c6557e7fSMartin Schwidefsky * 6c6557e7fSMartin Schwidefsky * Derived from "include/asm-i386/spinlock.h" 7c6557e7fSMartin Schwidefsky */ 8c6557e7fSMartin Schwidefsky 9c6557e7fSMartin Schwidefsky #ifndef __ASM_SPINLOCK_H 10c6557e7fSMartin Schwidefsky #define __ASM_SPINLOCK_H 11c6557e7fSMartin Schwidefsky 12c6557e7fSMartin Schwidefsky #include <linux/smp.h> 1302c503ffSMartin Schwidefsky #include <asm/atomic_ops.h> 14726328d9SPeter Zijlstra #include <asm/barrier.h> 15726328d9SPeter Zijlstra #include <asm/processor.h> 16c6557e7fSMartin Schwidefsky 176c8cd5bbSPhilipp Hachtmann #define SPINLOCK_LOCKVAL (S390_lowcore.spinlock_lockval) 186c8cd5bbSPhilipp Hachtmann 19638ad34aSMartin Schwidefsky extern int spin_retry; 20638ad34aSMartin Schwidefsky 21760928c0SChristian Borntraeger #ifndef CONFIG_SMP 22760928c0SChristian Borntraeger static inline bool arch_vcpu_is_preempted(int cpu) { return false; } 23760928c0SChristian Borntraeger #else 24760928c0SChristian Borntraeger bool arch_vcpu_is_preempted(int cpu); 25760928c0SChristian Borntraeger #endif 26760928c0SChristian Borntraeger 27760928c0SChristian Borntraeger #define vcpu_is_preempted arch_vcpu_is_preempted 28760928c0SChristian Borntraeger 29c6557e7fSMartin Schwidefsky /* 30c6557e7fSMartin Schwidefsky * Simple spin lock operations. There are two variants, one clears IRQ's 31c6557e7fSMartin Schwidefsky * on the local processor, one does not. 32c6557e7fSMartin Schwidefsky * 33c6557e7fSMartin Schwidefsky * We make no fairness assumptions. They have a cost. 34c6557e7fSMartin Schwidefsky * 35c6557e7fSMartin Schwidefsky * (the type definitions are in asm/spinlock_types.h) 36c6557e7fSMartin Schwidefsky */ 37c6557e7fSMartin Schwidefsky 3802c503ffSMartin Schwidefsky void arch_lock_relax(int cpu); 39d59b93daSMartin Schwidefsky 405b3f683eSPhilipp Hachtmann void arch_spin_lock_wait(arch_spinlock_t *); 415b3f683eSPhilipp Hachtmann int arch_spin_trylock_retry(arch_spinlock_t *); 425b3f683eSPhilipp Hachtmann void arch_spin_lock_wait_flags(arch_spinlock_t *, unsigned long flags); 43c6557e7fSMartin Schwidefsky 44d59b93daSMartin Schwidefsky static inline void arch_spin_relax(arch_spinlock_t *lock) 45d59b93daSMartin Schwidefsky { 46d59b93daSMartin Schwidefsky arch_lock_relax(lock->lock); 47d59b93daSMartin Schwidefsky } 48d59b93daSMartin Schwidefsky 496c8cd5bbSPhilipp Hachtmann static inline u32 arch_spin_lockval(int cpu) 506c8cd5bbSPhilipp Hachtmann { 5181533803SMartin Schwidefsky return cpu + 1; 526c8cd5bbSPhilipp Hachtmann } 536c8cd5bbSPhilipp Hachtmann 54efc1d23bSHeiko Carstens static inline int arch_spin_value_unlocked(arch_spinlock_t lock) 55efc1d23bSHeiko Carstens { 565b3f683eSPhilipp Hachtmann return lock.lock == 0; 575b3f683eSPhilipp Hachtmann } 585b3f683eSPhilipp Hachtmann 595b3f683eSPhilipp Hachtmann static inline int arch_spin_is_locked(arch_spinlock_t *lp) 605b3f683eSPhilipp Hachtmann { 61187b5f41SChristian Borntraeger return READ_ONCE(lp->lock) != 0; 625b3f683eSPhilipp Hachtmann } 635b3f683eSPhilipp Hachtmann 645b3f683eSPhilipp Hachtmann static inline int arch_spin_trylock_once(arch_spinlock_t *lp) 655b3f683eSPhilipp Hachtmann { 66bae8f567SMartin Schwidefsky barrier(); 67bae8f567SMartin Schwidefsky return likely(arch_spin_value_unlocked(*lp) && 6802c503ffSMartin Schwidefsky __atomic_cmpxchg_bool(&lp->lock, 0, SPINLOCK_LOCKVAL)); 695b3f683eSPhilipp Hachtmann } 705b3f683eSPhilipp Hachtmann 710199c4e6SThomas Gleixner static inline void arch_spin_lock(arch_spinlock_t *lp) 72c6557e7fSMartin Schwidefsky { 73bae8f567SMartin Schwidefsky if (!arch_spin_trylock_once(lp)) 740199c4e6SThomas Gleixner arch_spin_lock_wait(lp); 75c6557e7fSMartin Schwidefsky } 76c6557e7fSMartin Schwidefsky 770199c4e6SThomas Gleixner static inline void arch_spin_lock_flags(arch_spinlock_t *lp, 78c6557e7fSMartin Schwidefsky unsigned long flags) 79c6557e7fSMartin Schwidefsky { 80bae8f567SMartin Schwidefsky if (!arch_spin_trylock_once(lp)) 810199c4e6SThomas Gleixner arch_spin_lock_wait_flags(lp, flags); 82c6557e7fSMartin Schwidefsky } 83c6557e7fSMartin Schwidefsky 840199c4e6SThomas Gleixner static inline int arch_spin_trylock(arch_spinlock_t *lp) 85c6557e7fSMartin Schwidefsky { 86bae8f567SMartin Schwidefsky if (!arch_spin_trylock_once(lp)) 870199c4e6SThomas Gleixner return arch_spin_trylock_retry(lp); 885b3f683eSPhilipp Hachtmann return 1; 89c6557e7fSMartin Schwidefsky } 90c6557e7fSMartin Schwidefsky 910199c4e6SThomas Gleixner static inline void arch_spin_unlock(arch_spinlock_t *lp) 92c6557e7fSMartin Schwidefsky { 9302c503ffSMartin Schwidefsky typecheck(int, lp->lock); 9444230282SHeiko Carstens asm volatile( 957f7e6e28SMartin Schwidefsky #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES 967f7e6e28SMartin Schwidefsky " .long 0xb2fa0070\n" /* NIAI 7 */ 977f7e6e28SMartin Schwidefsky #endif 9844230282SHeiko Carstens " st %1,%0\n" 997f7e6e28SMartin Schwidefsky : "=Q" (lp->lock) : "d" (0) : "cc", "memory"); 1005b3f683eSPhilipp Hachtmann } 1015b3f683eSPhilipp Hachtmann 102c6557e7fSMartin Schwidefsky /* 103c6557e7fSMartin Schwidefsky * Read-write spinlocks, allowing multiple readers 104c6557e7fSMartin Schwidefsky * but only one writer. 105c6557e7fSMartin Schwidefsky * 106c6557e7fSMartin Schwidefsky * NOTE! it is quite common to have readers in interrupts 107c6557e7fSMartin Schwidefsky * but no interrupt writers. For those circumstances we 108c6557e7fSMartin Schwidefsky * can "mix" irq-safe locks - any writer needs to get a 109c6557e7fSMartin Schwidefsky * irq-safe write-lock, but readers can get non-irqsafe 110c6557e7fSMartin Schwidefsky * read-locks. 111c6557e7fSMartin Schwidefsky */ 112c6557e7fSMartin Schwidefsky 113c6557e7fSMartin Schwidefsky /** 114c6557e7fSMartin Schwidefsky * read_can_lock - would read_trylock() succeed? 115c6557e7fSMartin Schwidefsky * @lock: the rwlock in question. 116c6557e7fSMartin Schwidefsky */ 117e5931943SThomas Gleixner #define arch_read_can_lock(x) ((int)(x)->lock >= 0) 118c6557e7fSMartin Schwidefsky 119c6557e7fSMartin Schwidefsky /** 120c6557e7fSMartin Schwidefsky * write_can_lock - would write_trylock() succeed? 121c6557e7fSMartin Schwidefsky * @lock: the rwlock in question. 122c6557e7fSMartin Schwidefsky */ 123e5931943SThomas Gleixner #define arch_write_can_lock(x) ((x)->lock == 0) 124c6557e7fSMartin Schwidefsky 1252684e73aSMartin Schwidefsky extern int _raw_read_trylock_retry(arch_rwlock_t *lp); 126fb3a6bbcSThomas Gleixner extern int _raw_write_trylock_retry(arch_rwlock_t *lp); 127c6557e7fSMartin Schwidefsky 1282684e73aSMartin Schwidefsky #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 1292684e73aSMartin Schwidefsky #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 1302684e73aSMartin Schwidefsky 131bae8f567SMartin Schwidefsky static inline int arch_read_trylock_once(arch_rwlock_t *rw) 132bae8f567SMartin Schwidefsky { 13302c503ffSMartin Schwidefsky int old = ACCESS_ONCE(rw->lock); 13402c503ffSMartin Schwidefsky return likely(old >= 0 && 13502c503ffSMartin Schwidefsky __atomic_cmpxchg_bool(&rw->lock, old, old + 1)); 136bae8f567SMartin Schwidefsky } 137bae8f567SMartin Schwidefsky 138bae8f567SMartin Schwidefsky static inline int arch_write_trylock_once(arch_rwlock_t *rw) 139bae8f567SMartin Schwidefsky { 14002c503ffSMartin Schwidefsky int old = ACCESS_ONCE(rw->lock); 141bae8f567SMartin Schwidefsky return likely(old == 0 && 14202c503ffSMartin Schwidefsky __atomic_cmpxchg_bool(&rw->lock, 0, 0x80000000)); 143bae8f567SMartin Schwidefsky } 144bae8f567SMartin Schwidefsky 145bbae71bfSMartin Schwidefsky #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES 146bbae71bfSMartin Schwidefsky 147bbae71bfSMartin Schwidefsky #define __RAW_OP_OR "lao" 148bbae71bfSMartin Schwidefsky #define __RAW_OP_AND "lan" 149bbae71bfSMartin Schwidefsky #define __RAW_OP_ADD "laa" 150bbae71bfSMartin Schwidefsky 151bbae71bfSMartin Schwidefsky #define __RAW_LOCK(ptr, op_val, op_string) \ 152bbae71bfSMartin Schwidefsky ({ \ 15302c503ffSMartin Schwidefsky int old_val; \ 154bbae71bfSMartin Schwidefsky \ 15502c503ffSMartin Schwidefsky typecheck(int *, ptr); \ 156bbae71bfSMartin Schwidefsky asm volatile( \ 157bbae71bfSMartin Schwidefsky op_string " %0,%2,%1\n" \ 158bbae71bfSMartin Schwidefsky "bcr 14,0\n" \ 159bbae71bfSMartin Schwidefsky : "=d" (old_val), "+Q" (*ptr) \ 160bbae71bfSMartin Schwidefsky : "d" (op_val) \ 161bbae71bfSMartin Schwidefsky : "cc", "memory"); \ 162bbae71bfSMartin Schwidefsky old_val; \ 163bbae71bfSMartin Schwidefsky }) 164bbae71bfSMartin Schwidefsky 165bbae71bfSMartin Schwidefsky #define __RAW_UNLOCK(ptr, op_val, op_string) \ 166bbae71bfSMartin Schwidefsky ({ \ 16702c503ffSMartin Schwidefsky int old_val; \ 168bbae71bfSMartin Schwidefsky \ 16902c503ffSMartin Schwidefsky typecheck(int *, ptr); \ 170bbae71bfSMartin Schwidefsky asm volatile( \ 171bbae71bfSMartin Schwidefsky op_string " %0,%2,%1\n" \ 172bbae71bfSMartin Schwidefsky : "=d" (old_val), "+Q" (*ptr) \ 173bbae71bfSMartin Schwidefsky : "d" (op_val) \ 174bbae71bfSMartin Schwidefsky : "cc", "memory"); \ 175bbae71bfSMartin Schwidefsky old_val; \ 176bbae71bfSMartin Schwidefsky }) 177bbae71bfSMartin Schwidefsky 178bbae71bfSMartin Schwidefsky extern void _raw_read_lock_wait(arch_rwlock_t *lp); 17902c503ffSMartin Schwidefsky extern void _raw_write_lock_wait(arch_rwlock_t *lp, int prev); 180bbae71bfSMartin Schwidefsky 181bbae71bfSMartin Schwidefsky static inline void arch_read_lock(arch_rwlock_t *rw) 182bbae71bfSMartin Schwidefsky { 18302c503ffSMartin Schwidefsky int old; 184bbae71bfSMartin Schwidefsky 185bbae71bfSMartin Schwidefsky old = __RAW_LOCK(&rw->lock, 1, __RAW_OP_ADD); 18602c503ffSMartin Schwidefsky if (old < 0) 187bbae71bfSMartin Schwidefsky _raw_read_lock_wait(rw); 188bbae71bfSMartin Schwidefsky } 189bbae71bfSMartin Schwidefsky 190bbae71bfSMartin Schwidefsky static inline void arch_read_unlock(arch_rwlock_t *rw) 191bbae71bfSMartin Schwidefsky { 192bbae71bfSMartin Schwidefsky __RAW_UNLOCK(&rw->lock, -1, __RAW_OP_ADD); 193bbae71bfSMartin Schwidefsky } 194bbae71bfSMartin Schwidefsky 195bbae71bfSMartin Schwidefsky static inline void arch_write_lock(arch_rwlock_t *rw) 196bbae71bfSMartin Schwidefsky { 19702c503ffSMartin Schwidefsky int old; 198bbae71bfSMartin Schwidefsky 199bbae71bfSMartin Schwidefsky old = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR); 200bbae71bfSMartin Schwidefsky if (old != 0) 201bbae71bfSMartin Schwidefsky _raw_write_lock_wait(rw, old); 202bbae71bfSMartin Schwidefsky rw->owner = SPINLOCK_LOCKVAL; 203bbae71bfSMartin Schwidefsky } 204bbae71bfSMartin Schwidefsky 205bbae71bfSMartin Schwidefsky static inline void arch_write_unlock(arch_rwlock_t *rw) 206bbae71bfSMartin Schwidefsky { 207bbae71bfSMartin Schwidefsky rw->owner = 0; 208bbae71bfSMartin Schwidefsky __RAW_UNLOCK(&rw->lock, 0x7fffffff, __RAW_OP_AND); 209bbae71bfSMartin Schwidefsky } 210bbae71bfSMartin Schwidefsky 211bbae71bfSMartin Schwidefsky #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 212bbae71bfSMartin Schwidefsky 213bbae71bfSMartin Schwidefsky extern void _raw_read_lock_wait(arch_rwlock_t *lp); 214bbae71bfSMartin Schwidefsky extern void _raw_write_lock_wait(arch_rwlock_t *lp); 215bbae71bfSMartin Schwidefsky 216e5931943SThomas Gleixner static inline void arch_read_lock(arch_rwlock_t *rw) 217c6557e7fSMartin Schwidefsky { 218bae8f567SMartin Schwidefsky if (!arch_read_trylock_once(rw)) 219c6557e7fSMartin Schwidefsky _raw_read_lock_wait(rw); 220c6557e7fSMartin Schwidefsky } 221c6557e7fSMartin Schwidefsky 222e5931943SThomas Gleixner static inline void arch_read_unlock(arch_rwlock_t *rw) 223c6557e7fSMartin Schwidefsky { 22402c503ffSMartin Schwidefsky int old; 225c6557e7fSMartin Schwidefsky 226c6557e7fSMartin Schwidefsky do { 2275b3f683eSPhilipp Hachtmann old = ACCESS_ONCE(rw->lock); 22802c503ffSMartin Schwidefsky } while (!__atomic_cmpxchg_bool(&rw->lock, old, old - 1)); 229c6557e7fSMartin Schwidefsky } 230c6557e7fSMartin Schwidefsky 231e5931943SThomas Gleixner static inline void arch_write_lock(arch_rwlock_t *rw) 232c6557e7fSMartin Schwidefsky { 233bae8f567SMartin Schwidefsky if (!arch_write_trylock_once(rw)) 234c6557e7fSMartin Schwidefsky _raw_write_lock_wait(rw); 235d59b93daSMartin Schwidefsky rw->owner = SPINLOCK_LOCKVAL; 236c6557e7fSMartin Schwidefsky } 237c6557e7fSMartin Schwidefsky 238e5931943SThomas Gleixner static inline void arch_write_unlock(arch_rwlock_t *rw) 239c6557e7fSMartin Schwidefsky { 24002c503ffSMartin Schwidefsky typecheck(int, rw->lock); 241d59b93daSMartin Schwidefsky 242d59b93daSMartin Schwidefsky rw->owner = 0; 24344230282SHeiko Carstens asm volatile( 24444230282SHeiko Carstens "st %1,%0\n" 24544230282SHeiko Carstens : "+Q" (rw->lock) 24644230282SHeiko Carstens : "d" (0) 24744230282SHeiko Carstens : "cc", "memory"); 248c6557e7fSMartin Schwidefsky } 249c6557e7fSMartin Schwidefsky 250bbae71bfSMartin Schwidefsky #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ 251bbae71bfSMartin Schwidefsky 252e5931943SThomas Gleixner static inline int arch_read_trylock(arch_rwlock_t *rw) 253c6557e7fSMartin Schwidefsky { 254bae8f567SMartin Schwidefsky if (!arch_read_trylock_once(rw)) 255c6557e7fSMartin Schwidefsky return _raw_read_trylock_retry(rw); 256bae8f567SMartin Schwidefsky return 1; 257c6557e7fSMartin Schwidefsky } 258c6557e7fSMartin Schwidefsky 259e5931943SThomas Gleixner static inline int arch_write_trylock(arch_rwlock_t *rw) 260c6557e7fSMartin Schwidefsky { 261d59b93daSMartin Schwidefsky if (!arch_write_trylock_once(rw) && !_raw_write_trylock_retry(rw)) 262d59b93daSMartin Schwidefsky return 0; 263d59b93daSMartin Schwidefsky rw->owner = SPINLOCK_LOCKVAL; 264bae8f567SMartin Schwidefsky return 1; 265c6557e7fSMartin Schwidefsky } 266c6557e7fSMartin Schwidefsky 267d59b93daSMartin Schwidefsky static inline void arch_read_relax(arch_rwlock_t *rw) 268d59b93daSMartin Schwidefsky { 269d59b93daSMartin Schwidefsky arch_lock_relax(rw->owner); 270d59b93daSMartin Schwidefsky } 271d59b93daSMartin Schwidefsky 272d59b93daSMartin Schwidefsky static inline void arch_write_relax(arch_rwlock_t *rw) 273d59b93daSMartin Schwidefsky { 274d59b93daSMartin Schwidefsky arch_lock_relax(rw->owner); 275d59b93daSMartin Schwidefsky } 276c6557e7fSMartin Schwidefsky 277c6557e7fSMartin Schwidefsky #endif /* __ASM_SPINLOCK_H */ 278