108e875c1SCatalin Marinas /* 208e875c1SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 308e875c1SCatalin Marinas * 408e875c1SCatalin Marinas * This program is free software; you can redistribute it and/or modify 508e875c1SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 608e875c1SCatalin Marinas * published by the Free Software Foundation. 708e875c1SCatalin Marinas * 808e875c1SCatalin Marinas * This program is distributed in the hope that it will be useful, 908e875c1SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 1008e875c1SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1108e875c1SCatalin Marinas * GNU General Public License for more details. 1208e875c1SCatalin Marinas * 1308e875c1SCatalin Marinas * You should have received a copy of the GNU General Public License 1408e875c1SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 1508e875c1SCatalin Marinas */ 1608e875c1SCatalin Marinas #ifndef __ASM_SPINLOCK_H 1708e875c1SCatalin Marinas #define __ASM_SPINLOCK_H 1808e875c1SCatalin Marinas 1981bb5c64SWill Deacon #include <asm/lse.h> 2008e875c1SCatalin Marinas #include <asm/spinlock_types.h> 2108e875c1SCatalin Marinas #include <asm/processor.h> 2208e875c1SCatalin Marinas 2308e875c1SCatalin Marinas /* 2408e875c1SCatalin Marinas * Spinlock implementation. 2508e875c1SCatalin Marinas * 2608e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 2708e875c1SCatalin Marinas * instructions. 2808e875c1SCatalin Marinas */ 29d86b8da0SWill Deacon static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) 30d86b8da0SWill Deacon { 31d86b8da0SWill Deacon unsigned int tmp; 32d86b8da0SWill Deacon arch_spinlock_t lockval; 33c56bdcacSWill Deacon u32 owner; 3408e875c1SCatalin Marinas 3538b850a7SWill Deacon /* 3638b850a7SWill Deacon * Ensure prior spin_lock operations to other locks have completed 3738b850a7SWill Deacon * on this CPU before we test whether "lock" is locked. 3838b850a7SWill Deacon */ 3938b850a7SWill Deacon smp_mb(); 40c56bdcacSWill Deacon owner = READ_ONCE(lock->owner) << 16; 4138b850a7SWill Deacon 42d86b8da0SWill Deacon asm volatile( 43d86b8da0SWill Deacon " sevl\n" 44d86b8da0SWill Deacon "1: wfe\n" 45d86b8da0SWill Deacon "2: ldaxr %w0, %2\n" 46c56bdcacSWill Deacon /* Is the lock free? */ 47d86b8da0SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 48c56bdcacSWill Deacon " cbz %w1, 3f\n" 49c56bdcacSWill Deacon /* Lock taken -- has there been a subsequent unlock->lock transition? */ 50c56bdcacSWill Deacon " eor %w1, %w3, %w0, lsl #16\n" 51c56bdcacSWill Deacon " cbz %w1, 1b\n" 52c56bdcacSWill Deacon /* 53c56bdcacSWill Deacon * The owner has been updated, so there was an unlock->lock 54c56bdcacSWill Deacon * transition that we missed. That means we can rely on the 55c56bdcacSWill Deacon * store-release of the unlock operation paired with the 56c56bdcacSWill Deacon * load-acquire of the lock operation to publish any of our 57c56bdcacSWill Deacon * previous stores to the new lock owner and therefore don't 58c56bdcacSWill Deacon * need to bother with the writeback below. 59c56bdcacSWill Deacon */ 60c56bdcacSWill Deacon " b 4f\n" 61c56bdcacSWill Deacon "3:\n" 62c56bdcacSWill Deacon /* 63c56bdcacSWill Deacon * Serialise against any concurrent lockers by writing back the 64c56bdcacSWill Deacon * unlocked lock value 65c56bdcacSWill Deacon */ 66d86b8da0SWill Deacon ARM64_LSE_ATOMIC_INSN( 67d86b8da0SWill Deacon /* LL/SC */ 68d86b8da0SWill Deacon " stxr %w1, %w0, %2\n" 6905492f2fSWill Deacon __nops(2), 703a5facd0SWill Deacon /* LSE atomics */ 713a5facd0SWill Deacon " mov %w1, %w0\n" 723a5facd0SWill Deacon " cas %w0, %w0, %2\n" 733a5facd0SWill Deacon " eor %w1, %w1, %w0\n") 74c56bdcacSWill Deacon /* Somebody else wrote to the lock, GOTO 10 and reload the value */ 753a5facd0SWill Deacon " cbnz %w1, 2b\n" 76c56bdcacSWill Deacon "4:" 77d86b8da0SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 78c56bdcacSWill Deacon : "r" (owner) 79d86b8da0SWill Deacon : "memory"); 80d86b8da0SWill Deacon } 8108e875c1SCatalin Marinas 8208e875c1SCatalin Marinas #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 8308e875c1SCatalin Marinas 8408e875c1SCatalin Marinas static inline void arch_spin_lock(arch_spinlock_t *lock) 8508e875c1SCatalin Marinas { 8608e875c1SCatalin Marinas unsigned int tmp; 8752ea2a56SWill Deacon arch_spinlock_t lockval, newval; 8808e875c1SCatalin Marinas 8908e875c1SCatalin Marinas asm volatile( 9052ea2a56SWill Deacon /* Atomically increment the next ticket. */ 9181bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 9281bb5c64SWill Deacon /* LL/SC */ 9352ea2a56SWill Deacon " prfm pstl1strm, %3\n" 9452ea2a56SWill Deacon "1: ldaxr %w0, %3\n" 9552ea2a56SWill Deacon " add %w1, %w0, %w5\n" 9652ea2a56SWill Deacon " stxr %w2, %w1, %3\n" 9781bb5c64SWill Deacon " cbnz %w2, 1b\n", 9881bb5c64SWill Deacon /* LSE atomics */ 9981bb5c64SWill Deacon " mov %w2, %w5\n" 10081bb5c64SWill Deacon " ldadda %w2, %w0, %3\n" 10105492f2fSWill Deacon __nops(3) 10281bb5c64SWill Deacon ) 10381bb5c64SWill Deacon 10452ea2a56SWill Deacon /* Did we get the lock? */ 10552ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 10652ea2a56SWill Deacon " cbz %w1, 3f\n" 10752ea2a56SWill Deacon /* 10852ea2a56SWill Deacon * No: spin on the owner. Send a local event to avoid missing an 10952ea2a56SWill Deacon * unlock before the exclusive load. 11052ea2a56SWill Deacon */ 11108e875c1SCatalin Marinas " sevl\n" 11252ea2a56SWill Deacon "2: wfe\n" 11352ea2a56SWill Deacon " ldaxrh %w2, %4\n" 11452ea2a56SWill Deacon " eor %w1, %w2, %w0, lsr #16\n" 11552ea2a56SWill Deacon " cbnz %w1, 2b\n" 11652ea2a56SWill Deacon /* We got the lock. Critical section starts here. */ 11752ea2a56SWill Deacon "3:" 11852ea2a56SWill Deacon : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) 11952ea2a56SWill Deacon : "Q" (lock->owner), "I" (1 << TICKET_SHIFT) 12052ea2a56SWill Deacon : "memory"); 12108e875c1SCatalin Marinas } 12208e875c1SCatalin Marinas 12308e875c1SCatalin Marinas static inline int arch_spin_trylock(arch_spinlock_t *lock) 12408e875c1SCatalin Marinas { 12508e875c1SCatalin Marinas unsigned int tmp; 12652ea2a56SWill Deacon arch_spinlock_t lockval; 12708e875c1SCatalin Marinas 12881bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 12981bb5c64SWill Deacon /* LL/SC */ 13052ea2a56SWill Deacon " prfm pstl1strm, %2\n" 13152ea2a56SWill Deacon "1: ldaxr %w0, %2\n" 13252ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 13352ea2a56SWill Deacon " cbnz %w1, 2f\n" 13452ea2a56SWill Deacon " add %w0, %w0, %3\n" 13552ea2a56SWill Deacon " stxr %w1, %w0, %2\n" 13652ea2a56SWill Deacon " cbnz %w1, 1b\n" 13781bb5c64SWill Deacon "2:", 13881bb5c64SWill Deacon /* LSE atomics */ 13981bb5c64SWill Deacon " ldr %w0, %2\n" 14081bb5c64SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 14181bb5c64SWill Deacon " cbnz %w1, 1f\n" 14281bb5c64SWill Deacon " add %w1, %w0, %3\n" 14381bb5c64SWill Deacon " casa %w0, %w1, %2\n" 14481bb5c64SWill Deacon " and %w1, %w1, #0xffff\n" 14581bb5c64SWill Deacon " eor %w1, %w1, %w0, lsr #16\n" 14681bb5c64SWill Deacon "1:") 14752ea2a56SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 14852ea2a56SWill Deacon : "I" (1 << TICKET_SHIFT) 14952ea2a56SWill Deacon : "memory"); 15008e875c1SCatalin Marinas 15108e875c1SCatalin Marinas return !tmp; 15208e875c1SCatalin Marinas } 15308e875c1SCatalin Marinas 15408e875c1SCatalin Marinas static inline void arch_spin_unlock(arch_spinlock_t *lock) 15508e875c1SCatalin Marinas { 15681bb5c64SWill Deacon unsigned long tmp; 15781bb5c64SWill Deacon 15881bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 15981bb5c64SWill Deacon /* LL/SC */ 160c1d7cd22SWill Deacon " ldrh %w1, %0\n" 16181bb5c64SWill Deacon " add %w1, %w1, #1\n" 16281bb5c64SWill Deacon " stlrh %w1, %0", 16381bb5c64SWill Deacon /* LSE atomics */ 16481bb5c64SWill Deacon " mov %w1, #1\n" 16505492f2fSWill Deacon " staddlh %w1, %0\n" 16605492f2fSWill Deacon __nops(1)) 16781bb5c64SWill Deacon : "=Q" (lock->owner), "=&r" (tmp) 16881bb5c64SWill Deacon : 16952ea2a56SWill Deacon : "memory"); 17008e875c1SCatalin Marinas } 17108e875c1SCatalin Marinas 1725686b06cSWill Deacon static inline int arch_spin_value_unlocked(arch_spinlock_t lock) 1735686b06cSWill Deacon { 1745686b06cSWill Deacon return lock.owner == lock.next; 1755686b06cSWill Deacon } 1765686b06cSWill Deacon 17752ea2a56SWill Deacon static inline int arch_spin_is_locked(arch_spinlock_t *lock) 17852ea2a56SWill Deacon { 17938b850a7SWill Deacon smp_mb(); /* See arch_spin_unlock_wait */ 180af2e7aaeSChristian Borntraeger return !arch_spin_value_unlocked(READ_ONCE(*lock)); 18152ea2a56SWill Deacon } 18252ea2a56SWill Deacon 18352ea2a56SWill Deacon static inline int arch_spin_is_contended(arch_spinlock_t *lock) 18452ea2a56SWill Deacon { 185af2e7aaeSChristian Borntraeger arch_spinlock_t lockval = READ_ONCE(*lock); 18652ea2a56SWill Deacon return (lockval.next - lockval.owner) > 1; 18752ea2a56SWill Deacon } 18852ea2a56SWill Deacon #define arch_spin_is_contended arch_spin_is_contended 18952ea2a56SWill Deacon 19008e875c1SCatalin Marinas /* 19108e875c1SCatalin Marinas * Write lock implementation. 19208e875c1SCatalin Marinas * 19308e875c1SCatalin Marinas * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is 19408e875c1SCatalin Marinas * exclusively held. 19508e875c1SCatalin Marinas * 19608e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 19708e875c1SCatalin Marinas * instructions. 19808e875c1SCatalin Marinas */ 19908e875c1SCatalin Marinas 20008e875c1SCatalin Marinas static inline void arch_write_lock(arch_rwlock_t *rw) 20108e875c1SCatalin Marinas { 20208e875c1SCatalin Marinas unsigned int tmp; 20308e875c1SCatalin Marinas 20481bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 20581bb5c64SWill Deacon /* LL/SC */ 20608e875c1SCatalin Marinas " sevl\n" 20708e875c1SCatalin Marinas "1: wfe\n" 2083a0310ebSWill Deacon "2: ldaxr %w0, %1\n" 20908e875c1SCatalin Marinas " cbnz %w0, 1b\n" 2103a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 21108e875c1SCatalin Marinas " cbnz %w0, 2b\n" 21205492f2fSWill Deacon __nops(1), 21381bb5c64SWill Deacon /* LSE atomics */ 21481bb5c64SWill Deacon "1: mov %w0, wzr\n" 21581bb5c64SWill Deacon "2: casa %w0, %w2, %1\n" 21681bb5c64SWill Deacon " cbz %w0, 3f\n" 21781bb5c64SWill Deacon " ldxr %w0, %1\n" 21881bb5c64SWill Deacon " cbz %w0, 2b\n" 21981bb5c64SWill Deacon " wfe\n" 22081bb5c64SWill Deacon " b 1b\n" 22181bb5c64SWill Deacon "3:") 2223a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 2233a0310ebSWill Deacon : "r" (0x80000000) 22495c41896SWill Deacon : "memory"); 22508e875c1SCatalin Marinas } 22608e875c1SCatalin Marinas 22708e875c1SCatalin Marinas static inline int arch_write_trylock(arch_rwlock_t *rw) 22808e875c1SCatalin Marinas { 22908e875c1SCatalin Marinas unsigned int tmp; 23008e875c1SCatalin Marinas 23181bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 23281bb5c64SWill Deacon /* LL/SC */ 2339511ca19SWill Deacon "1: ldaxr %w0, %1\n" 2349511ca19SWill Deacon " cbnz %w0, 2f\n" 2353a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 2369511ca19SWill Deacon " cbnz %w0, 1b\n" 23781bb5c64SWill Deacon "2:", 23881bb5c64SWill Deacon /* LSE atomics */ 23981bb5c64SWill Deacon " mov %w0, wzr\n" 24081bb5c64SWill Deacon " casa %w0, %w2, %1\n" 24105492f2fSWill Deacon __nops(2)) 2423a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 2433a0310ebSWill Deacon : "r" (0x80000000) 24495c41896SWill Deacon : "memory"); 24508e875c1SCatalin Marinas 24608e875c1SCatalin Marinas return !tmp; 24708e875c1SCatalin Marinas } 24808e875c1SCatalin Marinas 24908e875c1SCatalin Marinas static inline void arch_write_unlock(arch_rwlock_t *rw) 25008e875c1SCatalin Marinas { 25181bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 25281bb5c64SWill Deacon " stlr wzr, %0", 25381bb5c64SWill Deacon " swpl wzr, wzr, %0") 25481bb5c64SWill Deacon : "=Q" (rw->lock) :: "memory"); 25508e875c1SCatalin Marinas } 25608e875c1SCatalin Marinas 25708e875c1SCatalin Marinas /* write_can_lock - would write_trylock() succeed? */ 25808e875c1SCatalin Marinas #define arch_write_can_lock(x) ((x)->lock == 0) 25908e875c1SCatalin Marinas 26008e875c1SCatalin Marinas /* 26108e875c1SCatalin Marinas * Read lock implementation. 26208e875c1SCatalin Marinas * 26308e875c1SCatalin Marinas * It exclusively loads the lock value, increments it and stores the new value 26408e875c1SCatalin Marinas * back if positive and the CPU still exclusively owns the location. If the 26508e875c1SCatalin Marinas * value is negative, the lock is already held. 26608e875c1SCatalin Marinas * 26708e875c1SCatalin Marinas * During unlocking there may be multiple active read locks but no write lock. 26808e875c1SCatalin Marinas * 26908e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 27008e875c1SCatalin Marinas * instructions. 27181bb5c64SWill Deacon * 27281bb5c64SWill Deacon * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC 27381bb5c64SWill Deacon * and LSE implementations may exhibit different behaviour (although this 27481bb5c64SWill Deacon * will have no effect on lockdep). 27508e875c1SCatalin Marinas */ 27608e875c1SCatalin Marinas static inline void arch_read_lock(arch_rwlock_t *rw) 27708e875c1SCatalin Marinas { 27808e875c1SCatalin Marinas unsigned int tmp, tmp2; 27908e875c1SCatalin Marinas 28008e875c1SCatalin Marinas asm volatile( 28108e875c1SCatalin Marinas " sevl\n" 28281bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 28381bb5c64SWill Deacon /* LL/SC */ 28408e875c1SCatalin Marinas "1: wfe\n" 2853a0310ebSWill Deacon "2: ldaxr %w0, %2\n" 28608e875c1SCatalin Marinas " add %w0, %w0, #1\n" 28708e875c1SCatalin Marinas " tbnz %w0, #31, 1b\n" 2883a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 28905492f2fSWill Deacon " cbnz %w1, 2b\n" 29005492f2fSWill Deacon __nops(1), 29181bb5c64SWill Deacon /* LSE atomics */ 29281bb5c64SWill Deacon "1: wfe\n" 29381bb5c64SWill Deacon "2: ldxr %w0, %2\n" 29481bb5c64SWill Deacon " adds %w1, %w0, #1\n" 29581bb5c64SWill Deacon " tbnz %w1, #31, 1b\n" 29681bb5c64SWill Deacon " casa %w0, %w1, %2\n" 29781bb5c64SWill Deacon " sbc %w0, %w1, %w0\n" 29881bb5c64SWill Deacon " cbnz %w0, 2b") 2993a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 3003a0310ebSWill Deacon : 30181bb5c64SWill Deacon : "cc", "memory"); 30208e875c1SCatalin Marinas } 30308e875c1SCatalin Marinas 30408e875c1SCatalin Marinas static inline void arch_read_unlock(arch_rwlock_t *rw) 30508e875c1SCatalin Marinas { 30608e875c1SCatalin Marinas unsigned int tmp, tmp2; 30708e875c1SCatalin Marinas 30881bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 30981bb5c64SWill Deacon /* LL/SC */ 3103a0310ebSWill Deacon "1: ldxr %w0, %2\n" 31108e875c1SCatalin Marinas " sub %w0, %w0, #1\n" 3123a0310ebSWill Deacon " stlxr %w1, %w0, %2\n" 31381bb5c64SWill Deacon " cbnz %w1, 1b", 31481bb5c64SWill Deacon /* LSE atomics */ 31581bb5c64SWill Deacon " movn %w0, #0\n" 31605492f2fSWill Deacon " staddl %w0, %2\n" 31705492f2fSWill Deacon __nops(2)) 3183a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 3193a0310ebSWill Deacon : 32095c41896SWill Deacon : "memory"); 32108e875c1SCatalin Marinas } 32208e875c1SCatalin Marinas 32308e875c1SCatalin Marinas static inline int arch_read_trylock(arch_rwlock_t *rw) 32408e875c1SCatalin Marinas { 32581bb5c64SWill Deacon unsigned int tmp, tmp2; 32608e875c1SCatalin Marinas 32781bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 32881bb5c64SWill Deacon /* LL/SC */ 32981bb5c64SWill Deacon " mov %w1, #1\n" 3309511ca19SWill Deacon "1: ldaxr %w0, %2\n" 33108e875c1SCatalin Marinas " add %w0, %w0, #1\n" 3329511ca19SWill Deacon " tbnz %w0, #31, 2f\n" 3333a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 3349511ca19SWill Deacon " cbnz %w1, 1b\n" 33581bb5c64SWill Deacon "2:", 33681bb5c64SWill Deacon /* LSE atomics */ 33781bb5c64SWill Deacon " ldr %w0, %2\n" 33881bb5c64SWill Deacon " adds %w1, %w0, #1\n" 33981bb5c64SWill Deacon " tbnz %w1, #31, 1f\n" 34081bb5c64SWill Deacon " casa %w0, %w1, %2\n" 34181bb5c64SWill Deacon " sbc %w1, %w1, %w0\n" 34205492f2fSWill Deacon __nops(1) 34381bb5c64SWill Deacon "1:") 34481bb5c64SWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 3453a0310ebSWill Deacon : 34681bb5c64SWill Deacon : "cc", "memory"); 34708e875c1SCatalin Marinas 34808e875c1SCatalin Marinas return !tmp2; 34908e875c1SCatalin Marinas } 35008e875c1SCatalin Marinas 35108e875c1SCatalin Marinas /* read_can_lock - would read_trylock() succeed? */ 35208e875c1SCatalin Marinas #define arch_read_can_lock(x) ((x)->lock < 0x80000000) 35308e875c1SCatalin Marinas 35408e875c1SCatalin Marinas #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 35508e875c1SCatalin Marinas #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 35608e875c1SCatalin Marinas 35708e875c1SCatalin Marinas #define arch_spin_relax(lock) cpu_relax() 35808e875c1SCatalin Marinas #define arch_read_relax(lock) cpu_relax() 35908e875c1SCatalin Marinas #define arch_write_relax(lock) cpu_relax() 36008e875c1SCatalin Marinas 361872c63fbSWill Deacon /* 362872c63fbSWill Deacon * Accesses appearing in program order before a spin_lock() operation 363872c63fbSWill Deacon * can be reordered with accesses inside the critical section, by virtue 364872c63fbSWill Deacon * of arch_spin_lock being constructed using acquire semantics. 365872c63fbSWill Deacon * 366872c63fbSWill Deacon * In cases where this is problematic (e.g. try_to_wake_up), an 367872c63fbSWill Deacon * smp_mb__before_spinlock() can restore the required ordering. 368872c63fbSWill Deacon */ 369872c63fbSWill Deacon #define smp_mb__before_spinlock() smp_mb() 370d89e588cSPeter Zijlstra /* See include/linux/spinlock.h */ 371d89e588cSPeter Zijlstra #define smp_mb__after_spinlock() smp_mb() 372872c63fbSWill Deacon 37308e875c1SCatalin Marinas #endif /* __ASM_SPINLOCK_H */ 374