108e875c1SCatalin Marinas /* 208e875c1SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 308e875c1SCatalin Marinas * 408e875c1SCatalin Marinas * This program is free software; you can redistribute it and/or modify 508e875c1SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 608e875c1SCatalin Marinas * published by the Free Software Foundation. 708e875c1SCatalin Marinas * 808e875c1SCatalin Marinas * This program is distributed in the hope that it will be useful, 908e875c1SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 1008e875c1SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1108e875c1SCatalin Marinas * GNU General Public License for more details. 1208e875c1SCatalin Marinas * 1308e875c1SCatalin Marinas * You should have received a copy of the GNU General Public License 1408e875c1SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 1508e875c1SCatalin Marinas */ 1608e875c1SCatalin Marinas #ifndef __ASM_SPINLOCK_H 1708e875c1SCatalin Marinas #define __ASM_SPINLOCK_H 1808e875c1SCatalin Marinas 1981bb5c64SWill Deacon #include <asm/lse.h> 2008e875c1SCatalin Marinas #include <asm/spinlock_types.h> 2108e875c1SCatalin Marinas #include <asm/processor.h> 2208e875c1SCatalin Marinas 2308e875c1SCatalin Marinas /* 2408e875c1SCatalin Marinas * Spinlock implementation. 2508e875c1SCatalin Marinas * 2608e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 2708e875c1SCatalin Marinas * instructions. 2808e875c1SCatalin Marinas */ 29d86b8da0SWill Deacon static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) 30d86b8da0SWill Deacon { 31d86b8da0SWill Deacon unsigned int tmp; 32d86b8da0SWill Deacon arch_spinlock_t lockval; 3308e875c1SCatalin Marinas 3438b850a7SWill Deacon /* 3538b850a7SWill Deacon * Ensure prior spin_lock operations to other locks have completed 3638b850a7SWill Deacon * on this CPU before we test whether "lock" is locked. 3738b850a7SWill Deacon */ 3838b850a7SWill Deacon smp_mb(); 3938b850a7SWill Deacon 40d86b8da0SWill Deacon asm volatile( 41d86b8da0SWill Deacon " sevl\n" 42d86b8da0SWill Deacon "1: wfe\n" 43d86b8da0SWill Deacon "2: ldaxr %w0, %2\n" 44d86b8da0SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 45d86b8da0SWill Deacon " cbnz %w1, 1b\n" 46d86b8da0SWill Deacon ARM64_LSE_ATOMIC_INSN( 47d86b8da0SWill Deacon /* LL/SC */ 48d86b8da0SWill Deacon " stxr %w1, %w0, %2\n" 49d86b8da0SWill Deacon " cbnz %w1, 2b\n", /* Serialise against any concurrent lockers */ 50d86b8da0SWill Deacon /* LSE atomics */ 51d86b8da0SWill Deacon " nop\n" 52d86b8da0SWill Deacon " nop\n") 53d86b8da0SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 54d86b8da0SWill Deacon : 55d86b8da0SWill Deacon : "memory"); 56d86b8da0SWill Deacon } 5708e875c1SCatalin Marinas 5808e875c1SCatalin Marinas #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 5908e875c1SCatalin Marinas 6008e875c1SCatalin Marinas static inline void arch_spin_lock(arch_spinlock_t *lock) 6108e875c1SCatalin Marinas { 6208e875c1SCatalin Marinas unsigned int tmp; 6352ea2a56SWill Deacon arch_spinlock_t lockval, newval; 6408e875c1SCatalin Marinas 6508e875c1SCatalin Marinas asm volatile( 6652ea2a56SWill Deacon /* Atomically increment the next ticket. */ 6781bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 6881bb5c64SWill Deacon /* LL/SC */ 6952ea2a56SWill Deacon " prfm pstl1strm, %3\n" 7052ea2a56SWill Deacon "1: ldaxr %w0, %3\n" 7152ea2a56SWill Deacon " add %w1, %w0, %w5\n" 7252ea2a56SWill Deacon " stxr %w2, %w1, %3\n" 7381bb5c64SWill Deacon " cbnz %w2, 1b\n", 7481bb5c64SWill Deacon /* LSE atomics */ 7581bb5c64SWill Deacon " mov %w2, %w5\n" 7681bb5c64SWill Deacon " ldadda %w2, %w0, %3\n" 7781bb5c64SWill Deacon " nop\n" 7881bb5c64SWill Deacon " nop\n" 7981bb5c64SWill Deacon " nop\n" 8081bb5c64SWill Deacon ) 8181bb5c64SWill Deacon 8252ea2a56SWill Deacon /* Did we get the lock? */ 8352ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 8452ea2a56SWill Deacon " cbz %w1, 3f\n" 8552ea2a56SWill Deacon /* 8652ea2a56SWill Deacon * No: spin on the owner. Send a local event to avoid missing an 8752ea2a56SWill Deacon * unlock before the exclusive load. 8852ea2a56SWill Deacon */ 8908e875c1SCatalin Marinas " sevl\n" 9052ea2a56SWill Deacon "2: wfe\n" 9152ea2a56SWill Deacon " ldaxrh %w2, %4\n" 9252ea2a56SWill Deacon " eor %w1, %w2, %w0, lsr #16\n" 9352ea2a56SWill Deacon " cbnz %w1, 2b\n" 9452ea2a56SWill Deacon /* We got the lock. Critical section starts here. */ 9552ea2a56SWill Deacon "3:" 9652ea2a56SWill Deacon : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) 9752ea2a56SWill Deacon : "Q" (lock->owner), "I" (1 << TICKET_SHIFT) 9852ea2a56SWill Deacon : "memory"); 9908e875c1SCatalin Marinas } 10008e875c1SCatalin Marinas 10108e875c1SCatalin Marinas static inline int arch_spin_trylock(arch_spinlock_t *lock) 10208e875c1SCatalin Marinas { 10308e875c1SCatalin Marinas unsigned int tmp; 10452ea2a56SWill Deacon arch_spinlock_t lockval; 10508e875c1SCatalin Marinas 10681bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 10781bb5c64SWill Deacon /* LL/SC */ 10852ea2a56SWill Deacon " prfm pstl1strm, %2\n" 10952ea2a56SWill Deacon "1: ldaxr %w0, %2\n" 11052ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 11152ea2a56SWill Deacon " cbnz %w1, 2f\n" 11252ea2a56SWill Deacon " add %w0, %w0, %3\n" 11352ea2a56SWill Deacon " stxr %w1, %w0, %2\n" 11452ea2a56SWill Deacon " cbnz %w1, 1b\n" 11581bb5c64SWill Deacon "2:", 11681bb5c64SWill Deacon /* LSE atomics */ 11781bb5c64SWill Deacon " ldr %w0, %2\n" 11881bb5c64SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 11981bb5c64SWill Deacon " cbnz %w1, 1f\n" 12081bb5c64SWill Deacon " add %w1, %w0, %3\n" 12181bb5c64SWill Deacon " casa %w0, %w1, %2\n" 12281bb5c64SWill Deacon " and %w1, %w1, #0xffff\n" 12381bb5c64SWill Deacon " eor %w1, %w1, %w0, lsr #16\n" 12481bb5c64SWill Deacon "1:") 12552ea2a56SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 12652ea2a56SWill Deacon : "I" (1 << TICKET_SHIFT) 12752ea2a56SWill Deacon : "memory"); 12808e875c1SCatalin Marinas 12908e875c1SCatalin Marinas return !tmp; 13008e875c1SCatalin Marinas } 13108e875c1SCatalin Marinas 13208e875c1SCatalin Marinas static inline void arch_spin_unlock(arch_spinlock_t *lock) 13308e875c1SCatalin Marinas { 13481bb5c64SWill Deacon unsigned long tmp; 13581bb5c64SWill Deacon 13681bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 13781bb5c64SWill Deacon /* LL/SC */ 138c1d7cd22SWill Deacon " ldrh %w1, %0\n" 13981bb5c64SWill Deacon " add %w1, %w1, #1\n" 14081bb5c64SWill Deacon " stlrh %w1, %0", 14181bb5c64SWill Deacon /* LSE atomics */ 14281bb5c64SWill Deacon " mov %w1, #1\n" 14381bb5c64SWill Deacon " nop\n" 14481bb5c64SWill Deacon " staddlh %w1, %0") 14581bb5c64SWill Deacon : "=Q" (lock->owner), "=&r" (tmp) 14681bb5c64SWill Deacon : 14752ea2a56SWill Deacon : "memory"); 14808e875c1SCatalin Marinas } 14908e875c1SCatalin Marinas 1505686b06cSWill Deacon static inline int arch_spin_value_unlocked(arch_spinlock_t lock) 1515686b06cSWill Deacon { 1525686b06cSWill Deacon return lock.owner == lock.next; 1535686b06cSWill Deacon } 1545686b06cSWill Deacon 15552ea2a56SWill Deacon static inline int arch_spin_is_locked(arch_spinlock_t *lock) 15652ea2a56SWill Deacon { 15738b850a7SWill Deacon smp_mb(); /* See arch_spin_unlock_wait */ 158af2e7aaeSChristian Borntraeger return !arch_spin_value_unlocked(READ_ONCE(*lock)); 15952ea2a56SWill Deacon } 16052ea2a56SWill Deacon 16152ea2a56SWill Deacon static inline int arch_spin_is_contended(arch_spinlock_t *lock) 16252ea2a56SWill Deacon { 163af2e7aaeSChristian Borntraeger arch_spinlock_t lockval = READ_ONCE(*lock); 16452ea2a56SWill Deacon return (lockval.next - lockval.owner) > 1; 16552ea2a56SWill Deacon } 16652ea2a56SWill Deacon #define arch_spin_is_contended arch_spin_is_contended 16752ea2a56SWill Deacon 16808e875c1SCatalin Marinas /* 16908e875c1SCatalin Marinas * Write lock implementation. 17008e875c1SCatalin Marinas * 17108e875c1SCatalin Marinas * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is 17208e875c1SCatalin Marinas * exclusively held. 17308e875c1SCatalin Marinas * 17408e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 17508e875c1SCatalin Marinas * instructions. 17608e875c1SCatalin Marinas */ 17708e875c1SCatalin Marinas 17808e875c1SCatalin Marinas static inline void arch_write_lock(arch_rwlock_t *rw) 17908e875c1SCatalin Marinas { 18008e875c1SCatalin Marinas unsigned int tmp; 18108e875c1SCatalin Marinas 18281bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 18381bb5c64SWill Deacon /* LL/SC */ 18408e875c1SCatalin Marinas " sevl\n" 18508e875c1SCatalin Marinas "1: wfe\n" 1863a0310ebSWill Deacon "2: ldaxr %w0, %1\n" 18708e875c1SCatalin Marinas " cbnz %w0, 1b\n" 1883a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 18908e875c1SCatalin Marinas " cbnz %w0, 2b\n" 19081bb5c64SWill Deacon " nop", 19181bb5c64SWill Deacon /* LSE atomics */ 19281bb5c64SWill Deacon "1: mov %w0, wzr\n" 19381bb5c64SWill Deacon "2: casa %w0, %w2, %1\n" 19481bb5c64SWill Deacon " cbz %w0, 3f\n" 19581bb5c64SWill Deacon " ldxr %w0, %1\n" 19681bb5c64SWill Deacon " cbz %w0, 2b\n" 19781bb5c64SWill Deacon " wfe\n" 19881bb5c64SWill Deacon " b 1b\n" 19981bb5c64SWill Deacon "3:") 2003a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 2013a0310ebSWill Deacon : "r" (0x80000000) 20295c41896SWill Deacon : "memory"); 20308e875c1SCatalin Marinas } 20408e875c1SCatalin Marinas 20508e875c1SCatalin Marinas static inline int arch_write_trylock(arch_rwlock_t *rw) 20608e875c1SCatalin Marinas { 20708e875c1SCatalin Marinas unsigned int tmp; 20808e875c1SCatalin Marinas 20981bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 21081bb5c64SWill Deacon /* LL/SC */ 2119511ca19SWill Deacon "1: ldaxr %w0, %1\n" 2129511ca19SWill Deacon " cbnz %w0, 2f\n" 2133a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 2149511ca19SWill Deacon " cbnz %w0, 1b\n" 21581bb5c64SWill Deacon "2:", 21681bb5c64SWill Deacon /* LSE atomics */ 21781bb5c64SWill Deacon " mov %w0, wzr\n" 21881bb5c64SWill Deacon " casa %w0, %w2, %1\n" 21981bb5c64SWill Deacon " nop\n" 22081bb5c64SWill Deacon " nop") 2213a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 2223a0310ebSWill Deacon : "r" (0x80000000) 22395c41896SWill Deacon : "memory"); 22408e875c1SCatalin Marinas 22508e875c1SCatalin Marinas return !tmp; 22608e875c1SCatalin Marinas } 22708e875c1SCatalin Marinas 22808e875c1SCatalin Marinas static inline void arch_write_unlock(arch_rwlock_t *rw) 22908e875c1SCatalin Marinas { 23081bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 23181bb5c64SWill Deacon " stlr wzr, %0", 23281bb5c64SWill Deacon " swpl wzr, wzr, %0") 23381bb5c64SWill Deacon : "=Q" (rw->lock) :: "memory"); 23408e875c1SCatalin Marinas } 23508e875c1SCatalin Marinas 23608e875c1SCatalin Marinas /* write_can_lock - would write_trylock() succeed? */ 23708e875c1SCatalin Marinas #define arch_write_can_lock(x) ((x)->lock == 0) 23808e875c1SCatalin Marinas 23908e875c1SCatalin Marinas /* 24008e875c1SCatalin Marinas * Read lock implementation. 24108e875c1SCatalin Marinas * 24208e875c1SCatalin Marinas * It exclusively loads the lock value, increments it and stores the new value 24308e875c1SCatalin Marinas * back if positive and the CPU still exclusively owns the location. If the 24408e875c1SCatalin Marinas * value is negative, the lock is already held. 24508e875c1SCatalin Marinas * 24608e875c1SCatalin Marinas * During unlocking there may be multiple active read locks but no write lock. 24708e875c1SCatalin Marinas * 24808e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 24908e875c1SCatalin Marinas * instructions. 25081bb5c64SWill Deacon * 25181bb5c64SWill Deacon * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC 25281bb5c64SWill Deacon * and LSE implementations may exhibit different behaviour (although this 25381bb5c64SWill Deacon * will have no effect on lockdep). 25408e875c1SCatalin Marinas */ 25508e875c1SCatalin Marinas static inline void arch_read_lock(arch_rwlock_t *rw) 25608e875c1SCatalin Marinas { 25708e875c1SCatalin Marinas unsigned int tmp, tmp2; 25808e875c1SCatalin Marinas 25908e875c1SCatalin Marinas asm volatile( 26008e875c1SCatalin Marinas " sevl\n" 26181bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 26281bb5c64SWill Deacon /* LL/SC */ 26308e875c1SCatalin Marinas "1: wfe\n" 2643a0310ebSWill Deacon "2: ldaxr %w0, %2\n" 26508e875c1SCatalin Marinas " add %w0, %w0, #1\n" 26608e875c1SCatalin Marinas " tbnz %w0, #31, 1b\n" 2673a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 26881bb5c64SWill Deacon " nop\n" 26981bb5c64SWill Deacon " cbnz %w1, 2b", 27081bb5c64SWill Deacon /* LSE atomics */ 27181bb5c64SWill Deacon "1: wfe\n" 27281bb5c64SWill Deacon "2: ldxr %w0, %2\n" 27381bb5c64SWill Deacon " adds %w1, %w0, #1\n" 27481bb5c64SWill Deacon " tbnz %w1, #31, 1b\n" 27581bb5c64SWill Deacon " casa %w0, %w1, %2\n" 27681bb5c64SWill Deacon " sbc %w0, %w1, %w0\n" 27781bb5c64SWill Deacon " cbnz %w0, 2b") 2783a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 2793a0310ebSWill Deacon : 28081bb5c64SWill Deacon : "cc", "memory"); 28108e875c1SCatalin Marinas } 28208e875c1SCatalin Marinas 28308e875c1SCatalin Marinas static inline void arch_read_unlock(arch_rwlock_t *rw) 28408e875c1SCatalin Marinas { 28508e875c1SCatalin Marinas unsigned int tmp, tmp2; 28608e875c1SCatalin Marinas 28781bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 28881bb5c64SWill Deacon /* LL/SC */ 2893a0310ebSWill Deacon "1: ldxr %w0, %2\n" 29008e875c1SCatalin Marinas " sub %w0, %w0, #1\n" 2913a0310ebSWill Deacon " stlxr %w1, %w0, %2\n" 29281bb5c64SWill Deacon " cbnz %w1, 1b", 29381bb5c64SWill Deacon /* LSE atomics */ 29481bb5c64SWill Deacon " movn %w0, #0\n" 29581bb5c64SWill Deacon " nop\n" 29681bb5c64SWill Deacon " nop\n" 29781bb5c64SWill Deacon " staddl %w0, %2") 2983a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 2993a0310ebSWill Deacon : 30095c41896SWill Deacon : "memory"); 30108e875c1SCatalin Marinas } 30208e875c1SCatalin Marinas 30308e875c1SCatalin Marinas static inline int arch_read_trylock(arch_rwlock_t *rw) 30408e875c1SCatalin Marinas { 30581bb5c64SWill Deacon unsigned int tmp, tmp2; 30608e875c1SCatalin Marinas 30781bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 30881bb5c64SWill Deacon /* LL/SC */ 30981bb5c64SWill Deacon " mov %w1, #1\n" 3109511ca19SWill Deacon "1: ldaxr %w0, %2\n" 31108e875c1SCatalin Marinas " add %w0, %w0, #1\n" 3129511ca19SWill Deacon " tbnz %w0, #31, 2f\n" 3133a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 3149511ca19SWill Deacon " cbnz %w1, 1b\n" 31581bb5c64SWill Deacon "2:", 31681bb5c64SWill Deacon /* LSE atomics */ 31781bb5c64SWill Deacon " ldr %w0, %2\n" 31881bb5c64SWill Deacon " adds %w1, %w0, #1\n" 31981bb5c64SWill Deacon " tbnz %w1, #31, 1f\n" 32081bb5c64SWill Deacon " casa %w0, %w1, %2\n" 32181bb5c64SWill Deacon " sbc %w1, %w1, %w0\n" 32281bb5c64SWill Deacon " nop\n" 32381bb5c64SWill Deacon "1:") 32481bb5c64SWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 3253a0310ebSWill Deacon : 32681bb5c64SWill Deacon : "cc", "memory"); 32708e875c1SCatalin Marinas 32808e875c1SCatalin Marinas return !tmp2; 32908e875c1SCatalin Marinas } 33008e875c1SCatalin Marinas 33108e875c1SCatalin Marinas /* read_can_lock - would read_trylock() succeed? */ 33208e875c1SCatalin Marinas #define arch_read_can_lock(x) ((x)->lock < 0x80000000) 33308e875c1SCatalin Marinas 33408e875c1SCatalin Marinas #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 33508e875c1SCatalin Marinas #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 33608e875c1SCatalin Marinas 33708e875c1SCatalin Marinas #define arch_spin_relax(lock) cpu_relax() 33808e875c1SCatalin Marinas #define arch_read_relax(lock) cpu_relax() 33908e875c1SCatalin Marinas #define arch_write_relax(lock) cpu_relax() 34008e875c1SCatalin Marinas 34108e875c1SCatalin Marinas #endif /* __ASM_SPINLOCK_H */ 342