108e875c1SCatalin Marinas /* 208e875c1SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 308e875c1SCatalin Marinas * 408e875c1SCatalin Marinas * This program is free software; you can redistribute it and/or modify 508e875c1SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 608e875c1SCatalin Marinas * published by the Free Software Foundation. 708e875c1SCatalin Marinas * 808e875c1SCatalin Marinas * This program is distributed in the hope that it will be useful, 908e875c1SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 1008e875c1SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1108e875c1SCatalin Marinas * GNU General Public License for more details. 1208e875c1SCatalin Marinas * 1308e875c1SCatalin Marinas * You should have received a copy of the GNU General Public License 1408e875c1SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 1508e875c1SCatalin Marinas */ 1608e875c1SCatalin Marinas #ifndef __ASM_SPINLOCK_H 1708e875c1SCatalin Marinas #define __ASM_SPINLOCK_H 1808e875c1SCatalin Marinas 1981bb5c64SWill Deacon #include <asm/lse.h> 2008e875c1SCatalin Marinas #include <asm/spinlock_types.h> 2108e875c1SCatalin Marinas #include <asm/processor.h> 2208e875c1SCatalin Marinas 2308e875c1SCatalin Marinas /* 2408e875c1SCatalin Marinas * Spinlock implementation. 2508e875c1SCatalin Marinas * 2608e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 2708e875c1SCatalin Marinas * instructions. 2808e875c1SCatalin Marinas */ 2908e875c1SCatalin Marinas 3008e875c1SCatalin Marinas #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 3108e875c1SCatalin Marinas 3208e875c1SCatalin Marinas static inline void arch_spin_lock(arch_spinlock_t *lock) 3308e875c1SCatalin Marinas { 3408e875c1SCatalin Marinas unsigned int tmp; 3552ea2a56SWill Deacon arch_spinlock_t lockval, newval; 3608e875c1SCatalin Marinas 3708e875c1SCatalin Marinas asm volatile( 3852ea2a56SWill Deacon /* Atomically increment the next ticket. */ 3981bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 4081bb5c64SWill Deacon /* LL/SC */ 4152ea2a56SWill Deacon " prfm pstl1strm, %3\n" 4252ea2a56SWill Deacon "1: ldaxr %w0, %3\n" 4352ea2a56SWill Deacon " add %w1, %w0, %w5\n" 4452ea2a56SWill Deacon " stxr %w2, %w1, %3\n" 4581bb5c64SWill Deacon " cbnz %w2, 1b\n", 4681bb5c64SWill Deacon /* LSE atomics */ 4781bb5c64SWill Deacon " mov %w2, %w5\n" 4881bb5c64SWill Deacon " ldadda %w2, %w0, %3\n" 4905492f2fSWill Deacon __nops(3) 5081bb5c64SWill Deacon ) 5181bb5c64SWill Deacon 5252ea2a56SWill Deacon /* Did we get the lock? */ 5352ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 5452ea2a56SWill Deacon " cbz %w1, 3f\n" 5552ea2a56SWill Deacon /* 5652ea2a56SWill Deacon * No: spin on the owner. Send a local event to avoid missing an 5752ea2a56SWill Deacon * unlock before the exclusive load. 5852ea2a56SWill Deacon */ 5908e875c1SCatalin Marinas " sevl\n" 6052ea2a56SWill Deacon "2: wfe\n" 6152ea2a56SWill Deacon " ldaxrh %w2, %4\n" 6252ea2a56SWill Deacon " eor %w1, %w2, %w0, lsr #16\n" 6352ea2a56SWill Deacon " cbnz %w1, 2b\n" 6452ea2a56SWill Deacon /* We got the lock. Critical section starts here. */ 6552ea2a56SWill Deacon "3:" 6652ea2a56SWill Deacon : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) 6752ea2a56SWill Deacon : "Q" (lock->owner), "I" (1 << TICKET_SHIFT) 6852ea2a56SWill Deacon : "memory"); 6908e875c1SCatalin Marinas } 7008e875c1SCatalin Marinas 7108e875c1SCatalin Marinas static inline int arch_spin_trylock(arch_spinlock_t *lock) 7208e875c1SCatalin Marinas { 7308e875c1SCatalin Marinas unsigned int tmp; 7452ea2a56SWill Deacon arch_spinlock_t lockval; 7508e875c1SCatalin Marinas 7681bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 7781bb5c64SWill Deacon /* LL/SC */ 7852ea2a56SWill Deacon " prfm pstl1strm, %2\n" 7952ea2a56SWill Deacon "1: ldaxr %w0, %2\n" 8052ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 8152ea2a56SWill Deacon " cbnz %w1, 2f\n" 8252ea2a56SWill Deacon " add %w0, %w0, %3\n" 8352ea2a56SWill Deacon " stxr %w1, %w0, %2\n" 8452ea2a56SWill Deacon " cbnz %w1, 1b\n" 8581bb5c64SWill Deacon "2:", 8681bb5c64SWill Deacon /* LSE atomics */ 8781bb5c64SWill Deacon " ldr %w0, %2\n" 8881bb5c64SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 8981bb5c64SWill Deacon " cbnz %w1, 1f\n" 9081bb5c64SWill Deacon " add %w1, %w0, %3\n" 9181bb5c64SWill Deacon " casa %w0, %w1, %2\n" 9281bb5c64SWill Deacon " and %w1, %w1, #0xffff\n" 9381bb5c64SWill Deacon " eor %w1, %w1, %w0, lsr #16\n" 9481bb5c64SWill Deacon "1:") 9552ea2a56SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 9652ea2a56SWill Deacon : "I" (1 << TICKET_SHIFT) 9752ea2a56SWill Deacon : "memory"); 9808e875c1SCatalin Marinas 9908e875c1SCatalin Marinas return !tmp; 10008e875c1SCatalin Marinas } 10108e875c1SCatalin Marinas 10208e875c1SCatalin Marinas static inline void arch_spin_unlock(arch_spinlock_t *lock) 10308e875c1SCatalin Marinas { 10481bb5c64SWill Deacon unsigned long tmp; 10581bb5c64SWill Deacon 10681bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 10781bb5c64SWill Deacon /* LL/SC */ 108c1d7cd22SWill Deacon " ldrh %w1, %0\n" 10981bb5c64SWill Deacon " add %w1, %w1, #1\n" 11081bb5c64SWill Deacon " stlrh %w1, %0", 11181bb5c64SWill Deacon /* LSE atomics */ 11281bb5c64SWill Deacon " mov %w1, #1\n" 11305492f2fSWill Deacon " staddlh %w1, %0\n" 11405492f2fSWill Deacon __nops(1)) 11581bb5c64SWill Deacon : "=Q" (lock->owner), "=&r" (tmp) 11681bb5c64SWill Deacon : 11752ea2a56SWill Deacon : "memory"); 11808e875c1SCatalin Marinas } 11908e875c1SCatalin Marinas 1205686b06cSWill Deacon static inline int arch_spin_value_unlocked(arch_spinlock_t lock) 1215686b06cSWill Deacon { 1225686b06cSWill Deacon return lock.owner == lock.next; 1235686b06cSWill Deacon } 1245686b06cSWill Deacon 12552ea2a56SWill Deacon static inline int arch_spin_is_locked(arch_spinlock_t *lock) 12652ea2a56SWill Deacon { 127952111d7SPaul E. McKenney /* 128952111d7SPaul E. McKenney * Ensure prior spin_lock operations to other locks have completed 129952111d7SPaul E. McKenney * on this CPU before we test whether "lock" is locked. 130952111d7SPaul E. McKenney */ 131952111d7SPaul E. McKenney smp_mb(); /* ^^^ */ 132af2e7aaeSChristian Borntraeger return !arch_spin_value_unlocked(READ_ONCE(*lock)); 13352ea2a56SWill Deacon } 13452ea2a56SWill Deacon 13552ea2a56SWill Deacon static inline int arch_spin_is_contended(arch_spinlock_t *lock) 13652ea2a56SWill Deacon { 137af2e7aaeSChristian Borntraeger arch_spinlock_t lockval = READ_ONCE(*lock); 13852ea2a56SWill Deacon return (lockval.next - lockval.owner) > 1; 13952ea2a56SWill Deacon } 14052ea2a56SWill Deacon #define arch_spin_is_contended arch_spin_is_contended 14152ea2a56SWill Deacon 14208e875c1SCatalin Marinas /* 14308e875c1SCatalin Marinas * Write lock implementation. 14408e875c1SCatalin Marinas * 14508e875c1SCatalin Marinas * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is 14608e875c1SCatalin Marinas * exclusively held. 14708e875c1SCatalin Marinas * 14808e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 14908e875c1SCatalin Marinas * instructions. 15008e875c1SCatalin Marinas */ 15108e875c1SCatalin Marinas 15208e875c1SCatalin Marinas static inline void arch_write_lock(arch_rwlock_t *rw) 15308e875c1SCatalin Marinas { 15408e875c1SCatalin Marinas unsigned int tmp; 15508e875c1SCatalin Marinas 15681bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 15781bb5c64SWill Deacon /* LL/SC */ 15808e875c1SCatalin Marinas " sevl\n" 15908e875c1SCatalin Marinas "1: wfe\n" 1603a0310ebSWill Deacon "2: ldaxr %w0, %1\n" 16108e875c1SCatalin Marinas " cbnz %w0, 1b\n" 1623a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 16308e875c1SCatalin Marinas " cbnz %w0, 2b\n" 16405492f2fSWill Deacon __nops(1), 16581bb5c64SWill Deacon /* LSE atomics */ 16681bb5c64SWill Deacon "1: mov %w0, wzr\n" 16781bb5c64SWill Deacon "2: casa %w0, %w2, %1\n" 16881bb5c64SWill Deacon " cbz %w0, 3f\n" 16981bb5c64SWill Deacon " ldxr %w0, %1\n" 17081bb5c64SWill Deacon " cbz %w0, 2b\n" 17181bb5c64SWill Deacon " wfe\n" 17281bb5c64SWill Deacon " b 1b\n" 17381bb5c64SWill Deacon "3:") 1743a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 1753a0310ebSWill Deacon : "r" (0x80000000) 17695c41896SWill Deacon : "memory"); 17708e875c1SCatalin Marinas } 17808e875c1SCatalin Marinas 17908e875c1SCatalin Marinas static inline int arch_write_trylock(arch_rwlock_t *rw) 18008e875c1SCatalin Marinas { 18108e875c1SCatalin Marinas unsigned int tmp; 18208e875c1SCatalin Marinas 18381bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 18481bb5c64SWill Deacon /* LL/SC */ 1859511ca19SWill Deacon "1: ldaxr %w0, %1\n" 1869511ca19SWill Deacon " cbnz %w0, 2f\n" 1873a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 1889511ca19SWill Deacon " cbnz %w0, 1b\n" 18981bb5c64SWill Deacon "2:", 19081bb5c64SWill Deacon /* LSE atomics */ 19181bb5c64SWill Deacon " mov %w0, wzr\n" 19281bb5c64SWill Deacon " casa %w0, %w2, %1\n" 19305492f2fSWill Deacon __nops(2)) 1943a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 1953a0310ebSWill Deacon : "r" (0x80000000) 19695c41896SWill Deacon : "memory"); 19708e875c1SCatalin Marinas 19808e875c1SCatalin Marinas return !tmp; 19908e875c1SCatalin Marinas } 20008e875c1SCatalin Marinas 20108e875c1SCatalin Marinas static inline void arch_write_unlock(arch_rwlock_t *rw) 20208e875c1SCatalin Marinas { 20381bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 20481bb5c64SWill Deacon " stlr wzr, %0", 20581bb5c64SWill Deacon " swpl wzr, wzr, %0") 20681bb5c64SWill Deacon : "=Q" (rw->lock) :: "memory"); 20708e875c1SCatalin Marinas } 20808e875c1SCatalin Marinas 20908e875c1SCatalin Marinas /* write_can_lock - would write_trylock() succeed? */ 21008e875c1SCatalin Marinas #define arch_write_can_lock(x) ((x)->lock == 0) 21108e875c1SCatalin Marinas 21208e875c1SCatalin Marinas /* 21308e875c1SCatalin Marinas * Read lock implementation. 21408e875c1SCatalin Marinas * 21508e875c1SCatalin Marinas * It exclusively loads the lock value, increments it and stores the new value 21608e875c1SCatalin Marinas * back if positive and the CPU still exclusively owns the location. If the 21708e875c1SCatalin Marinas * value is negative, the lock is already held. 21808e875c1SCatalin Marinas * 21908e875c1SCatalin Marinas * During unlocking there may be multiple active read locks but no write lock. 22008e875c1SCatalin Marinas * 22108e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 22208e875c1SCatalin Marinas * instructions. 22381bb5c64SWill Deacon * 22481bb5c64SWill Deacon * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC 22581bb5c64SWill Deacon * and LSE implementations may exhibit different behaviour (although this 22681bb5c64SWill Deacon * will have no effect on lockdep). 22708e875c1SCatalin Marinas */ 22808e875c1SCatalin Marinas static inline void arch_read_lock(arch_rwlock_t *rw) 22908e875c1SCatalin Marinas { 23008e875c1SCatalin Marinas unsigned int tmp, tmp2; 23108e875c1SCatalin Marinas 23208e875c1SCatalin Marinas asm volatile( 23308e875c1SCatalin Marinas " sevl\n" 23481bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 23581bb5c64SWill Deacon /* LL/SC */ 23608e875c1SCatalin Marinas "1: wfe\n" 2373a0310ebSWill Deacon "2: ldaxr %w0, %2\n" 23808e875c1SCatalin Marinas " add %w0, %w0, #1\n" 23908e875c1SCatalin Marinas " tbnz %w0, #31, 1b\n" 2403a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 24105492f2fSWill Deacon " cbnz %w1, 2b\n" 24205492f2fSWill Deacon __nops(1), 24381bb5c64SWill Deacon /* LSE atomics */ 24481bb5c64SWill Deacon "1: wfe\n" 24581bb5c64SWill Deacon "2: ldxr %w0, %2\n" 24681bb5c64SWill Deacon " adds %w1, %w0, #1\n" 24781bb5c64SWill Deacon " tbnz %w1, #31, 1b\n" 24881bb5c64SWill Deacon " casa %w0, %w1, %2\n" 24981bb5c64SWill Deacon " sbc %w0, %w1, %w0\n" 25081bb5c64SWill Deacon " cbnz %w0, 2b") 2513a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 2523a0310ebSWill Deacon : 25381bb5c64SWill Deacon : "cc", "memory"); 25408e875c1SCatalin Marinas } 25508e875c1SCatalin Marinas 25608e875c1SCatalin Marinas static inline void arch_read_unlock(arch_rwlock_t *rw) 25708e875c1SCatalin Marinas { 25808e875c1SCatalin Marinas unsigned int tmp, tmp2; 25908e875c1SCatalin Marinas 26081bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 26181bb5c64SWill Deacon /* LL/SC */ 2623a0310ebSWill Deacon "1: ldxr %w0, %2\n" 26308e875c1SCatalin Marinas " sub %w0, %w0, #1\n" 2643a0310ebSWill Deacon " stlxr %w1, %w0, %2\n" 26581bb5c64SWill Deacon " cbnz %w1, 1b", 26681bb5c64SWill Deacon /* LSE atomics */ 26781bb5c64SWill Deacon " movn %w0, #0\n" 26805492f2fSWill Deacon " staddl %w0, %2\n" 26905492f2fSWill Deacon __nops(2)) 2703a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 2713a0310ebSWill Deacon : 27295c41896SWill Deacon : "memory"); 27308e875c1SCatalin Marinas } 27408e875c1SCatalin Marinas 27508e875c1SCatalin Marinas static inline int arch_read_trylock(arch_rwlock_t *rw) 27608e875c1SCatalin Marinas { 27781bb5c64SWill Deacon unsigned int tmp, tmp2; 27808e875c1SCatalin Marinas 27981bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 28081bb5c64SWill Deacon /* LL/SC */ 28181bb5c64SWill Deacon " mov %w1, #1\n" 2829511ca19SWill Deacon "1: ldaxr %w0, %2\n" 28308e875c1SCatalin Marinas " add %w0, %w0, #1\n" 2849511ca19SWill Deacon " tbnz %w0, #31, 2f\n" 2853a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 2869511ca19SWill Deacon " cbnz %w1, 1b\n" 28781bb5c64SWill Deacon "2:", 28881bb5c64SWill Deacon /* LSE atomics */ 28981bb5c64SWill Deacon " ldr %w0, %2\n" 29081bb5c64SWill Deacon " adds %w1, %w0, #1\n" 29181bb5c64SWill Deacon " tbnz %w1, #31, 1f\n" 29281bb5c64SWill Deacon " casa %w0, %w1, %2\n" 29381bb5c64SWill Deacon " sbc %w1, %w1, %w0\n" 29405492f2fSWill Deacon __nops(1) 29581bb5c64SWill Deacon "1:") 29681bb5c64SWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 2973a0310ebSWill Deacon : 29881bb5c64SWill Deacon : "cc", "memory"); 29908e875c1SCatalin Marinas 30008e875c1SCatalin Marinas return !tmp2; 30108e875c1SCatalin Marinas } 30208e875c1SCatalin Marinas 30308e875c1SCatalin Marinas /* read_can_lock - would read_trylock() succeed? */ 30408e875c1SCatalin Marinas #define arch_read_can_lock(x) ((x)->lock < 0x80000000) 30508e875c1SCatalin Marinas 30608e875c1SCatalin Marinas #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 30708e875c1SCatalin Marinas #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 30808e875c1SCatalin Marinas 30908e875c1SCatalin Marinas #define arch_spin_relax(lock) cpu_relax() 31008e875c1SCatalin Marinas #define arch_read_relax(lock) cpu_relax() 31108e875c1SCatalin Marinas #define arch_write_relax(lock) cpu_relax() 31208e875c1SCatalin Marinas 313872c63fbSWill Deacon /* 314872c63fbSWill Deacon * Accesses appearing in program order before a spin_lock() operation 315872c63fbSWill Deacon * can be reordered with accesses inside the critical section, by virtue 316872c63fbSWill Deacon * of arch_spin_lock being constructed using acquire semantics. 317872c63fbSWill Deacon * 318872c63fbSWill Deacon * In cases where this is problematic (e.g. try_to_wake_up), an 319872c63fbSWill Deacon * smp_mb__before_spinlock() can restore the required ordering. 320872c63fbSWill Deacon */ 321872c63fbSWill Deacon #define smp_mb__before_spinlock() smp_mb() 322872c63fbSWill Deacon 32308e875c1SCatalin Marinas #endif /* __ASM_SPINLOCK_H */ 324