108e875c1SCatalin Marinas /* 208e875c1SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 308e875c1SCatalin Marinas * 408e875c1SCatalin Marinas * This program is free software; you can redistribute it and/or modify 508e875c1SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 608e875c1SCatalin Marinas * published by the Free Software Foundation. 708e875c1SCatalin Marinas * 808e875c1SCatalin Marinas * This program is distributed in the hope that it will be useful, 908e875c1SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 1008e875c1SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1108e875c1SCatalin Marinas * GNU General Public License for more details. 1208e875c1SCatalin Marinas * 1308e875c1SCatalin Marinas * You should have received a copy of the GNU General Public License 1408e875c1SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 1508e875c1SCatalin Marinas */ 1608e875c1SCatalin Marinas #ifndef __ASM_SPINLOCK_H 1708e875c1SCatalin Marinas #define __ASM_SPINLOCK_H 1808e875c1SCatalin Marinas 1981bb5c64SWill Deacon #include <asm/lse.h> 2008e875c1SCatalin Marinas #include <asm/spinlock_types.h> 2108e875c1SCatalin Marinas #include <asm/processor.h> 2208e875c1SCatalin Marinas 2308e875c1SCatalin Marinas /* 2408e875c1SCatalin Marinas * Spinlock implementation. 2508e875c1SCatalin Marinas * 2608e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 2708e875c1SCatalin Marinas * instructions. 2808e875c1SCatalin Marinas */ 2908e875c1SCatalin Marinas 3008e875c1SCatalin Marinas #define arch_spin_unlock_wait(lock) \ 3108e875c1SCatalin Marinas do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) 3208e875c1SCatalin Marinas 3308e875c1SCatalin Marinas #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 3408e875c1SCatalin Marinas 3508e875c1SCatalin Marinas static inline void arch_spin_lock(arch_spinlock_t *lock) 3608e875c1SCatalin Marinas { 3708e875c1SCatalin Marinas unsigned int tmp; 3852ea2a56SWill Deacon arch_spinlock_t lockval, newval; 3908e875c1SCatalin Marinas 4008e875c1SCatalin Marinas asm volatile( 4152ea2a56SWill Deacon /* Atomically increment the next ticket. */ 4281bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 4381bb5c64SWill Deacon /* LL/SC */ 4452ea2a56SWill Deacon " prfm pstl1strm, %3\n" 4552ea2a56SWill Deacon "1: ldaxr %w0, %3\n" 4652ea2a56SWill Deacon " add %w1, %w0, %w5\n" 4752ea2a56SWill Deacon " stxr %w2, %w1, %3\n" 4881bb5c64SWill Deacon " cbnz %w2, 1b\n", 4981bb5c64SWill Deacon /* LSE atomics */ 5081bb5c64SWill Deacon " mov %w2, %w5\n" 5181bb5c64SWill Deacon " ldadda %w2, %w0, %3\n" 5281bb5c64SWill Deacon " nop\n" 5381bb5c64SWill Deacon " nop\n" 5481bb5c64SWill Deacon " nop\n" 5581bb5c64SWill Deacon ) 5681bb5c64SWill Deacon 5752ea2a56SWill Deacon /* Did we get the lock? */ 5852ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 5952ea2a56SWill Deacon " cbz %w1, 3f\n" 6052ea2a56SWill Deacon /* 6152ea2a56SWill Deacon * No: spin on the owner. Send a local event to avoid missing an 6252ea2a56SWill Deacon * unlock before the exclusive load. 6352ea2a56SWill Deacon */ 6408e875c1SCatalin Marinas " sevl\n" 6552ea2a56SWill Deacon "2: wfe\n" 6652ea2a56SWill Deacon " ldaxrh %w2, %4\n" 6752ea2a56SWill Deacon " eor %w1, %w2, %w0, lsr #16\n" 6852ea2a56SWill Deacon " cbnz %w1, 2b\n" 6952ea2a56SWill Deacon /* We got the lock. Critical section starts here. */ 7052ea2a56SWill Deacon "3:" 7152ea2a56SWill Deacon : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) 7252ea2a56SWill Deacon : "Q" (lock->owner), "I" (1 << TICKET_SHIFT) 7352ea2a56SWill Deacon : "memory"); 7408e875c1SCatalin Marinas } 7508e875c1SCatalin Marinas 7608e875c1SCatalin Marinas static inline int arch_spin_trylock(arch_spinlock_t *lock) 7708e875c1SCatalin Marinas { 7808e875c1SCatalin Marinas unsigned int tmp; 7952ea2a56SWill Deacon arch_spinlock_t lockval; 8008e875c1SCatalin Marinas 8181bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 8281bb5c64SWill Deacon /* LL/SC */ 8352ea2a56SWill Deacon " prfm pstl1strm, %2\n" 8452ea2a56SWill Deacon "1: ldaxr %w0, %2\n" 8552ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 8652ea2a56SWill Deacon " cbnz %w1, 2f\n" 8752ea2a56SWill Deacon " add %w0, %w0, %3\n" 8852ea2a56SWill Deacon " stxr %w1, %w0, %2\n" 8952ea2a56SWill Deacon " cbnz %w1, 1b\n" 9081bb5c64SWill Deacon "2:", 9181bb5c64SWill Deacon /* LSE atomics */ 9281bb5c64SWill Deacon " ldr %w0, %2\n" 9381bb5c64SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 9481bb5c64SWill Deacon " cbnz %w1, 1f\n" 9581bb5c64SWill Deacon " add %w1, %w0, %3\n" 9681bb5c64SWill Deacon " casa %w0, %w1, %2\n" 9781bb5c64SWill Deacon " and %w1, %w1, #0xffff\n" 9881bb5c64SWill Deacon " eor %w1, %w1, %w0, lsr #16\n" 9981bb5c64SWill Deacon "1:") 10052ea2a56SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 10152ea2a56SWill Deacon : "I" (1 << TICKET_SHIFT) 10252ea2a56SWill Deacon : "memory"); 10308e875c1SCatalin Marinas 10408e875c1SCatalin Marinas return !tmp; 10508e875c1SCatalin Marinas } 10608e875c1SCatalin Marinas 10708e875c1SCatalin Marinas static inline void arch_spin_unlock(arch_spinlock_t *lock) 10808e875c1SCatalin Marinas { 10981bb5c64SWill Deacon unsigned long tmp; 11081bb5c64SWill Deacon 11181bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 11281bb5c64SWill Deacon /* LL/SC */ 113c1d7cd22SWill Deacon " ldrh %w1, %0\n" 11481bb5c64SWill Deacon " add %w1, %w1, #1\n" 11581bb5c64SWill Deacon " stlrh %w1, %0", 11681bb5c64SWill Deacon /* LSE atomics */ 11781bb5c64SWill Deacon " mov %w1, #1\n" 11881bb5c64SWill Deacon " nop\n" 11981bb5c64SWill Deacon " staddlh %w1, %0") 12081bb5c64SWill Deacon : "=Q" (lock->owner), "=&r" (tmp) 12181bb5c64SWill Deacon : 12252ea2a56SWill Deacon : "memory"); 12308e875c1SCatalin Marinas } 12408e875c1SCatalin Marinas 1255686b06cSWill Deacon static inline int arch_spin_value_unlocked(arch_spinlock_t lock) 1265686b06cSWill Deacon { 1275686b06cSWill Deacon return lock.owner == lock.next; 1285686b06cSWill Deacon } 1295686b06cSWill Deacon 13052ea2a56SWill Deacon static inline int arch_spin_is_locked(arch_spinlock_t *lock) 13152ea2a56SWill Deacon { 132af2e7aaeSChristian Borntraeger return !arch_spin_value_unlocked(READ_ONCE(*lock)); 13352ea2a56SWill Deacon } 13452ea2a56SWill Deacon 13552ea2a56SWill Deacon static inline int arch_spin_is_contended(arch_spinlock_t *lock) 13652ea2a56SWill Deacon { 137af2e7aaeSChristian Borntraeger arch_spinlock_t lockval = READ_ONCE(*lock); 13852ea2a56SWill Deacon return (lockval.next - lockval.owner) > 1; 13952ea2a56SWill Deacon } 14052ea2a56SWill Deacon #define arch_spin_is_contended arch_spin_is_contended 14152ea2a56SWill Deacon 14208e875c1SCatalin Marinas /* 14308e875c1SCatalin Marinas * Write lock implementation. 14408e875c1SCatalin Marinas * 14508e875c1SCatalin Marinas * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is 14608e875c1SCatalin Marinas * exclusively held. 14708e875c1SCatalin Marinas * 14808e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 14908e875c1SCatalin Marinas * instructions. 15008e875c1SCatalin Marinas */ 15108e875c1SCatalin Marinas 15208e875c1SCatalin Marinas static inline void arch_write_lock(arch_rwlock_t *rw) 15308e875c1SCatalin Marinas { 15408e875c1SCatalin Marinas unsigned int tmp; 15508e875c1SCatalin Marinas 15681bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 15781bb5c64SWill Deacon /* LL/SC */ 15808e875c1SCatalin Marinas " sevl\n" 15908e875c1SCatalin Marinas "1: wfe\n" 1603a0310ebSWill Deacon "2: ldaxr %w0, %1\n" 16108e875c1SCatalin Marinas " cbnz %w0, 1b\n" 1623a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 16308e875c1SCatalin Marinas " cbnz %w0, 2b\n" 16481bb5c64SWill Deacon " nop", 16581bb5c64SWill Deacon /* LSE atomics */ 16681bb5c64SWill Deacon "1: mov %w0, wzr\n" 16781bb5c64SWill Deacon "2: casa %w0, %w2, %1\n" 16881bb5c64SWill Deacon " cbz %w0, 3f\n" 16981bb5c64SWill Deacon " ldxr %w0, %1\n" 17081bb5c64SWill Deacon " cbz %w0, 2b\n" 17181bb5c64SWill Deacon " wfe\n" 17281bb5c64SWill Deacon " b 1b\n" 17381bb5c64SWill Deacon "3:") 1743a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 1753a0310ebSWill Deacon : "r" (0x80000000) 17695c41896SWill Deacon : "memory"); 17708e875c1SCatalin Marinas } 17808e875c1SCatalin Marinas 17908e875c1SCatalin Marinas static inline int arch_write_trylock(arch_rwlock_t *rw) 18008e875c1SCatalin Marinas { 18108e875c1SCatalin Marinas unsigned int tmp; 18208e875c1SCatalin Marinas 18381bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 18481bb5c64SWill Deacon /* LL/SC */ 1859511ca19SWill Deacon "1: ldaxr %w0, %1\n" 1869511ca19SWill Deacon " cbnz %w0, 2f\n" 1873a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 1889511ca19SWill Deacon " cbnz %w0, 1b\n" 18981bb5c64SWill Deacon "2:", 19081bb5c64SWill Deacon /* LSE atomics */ 19181bb5c64SWill Deacon " mov %w0, wzr\n" 19281bb5c64SWill Deacon " casa %w0, %w2, %1\n" 19381bb5c64SWill Deacon " nop\n" 19481bb5c64SWill Deacon " nop") 1953a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 1963a0310ebSWill Deacon : "r" (0x80000000) 19795c41896SWill Deacon : "memory"); 19808e875c1SCatalin Marinas 19908e875c1SCatalin Marinas return !tmp; 20008e875c1SCatalin Marinas } 20108e875c1SCatalin Marinas 20208e875c1SCatalin Marinas static inline void arch_write_unlock(arch_rwlock_t *rw) 20308e875c1SCatalin Marinas { 20481bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 20581bb5c64SWill Deacon " stlr wzr, %0", 20681bb5c64SWill Deacon " swpl wzr, wzr, %0") 20781bb5c64SWill Deacon : "=Q" (rw->lock) :: "memory"); 20808e875c1SCatalin Marinas } 20908e875c1SCatalin Marinas 21008e875c1SCatalin Marinas /* write_can_lock - would write_trylock() succeed? */ 21108e875c1SCatalin Marinas #define arch_write_can_lock(x) ((x)->lock == 0) 21208e875c1SCatalin Marinas 21308e875c1SCatalin Marinas /* 21408e875c1SCatalin Marinas * Read lock implementation. 21508e875c1SCatalin Marinas * 21608e875c1SCatalin Marinas * It exclusively loads the lock value, increments it and stores the new value 21708e875c1SCatalin Marinas * back if positive and the CPU still exclusively owns the location. If the 21808e875c1SCatalin Marinas * value is negative, the lock is already held. 21908e875c1SCatalin Marinas * 22008e875c1SCatalin Marinas * During unlocking there may be multiple active read locks but no write lock. 22108e875c1SCatalin Marinas * 22208e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 22308e875c1SCatalin Marinas * instructions. 22481bb5c64SWill Deacon * 22581bb5c64SWill Deacon * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC 22681bb5c64SWill Deacon * and LSE implementations may exhibit different behaviour (although this 22781bb5c64SWill Deacon * will have no effect on lockdep). 22808e875c1SCatalin Marinas */ 22908e875c1SCatalin Marinas static inline void arch_read_lock(arch_rwlock_t *rw) 23008e875c1SCatalin Marinas { 23108e875c1SCatalin Marinas unsigned int tmp, tmp2; 23208e875c1SCatalin Marinas 23308e875c1SCatalin Marinas asm volatile( 23408e875c1SCatalin Marinas " sevl\n" 23581bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 23681bb5c64SWill Deacon /* LL/SC */ 23708e875c1SCatalin Marinas "1: wfe\n" 2383a0310ebSWill Deacon "2: ldaxr %w0, %2\n" 23908e875c1SCatalin Marinas " add %w0, %w0, #1\n" 24008e875c1SCatalin Marinas " tbnz %w0, #31, 1b\n" 2413a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 24281bb5c64SWill Deacon " nop\n" 24381bb5c64SWill Deacon " cbnz %w1, 2b", 24481bb5c64SWill Deacon /* LSE atomics */ 24581bb5c64SWill Deacon "1: wfe\n" 24681bb5c64SWill Deacon "2: ldxr %w0, %2\n" 24781bb5c64SWill Deacon " adds %w1, %w0, #1\n" 24881bb5c64SWill Deacon " tbnz %w1, #31, 1b\n" 24981bb5c64SWill Deacon " casa %w0, %w1, %2\n" 25081bb5c64SWill Deacon " sbc %w0, %w1, %w0\n" 25181bb5c64SWill Deacon " cbnz %w0, 2b") 2523a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 2533a0310ebSWill Deacon : 25481bb5c64SWill Deacon : "cc", "memory"); 25508e875c1SCatalin Marinas } 25608e875c1SCatalin Marinas 25708e875c1SCatalin Marinas static inline void arch_read_unlock(arch_rwlock_t *rw) 25808e875c1SCatalin Marinas { 25908e875c1SCatalin Marinas unsigned int tmp, tmp2; 26008e875c1SCatalin Marinas 26181bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 26281bb5c64SWill Deacon /* LL/SC */ 2633a0310ebSWill Deacon "1: ldxr %w0, %2\n" 26408e875c1SCatalin Marinas " sub %w0, %w0, #1\n" 2653a0310ebSWill Deacon " stlxr %w1, %w0, %2\n" 26681bb5c64SWill Deacon " cbnz %w1, 1b", 26781bb5c64SWill Deacon /* LSE atomics */ 26881bb5c64SWill Deacon " movn %w0, #0\n" 26981bb5c64SWill Deacon " nop\n" 27081bb5c64SWill Deacon " nop\n" 27181bb5c64SWill Deacon " staddl %w0, %2") 2723a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 2733a0310ebSWill Deacon : 27495c41896SWill Deacon : "memory"); 27508e875c1SCatalin Marinas } 27608e875c1SCatalin Marinas 27708e875c1SCatalin Marinas static inline int arch_read_trylock(arch_rwlock_t *rw) 27808e875c1SCatalin Marinas { 27981bb5c64SWill Deacon unsigned int tmp, tmp2; 28008e875c1SCatalin Marinas 28181bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 28281bb5c64SWill Deacon /* LL/SC */ 28381bb5c64SWill Deacon " mov %w1, #1\n" 2849511ca19SWill Deacon "1: ldaxr %w0, %2\n" 28508e875c1SCatalin Marinas " add %w0, %w0, #1\n" 2869511ca19SWill Deacon " tbnz %w0, #31, 2f\n" 2873a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 2889511ca19SWill Deacon " cbnz %w1, 1b\n" 28981bb5c64SWill Deacon "2:", 29081bb5c64SWill Deacon /* LSE atomics */ 29181bb5c64SWill Deacon " ldr %w0, %2\n" 29281bb5c64SWill Deacon " adds %w1, %w0, #1\n" 29381bb5c64SWill Deacon " tbnz %w1, #31, 1f\n" 29481bb5c64SWill Deacon " casa %w0, %w1, %2\n" 29581bb5c64SWill Deacon " sbc %w1, %w1, %w0\n" 29681bb5c64SWill Deacon " nop\n" 29781bb5c64SWill Deacon "1:") 29881bb5c64SWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 2993a0310ebSWill Deacon : 30081bb5c64SWill Deacon : "cc", "memory"); 30108e875c1SCatalin Marinas 30208e875c1SCatalin Marinas return !tmp2; 30308e875c1SCatalin Marinas } 30408e875c1SCatalin Marinas 30508e875c1SCatalin Marinas /* read_can_lock - would read_trylock() succeed? */ 30608e875c1SCatalin Marinas #define arch_read_can_lock(x) ((x)->lock < 0x80000000) 30708e875c1SCatalin Marinas 30808e875c1SCatalin Marinas #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 30908e875c1SCatalin Marinas #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 31008e875c1SCatalin Marinas 31108e875c1SCatalin Marinas #define arch_spin_relax(lock) cpu_relax() 31208e875c1SCatalin Marinas #define arch_read_relax(lock) cpu_relax() 31308e875c1SCatalin Marinas #define arch_write_relax(lock) cpu_relax() 31408e875c1SCatalin Marinas 31508e875c1SCatalin Marinas #endif /* __ASM_SPINLOCK_H */ 316