108e875c1SCatalin Marinas /* 208e875c1SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 308e875c1SCatalin Marinas * 408e875c1SCatalin Marinas * This program is free software; you can redistribute it and/or modify 508e875c1SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 608e875c1SCatalin Marinas * published by the Free Software Foundation. 708e875c1SCatalin Marinas * 808e875c1SCatalin Marinas * This program is distributed in the hope that it will be useful, 908e875c1SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 1008e875c1SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1108e875c1SCatalin Marinas * GNU General Public License for more details. 1208e875c1SCatalin Marinas * 1308e875c1SCatalin Marinas * You should have received a copy of the GNU General Public License 1408e875c1SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 1508e875c1SCatalin Marinas */ 1608e875c1SCatalin Marinas #ifndef __ASM_SPINLOCK_H 1708e875c1SCatalin Marinas #define __ASM_SPINLOCK_H 1808e875c1SCatalin Marinas 1981bb5c64SWill Deacon #include <asm/lse.h> 2008e875c1SCatalin Marinas #include <asm/spinlock_types.h> 2108e875c1SCatalin Marinas #include <asm/processor.h> 2208e875c1SCatalin Marinas 2308e875c1SCatalin Marinas /* 2408e875c1SCatalin Marinas * Spinlock implementation. 2508e875c1SCatalin Marinas * 2608e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 2708e875c1SCatalin Marinas * instructions. 2808e875c1SCatalin Marinas */ 29d86b8da0SWill Deacon static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) 30d86b8da0SWill Deacon { 31d86b8da0SWill Deacon unsigned int tmp; 32d86b8da0SWill Deacon arch_spinlock_t lockval; 33c56bdcacSWill Deacon u32 owner; 3408e875c1SCatalin Marinas 3538b850a7SWill Deacon /* 3638b850a7SWill Deacon * Ensure prior spin_lock operations to other locks have completed 3738b850a7SWill Deacon * on this CPU before we test whether "lock" is locked. 3838b850a7SWill Deacon */ 3938b850a7SWill Deacon smp_mb(); 40c56bdcacSWill Deacon owner = READ_ONCE(lock->owner) << 16; 4138b850a7SWill Deacon 42d86b8da0SWill Deacon asm volatile( 43d86b8da0SWill Deacon " sevl\n" 44d86b8da0SWill Deacon "1: wfe\n" 45d86b8da0SWill Deacon "2: ldaxr %w0, %2\n" 46c56bdcacSWill Deacon /* Is the lock free? */ 47d86b8da0SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 48c56bdcacSWill Deacon " cbz %w1, 3f\n" 49c56bdcacSWill Deacon /* Lock taken -- has there been a subsequent unlock->lock transition? */ 50c56bdcacSWill Deacon " eor %w1, %w3, %w0, lsl #16\n" 51c56bdcacSWill Deacon " cbz %w1, 1b\n" 52c56bdcacSWill Deacon /* 53c56bdcacSWill Deacon * The owner has been updated, so there was an unlock->lock 54c56bdcacSWill Deacon * transition that we missed. That means we can rely on the 55c56bdcacSWill Deacon * store-release of the unlock operation paired with the 56c56bdcacSWill Deacon * load-acquire of the lock operation to publish any of our 57c56bdcacSWill Deacon * previous stores to the new lock owner and therefore don't 58c56bdcacSWill Deacon * need to bother with the writeback below. 59c56bdcacSWill Deacon */ 60c56bdcacSWill Deacon " b 4f\n" 61c56bdcacSWill Deacon "3:\n" 62c56bdcacSWill Deacon /* 63c56bdcacSWill Deacon * Serialise against any concurrent lockers by writing back the 64c56bdcacSWill Deacon * unlocked lock value 65c56bdcacSWill Deacon */ 66d86b8da0SWill Deacon ARM64_LSE_ATOMIC_INSN( 67d86b8da0SWill Deacon /* LL/SC */ 68d86b8da0SWill Deacon " stxr %w1, %w0, %2\n" 69d86b8da0SWill Deacon " nop\n" 703a5facd0SWill Deacon " nop\n", 713a5facd0SWill Deacon /* LSE atomics */ 723a5facd0SWill Deacon " mov %w1, %w0\n" 733a5facd0SWill Deacon " cas %w0, %w0, %2\n" 743a5facd0SWill Deacon " eor %w1, %w1, %w0\n") 75c56bdcacSWill Deacon /* Somebody else wrote to the lock, GOTO 10 and reload the value */ 763a5facd0SWill Deacon " cbnz %w1, 2b\n" 77c56bdcacSWill Deacon "4:" 78d86b8da0SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 79c56bdcacSWill Deacon : "r" (owner) 80d86b8da0SWill Deacon : "memory"); 81d86b8da0SWill Deacon } 8208e875c1SCatalin Marinas 8308e875c1SCatalin Marinas #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 8408e875c1SCatalin Marinas 8508e875c1SCatalin Marinas static inline void arch_spin_lock(arch_spinlock_t *lock) 8608e875c1SCatalin Marinas { 8708e875c1SCatalin Marinas unsigned int tmp; 8852ea2a56SWill Deacon arch_spinlock_t lockval, newval; 8908e875c1SCatalin Marinas 9008e875c1SCatalin Marinas asm volatile( 9152ea2a56SWill Deacon /* Atomically increment the next ticket. */ 9281bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 9381bb5c64SWill Deacon /* LL/SC */ 9452ea2a56SWill Deacon " prfm pstl1strm, %3\n" 9552ea2a56SWill Deacon "1: ldaxr %w0, %3\n" 9652ea2a56SWill Deacon " add %w1, %w0, %w5\n" 9752ea2a56SWill Deacon " stxr %w2, %w1, %3\n" 9881bb5c64SWill Deacon " cbnz %w2, 1b\n", 9981bb5c64SWill Deacon /* LSE atomics */ 10081bb5c64SWill Deacon " mov %w2, %w5\n" 10181bb5c64SWill Deacon " ldadda %w2, %w0, %3\n" 10281bb5c64SWill Deacon " nop\n" 10381bb5c64SWill Deacon " nop\n" 10481bb5c64SWill Deacon " nop\n" 10581bb5c64SWill Deacon ) 10681bb5c64SWill Deacon 10752ea2a56SWill Deacon /* Did we get the lock? */ 10852ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 10952ea2a56SWill Deacon " cbz %w1, 3f\n" 11052ea2a56SWill Deacon /* 11152ea2a56SWill Deacon * No: spin on the owner. Send a local event to avoid missing an 11252ea2a56SWill Deacon * unlock before the exclusive load. 11352ea2a56SWill Deacon */ 11408e875c1SCatalin Marinas " sevl\n" 11552ea2a56SWill Deacon "2: wfe\n" 11652ea2a56SWill Deacon " ldaxrh %w2, %4\n" 11752ea2a56SWill Deacon " eor %w1, %w2, %w0, lsr #16\n" 11852ea2a56SWill Deacon " cbnz %w1, 2b\n" 11952ea2a56SWill Deacon /* We got the lock. Critical section starts here. */ 12052ea2a56SWill Deacon "3:" 12152ea2a56SWill Deacon : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) 12252ea2a56SWill Deacon : "Q" (lock->owner), "I" (1 << TICKET_SHIFT) 12352ea2a56SWill Deacon : "memory"); 12408e875c1SCatalin Marinas } 12508e875c1SCatalin Marinas 12608e875c1SCatalin Marinas static inline int arch_spin_trylock(arch_spinlock_t *lock) 12708e875c1SCatalin Marinas { 12808e875c1SCatalin Marinas unsigned int tmp; 12952ea2a56SWill Deacon arch_spinlock_t lockval; 13008e875c1SCatalin Marinas 13181bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 13281bb5c64SWill Deacon /* LL/SC */ 13352ea2a56SWill Deacon " prfm pstl1strm, %2\n" 13452ea2a56SWill Deacon "1: ldaxr %w0, %2\n" 13552ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 13652ea2a56SWill Deacon " cbnz %w1, 2f\n" 13752ea2a56SWill Deacon " add %w0, %w0, %3\n" 13852ea2a56SWill Deacon " stxr %w1, %w0, %2\n" 13952ea2a56SWill Deacon " cbnz %w1, 1b\n" 14081bb5c64SWill Deacon "2:", 14181bb5c64SWill Deacon /* LSE atomics */ 14281bb5c64SWill Deacon " ldr %w0, %2\n" 14381bb5c64SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 14481bb5c64SWill Deacon " cbnz %w1, 1f\n" 14581bb5c64SWill Deacon " add %w1, %w0, %3\n" 14681bb5c64SWill Deacon " casa %w0, %w1, %2\n" 14781bb5c64SWill Deacon " and %w1, %w1, #0xffff\n" 14881bb5c64SWill Deacon " eor %w1, %w1, %w0, lsr #16\n" 14981bb5c64SWill Deacon "1:") 15052ea2a56SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 15152ea2a56SWill Deacon : "I" (1 << TICKET_SHIFT) 15252ea2a56SWill Deacon : "memory"); 15308e875c1SCatalin Marinas 15408e875c1SCatalin Marinas return !tmp; 15508e875c1SCatalin Marinas } 15608e875c1SCatalin Marinas 15708e875c1SCatalin Marinas static inline void arch_spin_unlock(arch_spinlock_t *lock) 15808e875c1SCatalin Marinas { 15981bb5c64SWill Deacon unsigned long tmp; 16081bb5c64SWill Deacon 16181bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 16281bb5c64SWill Deacon /* LL/SC */ 163c1d7cd22SWill Deacon " ldrh %w1, %0\n" 16481bb5c64SWill Deacon " add %w1, %w1, #1\n" 16581bb5c64SWill Deacon " stlrh %w1, %0", 16681bb5c64SWill Deacon /* LSE atomics */ 16781bb5c64SWill Deacon " mov %w1, #1\n" 16881bb5c64SWill Deacon " nop\n" 16981bb5c64SWill Deacon " staddlh %w1, %0") 17081bb5c64SWill Deacon : "=Q" (lock->owner), "=&r" (tmp) 17181bb5c64SWill Deacon : 17252ea2a56SWill Deacon : "memory"); 17308e875c1SCatalin Marinas } 17408e875c1SCatalin Marinas 1755686b06cSWill Deacon static inline int arch_spin_value_unlocked(arch_spinlock_t lock) 1765686b06cSWill Deacon { 1775686b06cSWill Deacon return lock.owner == lock.next; 1785686b06cSWill Deacon } 1795686b06cSWill Deacon 18052ea2a56SWill Deacon static inline int arch_spin_is_locked(arch_spinlock_t *lock) 18152ea2a56SWill Deacon { 18238b850a7SWill Deacon smp_mb(); /* See arch_spin_unlock_wait */ 183af2e7aaeSChristian Borntraeger return !arch_spin_value_unlocked(READ_ONCE(*lock)); 18452ea2a56SWill Deacon } 18552ea2a56SWill Deacon 18652ea2a56SWill Deacon static inline int arch_spin_is_contended(arch_spinlock_t *lock) 18752ea2a56SWill Deacon { 188af2e7aaeSChristian Borntraeger arch_spinlock_t lockval = READ_ONCE(*lock); 18952ea2a56SWill Deacon return (lockval.next - lockval.owner) > 1; 19052ea2a56SWill Deacon } 19152ea2a56SWill Deacon #define arch_spin_is_contended arch_spin_is_contended 19252ea2a56SWill Deacon 19308e875c1SCatalin Marinas /* 19408e875c1SCatalin Marinas * Write lock implementation. 19508e875c1SCatalin Marinas * 19608e875c1SCatalin Marinas * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is 19708e875c1SCatalin Marinas * exclusively held. 19808e875c1SCatalin Marinas * 19908e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 20008e875c1SCatalin Marinas * instructions. 20108e875c1SCatalin Marinas */ 20208e875c1SCatalin Marinas 20308e875c1SCatalin Marinas static inline void arch_write_lock(arch_rwlock_t *rw) 20408e875c1SCatalin Marinas { 20508e875c1SCatalin Marinas unsigned int tmp; 20608e875c1SCatalin Marinas 20781bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 20881bb5c64SWill Deacon /* LL/SC */ 20908e875c1SCatalin Marinas " sevl\n" 21008e875c1SCatalin Marinas "1: wfe\n" 2113a0310ebSWill Deacon "2: ldaxr %w0, %1\n" 21208e875c1SCatalin Marinas " cbnz %w0, 1b\n" 2133a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 21408e875c1SCatalin Marinas " cbnz %w0, 2b\n" 21581bb5c64SWill Deacon " nop", 21681bb5c64SWill Deacon /* LSE atomics */ 21781bb5c64SWill Deacon "1: mov %w0, wzr\n" 21881bb5c64SWill Deacon "2: casa %w0, %w2, %1\n" 21981bb5c64SWill Deacon " cbz %w0, 3f\n" 22081bb5c64SWill Deacon " ldxr %w0, %1\n" 22181bb5c64SWill Deacon " cbz %w0, 2b\n" 22281bb5c64SWill Deacon " wfe\n" 22381bb5c64SWill Deacon " b 1b\n" 22481bb5c64SWill Deacon "3:") 2253a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 2263a0310ebSWill Deacon : "r" (0x80000000) 22795c41896SWill Deacon : "memory"); 22808e875c1SCatalin Marinas } 22908e875c1SCatalin Marinas 23008e875c1SCatalin Marinas static inline int arch_write_trylock(arch_rwlock_t *rw) 23108e875c1SCatalin Marinas { 23208e875c1SCatalin Marinas unsigned int tmp; 23308e875c1SCatalin Marinas 23481bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 23581bb5c64SWill Deacon /* LL/SC */ 2369511ca19SWill Deacon "1: ldaxr %w0, %1\n" 2379511ca19SWill Deacon " cbnz %w0, 2f\n" 2383a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 2399511ca19SWill Deacon " cbnz %w0, 1b\n" 24081bb5c64SWill Deacon "2:", 24181bb5c64SWill Deacon /* LSE atomics */ 24281bb5c64SWill Deacon " mov %w0, wzr\n" 24381bb5c64SWill Deacon " casa %w0, %w2, %1\n" 24481bb5c64SWill Deacon " nop\n" 24581bb5c64SWill Deacon " nop") 2463a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 2473a0310ebSWill Deacon : "r" (0x80000000) 24895c41896SWill Deacon : "memory"); 24908e875c1SCatalin Marinas 25008e875c1SCatalin Marinas return !tmp; 25108e875c1SCatalin Marinas } 25208e875c1SCatalin Marinas 25308e875c1SCatalin Marinas static inline void arch_write_unlock(arch_rwlock_t *rw) 25408e875c1SCatalin Marinas { 25581bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 25681bb5c64SWill Deacon " stlr wzr, %0", 25781bb5c64SWill Deacon " swpl wzr, wzr, %0") 25881bb5c64SWill Deacon : "=Q" (rw->lock) :: "memory"); 25908e875c1SCatalin Marinas } 26008e875c1SCatalin Marinas 26108e875c1SCatalin Marinas /* write_can_lock - would write_trylock() succeed? */ 26208e875c1SCatalin Marinas #define arch_write_can_lock(x) ((x)->lock == 0) 26308e875c1SCatalin Marinas 26408e875c1SCatalin Marinas /* 26508e875c1SCatalin Marinas * Read lock implementation. 26608e875c1SCatalin Marinas * 26708e875c1SCatalin Marinas * It exclusively loads the lock value, increments it and stores the new value 26808e875c1SCatalin Marinas * back if positive and the CPU still exclusively owns the location. If the 26908e875c1SCatalin Marinas * value is negative, the lock is already held. 27008e875c1SCatalin Marinas * 27108e875c1SCatalin Marinas * During unlocking there may be multiple active read locks but no write lock. 27208e875c1SCatalin Marinas * 27308e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 27408e875c1SCatalin Marinas * instructions. 27581bb5c64SWill Deacon * 27681bb5c64SWill Deacon * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC 27781bb5c64SWill Deacon * and LSE implementations may exhibit different behaviour (although this 27881bb5c64SWill Deacon * will have no effect on lockdep). 27908e875c1SCatalin Marinas */ 28008e875c1SCatalin Marinas static inline void arch_read_lock(arch_rwlock_t *rw) 28108e875c1SCatalin Marinas { 28208e875c1SCatalin Marinas unsigned int tmp, tmp2; 28308e875c1SCatalin Marinas 28408e875c1SCatalin Marinas asm volatile( 28508e875c1SCatalin Marinas " sevl\n" 28681bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 28781bb5c64SWill Deacon /* LL/SC */ 28808e875c1SCatalin Marinas "1: wfe\n" 2893a0310ebSWill Deacon "2: ldaxr %w0, %2\n" 29008e875c1SCatalin Marinas " add %w0, %w0, #1\n" 29108e875c1SCatalin Marinas " tbnz %w0, #31, 1b\n" 2923a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 29381bb5c64SWill Deacon " nop\n" 29481bb5c64SWill Deacon " cbnz %w1, 2b", 29581bb5c64SWill Deacon /* LSE atomics */ 29681bb5c64SWill Deacon "1: wfe\n" 29781bb5c64SWill Deacon "2: ldxr %w0, %2\n" 29881bb5c64SWill Deacon " adds %w1, %w0, #1\n" 29981bb5c64SWill Deacon " tbnz %w1, #31, 1b\n" 30081bb5c64SWill Deacon " casa %w0, %w1, %2\n" 30181bb5c64SWill Deacon " sbc %w0, %w1, %w0\n" 30281bb5c64SWill Deacon " cbnz %w0, 2b") 3033a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 3043a0310ebSWill Deacon : 30581bb5c64SWill Deacon : "cc", "memory"); 30608e875c1SCatalin Marinas } 30708e875c1SCatalin Marinas 30808e875c1SCatalin Marinas static inline void arch_read_unlock(arch_rwlock_t *rw) 30908e875c1SCatalin Marinas { 31008e875c1SCatalin Marinas unsigned int tmp, tmp2; 31108e875c1SCatalin Marinas 31281bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 31381bb5c64SWill Deacon /* LL/SC */ 3143a0310ebSWill Deacon "1: ldxr %w0, %2\n" 31508e875c1SCatalin Marinas " sub %w0, %w0, #1\n" 3163a0310ebSWill Deacon " stlxr %w1, %w0, %2\n" 31781bb5c64SWill Deacon " cbnz %w1, 1b", 31881bb5c64SWill Deacon /* LSE atomics */ 31981bb5c64SWill Deacon " movn %w0, #0\n" 32081bb5c64SWill Deacon " nop\n" 32181bb5c64SWill Deacon " nop\n" 32281bb5c64SWill Deacon " staddl %w0, %2") 3233a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 3243a0310ebSWill Deacon : 32595c41896SWill Deacon : "memory"); 32608e875c1SCatalin Marinas } 32708e875c1SCatalin Marinas 32808e875c1SCatalin Marinas static inline int arch_read_trylock(arch_rwlock_t *rw) 32908e875c1SCatalin Marinas { 33081bb5c64SWill Deacon unsigned int tmp, tmp2; 33108e875c1SCatalin Marinas 33281bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 33381bb5c64SWill Deacon /* LL/SC */ 33481bb5c64SWill Deacon " mov %w1, #1\n" 3359511ca19SWill Deacon "1: ldaxr %w0, %2\n" 33608e875c1SCatalin Marinas " add %w0, %w0, #1\n" 3379511ca19SWill Deacon " tbnz %w0, #31, 2f\n" 3383a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 3399511ca19SWill Deacon " cbnz %w1, 1b\n" 34081bb5c64SWill Deacon "2:", 34181bb5c64SWill Deacon /* LSE atomics */ 34281bb5c64SWill Deacon " ldr %w0, %2\n" 34381bb5c64SWill Deacon " adds %w1, %w0, #1\n" 34481bb5c64SWill Deacon " tbnz %w1, #31, 1f\n" 34581bb5c64SWill Deacon " casa %w0, %w1, %2\n" 34681bb5c64SWill Deacon " sbc %w1, %w1, %w0\n" 34781bb5c64SWill Deacon " nop\n" 34881bb5c64SWill Deacon "1:") 34981bb5c64SWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 3503a0310ebSWill Deacon : 35181bb5c64SWill Deacon : "cc", "memory"); 35208e875c1SCatalin Marinas 35308e875c1SCatalin Marinas return !tmp2; 35408e875c1SCatalin Marinas } 35508e875c1SCatalin Marinas 35608e875c1SCatalin Marinas /* read_can_lock - would read_trylock() succeed? */ 35708e875c1SCatalin Marinas #define arch_read_can_lock(x) ((x)->lock < 0x80000000) 35808e875c1SCatalin Marinas 35908e875c1SCatalin Marinas #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 36008e875c1SCatalin Marinas #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 36108e875c1SCatalin Marinas 36208e875c1SCatalin Marinas #define arch_spin_relax(lock) cpu_relax() 36308e875c1SCatalin Marinas #define arch_read_relax(lock) cpu_relax() 36408e875c1SCatalin Marinas #define arch_write_relax(lock) cpu_relax() 36508e875c1SCatalin Marinas 366872c63fbSWill Deacon /* 367872c63fbSWill Deacon * Accesses appearing in program order before a spin_lock() operation 368872c63fbSWill Deacon * can be reordered with accesses inside the critical section, by virtue 369872c63fbSWill Deacon * of arch_spin_lock being constructed using acquire semantics. 370872c63fbSWill Deacon * 371872c63fbSWill Deacon * In cases where this is problematic (e.g. try_to_wake_up), an 372872c63fbSWill Deacon * smp_mb__before_spinlock() can restore the required ordering. 373872c63fbSWill Deacon */ 374872c63fbSWill Deacon #define smp_mb__before_spinlock() smp_mb() 375872c63fbSWill Deacon 37608e875c1SCatalin Marinas #endif /* __ASM_SPINLOCK_H */ 377