108e875c1SCatalin Marinas /* 208e875c1SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 308e875c1SCatalin Marinas * 408e875c1SCatalin Marinas * This program is free software; you can redistribute it and/or modify 508e875c1SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 608e875c1SCatalin Marinas * published by the Free Software Foundation. 708e875c1SCatalin Marinas * 808e875c1SCatalin Marinas * This program is distributed in the hope that it will be useful, 908e875c1SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 1008e875c1SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1108e875c1SCatalin Marinas * GNU General Public License for more details. 1208e875c1SCatalin Marinas * 1308e875c1SCatalin Marinas * You should have received a copy of the GNU General Public License 1408e875c1SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 1508e875c1SCatalin Marinas */ 1608e875c1SCatalin Marinas #ifndef __ASM_SPINLOCK_H 1708e875c1SCatalin Marinas #define __ASM_SPINLOCK_H 1808e875c1SCatalin Marinas 1981bb5c64SWill Deacon #include <asm/lse.h> 2008e875c1SCatalin Marinas #include <asm/spinlock_types.h> 2108e875c1SCatalin Marinas #include <asm/processor.h> 2208e875c1SCatalin Marinas 2308e875c1SCatalin Marinas /* 2408e875c1SCatalin Marinas * Spinlock implementation. 2508e875c1SCatalin Marinas * 2608e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 2708e875c1SCatalin Marinas * instructions. 2808e875c1SCatalin Marinas */ 29d86b8da0SWill Deacon static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) 30d86b8da0SWill Deacon { 31d86b8da0SWill Deacon unsigned int tmp; 32d86b8da0SWill Deacon arch_spinlock_t lockval; 3308e875c1SCatalin Marinas 3438b850a7SWill Deacon /* 3538b850a7SWill Deacon * Ensure prior spin_lock operations to other locks have completed 3638b850a7SWill Deacon * on this CPU before we test whether "lock" is locked. 3738b850a7SWill Deacon */ 3838b850a7SWill Deacon smp_mb(); 3938b850a7SWill Deacon 40d86b8da0SWill Deacon asm volatile( 41d86b8da0SWill Deacon " sevl\n" 42d86b8da0SWill Deacon "1: wfe\n" 43d86b8da0SWill Deacon "2: ldaxr %w0, %2\n" 44d86b8da0SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 45d86b8da0SWill Deacon " cbnz %w1, 1b\n" 463a5facd0SWill Deacon /* Serialise against any concurrent lockers */ 47d86b8da0SWill Deacon ARM64_LSE_ATOMIC_INSN( 48d86b8da0SWill Deacon /* LL/SC */ 49d86b8da0SWill Deacon " stxr %w1, %w0, %2\n" 50d86b8da0SWill Deacon " nop\n" 513a5facd0SWill Deacon " nop\n", 523a5facd0SWill Deacon /* LSE atomics */ 533a5facd0SWill Deacon " mov %w1, %w0\n" 543a5facd0SWill Deacon " cas %w0, %w0, %2\n" 553a5facd0SWill Deacon " eor %w1, %w1, %w0\n") 563a5facd0SWill Deacon " cbnz %w1, 2b\n" 57d86b8da0SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 58d86b8da0SWill Deacon : 59d86b8da0SWill Deacon : "memory"); 60d86b8da0SWill Deacon } 6108e875c1SCatalin Marinas 6208e875c1SCatalin Marinas #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 6308e875c1SCatalin Marinas 6408e875c1SCatalin Marinas static inline void arch_spin_lock(arch_spinlock_t *lock) 6508e875c1SCatalin Marinas { 6608e875c1SCatalin Marinas unsigned int tmp; 6752ea2a56SWill Deacon arch_spinlock_t lockval, newval; 6808e875c1SCatalin Marinas 6908e875c1SCatalin Marinas asm volatile( 7052ea2a56SWill Deacon /* Atomically increment the next ticket. */ 7181bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 7281bb5c64SWill Deacon /* LL/SC */ 7352ea2a56SWill Deacon " prfm pstl1strm, %3\n" 7452ea2a56SWill Deacon "1: ldaxr %w0, %3\n" 7552ea2a56SWill Deacon " add %w1, %w0, %w5\n" 7652ea2a56SWill Deacon " stxr %w2, %w1, %3\n" 7781bb5c64SWill Deacon " cbnz %w2, 1b\n", 7881bb5c64SWill Deacon /* LSE atomics */ 7981bb5c64SWill Deacon " mov %w2, %w5\n" 8081bb5c64SWill Deacon " ldadda %w2, %w0, %3\n" 8181bb5c64SWill Deacon " nop\n" 8281bb5c64SWill Deacon " nop\n" 8381bb5c64SWill Deacon " nop\n" 8481bb5c64SWill Deacon ) 8581bb5c64SWill Deacon 8652ea2a56SWill Deacon /* Did we get the lock? */ 8752ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 8852ea2a56SWill Deacon " cbz %w1, 3f\n" 8952ea2a56SWill Deacon /* 9052ea2a56SWill Deacon * No: spin on the owner. Send a local event to avoid missing an 9152ea2a56SWill Deacon * unlock before the exclusive load. 9252ea2a56SWill Deacon */ 9308e875c1SCatalin Marinas " sevl\n" 9452ea2a56SWill Deacon "2: wfe\n" 9552ea2a56SWill Deacon " ldaxrh %w2, %4\n" 9652ea2a56SWill Deacon " eor %w1, %w2, %w0, lsr #16\n" 9752ea2a56SWill Deacon " cbnz %w1, 2b\n" 9852ea2a56SWill Deacon /* We got the lock. Critical section starts here. */ 9952ea2a56SWill Deacon "3:" 10052ea2a56SWill Deacon : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) 10152ea2a56SWill Deacon : "Q" (lock->owner), "I" (1 << TICKET_SHIFT) 10252ea2a56SWill Deacon : "memory"); 10308e875c1SCatalin Marinas } 10408e875c1SCatalin Marinas 10508e875c1SCatalin Marinas static inline int arch_spin_trylock(arch_spinlock_t *lock) 10608e875c1SCatalin Marinas { 10708e875c1SCatalin Marinas unsigned int tmp; 10852ea2a56SWill Deacon arch_spinlock_t lockval; 10908e875c1SCatalin Marinas 11081bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 11181bb5c64SWill Deacon /* LL/SC */ 11252ea2a56SWill Deacon " prfm pstl1strm, %2\n" 11352ea2a56SWill Deacon "1: ldaxr %w0, %2\n" 11452ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 11552ea2a56SWill Deacon " cbnz %w1, 2f\n" 11652ea2a56SWill Deacon " add %w0, %w0, %3\n" 11752ea2a56SWill Deacon " stxr %w1, %w0, %2\n" 11852ea2a56SWill Deacon " cbnz %w1, 1b\n" 11981bb5c64SWill Deacon "2:", 12081bb5c64SWill Deacon /* LSE atomics */ 12181bb5c64SWill Deacon " ldr %w0, %2\n" 12281bb5c64SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 12381bb5c64SWill Deacon " cbnz %w1, 1f\n" 12481bb5c64SWill Deacon " add %w1, %w0, %3\n" 12581bb5c64SWill Deacon " casa %w0, %w1, %2\n" 12681bb5c64SWill Deacon " and %w1, %w1, #0xffff\n" 12781bb5c64SWill Deacon " eor %w1, %w1, %w0, lsr #16\n" 12881bb5c64SWill Deacon "1:") 12952ea2a56SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 13052ea2a56SWill Deacon : "I" (1 << TICKET_SHIFT) 13152ea2a56SWill Deacon : "memory"); 13208e875c1SCatalin Marinas 13308e875c1SCatalin Marinas return !tmp; 13408e875c1SCatalin Marinas } 13508e875c1SCatalin Marinas 13608e875c1SCatalin Marinas static inline void arch_spin_unlock(arch_spinlock_t *lock) 13708e875c1SCatalin Marinas { 13881bb5c64SWill Deacon unsigned long tmp; 13981bb5c64SWill Deacon 14081bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 14181bb5c64SWill Deacon /* LL/SC */ 142c1d7cd22SWill Deacon " ldrh %w1, %0\n" 14381bb5c64SWill Deacon " add %w1, %w1, #1\n" 14481bb5c64SWill Deacon " stlrh %w1, %0", 14581bb5c64SWill Deacon /* LSE atomics */ 14681bb5c64SWill Deacon " mov %w1, #1\n" 14781bb5c64SWill Deacon " nop\n" 14881bb5c64SWill Deacon " staddlh %w1, %0") 14981bb5c64SWill Deacon : "=Q" (lock->owner), "=&r" (tmp) 15081bb5c64SWill Deacon : 15152ea2a56SWill Deacon : "memory"); 15208e875c1SCatalin Marinas } 15308e875c1SCatalin Marinas 1545686b06cSWill Deacon static inline int arch_spin_value_unlocked(arch_spinlock_t lock) 1555686b06cSWill Deacon { 1565686b06cSWill Deacon return lock.owner == lock.next; 1575686b06cSWill Deacon } 1585686b06cSWill Deacon 15952ea2a56SWill Deacon static inline int arch_spin_is_locked(arch_spinlock_t *lock) 16052ea2a56SWill Deacon { 16138b850a7SWill Deacon smp_mb(); /* See arch_spin_unlock_wait */ 162af2e7aaeSChristian Borntraeger return !arch_spin_value_unlocked(READ_ONCE(*lock)); 16352ea2a56SWill Deacon } 16452ea2a56SWill Deacon 16552ea2a56SWill Deacon static inline int arch_spin_is_contended(arch_spinlock_t *lock) 16652ea2a56SWill Deacon { 167af2e7aaeSChristian Borntraeger arch_spinlock_t lockval = READ_ONCE(*lock); 16852ea2a56SWill Deacon return (lockval.next - lockval.owner) > 1; 16952ea2a56SWill Deacon } 17052ea2a56SWill Deacon #define arch_spin_is_contended arch_spin_is_contended 17152ea2a56SWill Deacon 17208e875c1SCatalin Marinas /* 17308e875c1SCatalin Marinas * Write lock implementation. 17408e875c1SCatalin Marinas * 17508e875c1SCatalin Marinas * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is 17608e875c1SCatalin Marinas * exclusively held. 17708e875c1SCatalin Marinas * 17808e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 17908e875c1SCatalin Marinas * instructions. 18008e875c1SCatalin Marinas */ 18108e875c1SCatalin Marinas 18208e875c1SCatalin Marinas static inline void arch_write_lock(arch_rwlock_t *rw) 18308e875c1SCatalin Marinas { 18408e875c1SCatalin Marinas unsigned int tmp; 18508e875c1SCatalin Marinas 18681bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 18781bb5c64SWill Deacon /* LL/SC */ 18808e875c1SCatalin Marinas " sevl\n" 18908e875c1SCatalin Marinas "1: wfe\n" 1903a0310ebSWill Deacon "2: ldaxr %w0, %1\n" 19108e875c1SCatalin Marinas " cbnz %w0, 1b\n" 1923a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 19308e875c1SCatalin Marinas " cbnz %w0, 2b\n" 19481bb5c64SWill Deacon " nop", 19581bb5c64SWill Deacon /* LSE atomics */ 19681bb5c64SWill Deacon "1: mov %w0, wzr\n" 19781bb5c64SWill Deacon "2: casa %w0, %w2, %1\n" 19881bb5c64SWill Deacon " cbz %w0, 3f\n" 19981bb5c64SWill Deacon " ldxr %w0, %1\n" 20081bb5c64SWill Deacon " cbz %w0, 2b\n" 20181bb5c64SWill Deacon " wfe\n" 20281bb5c64SWill Deacon " b 1b\n" 20381bb5c64SWill Deacon "3:") 2043a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 2053a0310ebSWill Deacon : "r" (0x80000000) 20695c41896SWill Deacon : "memory"); 20708e875c1SCatalin Marinas } 20808e875c1SCatalin Marinas 20908e875c1SCatalin Marinas static inline int arch_write_trylock(arch_rwlock_t *rw) 21008e875c1SCatalin Marinas { 21108e875c1SCatalin Marinas unsigned int tmp; 21208e875c1SCatalin Marinas 21381bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 21481bb5c64SWill Deacon /* LL/SC */ 2159511ca19SWill Deacon "1: ldaxr %w0, %1\n" 2169511ca19SWill Deacon " cbnz %w0, 2f\n" 2173a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 2189511ca19SWill Deacon " cbnz %w0, 1b\n" 21981bb5c64SWill Deacon "2:", 22081bb5c64SWill Deacon /* LSE atomics */ 22181bb5c64SWill Deacon " mov %w0, wzr\n" 22281bb5c64SWill Deacon " casa %w0, %w2, %1\n" 22381bb5c64SWill Deacon " nop\n" 22481bb5c64SWill Deacon " nop") 2253a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 2263a0310ebSWill Deacon : "r" (0x80000000) 22795c41896SWill Deacon : "memory"); 22808e875c1SCatalin Marinas 22908e875c1SCatalin Marinas return !tmp; 23008e875c1SCatalin Marinas } 23108e875c1SCatalin Marinas 23208e875c1SCatalin Marinas static inline void arch_write_unlock(arch_rwlock_t *rw) 23308e875c1SCatalin Marinas { 23481bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 23581bb5c64SWill Deacon " stlr wzr, %0", 23681bb5c64SWill Deacon " swpl wzr, wzr, %0") 23781bb5c64SWill Deacon : "=Q" (rw->lock) :: "memory"); 23808e875c1SCatalin Marinas } 23908e875c1SCatalin Marinas 24008e875c1SCatalin Marinas /* write_can_lock - would write_trylock() succeed? */ 24108e875c1SCatalin Marinas #define arch_write_can_lock(x) ((x)->lock == 0) 24208e875c1SCatalin Marinas 24308e875c1SCatalin Marinas /* 24408e875c1SCatalin Marinas * Read lock implementation. 24508e875c1SCatalin Marinas * 24608e875c1SCatalin Marinas * It exclusively loads the lock value, increments it and stores the new value 24708e875c1SCatalin Marinas * back if positive and the CPU still exclusively owns the location. If the 24808e875c1SCatalin Marinas * value is negative, the lock is already held. 24908e875c1SCatalin Marinas * 25008e875c1SCatalin Marinas * During unlocking there may be multiple active read locks but no write lock. 25108e875c1SCatalin Marinas * 25208e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 25308e875c1SCatalin Marinas * instructions. 25481bb5c64SWill Deacon * 25581bb5c64SWill Deacon * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC 25681bb5c64SWill Deacon * and LSE implementations may exhibit different behaviour (although this 25781bb5c64SWill Deacon * will have no effect on lockdep). 25808e875c1SCatalin Marinas */ 25908e875c1SCatalin Marinas static inline void arch_read_lock(arch_rwlock_t *rw) 26008e875c1SCatalin Marinas { 26108e875c1SCatalin Marinas unsigned int tmp, tmp2; 26208e875c1SCatalin Marinas 26308e875c1SCatalin Marinas asm volatile( 26408e875c1SCatalin Marinas " sevl\n" 26581bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 26681bb5c64SWill Deacon /* LL/SC */ 26708e875c1SCatalin Marinas "1: wfe\n" 2683a0310ebSWill Deacon "2: ldaxr %w0, %2\n" 26908e875c1SCatalin Marinas " add %w0, %w0, #1\n" 27008e875c1SCatalin Marinas " tbnz %w0, #31, 1b\n" 2713a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 27281bb5c64SWill Deacon " nop\n" 27381bb5c64SWill Deacon " cbnz %w1, 2b", 27481bb5c64SWill Deacon /* LSE atomics */ 27581bb5c64SWill Deacon "1: wfe\n" 27681bb5c64SWill Deacon "2: ldxr %w0, %2\n" 27781bb5c64SWill Deacon " adds %w1, %w0, #1\n" 27881bb5c64SWill Deacon " tbnz %w1, #31, 1b\n" 27981bb5c64SWill Deacon " casa %w0, %w1, %2\n" 28081bb5c64SWill Deacon " sbc %w0, %w1, %w0\n" 28181bb5c64SWill Deacon " cbnz %w0, 2b") 2823a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 2833a0310ebSWill Deacon : 28481bb5c64SWill Deacon : "cc", "memory"); 28508e875c1SCatalin Marinas } 28608e875c1SCatalin Marinas 28708e875c1SCatalin Marinas static inline void arch_read_unlock(arch_rwlock_t *rw) 28808e875c1SCatalin Marinas { 28908e875c1SCatalin Marinas unsigned int tmp, tmp2; 29008e875c1SCatalin Marinas 29181bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 29281bb5c64SWill Deacon /* LL/SC */ 2933a0310ebSWill Deacon "1: ldxr %w0, %2\n" 29408e875c1SCatalin Marinas " sub %w0, %w0, #1\n" 2953a0310ebSWill Deacon " stlxr %w1, %w0, %2\n" 29681bb5c64SWill Deacon " cbnz %w1, 1b", 29781bb5c64SWill Deacon /* LSE atomics */ 29881bb5c64SWill Deacon " movn %w0, #0\n" 29981bb5c64SWill Deacon " nop\n" 30081bb5c64SWill Deacon " nop\n" 30181bb5c64SWill Deacon " staddl %w0, %2") 3023a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 3033a0310ebSWill Deacon : 30495c41896SWill Deacon : "memory"); 30508e875c1SCatalin Marinas } 30608e875c1SCatalin Marinas 30708e875c1SCatalin Marinas static inline int arch_read_trylock(arch_rwlock_t *rw) 30808e875c1SCatalin Marinas { 30981bb5c64SWill Deacon unsigned int tmp, tmp2; 31008e875c1SCatalin Marinas 31181bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 31281bb5c64SWill Deacon /* LL/SC */ 31381bb5c64SWill Deacon " mov %w1, #1\n" 3149511ca19SWill Deacon "1: ldaxr %w0, %2\n" 31508e875c1SCatalin Marinas " add %w0, %w0, #1\n" 3169511ca19SWill Deacon " tbnz %w0, #31, 2f\n" 3173a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 3189511ca19SWill Deacon " cbnz %w1, 1b\n" 31981bb5c64SWill Deacon "2:", 32081bb5c64SWill Deacon /* LSE atomics */ 32181bb5c64SWill Deacon " ldr %w0, %2\n" 32281bb5c64SWill Deacon " adds %w1, %w0, #1\n" 32381bb5c64SWill Deacon " tbnz %w1, #31, 1f\n" 32481bb5c64SWill Deacon " casa %w0, %w1, %2\n" 32581bb5c64SWill Deacon " sbc %w1, %w1, %w0\n" 32681bb5c64SWill Deacon " nop\n" 32781bb5c64SWill Deacon "1:") 32881bb5c64SWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 3293a0310ebSWill Deacon : 33081bb5c64SWill Deacon : "cc", "memory"); 33108e875c1SCatalin Marinas 33208e875c1SCatalin Marinas return !tmp2; 33308e875c1SCatalin Marinas } 33408e875c1SCatalin Marinas 33508e875c1SCatalin Marinas /* read_can_lock - would read_trylock() succeed? */ 33608e875c1SCatalin Marinas #define arch_read_can_lock(x) ((x)->lock < 0x80000000) 33708e875c1SCatalin Marinas 33808e875c1SCatalin Marinas #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 33908e875c1SCatalin Marinas #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 34008e875c1SCatalin Marinas 34108e875c1SCatalin Marinas #define arch_spin_relax(lock) cpu_relax() 34208e875c1SCatalin Marinas #define arch_read_relax(lock) cpu_relax() 34308e875c1SCatalin Marinas #define arch_write_relax(lock) cpu_relax() 34408e875c1SCatalin Marinas 34508e875c1SCatalin Marinas #endif /* __ASM_SPINLOCK_H */ 346