108e875c1SCatalin Marinas /* 208e875c1SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 308e875c1SCatalin Marinas * 408e875c1SCatalin Marinas * This program is free software; you can redistribute it and/or modify 508e875c1SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 608e875c1SCatalin Marinas * published by the Free Software Foundation. 708e875c1SCatalin Marinas * 808e875c1SCatalin Marinas * This program is distributed in the hope that it will be useful, 908e875c1SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 1008e875c1SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1108e875c1SCatalin Marinas * GNU General Public License for more details. 1208e875c1SCatalin Marinas * 1308e875c1SCatalin Marinas * You should have received a copy of the GNU General Public License 1408e875c1SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 1508e875c1SCatalin Marinas */ 1608e875c1SCatalin Marinas #ifndef __ASM_SPINLOCK_H 1708e875c1SCatalin Marinas #define __ASM_SPINLOCK_H 1808e875c1SCatalin Marinas 1981bb5c64SWill Deacon #include <asm/lse.h> 2008e875c1SCatalin Marinas #include <asm/spinlock_types.h> 2108e875c1SCatalin Marinas #include <asm/processor.h> 2208e875c1SCatalin Marinas 2308e875c1SCatalin Marinas /* 2408e875c1SCatalin Marinas * Spinlock implementation. 2508e875c1SCatalin Marinas * 2608e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 2708e875c1SCatalin Marinas * instructions. 2808e875c1SCatalin Marinas */ 29d86b8da0SWill Deacon static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) 30d86b8da0SWill Deacon { 31d86b8da0SWill Deacon unsigned int tmp; 32d86b8da0SWill Deacon arch_spinlock_t lockval; 3308e875c1SCatalin Marinas 34d86b8da0SWill Deacon asm volatile( 35d86b8da0SWill Deacon " sevl\n" 36d86b8da0SWill Deacon "1: wfe\n" 37d86b8da0SWill Deacon "2: ldaxr %w0, %2\n" 38d86b8da0SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 39d86b8da0SWill Deacon " cbnz %w1, 1b\n" 40d86b8da0SWill Deacon ARM64_LSE_ATOMIC_INSN( 41d86b8da0SWill Deacon /* LL/SC */ 42d86b8da0SWill Deacon " stxr %w1, %w0, %2\n" 43d86b8da0SWill Deacon " cbnz %w1, 2b\n", /* Serialise against any concurrent lockers */ 44d86b8da0SWill Deacon /* LSE atomics */ 45d86b8da0SWill Deacon " nop\n" 46d86b8da0SWill Deacon " nop\n") 47d86b8da0SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 48d86b8da0SWill Deacon : 49d86b8da0SWill Deacon : "memory"); 50d86b8da0SWill Deacon } 5108e875c1SCatalin Marinas 5208e875c1SCatalin Marinas #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 5308e875c1SCatalin Marinas 5408e875c1SCatalin Marinas static inline void arch_spin_lock(arch_spinlock_t *lock) 5508e875c1SCatalin Marinas { 5608e875c1SCatalin Marinas unsigned int tmp; 5752ea2a56SWill Deacon arch_spinlock_t lockval, newval; 5808e875c1SCatalin Marinas 5908e875c1SCatalin Marinas asm volatile( 6052ea2a56SWill Deacon /* Atomically increment the next ticket. */ 6181bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 6281bb5c64SWill Deacon /* LL/SC */ 6352ea2a56SWill Deacon " prfm pstl1strm, %3\n" 6452ea2a56SWill Deacon "1: ldaxr %w0, %3\n" 6552ea2a56SWill Deacon " add %w1, %w0, %w5\n" 6652ea2a56SWill Deacon " stxr %w2, %w1, %3\n" 6781bb5c64SWill Deacon " cbnz %w2, 1b\n", 6881bb5c64SWill Deacon /* LSE atomics */ 6981bb5c64SWill Deacon " mov %w2, %w5\n" 7081bb5c64SWill Deacon " ldadda %w2, %w0, %3\n" 7181bb5c64SWill Deacon " nop\n" 7281bb5c64SWill Deacon " nop\n" 7381bb5c64SWill Deacon " nop\n" 7481bb5c64SWill Deacon ) 7581bb5c64SWill Deacon 7652ea2a56SWill Deacon /* Did we get the lock? */ 7752ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 7852ea2a56SWill Deacon " cbz %w1, 3f\n" 7952ea2a56SWill Deacon /* 8052ea2a56SWill Deacon * No: spin on the owner. Send a local event to avoid missing an 8152ea2a56SWill Deacon * unlock before the exclusive load. 8252ea2a56SWill Deacon */ 8308e875c1SCatalin Marinas " sevl\n" 8452ea2a56SWill Deacon "2: wfe\n" 8552ea2a56SWill Deacon " ldaxrh %w2, %4\n" 8652ea2a56SWill Deacon " eor %w1, %w2, %w0, lsr #16\n" 8752ea2a56SWill Deacon " cbnz %w1, 2b\n" 8852ea2a56SWill Deacon /* We got the lock. Critical section starts here. */ 8952ea2a56SWill Deacon "3:" 9052ea2a56SWill Deacon : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) 9152ea2a56SWill Deacon : "Q" (lock->owner), "I" (1 << TICKET_SHIFT) 9252ea2a56SWill Deacon : "memory"); 9308e875c1SCatalin Marinas } 9408e875c1SCatalin Marinas 9508e875c1SCatalin Marinas static inline int arch_spin_trylock(arch_spinlock_t *lock) 9608e875c1SCatalin Marinas { 9708e875c1SCatalin Marinas unsigned int tmp; 9852ea2a56SWill Deacon arch_spinlock_t lockval; 9908e875c1SCatalin Marinas 10081bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 10181bb5c64SWill Deacon /* LL/SC */ 10252ea2a56SWill Deacon " prfm pstl1strm, %2\n" 10352ea2a56SWill Deacon "1: ldaxr %w0, %2\n" 10452ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 10552ea2a56SWill Deacon " cbnz %w1, 2f\n" 10652ea2a56SWill Deacon " add %w0, %w0, %3\n" 10752ea2a56SWill Deacon " stxr %w1, %w0, %2\n" 10852ea2a56SWill Deacon " cbnz %w1, 1b\n" 10981bb5c64SWill Deacon "2:", 11081bb5c64SWill Deacon /* LSE atomics */ 11181bb5c64SWill Deacon " ldr %w0, %2\n" 11281bb5c64SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 11381bb5c64SWill Deacon " cbnz %w1, 1f\n" 11481bb5c64SWill Deacon " add %w1, %w0, %3\n" 11581bb5c64SWill Deacon " casa %w0, %w1, %2\n" 11681bb5c64SWill Deacon " and %w1, %w1, #0xffff\n" 11781bb5c64SWill Deacon " eor %w1, %w1, %w0, lsr #16\n" 11881bb5c64SWill Deacon "1:") 11952ea2a56SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 12052ea2a56SWill Deacon : "I" (1 << TICKET_SHIFT) 12152ea2a56SWill Deacon : "memory"); 12208e875c1SCatalin Marinas 12308e875c1SCatalin Marinas return !tmp; 12408e875c1SCatalin Marinas } 12508e875c1SCatalin Marinas 12608e875c1SCatalin Marinas static inline void arch_spin_unlock(arch_spinlock_t *lock) 12708e875c1SCatalin Marinas { 12881bb5c64SWill Deacon unsigned long tmp; 12981bb5c64SWill Deacon 13081bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 13181bb5c64SWill Deacon /* LL/SC */ 132c1d7cd22SWill Deacon " ldrh %w1, %0\n" 13381bb5c64SWill Deacon " add %w1, %w1, #1\n" 13481bb5c64SWill Deacon " stlrh %w1, %0", 13581bb5c64SWill Deacon /* LSE atomics */ 13681bb5c64SWill Deacon " mov %w1, #1\n" 13781bb5c64SWill Deacon " nop\n" 13881bb5c64SWill Deacon " staddlh %w1, %0") 13981bb5c64SWill Deacon : "=Q" (lock->owner), "=&r" (tmp) 14081bb5c64SWill Deacon : 14152ea2a56SWill Deacon : "memory"); 14208e875c1SCatalin Marinas } 14308e875c1SCatalin Marinas 1445686b06cSWill Deacon static inline int arch_spin_value_unlocked(arch_spinlock_t lock) 1455686b06cSWill Deacon { 1465686b06cSWill Deacon return lock.owner == lock.next; 1475686b06cSWill Deacon } 1485686b06cSWill Deacon 14952ea2a56SWill Deacon static inline int arch_spin_is_locked(arch_spinlock_t *lock) 15052ea2a56SWill Deacon { 151af2e7aaeSChristian Borntraeger return !arch_spin_value_unlocked(READ_ONCE(*lock)); 15252ea2a56SWill Deacon } 15352ea2a56SWill Deacon 15452ea2a56SWill Deacon static inline int arch_spin_is_contended(arch_spinlock_t *lock) 15552ea2a56SWill Deacon { 156af2e7aaeSChristian Borntraeger arch_spinlock_t lockval = READ_ONCE(*lock); 15752ea2a56SWill Deacon return (lockval.next - lockval.owner) > 1; 15852ea2a56SWill Deacon } 15952ea2a56SWill Deacon #define arch_spin_is_contended arch_spin_is_contended 16052ea2a56SWill Deacon 16108e875c1SCatalin Marinas /* 16208e875c1SCatalin Marinas * Write lock implementation. 16308e875c1SCatalin Marinas * 16408e875c1SCatalin Marinas * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is 16508e875c1SCatalin Marinas * exclusively held. 16608e875c1SCatalin Marinas * 16708e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 16808e875c1SCatalin Marinas * instructions. 16908e875c1SCatalin Marinas */ 17008e875c1SCatalin Marinas 17108e875c1SCatalin Marinas static inline void arch_write_lock(arch_rwlock_t *rw) 17208e875c1SCatalin Marinas { 17308e875c1SCatalin Marinas unsigned int tmp; 17408e875c1SCatalin Marinas 17581bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 17681bb5c64SWill Deacon /* LL/SC */ 17708e875c1SCatalin Marinas " sevl\n" 17808e875c1SCatalin Marinas "1: wfe\n" 1793a0310ebSWill Deacon "2: ldaxr %w0, %1\n" 18008e875c1SCatalin Marinas " cbnz %w0, 1b\n" 1813a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 18208e875c1SCatalin Marinas " cbnz %w0, 2b\n" 18381bb5c64SWill Deacon " nop", 18481bb5c64SWill Deacon /* LSE atomics */ 18581bb5c64SWill Deacon "1: mov %w0, wzr\n" 18681bb5c64SWill Deacon "2: casa %w0, %w2, %1\n" 18781bb5c64SWill Deacon " cbz %w0, 3f\n" 18881bb5c64SWill Deacon " ldxr %w0, %1\n" 18981bb5c64SWill Deacon " cbz %w0, 2b\n" 19081bb5c64SWill Deacon " wfe\n" 19181bb5c64SWill Deacon " b 1b\n" 19281bb5c64SWill Deacon "3:") 1933a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 1943a0310ebSWill Deacon : "r" (0x80000000) 19595c41896SWill Deacon : "memory"); 19608e875c1SCatalin Marinas } 19708e875c1SCatalin Marinas 19808e875c1SCatalin Marinas static inline int arch_write_trylock(arch_rwlock_t *rw) 19908e875c1SCatalin Marinas { 20008e875c1SCatalin Marinas unsigned int tmp; 20108e875c1SCatalin Marinas 20281bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 20381bb5c64SWill Deacon /* LL/SC */ 2049511ca19SWill Deacon "1: ldaxr %w0, %1\n" 2059511ca19SWill Deacon " cbnz %w0, 2f\n" 2063a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 2079511ca19SWill Deacon " cbnz %w0, 1b\n" 20881bb5c64SWill Deacon "2:", 20981bb5c64SWill Deacon /* LSE atomics */ 21081bb5c64SWill Deacon " mov %w0, wzr\n" 21181bb5c64SWill Deacon " casa %w0, %w2, %1\n" 21281bb5c64SWill Deacon " nop\n" 21381bb5c64SWill Deacon " nop") 2143a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 2153a0310ebSWill Deacon : "r" (0x80000000) 21695c41896SWill Deacon : "memory"); 21708e875c1SCatalin Marinas 21808e875c1SCatalin Marinas return !tmp; 21908e875c1SCatalin Marinas } 22008e875c1SCatalin Marinas 22108e875c1SCatalin Marinas static inline void arch_write_unlock(arch_rwlock_t *rw) 22208e875c1SCatalin Marinas { 22381bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 22481bb5c64SWill Deacon " stlr wzr, %0", 22581bb5c64SWill Deacon " swpl wzr, wzr, %0") 22681bb5c64SWill Deacon : "=Q" (rw->lock) :: "memory"); 22708e875c1SCatalin Marinas } 22808e875c1SCatalin Marinas 22908e875c1SCatalin Marinas /* write_can_lock - would write_trylock() succeed? */ 23008e875c1SCatalin Marinas #define arch_write_can_lock(x) ((x)->lock == 0) 23108e875c1SCatalin Marinas 23208e875c1SCatalin Marinas /* 23308e875c1SCatalin Marinas * Read lock implementation. 23408e875c1SCatalin Marinas * 23508e875c1SCatalin Marinas * It exclusively loads the lock value, increments it and stores the new value 23608e875c1SCatalin Marinas * back if positive and the CPU still exclusively owns the location. If the 23708e875c1SCatalin Marinas * value is negative, the lock is already held. 23808e875c1SCatalin Marinas * 23908e875c1SCatalin Marinas * During unlocking there may be multiple active read locks but no write lock. 24008e875c1SCatalin Marinas * 24108e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 24208e875c1SCatalin Marinas * instructions. 24381bb5c64SWill Deacon * 24481bb5c64SWill Deacon * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC 24581bb5c64SWill Deacon * and LSE implementations may exhibit different behaviour (although this 24681bb5c64SWill Deacon * will have no effect on lockdep). 24708e875c1SCatalin Marinas */ 24808e875c1SCatalin Marinas static inline void arch_read_lock(arch_rwlock_t *rw) 24908e875c1SCatalin Marinas { 25008e875c1SCatalin Marinas unsigned int tmp, tmp2; 25108e875c1SCatalin Marinas 25208e875c1SCatalin Marinas asm volatile( 25308e875c1SCatalin Marinas " sevl\n" 25481bb5c64SWill Deacon ARM64_LSE_ATOMIC_INSN( 25581bb5c64SWill Deacon /* LL/SC */ 25608e875c1SCatalin Marinas "1: wfe\n" 2573a0310ebSWill Deacon "2: ldaxr %w0, %2\n" 25808e875c1SCatalin Marinas " add %w0, %w0, #1\n" 25908e875c1SCatalin Marinas " tbnz %w0, #31, 1b\n" 2603a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 26181bb5c64SWill Deacon " nop\n" 26281bb5c64SWill Deacon " cbnz %w1, 2b", 26381bb5c64SWill Deacon /* LSE atomics */ 26481bb5c64SWill Deacon "1: wfe\n" 26581bb5c64SWill Deacon "2: ldxr %w0, %2\n" 26681bb5c64SWill Deacon " adds %w1, %w0, #1\n" 26781bb5c64SWill Deacon " tbnz %w1, #31, 1b\n" 26881bb5c64SWill Deacon " casa %w0, %w1, %2\n" 26981bb5c64SWill Deacon " sbc %w0, %w1, %w0\n" 27081bb5c64SWill Deacon " cbnz %w0, 2b") 2713a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 2723a0310ebSWill Deacon : 27381bb5c64SWill Deacon : "cc", "memory"); 27408e875c1SCatalin Marinas } 27508e875c1SCatalin Marinas 27608e875c1SCatalin Marinas static inline void arch_read_unlock(arch_rwlock_t *rw) 27708e875c1SCatalin Marinas { 27808e875c1SCatalin Marinas unsigned int tmp, tmp2; 27908e875c1SCatalin Marinas 28081bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 28181bb5c64SWill Deacon /* LL/SC */ 2823a0310ebSWill Deacon "1: ldxr %w0, %2\n" 28308e875c1SCatalin Marinas " sub %w0, %w0, #1\n" 2843a0310ebSWill Deacon " stlxr %w1, %w0, %2\n" 28581bb5c64SWill Deacon " cbnz %w1, 1b", 28681bb5c64SWill Deacon /* LSE atomics */ 28781bb5c64SWill Deacon " movn %w0, #0\n" 28881bb5c64SWill Deacon " nop\n" 28981bb5c64SWill Deacon " nop\n" 29081bb5c64SWill Deacon " staddl %w0, %2") 2913a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 2923a0310ebSWill Deacon : 29395c41896SWill Deacon : "memory"); 29408e875c1SCatalin Marinas } 29508e875c1SCatalin Marinas 29608e875c1SCatalin Marinas static inline int arch_read_trylock(arch_rwlock_t *rw) 29708e875c1SCatalin Marinas { 29881bb5c64SWill Deacon unsigned int tmp, tmp2; 29908e875c1SCatalin Marinas 30081bb5c64SWill Deacon asm volatile(ARM64_LSE_ATOMIC_INSN( 30181bb5c64SWill Deacon /* LL/SC */ 30281bb5c64SWill Deacon " mov %w1, #1\n" 3039511ca19SWill Deacon "1: ldaxr %w0, %2\n" 30408e875c1SCatalin Marinas " add %w0, %w0, #1\n" 3059511ca19SWill Deacon " tbnz %w0, #31, 2f\n" 3063a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 3079511ca19SWill Deacon " cbnz %w1, 1b\n" 30881bb5c64SWill Deacon "2:", 30981bb5c64SWill Deacon /* LSE atomics */ 31081bb5c64SWill Deacon " ldr %w0, %2\n" 31181bb5c64SWill Deacon " adds %w1, %w0, #1\n" 31281bb5c64SWill Deacon " tbnz %w1, #31, 1f\n" 31381bb5c64SWill Deacon " casa %w0, %w1, %2\n" 31481bb5c64SWill Deacon " sbc %w1, %w1, %w0\n" 31581bb5c64SWill Deacon " nop\n" 31681bb5c64SWill Deacon "1:") 31781bb5c64SWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 3183a0310ebSWill Deacon : 31981bb5c64SWill Deacon : "cc", "memory"); 32008e875c1SCatalin Marinas 32108e875c1SCatalin Marinas return !tmp2; 32208e875c1SCatalin Marinas } 32308e875c1SCatalin Marinas 32408e875c1SCatalin Marinas /* read_can_lock - would read_trylock() succeed? */ 32508e875c1SCatalin Marinas #define arch_read_can_lock(x) ((x)->lock < 0x80000000) 32608e875c1SCatalin Marinas 32708e875c1SCatalin Marinas #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 32808e875c1SCatalin Marinas #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 32908e875c1SCatalin Marinas 33008e875c1SCatalin Marinas #define arch_spin_relax(lock) cpu_relax() 33108e875c1SCatalin Marinas #define arch_read_relax(lock) cpu_relax() 33208e875c1SCatalin Marinas #define arch_write_relax(lock) cpu_relax() 33308e875c1SCatalin Marinas 33408e875c1SCatalin Marinas #endif /* __ASM_SPINLOCK_H */ 335