108e875c1SCatalin Marinas /* 208e875c1SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 308e875c1SCatalin Marinas * 408e875c1SCatalin Marinas * This program is free software; you can redistribute it and/or modify 508e875c1SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 608e875c1SCatalin Marinas * published by the Free Software Foundation. 708e875c1SCatalin Marinas * 808e875c1SCatalin Marinas * This program is distributed in the hope that it will be useful, 908e875c1SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 1008e875c1SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1108e875c1SCatalin Marinas * GNU General Public License for more details. 1208e875c1SCatalin Marinas * 1308e875c1SCatalin Marinas * You should have received a copy of the GNU General Public License 1408e875c1SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 1508e875c1SCatalin Marinas */ 1608e875c1SCatalin Marinas #ifndef __ASM_SPINLOCK_H 1708e875c1SCatalin Marinas #define __ASM_SPINLOCK_H 1808e875c1SCatalin Marinas 1908e875c1SCatalin Marinas #include <asm/spinlock_types.h> 2008e875c1SCatalin Marinas #include <asm/processor.h> 2108e875c1SCatalin Marinas 2208e875c1SCatalin Marinas /* 2308e875c1SCatalin Marinas * Spinlock implementation. 2408e875c1SCatalin Marinas * 2508e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 2608e875c1SCatalin Marinas * instructions. 2708e875c1SCatalin Marinas */ 2808e875c1SCatalin Marinas 2908e875c1SCatalin Marinas #define arch_spin_unlock_wait(lock) \ 3008e875c1SCatalin Marinas do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) 3108e875c1SCatalin Marinas 3208e875c1SCatalin Marinas #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 3308e875c1SCatalin Marinas 3408e875c1SCatalin Marinas static inline void arch_spin_lock(arch_spinlock_t *lock) 3508e875c1SCatalin Marinas { 3608e875c1SCatalin Marinas unsigned int tmp; 3752ea2a56SWill Deacon arch_spinlock_t lockval, newval; 3808e875c1SCatalin Marinas 3908e875c1SCatalin Marinas asm volatile( 4052ea2a56SWill Deacon /* Atomically increment the next ticket. */ 4152ea2a56SWill Deacon " prfm pstl1strm, %3\n" 4252ea2a56SWill Deacon "1: ldaxr %w0, %3\n" 4352ea2a56SWill Deacon " add %w1, %w0, %w5\n" 4452ea2a56SWill Deacon " stxr %w2, %w1, %3\n" 4552ea2a56SWill Deacon " cbnz %w2, 1b\n" 4652ea2a56SWill Deacon /* Did we get the lock? */ 4752ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 4852ea2a56SWill Deacon " cbz %w1, 3f\n" 4952ea2a56SWill Deacon /* 5052ea2a56SWill Deacon * No: spin on the owner. Send a local event to avoid missing an 5152ea2a56SWill Deacon * unlock before the exclusive load. 5252ea2a56SWill Deacon */ 5308e875c1SCatalin Marinas " sevl\n" 5452ea2a56SWill Deacon "2: wfe\n" 5552ea2a56SWill Deacon " ldaxrh %w2, %4\n" 5652ea2a56SWill Deacon " eor %w1, %w2, %w0, lsr #16\n" 5752ea2a56SWill Deacon " cbnz %w1, 2b\n" 5852ea2a56SWill Deacon /* We got the lock. Critical section starts here. */ 5952ea2a56SWill Deacon "3:" 6052ea2a56SWill Deacon : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock) 6152ea2a56SWill Deacon : "Q" (lock->owner), "I" (1 << TICKET_SHIFT) 6252ea2a56SWill Deacon : "memory"); 6308e875c1SCatalin Marinas } 6408e875c1SCatalin Marinas 6508e875c1SCatalin Marinas static inline int arch_spin_trylock(arch_spinlock_t *lock) 6608e875c1SCatalin Marinas { 6708e875c1SCatalin Marinas unsigned int tmp; 6852ea2a56SWill Deacon arch_spinlock_t lockval; 6908e875c1SCatalin Marinas 7008e875c1SCatalin Marinas asm volatile( 7152ea2a56SWill Deacon " prfm pstl1strm, %2\n" 7252ea2a56SWill Deacon "1: ldaxr %w0, %2\n" 7352ea2a56SWill Deacon " eor %w1, %w0, %w0, ror #16\n" 7452ea2a56SWill Deacon " cbnz %w1, 2f\n" 7552ea2a56SWill Deacon " add %w0, %w0, %3\n" 7652ea2a56SWill Deacon " stxr %w1, %w0, %2\n" 7752ea2a56SWill Deacon " cbnz %w1, 1b\n" 7852ea2a56SWill Deacon "2:" 7952ea2a56SWill Deacon : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) 8052ea2a56SWill Deacon : "I" (1 << TICKET_SHIFT) 8152ea2a56SWill Deacon : "memory"); 8208e875c1SCatalin Marinas 8308e875c1SCatalin Marinas return !tmp; 8408e875c1SCatalin Marinas } 8508e875c1SCatalin Marinas 8608e875c1SCatalin Marinas static inline void arch_spin_unlock(arch_spinlock_t *lock) 8708e875c1SCatalin Marinas { 8808e875c1SCatalin Marinas asm volatile( 8952ea2a56SWill Deacon " stlrh %w1, %0\n" 9052ea2a56SWill Deacon : "=Q" (lock->owner) 9152ea2a56SWill Deacon : "r" (lock->owner + 1) 9252ea2a56SWill Deacon : "memory"); 9308e875c1SCatalin Marinas } 9408e875c1SCatalin Marinas 9552ea2a56SWill Deacon static inline int arch_spin_is_locked(arch_spinlock_t *lock) 9652ea2a56SWill Deacon { 9752ea2a56SWill Deacon arch_spinlock_t lockval = ACCESS_ONCE(*lock); 9852ea2a56SWill Deacon return lockval.owner != lockval.next; 9952ea2a56SWill Deacon } 10052ea2a56SWill Deacon 10152ea2a56SWill Deacon static inline int arch_spin_is_contended(arch_spinlock_t *lock) 10252ea2a56SWill Deacon { 10352ea2a56SWill Deacon arch_spinlock_t lockval = ACCESS_ONCE(*lock); 10452ea2a56SWill Deacon return (lockval.next - lockval.owner) > 1; 10552ea2a56SWill Deacon } 10652ea2a56SWill Deacon #define arch_spin_is_contended arch_spin_is_contended 10752ea2a56SWill Deacon 10808e875c1SCatalin Marinas /* 10908e875c1SCatalin Marinas * Write lock implementation. 11008e875c1SCatalin Marinas * 11108e875c1SCatalin Marinas * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is 11208e875c1SCatalin Marinas * exclusively held. 11308e875c1SCatalin Marinas * 11408e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 11508e875c1SCatalin Marinas * instructions. 11608e875c1SCatalin Marinas */ 11708e875c1SCatalin Marinas 11808e875c1SCatalin Marinas static inline void arch_write_lock(arch_rwlock_t *rw) 11908e875c1SCatalin Marinas { 12008e875c1SCatalin Marinas unsigned int tmp; 12108e875c1SCatalin Marinas 12208e875c1SCatalin Marinas asm volatile( 12308e875c1SCatalin Marinas " sevl\n" 12408e875c1SCatalin Marinas "1: wfe\n" 1253a0310ebSWill Deacon "2: ldaxr %w0, %1\n" 12608e875c1SCatalin Marinas " cbnz %w0, 1b\n" 1273a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 12808e875c1SCatalin Marinas " cbnz %w0, 2b\n" 1293a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 1303a0310ebSWill Deacon : "r" (0x80000000) 1313a0310ebSWill Deacon : "cc", "memory"); 13208e875c1SCatalin Marinas } 13308e875c1SCatalin Marinas 13408e875c1SCatalin Marinas static inline int arch_write_trylock(arch_rwlock_t *rw) 13508e875c1SCatalin Marinas { 13608e875c1SCatalin Marinas unsigned int tmp; 13708e875c1SCatalin Marinas 13808e875c1SCatalin Marinas asm volatile( 1393a0310ebSWill Deacon " ldaxr %w0, %1\n" 14008e875c1SCatalin Marinas " cbnz %w0, 1f\n" 1413a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 14208e875c1SCatalin Marinas "1:\n" 1433a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 1443a0310ebSWill Deacon : "r" (0x80000000) 1453a0310ebSWill Deacon : "cc", "memory"); 14608e875c1SCatalin Marinas 14708e875c1SCatalin Marinas return !tmp; 14808e875c1SCatalin Marinas } 14908e875c1SCatalin Marinas 15008e875c1SCatalin Marinas static inline void arch_write_unlock(arch_rwlock_t *rw) 15108e875c1SCatalin Marinas { 15208e875c1SCatalin Marinas asm volatile( 1533a0310ebSWill Deacon " stlr %w1, %0\n" 1543a0310ebSWill Deacon : "=Q" (rw->lock) : "r" (0) : "memory"); 15508e875c1SCatalin Marinas } 15608e875c1SCatalin Marinas 15708e875c1SCatalin Marinas /* write_can_lock - would write_trylock() succeed? */ 15808e875c1SCatalin Marinas #define arch_write_can_lock(x) ((x)->lock == 0) 15908e875c1SCatalin Marinas 16008e875c1SCatalin Marinas /* 16108e875c1SCatalin Marinas * Read lock implementation. 16208e875c1SCatalin Marinas * 16308e875c1SCatalin Marinas * It exclusively loads the lock value, increments it and stores the new value 16408e875c1SCatalin Marinas * back if positive and the CPU still exclusively owns the location. If the 16508e875c1SCatalin Marinas * value is negative, the lock is already held. 16608e875c1SCatalin Marinas * 16708e875c1SCatalin Marinas * During unlocking there may be multiple active read locks but no write lock. 16808e875c1SCatalin Marinas * 16908e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 17008e875c1SCatalin Marinas * instructions. 17108e875c1SCatalin Marinas */ 17208e875c1SCatalin Marinas static inline void arch_read_lock(arch_rwlock_t *rw) 17308e875c1SCatalin Marinas { 17408e875c1SCatalin Marinas unsigned int tmp, tmp2; 17508e875c1SCatalin Marinas 17608e875c1SCatalin Marinas asm volatile( 17708e875c1SCatalin Marinas " sevl\n" 17808e875c1SCatalin Marinas "1: wfe\n" 1793a0310ebSWill Deacon "2: ldaxr %w0, %2\n" 18008e875c1SCatalin Marinas " add %w0, %w0, #1\n" 18108e875c1SCatalin Marinas " tbnz %w0, #31, 1b\n" 1823a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 18308e875c1SCatalin Marinas " cbnz %w1, 2b\n" 1843a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 1853a0310ebSWill Deacon : 1863a0310ebSWill Deacon : "cc", "memory"); 18708e875c1SCatalin Marinas } 18808e875c1SCatalin Marinas 18908e875c1SCatalin Marinas static inline void arch_read_unlock(arch_rwlock_t *rw) 19008e875c1SCatalin Marinas { 19108e875c1SCatalin Marinas unsigned int tmp, tmp2; 19208e875c1SCatalin Marinas 19308e875c1SCatalin Marinas asm volatile( 1943a0310ebSWill Deacon "1: ldxr %w0, %2\n" 19508e875c1SCatalin Marinas " sub %w0, %w0, #1\n" 1963a0310ebSWill Deacon " stlxr %w1, %w0, %2\n" 19708e875c1SCatalin Marinas " cbnz %w1, 1b\n" 1983a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 1993a0310ebSWill Deacon : 2003a0310ebSWill Deacon : "cc", "memory"); 20108e875c1SCatalin Marinas } 20208e875c1SCatalin Marinas 20308e875c1SCatalin Marinas static inline int arch_read_trylock(arch_rwlock_t *rw) 20408e875c1SCatalin Marinas { 20508e875c1SCatalin Marinas unsigned int tmp, tmp2 = 1; 20608e875c1SCatalin Marinas 20708e875c1SCatalin Marinas asm volatile( 2083a0310ebSWill Deacon " ldaxr %w0, %2\n" 20908e875c1SCatalin Marinas " add %w0, %w0, #1\n" 21008e875c1SCatalin Marinas " tbnz %w0, #31, 1f\n" 2113a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 21208e875c1SCatalin Marinas "1:\n" 2133a0310ebSWill Deacon : "=&r" (tmp), "+r" (tmp2), "+Q" (rw->lock) 2143a0310ebSWill Deacon : 2153a0310ebSWill Deacon : "cc", "memory"); 21608e875c1SCatalin Marinas 21708e875c1SCatalin Marinas return !tmp2; 21808e875c1SCatalin Marinas } 21908e875c1SCatalin Marinas 22008e875c1SCatalin Marinas /* read_can_lock - would read_trylock() succeed? */ 22108e875c1SCatalin Marinas #define arch_read_can_lock(x) ((x)->lock < 0x80000000) 22208e875c1SCatalin Marinas 22308e875c1SCatalin Marinas #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 22408e875c1SCatalin Marinas #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 22508e875c1SCatalin Marinas 22608e875c1SCatalin Marinas #define arch_spin_relax(lock) cpu_relax() 22708e875c1SCatalin Marinas #define arch_read_relax(lock) cpu_relax() 22808e875c1SCatalin Marinas #define arch_write_relax(lock) cpu_relax() 22908e875c1SCatalin Marinas 23008e875c1SCatalin Marinas #endif /* __ASM_SPINLOCK_H */ 231