108e875c1SCatalin Marinas /* 208e875c1SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 308e875c1SCatalin Marinas * 408e875c1SCatalin Marinas * This program is free software; you can redistribute it and/or modify 508e875c1SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 608e875c1SCatalin Marinas * published by the Free Software Foundation. 708e875c1SCatalin Marinas * 808e875c1SCatalin Marinas * This program is distributed in the hope that it will be useful, 908e875c1SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 1008e875c1SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1108e875c1SCatalin Marinas * GNU General Public License for more details. 1208e875c1SCatalin Marinas * 1308e875c1SCatalin Marinas * You should have received a copy of the GNU General Public License 1408e875c1SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 1508e875c1SCatalin Marinas */ 1608e875c1SCatalin Marinas #ifndef __ASM_SPINLOCK_H 1708e875c1SCatalin Marinas #define __ASM_SPINLOCK_H 1808e875c1SCatalin Marinas 1908e875c1SCatalin Marinas #include <asm/spinlock_types.h> 2008e875c1SCatalin Marinas #include <asm/processor.h> 2108e875c1SCatalin Marinas 2208e875c1SCatalin Marinas /* 2308e875c1SCatalin Marinas * Spinlock implementation. 2408e875c1SCatalin Marinas * 2508e875c1SCatalin Marinas * The old value is read exclusively and the new one, if unlocked, is written 2608e875c1SCatalin Marinas * exclusively. In case of failure, the loop is restarted. 2708e875c1SCatalin Marinas * 2808e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 2908e875c1SCatalin Marinas * instructions. 3008e875c1SCatalin Marinas * 3108e875c1SCatalin Marinas * Unlocked value: 0 3208e875c1SCatalin Marinas * Locked value: 1 3308e875c1SCatalin Marinas */ 3408e875c1SCatalin Marinas 3508e875c1SCatalin Marinas #define arch_spin_is_locked(x) ((x)->lock != 0) 3608e875c1SCatalin Marinas #define arch_spin_unlock_wait(lock) \ 3708e875c1SCatalin Marinas do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) 3808e875c1SCatalin Marinas 3908e875c1SCatalin Marinas #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) 4008e875c1SCatalin Marinas 4108e875c1SCatalin Marinas static inline void arch_spin_lock(arch_spinlock_t *lock) 4208e875c1SCatalin Marinas { 4308e875c1SCatalin Marinas unsigned int tmp; 4408e875c1SCatalin Marinas 4508e875c1SCatalin Marinas asm volatile( 4608e875c1SCatalin Marinas " sevl\n" 4708e875c1SCatalin Marinas "1: wfe\n" 483a0310ebSWill Deacon "2: ldaxr %w0, %1\n" 4908e875c1SCatalin Marinas " cbnz %w0, 1b\n" 503a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 5108e875c1SCatalin Marinas " cbnz %w0, 2b\n" 523a0310ebSWill Deacon : "=&r" (tmp), "+Q" (lock->lock) 533a0310ebSWill Deacon : "r" (1) 543a0310ebSWill Deacon : "cc", "memory"); 5508e875c1SCatalin Marinas } 5608e875c1SCatalin Marinas 5708e875c1SCatalin Marinas static inline int arch_spin_trylock(arch_spinlock_t *lock) 5808e875c1SCatalin Marinas { 5908e875c1SCatalin Marinas unsigned int tmp; 6008e875c1SCatalin Marinas 6108e875c1SCatalin Marinas asm volatile( 623a0310ebSWill Deacon " ldaxr %w0, %1\n" 6308e875c1SCatalin Marinas " cbnz %w0, 1f\n" 643a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 6508e875c1SCatalin Marinas "1:\n" 663a0310ebSWill Deacon : "=&r" (tmp), "+Q" (lock->lock) 673a0310ebSWill Deacon : "r" (1) 683a0310ebSWill Deacon : "cc", "memory"); 6908e875c1SCatalin Marinas 7008e875c1SCatalin Marinas return !tmp; 7108e875c1SCatalin Marinas } 7208e875c1SCatalin Marinas 7308e875c1SCatalin Marinas static inline void arch_spin_unlock(arch_spinlock_t *lock) 7408e875c1SCatalin Marinas { 7508e875c1SCatalin Marinas asm volatile( 763a0310ebSWill Deacon " stlr %w1, %0\n" 773a0310ebSWill Deacon : "=Q" (lock->lock) : "r" (0) : "memory"); 7808e875c1SCatalin Marinas } 7908e875c1SCatalin Marinas 8008e875c1SCatalin Marinas /* 8108e875c1SCatalin Marinas * Write lock implementation. 8208e875c1SCatalin Marinas * 8308e875c1SCatalin Marinas * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is 8408e875c1SCatalin Marinas * exclusively held. 8508e875c1SCatalin Marinas * 8608e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 8708e875c1SCatalin Marinas * instructions. 8808e875c1SCatalin Marinas */ 8908e875c1SCatalin Marinas 9008e875c1SCatalin Marinas static inline void arch_write_lock(arch_rwlock_t *rw) 9108e875c1SCatalin Marinas { 9208e875c1SCatalin Marinas unsigned int tmp; 9308e875c1SCatalin Marinas 9408e875c1SCatalin Marinas asm volatile( 9508e875c1SCatalin Marinas " sevl\n" 9608e875c1SCatalin Marinas "1: wfe\n" 973a0310ebSWill Deacon "2: ldaxr %w0, %1\n" 9808e875c1SCatalin Marinas " cbnz %w0, 1b\n" 993a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 10008e875c1SCatalin Marinas " cbnz %w0, 2b\n" 1013a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 1023a0310ebSWill Deacon : "r" (0x80000000) 1033a0310ebSWill Deacon : "cc", "memory"); 10408e875c1SCatalin Marinas } 10508e875c1SCatalin Marinas 10608e875c1SCatalin Marinas static inline int arch_write_trylock(arch_rwlock_t *rw) 10708e875c1SCatalin Marinas { 10808e875c1SCatalin Marinas unsigned int tmp; 10908e875c1SCatalin Marinas 11008e875c1SCatalin Marinas asm volatile( 1113a0310ebSWill Deacon " ldaxr %w0, %1\n" 11208e875c1SCatalin Marinas " cbnz %w0, 1f\n" 1133a0310ebSWill Deacon " stxr %w0, %w2, %1\n" 11408e875c1SCatalin Marinas "1:\n" 1153a0310ebSWill Deacon : "=&r" (tmp), "+Q" (rw->lock) 1163a0310ebSWill Deacon : "r" (0x80000000) 1173a0310ebSWill Deacon : "cc", "memory"); 11808e875c1SCatalin Marinas 11908e875c1SCatalin Marinas return !tmp; 12008e875c1SCatalin Marinas } 12108e875c1SCatalin Marinas 12208e875c1SCatalin Marinas static inline void arch_write_unlock(arch_rwlock_t *rw) 12308e875c1SCatalin Marinas { 12408e875c1SCatalin Marinas asm volatile( 1253a0310ebSWill Deacon " stlr %w1, %0\n" 1263a0310ebSWill Deacon : "=Q" (rw->lock) : "r" (0) : "memory"); 12708e875c1SCatalin Marinas } 12808e875c1SCatalin Marinas 12908e875c1SCatalin Marinas /* write_can_lock - would write_trylock() succeed? */ 13008e875c1SCatalin Marinas #define arch_write_can_lock(x) ((x)->lock == 0) 13108e875c1SCatalin Marinas 13208e875c1SCatalin Marinas /* 13308e875c1SCatalin Marinas * Read lock implementation. 13408e875c1SCatalin Marinas * 13508e875c1SCatalin Marinas * It exclusively loads the lock value, increments it and stores the new value 13608e875c1SCatalin Marinas * back if positive and the CPU still exclusively owns the location. If the 13708e875c1SCatalin Marinas * value is negative, the lock is already held. 13808e875c1SCatalin Marinas * 13908e875c1SCatalin Marinas * During unlocking there may be multiple active read locks but no write lock. 14008e875c1SCatalin Marinas * 14108e875c1SCatalin Marinas * The memory barriers are implicit with the load-acquire and store-release 14208e875c1SCatalin Marinas * instructions. 14308e875c1SCatalin Marinas */ 14408e875c1SCatalin Marinas static inline void arch_read_lock(arch_rwlock_t *rw) 14508e875c1SCatalin Marinas { 14608e875c1SCatalin Marinas unsigned int tmp, tmp2; 14708e875c1SCatalin Marinas 14808e875c1SCatalin Marinas asm volatile( 14908e875c1SCatalin Marinas " sevl\n" 15008e875c1SCatalin Marinas "1: wfe\n" 1513a0310ebSWill Deacon "2: ldaxr %w0, %2\n" 15208e875c1SCatalin Marinas " add %w0, %w0, #1\n" 15308e875c1SCatalin Marinas " tbnz %w0, #31, 1b\n" 1543a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 15508e875c1SCatalin Marinas " cbnz %w1, 2b\n" 1563a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 1573a0310ebSWill Deacon : 1583a0310ebSWill Deacon : "cc", "memory"); 15908e875c1SCatalin Marinas } 16008e875c1SCatalin Marinas 16108e875c1SCatalin Marinas static inline void arch_read_unlock(arch_rwlock_t *rw) 16208e875c1SCatalin Marinas { 16308e875c1SCatalin Marinas unsigned int tmp, tmp2; 16408e875c1SCatalin Marinas 16508e875c1SCatalin Marinas asm volatile( 1663a0310ebSWill Deacon "1: ldxr %w0, %2\n" 16708e875c1SCatalin Marinas " sub %w0, %w0, #1\n" 1683a0310ebSWill Deacon " stlxr %w1, %w0, %2\n" 16908e875c1SCatalin Marinas " cbnz %w1, 1b\n" 1703a0310ebSWill Deacon : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) 1713a0310ebSWill Deacon : 1723a0310ebSWill Deacon : "cc", "memory"); 17308e875c1SCatalin Marinas } 17408e875c1SCatalin Marinas 17508e875c1SCatalin Marinas static inline int arch_read_trylock(arch_rwlock_t *rw) 17608e875c1SCatalin Marinas { 17708e875c1SCatalin Marinas unsigned int tmp, tmp2 = 1; 17808e875c1SCatalin Marinas 17908e875c1SCatalin Marinas asm volatile( 1803a0310ebSWill Deacon " ldaxr %w0, %2\n" 18108e875c1SCatalin Marinas " add %w0, %w0, #1\n" 18208e875c1SCatalin Marinas " tbnz %w0, #31, 1f\n" 1833a0310ebSWill Deacon " stxr %w1, %w0, %2\n" 18408e875c1SCatalin Marinas "1:\n" 1853a0310ebSWill Deacon : "=&r" (tmp), "+r" (tmp2), "+Q" (rw->lock) 1863a0310ebSWill Deacon : 1873a0310ebSWill Deacon : "cc", "memory"); 18808e875c1SCatalin Marinas 18908e875c1SCatalin Marinas return !tmp2; 19008e875c1SCatalin Marinas } 19108e875c1SCatalin Marinas 19208e875c1SCatalin Marinas /* read_can_lock - would read_trylock() succeed? */ 19308e875c1SCatalin Marinas #define arch_read_can_lock(x) ((x)->lock < 0x80000000) 19408e875c1SCatalin Marinas 19508e875c1SCatalin Marinas #define arch_read_lock_flags(lock, flags) arch_read_lock(lock) 19608e875c1SCatalin Marinas #define arch_write_lock_flags(lock, flags) arch_write_lock(lock) 19708e875c1SCatalin Marinas 19808e875c1SCatalin Marinas #define arch_spin_relax(lock) cpu_relax() 19908e875c1SCatalin Marinas #define arch_read_relax(lock) cpu_relax() 20008e875c1SCatalin Marinas #define arch_write_relax(lock) cpu_relax() 20108e875c1SCatalin Marinas 20208e875c1SCatalin Marinas #endif /* __ASM_SPINLOCK_H */ 203