1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 #ifndef __ASM_SPINLOCK_H 3 #define __ASM_SPINLOCK_H 4 #ifdef __KERNEL__ 5 6 /* 7 * Simple spin lock operations. 8 * 9 * Copyright (C) 2001-2004 Paul Mackerras <paulus@au.ibm.com>, IBM 10 * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM 11 * Copyright (C) 2002 Dave Engebretsen <engebret@us.ibm.com>, IBM 12 * Rework to support virtual processors 13 * 14 * Type of int is used as a full 64b word is not necessary. 15 * 16 * (the type definitions are in asm/spinlock_types.h) 17 */ 18 #include <linux/irqflags.h> 19 #ifdef CONFIG_PPC64 20 #include <asm/paca.h> 21 #include <asm/hvcall.h> 22 #endif 23 #include <asm/synch.h> 24 #include <asm/ppc-opcode.h> 25 #include <asm/asm-405.h> 26 27 #ifdef CONFIG_PPC64 28 /* use 0x800000yy when locked, where yy == CPU number */ 29 #ifdef __BIG_ENDIAN__ 30 #define LOCK_TOKEN (*(u32 *)(&get_paca()->lock_token)) 31 #else 32 #define LOCK_TOKEN (*(u32 *)(&get_paca()->paca_index)) 33 #endif 34 #else 35 #define LOCK_TOKEN 1 36 #endif 37 38 #ifdef CONFIG_PPC_PSERIES 39 DECLARE_STATIC_KEY_FALSE(shared_processor); 40 41 #define vcpu_is_preempted vcpu_is_preempted 42 static inline bool vcpu_is_preempted(int cpu) 43 { 44 if (!static_branch_unlikely(&shared_processor)) 45 return false; 46 return !!(be32_to_cpu(lppaca_of(cpu).yield_count) & 1); 47 } 48 #endif 49 50 static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) 51 { 52 return lock.slock == 0; 53 } 54 55 static inline int arch_spin_is_locked(arch_spinlock_t *lock) 56 { 57 smp_mb(); 58 return !arch_spin_value_unlocked(*lock); 59 } 60 61 /* 62 * This returns the old value in the lock, so we succeeded 63 * in getting the lock if the return value is 0. 64 */ 65 static inline unsigned long __arch_spin_trylock(arch_spinlock_t *lock) 66 { 67 unsigned long tmp, token; 68 69 token = LOCK_TOKEN; 70 __asm__ __volatile__( 71 "1: " PPC_LWARX(%0,0,%2,1) "\n\ 72 cmpwi 0,%0,0\n\ 73 bne- 2f\n\ 74 stwcx. %1,0,%2\n\ 75 bne- 1b\n" 76 PPC_ACQUIRE_BARRIER 77 "2:" 78 : "=&r" (tmp) 79 : "r" (token), "r" (&lock->slock) 80 : "cr0", "memory"); 81 82 return tmp; 83 } 84 85 static inline int arch_spin_trylock(arch_spinlock_t *lock) 86 { 87 return __arch_spin_trylock(lock) == 0; 88 } 89 90 /* 91 * On a system with shared processors (that is, where a physical 92 * processor is multiplexed between several virtual processors), 93 * there is no point spinning on a lock if the holder of the lock 94 * isn't currently scheduled on a physical processor. Instead 95 * we detect this situation and ask the hypervisor to give the 96 * rest of our timeslice to the lock holder. 97 * 98 * So that we can tell which virtual processor is holding a lock, 99 * we put 0x80000000 | smp_processor_id() in the lock when it is 100 * held. Conveniently, we have a word in the paca that holds this 101 * value. 102 */ 103 104 #if defined(CONFIG_PPC_SPLPAR) 105 /* We only yield to the hypervisor if we are in shared processor mode */ 106 void splpar_spin_yield(arch_spinlock_t *lock); 107 void splpar_rw_yield(arch_rwlock_t *lock); 108 #else /* SPLPAR */ 109 static inline void splpar_spin_yield(arch_spinlock_t *lock) {}; 110 static inline void splpar_rw_yield(arch_rwlock_t *lock) {}; 111 #endif 112 113 static inline bool is_shared_processor(void) 114 { 115 #ifdef CONFIG_PPC_SPLPAR 116 return static_branch_unlikely(&shared_processor); 117 #else 118 return false; 119 #endif 120 } 121 122 static inline void spin_yield(arch_spinlock_t *lock) 123 { 124 if (is_shared_processor()) 125 splpar_spin_yield(lock); 126 else 127 barrier(); 128 } 129 130 static inline void rw_yield(arch_rwlock_t *lock) 131 { 132 if (is_shared_processor()) 133 splpar_rw_yield(lock); 134 else 135 barrier(); 136 } 137 138 static inline void arch_spin_lock(arch_spinlock_t *lock) 139 { 140 while (1) { 141 if (likely(__arch_spin_trylock(lock) == 0)) 142 break; 143 do { 144 HMT_low(); 145 if (is_shared_processor()) 146 splpar_spin_yield(lock); 147 } while (unlikely(lock->slock != 0)); 148 HMT_medium(); 149 } 150 } 151 152 static inline 153 void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long flags) 154 { 155 unsigned long flags_dis; 156 157 while (1) { 158 if (likely(__arch_spin_trylock(lock) == 0)) 159 break; 160 local_save_flags(flags_dis); 161 local_irq_restore(flags); 162 do { 163 HMT_low(); 164 if (is_shared_processor()) 165 splpar_spin_yield(lock); 166 } while (unlikely(lock->slock != 0)); 167 HMT_medium(); 168 local_irq_restore(flags_dis); 169 } 170 } 171 #define arch_spin_lock_flags arch_spin_lock_flags 172 173 static inline void arch_spin_unlock(arch_spinlock_t *lock) 174 { 175 __asm__ __volatile__("# arch_spin_unlock\n\t" 176 PPC_RELEASE_BARRIER: : :"memory"); 177 lock->slock = 0; 178 } 179 180 /* 181 * Read-write spinlocks, allowing multiple readers 182 * but only one writer. 183 * 184 * NOTE! it is quite common to have readers in interrupts 185 * but no interrupt writers. For those circumstances we 186 * can "mix" irq-safe locks - any writer needs to get a 187 * irq-safe write-lock, but readers can get non-irqsafe 188 * read-locks. 189 */ 190 191 #ifdef CONFIG_PPC64 192 #define __DO_SIGN_EXTEND "extsw %0,%0\n" 193 #define WRLOCK_TOKEN LOCK_TOKEN /* it's negative */ 194 #else 195 #define __DO_SIGN_EXTEND 196 #define WRLOCK_TOKEN (-1) 197 #endif 198 199 /* 200 * This returns the old value in the lock + 1, 201 * so we got a read lock if the return value is > 0. 202 */ 203 static inline long __arch_read_trylock(arch_rwlock_t *rw) 204 { 205 long tmp; 206 207 __asm__ __volatile__( 208 "1: " PPC_LWARX(%0,0,%1,1) "\n" 209 __DO_SIGN_EXTEND 210 " addic. %0,%0,1\n\ 211 ble- 2f\n" 212 PPC405_ERR77(0,%1) 213 " stwcx. %0,0,%1\n\ 214 bne- 1b\n" 215 PPC_ACQUIRE_BARRIER 216 "2:" : "=&r" (tmp) 217 : "r" (&rw->lock) 218 : "cr0", "xer", "memory"); 219 220 return tmp; 221 } 222 223 /* 224 * This returns the old value in the lock, 225 * so we got the write lock if the return value is 0. 226 */ 227 static inline long __arch_write_trylock(arch_rwlock_t *rw) 228 { 229 long tmp, token; 230 231 token = WRLOCK_TOKEN; 232 __asm__ __volatile__( 233 "1: " PPC_LWARX(%0,0,%2,1) "\n\ 234 cmpwi 0,%0,0\n\ 235 bne- 2f\n" 236 PPC405_ERR77(0,%1) 237 " stwcx. %1,0,%2\n\ 238 bne- 1b\n" 239 PPC_ACQUIRE_BARRIER 240 "2:" : "=&r" (tmp) 241 : "r" (token), "r" (&rw->lock) 242 : "cr0", "memory"); 243 244 return tmp; 245 } 246 247 static inline void arch_read_lock(arch_rwlock_t *rw) 248 { 249 while (1) { 250 if (likely(__arch_read_trylock(rw) > 0)) 251 break; 252 do { 253 HMT_low(); 254 if (is_shared_processor()) 255 splpar_rw_yield(rw); 256 } while (unlikely(rw->lock < 0)); 257 HMT_medium(); 258 } 259 } 260 261 static inline void arch_write_lock(arch_rwlock_t *rw) 262 { 263 while (1) { 264 if (likely(__arch_write_trylock(rw) == 0)) 265 break; 266 do { 267 HMT_low(); 268 if (is_shared_processor()) 269 splpar_rw_yield(rw); 270 } while (unlikely(rw->lock != 0)); 271 HMT_medium(); 272 } 273 } 274 275 static inline int arch_read_trylock(arch_rwlock_t *rw) 276 { 277 return __arch_read_trylock(rw) > 0; 278 } 279 280 static inline int arch_write_trylock(arch_rwlock_t *rw) 281 { 282 return __arch_write_trylock(rw) == 0; 283 } 284 285 static inline void arch_read_unlock(arch_rwlock_t *rw) 286 { 287 long tmp; 288 289 __asm__ __volatile__( 290 "# read_unlock\n\t" 291 PPC_RELEASE_BARRIER 292 "1: lwarx %0,0,%1\n\ 293 addic %0,%0,-1\n" 294 PPC405_ERR77(0,%1) 295 " stwcx. %0,0,%1\n\ 296 bne- 1b" 297 : "=&r"(tmp) 298 : "r"(&rw->lock) 299 : "cr0", "xer", "memory"); 300 } 301 302 static inline void arch_write_unlock(arch_rwlock_t *rw) 303 { 304 __asm__ __volatile__("# write_unlock\n\t" 305 PPC_RELEASE_BARRIER: : :"memory"); 306 rw->lock = 0; 307 } 308 309 #define arch_spin_relax(lock) spin_yield(lock) 310 #define arch_read_relax(lock) rw_yield(lock) 311 #define arch_write_relax(lock) rw_yield(lock) 312 313 /* See include/linux/spinlock.h */ 314 #define smp_mb__after_spinlock() smp_mb() 315 316 #endif /* __KERNEL__ */ 317 #endif /* __ASM_SPINLOCK_H */ 318