1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (2004) Linus Torvalds 4 * 5 * Author: Zwane Mwaikambo <zwane@fsmlabs.com> 6 * 7 * Copyright (2004, 2005) Ingo Molnar 8 * 9 * This file contains the spinlock/rwlock implementations for the 10 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them) 11 * 12 * Note that some architectures have special knowledge about the 13 * stack frames of these functions in their profile_pc. If you 14 * change anything significant here that could change the stack 15 * frame contact the architecture maintainers. 16 */ 17 18 #include <linux/linkage.h> 19 #include <linux/preempt.h> 20 #include <linux/spinlock.h> 21 #include <linux/interrupt.h> 22 #include <linux/debug_locks.h> 23 #include <linux/export.h> 24 25 #ifdef CONFIG_MMIOWB 26 #ifndef arch_mmiowb_state 27 DEFINE_PER_CPU(struct mmiowb_state, __mmiowb_state); 28 EXPORT_PER_CPU_SYMBOL(__mmiowb_state); 29 #endif 30 #endif 31 32 /* 33 * If lockdep is enabled then we use the non-preemption spin-ops 34 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are 35 * not re-enabled during lock-acquire (which the preempt-spin-ops do): 36 */ 37 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) 38 /* 39 * The __lock_function inlines are taken from 40 * spinlock : include/linux/spinlock_api_smp.h 41 * rwlock : include/linux/rwlock_api_smp.h 42 */ 43 #else 44 45 /* 46 * Some architectures can relax in favour of the CPU owning the lock. 47 */ 48 #ifndef arch_read_relax 49 # define arch_read_relax(l) cpu_relax() 50 #endif 51 #ifndef arch_write_relax 52 # define arch_write_relax(l) cpu_relax() 53 #endif 54 #ifndef arch_spin_relax 55 # define arch_spin_relax(l) cpu_relax() 56 #endif 57 58 /* 59 * We build the __lock_function inlines here. They are too large for 60 * inlining all over the place, but here is only one user per function 61 * which embeds them into the calling _lock_function below. 62 * 63 * This could be a long-held lock. We both prepare to spin for a long 64 * time (making _this_ CPU preemptible if possible), and we also signal 65 * towards that other CPU that it should break the lock ASAP. 66 */ 67 #define BUILD_LOCK_OPS(op, locktype) \ 68 void __lockfunc __raw_##op##_lock(locktype##_t *lock) \ 69 { \ 70 for (;;) { \ 71 preempt_disable(); \ 72 if (likely(do_raw_##op##_trylock(lock))) \ 73 break; \ 74 preempt_enable(); \ 75 \ 76 arch_##op##_relax(&lock->raw_lock); \ 77 } \ 78 } \ 79 \ 80 unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \ 81 { \ 82 unsigned long flags; \ 83 \ 84 for (;;) { \ 85 preempt_disable(); \ 86 local_irq_save(flags); \ 87 if (likely(do_raw_##op##_trylock(lock))) \ 88 break; \ 89 local_irq_restore(flags); \ 90 preempt_enable(); \ 91 \ 92 arch_##op##_relax(&lock->raw_lock); \ 93 } \ 94 \ 95 return flags; \ 96 } \ 97 \ 98 void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock) \ 99 { \ 100 _raw_##op##_lock_irqsave(lock); \ 101 } \ 102 \ 103 void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ 104 { \ 105 unsigned long flags; \ 106 \ 107 /* */ \ 108 /* Careful: we must exclude softirqs too, hence the */ \ 109 /* irq-disabling. We use the generic preemption-aware */ \ 110 /* function: */ \ 111 /**/ \ 112 flags = _raw_##op##_lock_irqsave(lock); \ 113 local_bh_disable(); \ 114 local_irq_restore(flags); \ 115 } \ 116 117 /* 118 * Build preemption-friendly versions of the following 119 * lock-spinning functions: 120 * 121 * __[spin|read|write]_lock() 122 * __[spin|read|write]_lock_irq() 123 * __[spin|read|write]_lock_irqsave() 124 * __[spin|read|write]_lock_bh() 125 */ 126 BUILD_LOCK_OPS(spin, raw_spinlock); 127 128 #ifndef CONFIG_PREEMPT_RT 129 BUILD_LOCK_OPS(read, rwlock); 130 BUILD_LOCK_OPS(write, rwlock); 131 #endif 132 133 #endif 134 135 #ifndef CONFIG_INLINE_SPIN_TRYLOCK 136 noinline int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) 137 { 138 return __raw_spin_trylock(lock); 139 } 140 EXPORT_SYMBOL(_raw_spin_trylock); 141 #endif 142 143 #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH 144 noinline int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) 145 { 146 return __raw_spin_trylock_bh(lock); 147 } 148 EXPORT_SYMBOL(_raw_spin_trylock_bh); 149 #endif 150 151 #ifndef CONFIG_INLINE_SPIN_LOCK 152 noinline void __lockfunc _raw_spin_lock(raw_spinlock_t *lock) 153 { 154 __raw_spin_lock(lock); 155 } 156 EXPORT_SYMBOL(_raw_spin_lock); 157 #endif 158 159 #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE 160 noinline unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) 161 { 162 return __raw_spin_lock_irqsave(lock); 163 } 164 EXPORT_SYMBOL(_raw_spin_lock_irqsave); 165 #endif 166 167 #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ 168 noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) 169 { 170 __raw_spin_lock_irq(lock); 171 } 172 EXPORT_SYMBOL(_raw_spin_lock_irq); 173 #endif 174 175 #ifndef CONFIG_INLINE_SPIN_LOCK_BH 176 noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) 177 { 178 __raw_spin_lock_bh(lock); 179 } 180 EXPORT_SYMBOL(_raw_spin_lock_bh); 181 #endif 182 183 #ifdef CONFIG_UNINLINE_SPIN_UNLOCK 184 noinline void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) 185 { 186 __raw_spin_unlock(lock); 187 } 188 EXPORT_SYMBOL(_raw_spin_unlock); 189 #endif 190 191 #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE 192 noinline void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) 193 { 194 __raw_spin_unlock_irqrestore(lock, flags); 195 } 196 EXPORT_SYMBOL(_raw_spin_unlock_irqrestore); 197 #endif 198 199 #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ 200 noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) 201 { 202 __raw_spin_unlock_irq(lock); 203 } 204 EXPORT_SYMBOL(_raw_spin_unlock_irq); 205 #endif 206 207 #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH 208 noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) 209 { 210 __raw_spin_unlock_bh(lock); 211 } 212 EXPORT_SYMBOL(_raw_spin_unlock_bh); 213 #endif 214 215 #ifndef CONFIG_PREEMPT_RT 216 217 #ifndef CONFIG_INLINE_READ_TRYLOCK 218 noinline int __lockfunc _raw_read_trylock(rwlock_t *lock) 219 { 220 return __raw_read_trylock(lock); 221 } 222 EXPORT_SYMBOL(_raw_read_trylock); 223 #endif 224 225 #ifndef CONFIG_INLINE_READ_LOCK 226 noinline void __lockfunc _raw_read_lock(rwlock_t *lock) 227 { 228 __raw_read_lock(lock); 229 } 230 EXPORT_SYMBOL(_raw_read_lock); 231 #endif 232 233 #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE 234 noinline unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) 235 { 236 return __raw_read_lock_irqsave(lock); 237 } 238 EXPORT_SYMBOL(_raw_read_lock_irqsave); 239 #endif 240 241 #ifndef CONFIG_INLINE_READ_LOCK_IRQ 242 noinline void __lockfunc _raw_read_lock_irq(rwlock_t *lock) 243 { 244 __raw_read_lock_irq(lock); 245 } 246 EXPORT_SYMBOL(_raw_read_lock_irq); 247 #endif 248 249 #ifndef CONFIG_INLINE_READ_LOCK_BH 250 noinline void __lockfunc _raw_read_lock_bh(rwlock_t *lock) 251 { 252 __raw_read_lock_bh(lock); 253 } 254 EXPORT_SYMBOL(_raw_read_lock_bh); 255 #endif 256 257 #ifndef CONFIG_INLINE_READ_UNLOCK 258 noinline void __lockfunc _raw_read_unlock(rwlock_t *lock) 259 { 260 __raw_read_unlock(lock); 261 } 262 EXPORT_SYMBOL(_raw_read_unlock); 263 #endif 264 265 #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE 266 noinline void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) 267 { 268 __raw_read_unlock_irqrestore(lock, flags); 269 } 270 EXPORT_SYMBOL(_raw_read_unlock_irqrestore); 271 #endif 272 273 #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ 274 noinline void __lockfunc _raw_read_unlock_irq(rwlock_t *lock) 275 { 276 __raw_read_unlock_irq(lock); 277 } 278 EXPORT_SYMBOL(_raw_read_unlock_irq); 279 #endif 280 281 #ifndef CONFIG_INLINE_READ_UNLOCK_BH 282 noinline void __lockfunc _raw_read_unlock_bh(rwlock_t *lock) 283 { 284 __raw_read_unlock_bh(lock); 285 } 286 EXPORT_SYMBOL(_raw_read_unlock_bh); 287 #endif 288 289 #ifndef CONFIG_INLINE_WRITE_TRYLOCK 290 noinline int __lockfunc _raw_write_trylock(rwlock_t *lock) 291 { 292 return __raw_write_trylock(lock); 293 } 294 EXPORT_SYMBOL(_raw_write_trylock); 295 #endif 296 297 #ifndef CONFIG_INLINE_WRITE_LOCK 298 noinline void __lockfunc _raw_write_lock(rwlock_t *lock) 299 { 300 __raw_write_lock(lock); 301 } 302 EXPORT_SYMBOL(_raw_write_lock); 303 304 #ifndef CONFIG_DEBUG_LOCK_ALLOC 305 #define __raw_write_lock_nested(lock, subclass) __raw_write_lock(((void)(subclass), (lock))) 306 #endif 307 308 void __lockfunc _raw_write_lock_nested(rwlock_t *lock, int subclass) 309 { 310 __raw_write_lock_nested(lock, subclass); 311 } 312 EXPORT_SYMBOL(_raw_write_lock_nested); 313 #endif 314 315 #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE 316 noinline unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) 317 { 318 return __raw_write_lock_irqsave(lock); 319 } 320 EXPORT_SYMBOL(_raw_write_lock_irqsave); 321 #endif 322 323 #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ 324 noinline void __lockfunc _raw_write_lock_irq(rwlock_t *lock) 325 { 326 __raw_write_lock_irq(lock); 327 } 328 EXPORT_SYMBOL(_raw_write_lock_irq); 329 #endif 330 331 #ifndef CONFIG_INLINE_WRITE_LOCK_BH 332 noinline void __lockfunc _raw_write_lock_bh(rwlock_t *lock) 333 { 334 __raw_write_lock_bh(lock); 335 } 336 EXPORT_SYMBOL(_raw_write_lock_bh); 337 #endif 338 339 #ifndef CONFIG_INLINE_WRITE_UNLOCK 340 noinline void __lockfunc _raw_write_unlock(rwlock_t *lock) 341 { 342 __raw_write_unlock(lock); 343 } 344 EXPORT_SYMBOL(_raw_write_unlock); 345 #endif 346 347 #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE 348 noinline void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) 349 { 350 __raw_write_unlock_irqrestore(lock, flags); 351 } 352 EXPORT_SYMBOL(_raw_write_unlock_irqrestore); 353 #endif 354 355 #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ 356 noinline void __lockfunc _raw_write_unlock_irq(rwlock_t *lock) 357 { 358 __raw_write_unlock_irq(lock); 359 } 360 EXPORT_SYMBOL(_raw_write_unlock_irq); 361 #endif 362 363 #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH 364 noinline void __lockfunc _raw_write_unlock_bh(rwlock_t *lock) 365 { 366 __raw_write_unlock_bh(lock); 367 } 368 EXPORT_SYMBOL(_raw_write_unlock_bh); 369 #endif 370 371 #endif /* !CONFIG_PREEMPT_RT */ 372 373 #ifdef CONFIG_DEBUG_LOCK_ALLOC 374 375 void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass) 376 { 377 preempt_disable(); 378 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); 379 LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); 380 } 381 EXPORT_SYMBOL(_raw_spin_lock_nested); 382 383 unsigned long __lockfunc _raw_spin_lock_irqsave_nested(raw_spinlock_t *lock, 384 int subclass) 385 { 386 unsigned long flags; 387 388 local_irq_save(flags); 389 preempt_disable(); 390 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); 391 LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); 392 return flags; 393 } 394 EXPORT_SYMBOL(_raw_spin_lock_irqsave_nested); 395 396 void __lockfunc _raw_spin_lock_nest_lock(raw_spinlock_t *lock, 397 struct lockdep_map *nest_lock) 398 { 399 preempt_disable(); 400 spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); 401 LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); 402 } 403 EXPORT_SYMBOL(_raw_spin_lock_nest_lock); 404 405 #endif 406 407 notrace int in_lock_functions(unsigned long addr) 408 { 409 /* Linker adds these: start and end of __lockfunc functions */ 410 extern char __lock_text_start[], __lock_text_end[]; 411 412 return addr >= (unsigned long)__lock_text_start 413 && addr < (unsigned long)__lock_text_end; 414 } 415 EXPORT_SYMBOL(in_lock_functions); 416