1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * PREEMPT_RT substitution for spin/rw_locks 4 * 5 * spinlocks and rwlocks on RT are based on rtmutexes, with a few twists to 6 * resemble the non RT semantics: 7 * 8 * - Contrary to plain rtmutexes, spinlocks and rwlocks are state 9 * preserving. The task state is saved before blocking on the underlying 10 * rtmutex, and restored when the lock has been acquired. Regular wakeups 11 * during that time are redirected to the saved state so no wake up is 12 * missed. 13 * 14 * - Non RT spin/rwlocks disable preemption and eventually interrupts. 15 * Disabling preemption has the side effect of disabling migration and 16 * preventing RCU grace periods. 17 * 18 * The RT substitutions explicitly disable migration and take 19 * rcu_read_lock() across the lock held section. 20 */ 21 #include <linux/spinlock.h> 22 #include <linux/export.h> 23 24 #define RT_MUTEX_BUILD_SPINLOCKS 25 #include "rtmutex.c" 26 27 static __always_inline void rtlock_lock(struct rt_mutex_base *rtm) 28 { 29 if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current))) 30 rtlock_slowlock(rtm); 31 } 32 33 static __always_inline void __rt_spin_lock(spinlock_t *lock) 34 { 35 ___might_sleep(__FILE__, __LINE__, 0); 36 rtlock_lock(&lock->lock); 37 rcu_read_lock(); 38 migrate_disable(); 39 } 40 41 void __sched rt_spin_lock(spinlock_t *lock) 42 { 43 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); 44 __rt_spin_lock(lock); 45 } 46 EXPORT_SYMBOL(rt_spin_lock); 47 48 #ifdef CONFIG_DEBUG_LOCK_ALLOC 49 void __sched rt_spin_lock_nested(spinlock_t *lock, int subclass) 50 { 51 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); 52 __rt_spin_lock(lock); 53 } 54 EXPORT_SYMBOL(rt_spin_lock_nested); 55 56 void __sched rt_spin_lock_nest_lock(spinlock_t *lock, 57 struct lockdep_map *nest_lock) 58 { 59 spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); 60 __rt_spin_lock(lock); 61 } 62 EXPORT_SYMBOL(rt_spin_lock_nest_lock); 63 #endif 64 65 void __sched rt_spin_unlock(spinlock_t *lock) 66 { 67 spin_release(&lock->dep_map, _RET_IP_); 68 migrate_enable(); 69 rcu_read_unlock(); 70 71 if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL))) 72 rt_mutex_slowunlock(&lock->lock); 73 } 74 EXPORT_SYMBOL(rt_spin_unlock); 75 76 /* 77 * Wait for the lock to get unlocked: instead of polling for an unlock 78 * (like raw spinlocks do), lock and unlock, to force the kernel to 79 * schedule if there's contention: 80 */ 81 void __sched rt_spin_lock_unlock(spinlock_t *lock) 82 { 83 spin_lock(lock); 84 spin_unlock(lock); 85 } 86 EXPORT_SYMBOL(rt_spin_lock_unlock); 87 88 static __always_inline int __rt_spin_trylock(spinlock_t *lock) 89 { 90 int ret = 1; 91 92 if (unlikely(!rt_mutex_cmpxchg_acquire(&lock->lock, NULL, current))) 93 ret = rt_mutex_slowtrylock(&lock->lock); 94 95 if (ret) { 96 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); 97 rcu_read_lock(); 98 migrate_disable(); 99 } 100 return ret; 101 } 102 103 int __sched rt_spin_trylock(spinlock_t *lock) 104 { 105 return __rt_spin_trylock(lock); 106 } 107 EXPORT_SYMBOL(rt_spin_trylock); 108 109 int __sched rt_spin_trylock_bh(spinlock_t *lock) 110 { 111 int ret; 112 113 local_bh_disable(); 114 ret = __rt_spin_trylock(lock); 115 if (!ret) 116 local_bh_enable(); 117 return ret; 118 } 119 EXPORT_SYMBOL(rt_spin_trylock_bh); 120 121 #ifdef CONFIG_DEBUG_LOCK_ALLOC 122 void __rt_spin_lock_init(spinlock_t *lock, const char *name, 123 struct lock_class_key *key, bool percpu) 124 { 125 u8 type = percpu ? LD_LOCK_PERCPU : LD_LOCK_NORMAL; 126 127 debug_check_no_locks_freed((void *)lock, sizeof(*lock)); 128 lockdep_init_map_type(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG, 129 LD_WAIT_INV, type); 130 } 131 EXPORT_SYMBOL(__rt_spin_lock_init); 132 #endif 133 134 /* 135 * RT-specific reader/writer locks 136 */ 137 #define rwbase_set_and_save_current_state(state) \ 138 current_save_and_set_rtlock_wait_state() 139 140 #define rwbase_restore_current_state() \ 141 current_restore_rtlock_saved_state() 142 143 static __always_inline int 144 rwbase_rtmutex_lock_state(struct rt_mutex_base *rtm, unsigned int state) 145 { 146 if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current))) 147 rtlock_slowlock(rtm); 148 return 0; 149 } 150 151 static __always_inline int 152 rwbase_rtmutex_slowlock_locked(struct rt_mutex_base *rtm, unsigned int state) 153 { 154 rtlock_slowlock_locked(rtm); 155 return 0; 156 } 157 158 static __always_inline void rwbase_rtmutex_unlock(struct rt_mutex_base *rtm) 159 { 160 if (likely(rt_mutex_cmpxchg_acquire(rtm, current, NULL))) 161 return; 162 163 rt_mutex_slowunlock(rtm); 164 } 165 166 static __always_inline int rwbase_rtmutex_trylock(struct rt_mutex_base *rtm) 167 { 168 if (likely(rt_mutex_cmpxchg_acquire(rtm, NULL, current))) 169 return 1; 170 171 return rt_mutex_slowtrylock(rtm); 172 } 173 174 #define rwbase_signal_pending_state(state, current) (0) 175 176 #define rwbase_schedule() \ 177 schedule_rtlock() 178 179 #include "rwbase_rt.c" 180 /* 181 * The common functions which get wrapped into the rwlock API. 182 */ 183 int __sched rt_read_trylock(rwlock_t *rwlock) 184 { 185 int ret; 186 187 ret = rwbase_read_trylock(&rwlock->rwbase); 188 if (ret) { 189 rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_); 190 rcu_read_lock(); 191 migrate_disable(); 192 } 193 return ret; 194 } 195 EXPORT_SYMBOL(rt_read_trylock); 196 197 int __sched rt_write_trylock(rwlock_t *rwlock) 198 { 199 int ret; 200 201 ret = rwbase_write_trylock(&rwlock->rwbase); 202 if (ret) { 203 rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); 204 rcu_read_lock(); 205 migrate_disable(); 206 } 207 return ret; 208 } 209 EXPORT_SYMBOL(rt_write_trylock); 210 211 void __sched rt_read_lock(rwlock_t *rwlock) 212 { 213 ___might_sleep(__FILE__, __LINE__, 0); 214 rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_); 215 rwbase_read_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 216 rcu_read_lock(); 217 migrate_disable(); 218 } 219 EXPORT_SYMBOL(rt_read_lock); 220 221 void __sched rt_write_lock(rwlock_t *rwlock) 222 { 223 ___might_sleep(__FILE__, __LINE__, 0); 224 rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); 225 rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 226 rcu_read_lock(); 227 migrate_disable(); 228 } 229 EXPORT_SYMBOL(rt_write_lock); 230 231 void __sched rt_read_unlock(rwlock_t *rwlock) 232 { 233 rwlock_release(&rwlock->dep_map, _RET_IP_); 234 migrate_enable(); 235 rcu_read_unlock(); 236 rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT); 237 } 238 EXPORT_SYMBOL(rt_read_unlock); 239 240 void __sched rt_write_unlock(rwlock_t *rwlock) 241 { 242 rwlock_release(&rwlock->dep_map, _RET_IP_); 243 rcu_read_unlock(); 244 migrate_enable(); 245 rwbase_write_unlock(&rwlock->rwbase); 246 } 247 EXPORT_SYMBOL(rt_write_unlock); 248 249 int __sched rt_rwlock_is_contended(rwlock_t *rwlock) 250 { 251 return rw_base_is_contended(&rwlock->rwbase); 252 } 253 EXPORT_SYMBOL(rt_rwlock_is_contended); 254 255 #ifdef CONFIG_DEBUG_LOCK_ALLOC 256 void __rt_rwlock_init(rwlock_t *rwlock, const char *name, 257 struct lock_class_key *key) 258 { 259 debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock)); 260 lockdep_init_map_wait(&rwlock->dep_map, name, key, 0, LD_WAIT_CONFIG); 261 } 262 EXPORT_SYMBOL(__rt_rwlock_init); 263 #endif 264