1 /*
2  * Copyright 2005, Red Hat, Inc., Ingo Molnar
3  * Released under the General Public License (GPL).
4  *
5  * This file contains the spinlock/rwlock implementations for
6  * DEBUG_SPINLOCK.
7  */
8 
9 #include <linux/spinlock.h>
10 #include <linux/nmi.h>
11 #include <linux/interrupt.h>
12 #include <linux/debug_locks.h>
13 #include <linux/delay.h>
14 #include <linux/export.h>
15 
__raw_spin_lock_init(raw_spinlock_t * lock,const char * name,struct lock_class_key * key,short inner)16 void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
17 			  struct lock_class_key *key, short inner)
18 {
19 #ifdef CONFIG_DEBUG_LOCK_ALLOC
20 	/*
21 	 * Make sure we are not reinitializing a held lock:
22 	 */
23 	debug_check_no_locks_freed((void *)lock, sizeof(*lock));
24 	lockdep_init_map_wait(&lock->dep_map, name, key, 0, inner);
25 #endif
26 	lock->raw_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
27 	lock->magic = SPINLOCK_MAGIC;
28 	lock->owner = SPINLOCK_OWNER_INIT;
29 	lock->owner_cpu = -1;
30 }
31 
32 EXPORT_SYMBOL(__raw_spin_lock_init);
33 
34 #ifndef CONFIG_PREEMPT_RT
__rwlock_init(rwlock_t * lock,const char * name,struct lock_class_key * key)35 void __rwlock_init(rwlock_t *lock, const char *name,
36 		   struct lock_class_key *key)
37 {
38 #ifdef CONFIG_DEBUG_LOCK_ALLOC
39 	/*
40 	 * Make sure we are not reinitializing a held lock:
41 	 */
42 	debug_check_no_locks_freed((void *)lock, sizeof(*lock));
43 	lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG);
44 #endif
45 	lock->raw_lock = (arch_rwlock_t) __ARCH_RW_LOCK_UNLOCKED;
46 	lock->magic = RWLOCK_MAGIC;
47 	lock->owner = SPINLOCK_OWNER_INIT;
48 	lock->owner_cpu = -1;
49 }
50 
51 EXPORT_SYMBOL(__rwlock_init);
52 #endif
53 
spin_dump(raw_spinlock_t * lock,const char * msg)54 static void spin_dump(raw_spinlock_t *lock, const char *msg)
55 {
56 	struct task_struct *owner = READ_ONCE(lock->owner);
57 
58 	if (owner == SPINLOCK_OWNER_INIT)
59 		owner = NULL;
60 	printk(KERN_EMERG "BUG: spinlock %s on CPU#%d, %s/%d\n",
61 		msg, raw_smp_processor_id(),
62 		current->comm, task_pid_nr(current));
63 	printk(KERN_EMERG " lock: %pS, .magic: %08x, .owner: %s/%d, "
64 			".owner_cpu: %d\n",
65 		lock, READ_ONCE(lock->magic),
66 		owner ? owner->comm : "<none>",
67 		owner ? task_pid_nr(owner) : -1,
68 		READ_ONCE(lock->owner_cpu));
69 	dump_stack();
70 }
71 
spin_bug(raw_spinlock_t * lock,const char * msg)72 static void spin_bug(raw_spinlock_t *lock, const char *msg)
73 {
74 	if (!debug_locks_off())
75 		return;
76 
77 	spin_dump(lock, msg);
78 }
79 
80 #define SPIN_BUG_ON(cond, lock, msg) if (unlikely(cond)) spin_bug(lock, msg)
81 
82 static inline void
debug_spin_lock_before(raw_spinlock_t * lock)83 debug_spin_lock_before(raw_spinlock_t *lock)
84 {
85 	SPIN_BUG_ON(READ_ONCE(lock->magic) != SPINLOCK_MAGIC, lock, "bad magic");
86 	SPIN_BUG_ON(READ_ONCE(lock->owner) == current, lock, "recursion");
87 	SPIN_BUG_ON(READ_ONCE(lock->owner_cpu) == raw_smp_processor_id(),
88 							lock, "cpu recursion");
89 }
90 
debug_spin_lock_after(raw_spinlock_t * lock)91 static inline void debug_spin_lock_after(raw_spinlock_t *lock)
92 {
93 	WRITE_ONCE(lock->owner_cpu, raw_smp_processor_id());
94 	WRITE_ONCE(lock->owner, current);
95 }
96 
debug_spin_unlock(raw_spinlock_t * lock)97 static inline void debug_spin_unlock(raw_spinlock_t *lock)
98 {
99 	SPIN_BUG_ON(lock->magic != SPINLOCK_MAGIC, lock, "bad magic");
100 	SPIN_BUG_ON(!raw_spin_is_locked(lock), lock, "already unlocked");
101 	SPIN_BUG_ON(lock->owner != current, lock, "wrong owner");
102 	SPIN_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
103 							lock, "wrong CPU");
104 	WRITE_ONCE(lock->owner, SPINLOCK_OWNER_INIT);
105 	WRITE_ONCE(lock->owner_cpu, -1);
106 }
107 
108 /*
109  * We are now relying on the NMI watchdog to detect lockup instead of doing
110  * the detection here with an unfair lock which can cause problem of its own.
111  */
do_raw_spin_lock(raw_spinlock_t * lock)112 void do_raw_spin_lock(raw_spinlock_t *lock)
113 {
114 	debug_spin_lock_before(lock);
115 	arch_spin_lock(&lock->raw_lock);
116 	mmiowb_spin_lock();
117 	debug_spin_lock_after(lock);
118 }
119 
do_raw_spin_trylock(raw_spinlock_t * lock)120 int do_raw_spin_trylock(raw_spinlock_t *lock)
121 {
122 	int ret = arch_spin_trylock(&lock->raw_lock);
123 
124 	if (ret) {
125 		mmiowb_spin_lock();
126 		debug_spin_lock_after(lock);
127 	}
128 #ifndef CONFIG_SMP
129 	/*
130 	 * Must not happen on UP:
131 	 */
132 	SPIN_BUG_ON(!ret, lock, "trylock failure on UP");
133 #endif
134 	return ret;
135 }
136 
do_raw_spin_unlock(raw_spinlock_t * lock)137 void do_raw_spin_unlock(raw_spinlock_t *lock)
138 {
139 	mmiowb_spin_unlock();
140 	debug_spin_unlock(lock);
141 	arch_spin_unlock(&lock->raw_lock);
142 }
143 
144 #ifndef CONFIG_PREEMPT_RT
rwlock_bug(rwlock_t * lock,const char * msg)145 static void rwlock_bug(rwlock_t *lock, const char *msg)
146 {
147 	if (!debug_locks_off())
148 		return;
149 
150 	printk(KERN_EMERG "BUG: rwlock %s on CPU#%d, %s/%d, %p\n",
151 		msg, raw_smp_processor_id(), current->comm,
152 		task_pid_nr(current), lock);
153 	dump_stack();
154 }
155 
156 #define RWLOCK_BUG_ON(cond, lock, msg) if (unlikely(cond)) rwlock_bug(lock, msg)
157 
do_raw_read_lock(rwlock_t * lock)158 void do_raw_read_lock(rwlock_t *lock)
159 {
160 	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
161 	arch_read_lock(&lock->raw_lock);
162 }
163 
do_raw_read_trylock(rwlock_t * lock)164 int do_raw_read_trylock(rwlock_t *lock)
165 {
166 	int ret = arch_read_trylock(&lock->raw_lock);
167 
168 #ifndef CONFIG_SMP
169 	/*
170 	 * Must not happen on UP:
171 	 */
172 	RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
173 #endif
174 	return ret;
175 }
176 
do_raw_read_unlock(rwlock_t * lock)177 void do_raw_read_unlock(rwlock_t *lock)
178 {
179 	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
180 	arch_read_unlock(&lock->raw_lock);
181 }
182 
debug_write_lock_before(rwlock_t * lock)183 static inline void debug_write_lock_before(rwlock_t *lock)
184 {
185 	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
186 	RWLOCK_BUG_ON(lock->owner == current, lock, "recursion");
187 	RWLOCK_BUG_ON(lock->owner_cpu == raw_smp_processor_id(),
188 							lock, "cpu recursion");
189 }
190 
debug_write_lock_after(rwlock_t * lock)191 static inline void debug_write_lock_after(rwlock_t *lock)
192 {
193 	WRITE_ONCE(lock->owner_cpu, raw_smp_processor_id());
194 	WRITE_ONCE(lock->owner, current);
195 }
196 
debug_write_unlock(rwlock_t * lock)197 static inline void debug_write_unlock(rwlock_t *lock)
198 {
199 	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
200 	RWLOCK_BUG_ON(lock->owner != current, lock, "wrong owner");
201 	RWLOCK_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
202 							lock, "wrong CPU");
203 	WRITE_ONCE(lock->owner, SPINLOCK_OWNER_INIT);
204 	WRITE_ONCE(lock->owner_cpu, -1);
205 }
206 
do_raw_write_lock(rwlock_t * lock)207 void do_raw_write_lock(rwlock_t *lock)
208 {
209 	debug_write_lock_before(lock);
210 	arch_write_lock(&lock->raw_lock);
211 	debug_write_lock_after(lock);
212 }
213 
do_raw_write_trylock(rwlock_t * lock)214 int do_raw_write_trylock(rwlock_t *lock)
215 {
216 	int ret = arch_write_trylock(&lock->raw_lock);
217 
218 	if (ret)
219 		debug_write_lock_after(lock);
220 #ifndef CONFIG_SMP
221 	/*
222 	 * Must not happen on UP:
223 	 */
224 	RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
225 #endif
226 	return ret;
227 }
228 
do_raw_write_unlock(rwlock_t * lock)229 void do_raw_write_unlock(rwlock_t *lock)
230 {
231 	debug_write_unlock(lock);
232 	arch_write_unlock(&lock->raw_lock);
233 }
234 
235 #endif /* !CONFIG_PREEMPT_RT */
236