1 /*
2  * Copyright 2005, Red Hat, Inc., Ingo Molnar
3  * Released under the General Public License (GPL).
4  *
5  * This file contains the spinlock/rwlock implementations for
6  * DEBUG_SPINLOCK.
7  */
8 
9 #include <linux/spinlock.h>
10 #include <linux/nmi.h>
11 #include <linux/interrupt.h>
12 #include <linux/debug_locks.h>
13 #include <linux/delay.h>
14 #include <linux/export.h>
15 
16 void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
17 			  struct lock_class_key *key, short inner)
18 {
19 #ifdef CONFIG_DEBUG_LOCK_ALLOC
20 	/*
21 	 * Make sure we are not reinitializing a held lock:
22 	 */
23 	debug_check_no_locks_freed((void *)lock, sizeof(*lock));
24 	lockdep_init_map_wait(&lock->dep_map, name, key, 0, inner);
25 #endif
26 	lock->raw_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
27 	lock->magic = SPINLOCK_MAGIC;
28 	lock->owner = SPINLOCK_OWNER_INIT;
29 	lock->owner_cpu = -1;
30 }
31 
32 EXPORT_SYMBOL(__raw_spin_lock_init);
33 
34 void __rwlock_init(rwlock_t *lock, const char *name,
35 		   struct lock_class_key *key)
36 {
37 #ifdef CONFIG_DEBUG_LOCK_ALLOC
38 	/*
39 	 * Make sure we are not reinitializing a held lock:
40 	 */
41 	debug_check_no_locks_freed((void *)lock, sizeof(*lock));
42 	lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG);
43 #endif
44 	lock->raw_lock = (arch_rwlock_t) __ARCH_RW_LOCK_UNLOCKED;
45 	lock->magic = RWLOCK_MAGIC;
46 	lock->owner = SPINLOCK_OWNER_INIT;
47 	lock->owner_cpu = -1;
48 }
49 
50 EXPORT_SYMBOL(__rwlock_init);
51 
52 static void spin_dump(raw_spinlock_t *lock, const char *msg)
53 {
54 	struct task_struct *owner = READ_ONCE(lock->owner);
55 
56 	if (owner == SPINLOCK_OWNER_INIT)
57 		owner = NULL;
58 	printk(KERN_EMERG "BUG: spinlock %s on CPU#%d, %s/%d\n",
59 		msg, raw_smp_processor_id(),
60 		current->comm, task_pid_nr(current));
61 	printk(KERN_EMERG " lock: %pS, .magic: %08x, .owner: %s/%d, "
62 			".owner_cpu: %d\n",
63 		lock, READ_ONCE(lock->magic),
64 		owner ? owner->comm : "<none>",
65 		owner ? task_pid_nr(owner) : -1,
66 		READ_ONCE(lock->owner_cpu));
67 	dump_stack();
68 }
69 
70 static void spin_bug(raw_spinlock_t *lock, const char *msg)
71 {
72 	if (!debug_locks_off())
73 		return;
74 
75 	spin_dump(lock, msg);
76 }
77 
78 #define SPIN_BUG_ON(cond, lock, msg) if (unlikely(cond)) spin_bug(lock, msg)
79 
80 static inline void
81 debug_spin_lock_before(raw_spinlock_t *lock)
82 {
83 	SPIN_BUG_ON(READ_ONCE(lock->magic) != SPINLOCK_MAGIC, lock, "bad magic");
84 	SPIN_BUG_ON(READ_ONCE(lock->owner) == current, lock, "recursion");
85 	SPIN_BUG_ON(READ_ONCE(lock->owner_cpu) == raw_smp_processor_id(),
86 							lock, "cpu recursion");
87 }
88 
89 static inline void debug_spin_lock_after(raw_spinlock_t *lock)
90 {
91 	WRITE_ONCE(lock->owner_cpu, raw_smp_processor_id());
92 	WRITE_ONCE(lock->owner, current);
93 }
94 
95 static inline void debug_spin_unlock(raw_spinlock_t *lock)
96 {
97 	SPIN_BUG_ON(lock->magic != SPINLOCK_MAGIC, lock, "bad magic");
98 	SPIN_BUG_ON(!raw_spin_is_locked(lock), lock, "already unlocked");
99 	SPIN_BUG_ON(lock->owner != current, lock, "wrong owner");
100 	SPIN_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
101 							lock, "wrong CPU");
102 	WRITE_ONCE(lock->owner, SPINLOCK_OWNER_INIT);
103 	WRITE_ONCE(lock->owner_cpu, -1);
104 }
105 
106 /*
107  * We are now relying on the NMI watchdog to detect lockup instead of doing
108  * the detection here with an unfair lock which can cause problem of its own.
109  */
110 void do_raw_spin_lock(raw_spinlock_t *lock)
111 {
112 	debug_spin_lock_before(lock);
113 	arch_spin_lock(&lock->raw_lock);
114 	mmiowb_spin_lock();
115 	debug_spin_lock_after(lock);
116 }
117 
118 int do_raw_spin_trylock(raw_spinlock_t *lock)
119 {
120 	int ret = arch_spin_trylock(&lock->raw_lock);
121 
122 	if (ret) {
123 		mmiowb_spin_lock();
124 		debug_spin_lock_after(lock);
125 	}
126 #ifndef CONFIG_SMP
127 	/*
128 	 * Must not happen on UP:
129 	 */
130 	SPIN_BUG_ON(!ret, lock, "trylock failure on UP");
131 #endif
132 	return ret;
133 }
134 
135 void do_raw_spin_unlock(raw_spinlock_t *lock)
136 {
137 	mmiowb_spin_unlock();
138 	debug_spin_unlock(lock);
139 	arch_spin_unlock(&lock->raw_lock);
140 }
141 
142 static void rwlock_bug(rwlock_t *lock, const char *msg)
143 {
144 	if (!debug_locks_off())
145 		return;
146 
147 	printk(KERN_EMERG "BUG: rwlock %s on CPU#%d, %s/%d, %p\n",
148 		msg, raw_smp_processor_id(), current->comm,
149 		task_pid_nr(current), lock);
150 	dump_stack();
151 }
152 
153 #define RWLOCK_BUG_ON(cond, lock, msg) if (unlikely(cond)) rwlock_bug(lock, msg)
154 
155 void do_raw_read_lock(rwlock_t *lock)
156 {
157 	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
158 	arch_read_lock(&lock->raw_lock);
159 }
160 
161 int do_raw_read_trylock(rwlock_t *lock)
162 {
163 	int ret = arch_read_trylock(&lock->raw_lock);
164 
165 #ifndef CONFIG_SMP
166 	/*
167 	 * Must not happen on UP:
168 	 */
169 	RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
170 #endif
171 	return ret;
172 }
173 
174 void do_raw_read_unlock(rwlock_t *lock)
175 {
176 	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
177 	arch_read_unlock(&lock->raw_lock);
178 }
179 
180 static inline void debug_write_lock_before(rwlock_t *lock)
181 {
182 	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
183 	RWLOCK_BUG_ON(lock->owner == current, lock, "recursion");
184 	RWLOCK_BUG_ON(lock->owner_cpu == raw_smp_processor_id(),
185 							lock, "cpu recursion");
186 }
187 
188 static inline void debug_write_lock_after(rwlock_t *lock)
189 {
190 	WRITE_ONCE(lock->owner_cpu, raw_smp_processor_id());
191 	WRITE_ONCE(lock->owner, current);
192 }
193 
194 static inline void debug_write_unlock(rwlock_t *lock)
195 {
196 	RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
197 	RWLOCK_BUG_ON(lock->owner != current, lock, "wrong owner");
198 	RWLOCK_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
199 							lock, "wrong CPU");
200 	WRITE_ONCE(lock->owner, SPINLOCK_OWNER_INIT);
201 	WRITE_ONCE(lock->owner_cpu, -1);
202 }
203 
204 void do_raw_write_lock(rwlock_t *lock)
205 {
206 	debug_write_lock_before(lock);
207 	arch_write_lock(&lock->raw_lock);
208 	debug_write_lock_after(lock);
209 }
210 
211 int do_raw_write_trylock(rwlock_t *lock)
212 {
213 	int ret = arch_write_trylock(&lock->raw_lock);
214 
215 	if (ret)
216 		debug_write_lock_after(lock);
217 #ifndef CONFIG_SMP
218 	/*
219 	 * Must not happen on UP:
220 	 */
221 	RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
222 #endif
223 	return ret;
224 }
225 
226 void do_raw_write_unlock(rwlock_t *lock)
227 {
228 	debug_write_unlock(lock);
229 	arch_write_unlock(&lock->raw_lock);
230 }
231