xref: /openbmc/linux/fs/btrfs/locking.c (revision 9deff920) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2008 Oracle.  All rights reserved.
4  */
5 
6 #include <linux/sched.h>
7 #include <linux/pagemap.h>
8 #include <linux/spinlock.h>
9 #include <linux/page-flags.h>
10 #include <asm/bug.h>
11 #include "ctree.h"
12 #include "extent_io.h"
13 #include "locking.h"
14 
15 #ifdef CONFIG_BTRFS_DEBUG
16 static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb)
17 {
18 	WARN_ON(eb->spinning_writers);
19 	eb->spinning_writers++;
20 }
21 
22 static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb)
23 {
24 	WARN_ON(eb->spinning_writers != 1);
25 	eb->spinning_writers--;
26 }
27 
28 static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb)
29 {
30 	WARN_ON(eb->spinning_writers);
31 }
32 
33 static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb)
34 {
35 	atomic_inc(&eb->spinning_readers);
36 }
37 
38 static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb)
39 {
40 	WARN_ON(atomic_read(&eb->spinning_readers) == 0);
41 	atomic_dec(&eb->spinning_readers);
42 }
43 
44 static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb)
45 {
46 	atomic_inc(&eb->read_locks);
47 }
48 
49 static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb)
50 {
51 	atomic_dec(&eb->read_locks);
52 }
53 
54 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb)
55 {
56 	BUG_ON(!atomic_read(&eb->read_locks));
57 }
58 
59 static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb)
60 {
61 	eb->write_locks++;
62 }
63 
64 static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb)
65 {
66 	eb->write_locks--;
67 }
68 
69 void btrfs_assert_tree_locked(struct extent_buffer *eb)
70 {
71 	BUG_ON(!eb->write_locks);
72 }
73 
74 #else
75 static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb) { }
76 static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb) { }
77 static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb) { }
78 static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb) { }
79 static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb) { }
80 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb) { }
81 static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb) { }
82 static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb) { }
83 void btrfs_assert_tree_locked(struct extent_buffer *eb) { }
84 static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb) { }
85 static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb) { }
86 #endif
87 
88 void btrfs_set_lock_blocking_read(struct extent_buffer *eb)
89 {
90 	trace_btrfs_set_lock_blocking_read(eb);
91 	/*
92 	 * No lock is required.  The lock owner may change if we have a read
93 	 * lock, but it won't change to or away from us.  If we have the write
94 	 * lock, we are the owner and it'll never change.
95 	 */
96 	if (eb->lock_nested && current->pid == eb->lock_owner)
97 		return;
98 	btrfs_assert_tree_read_locked(eb);
99 	atomic_inc(&eb->blocking_readers);
100 	btrfs_assert_spinning_readers_put(eb);
101 	read_unlock(&eb->lock);
102 }
103 
104 void btrfs_set_lock_blocking_write(struct extent_buffer *eb)
105 {
106 	trace_btrfs_set_lock_blocking_write(eb);
107 	/*
108 	 * No lock is required.  The lock owner may change if we have a read
109 	 * lock, but it won't change to or away from us.  If we have the write
110 	 * lock, we are the owner and it'll never change.
111 	 */
112 	if (eb->lock_nested && current->pid == eb->lock_owner)
113 		return;
114 	if (eb->blocking_writers == 0) {
115 		btrfs_assert_spinning_writers_put(eb);
116 		btrfs_assert_tree_locked(eb);
117 		eb->blocking_writers++;
118 		write_unlock(&eb->lock);
119 	}
120 }
121 
122 void btrfs_clear_lock_blocking_read(struct extent_buffer *eb)
123 {
124 	trace_btrfs_clear_lock_blocking_read(eb);
125 	/*
126 	 * No lock is required.  The lock owner may change if we have a read
127 	 * lock, but it won't change to or away from us.  If we have the write
128 	 * lock, we are the owner and it'll never change.
129 	 */
130 	if (eb->lock_nested && current->pid == eb->lock_owner)
131 		return;
132 	BUG_ON(atomic_read(&eb->blocking_readers) == 0);
133 	read_lock(&eb->lock);
134 	btrfs_assert_spinning_readers_get(eb);
135 	/* atomic_dec_and_test implies a barrier */
136 	if (atomic_dec_and_test(&eb->blocking_readers))
137 		cond_wake_up_nomb(&eb->read_lock_wq);
138 }
139 
140 void btrfs_clear_lock_blocking_write(struct extent_buffer *eb)
141 {
142 	trace_btrfs_clear_lock_blocking_write(eb);
143 	/*
144 	 * no lock is required.  The lock owner may change if
145 	 * we have a read lock, but it won't change to or away
146 	 * from us.  If we have the write lock, we are the owner
147 	 * and it'll never change.
148 	 */
149 	if (eb->lock_nested && current->pid == eb->lock_owner)
150 		return;
151 	write_lock(&eb->lock);
152 	BUG_ON(eb->blocking_writers != 1);
153 	btrfs_assert_spinning_writers_get(eb);
154 	if (--eb->blocking_writers == 0)
155 		cond_wake_up(&eb->write_lock_wq);
156 }
157 
158 /*
159  * take a spinning read lock.  This will wait for any blocking
160  * writers
161  */
162 void btrfs_tree_read_lock(struct extent_buffer *eb)
163 {
164 	u64 start_ns = 0;
165 
166 	if (trace_btrfs_tree_read_lock_enabled())
167 		start_ns = ktime_get_ns();
168 again:
169 	read_lock(&eb->lock);
170 	BUG_ON(eb->blocking_writers == 0 &&
171 	       current->pid == eb->lock_owner);
172 	if (eb->blocking_writers && current->pid == eb->lock_owner) {
173 		/*
174 		 * This extent is already write-locked by our thread. We allow
175 		 * an additional read lock to be added because it's for the same
176 		 * thread. btrfs_find_all_roots() depends on this as it may be
177 		 * called on a partly (write-)locked tree.
178 		 */
179 		BUG_ON(eb->lock_nested);
180 		eb->lock_nested = true;
181 		read_unlock(&eb->lock);
182 		trace_btrfs_tree_read_lock(eb, start_ns);
183 		return;
184 	}
185 	if (eb->blocking_writers) {
186 		read_unlock(&eb->lock);
187 		wait_event(eb->write_lock_wq,
188 			   eb->blocking_writers == 0);
189 		goto again;
190 	}
191 	btrfs_assert_tree_read_locks_get(eb);
192 	btrfs_assert_spinning_readers_get(eb);
193 	trace_btrfs_tree_read_lock(eb, start_ns);
194 }
195 
196 /*
197  * take a spinning read lock.
198  * returns 1 if we get the read lock and 0 if we don't
199  * this won't wait for blocking writers
200  */
201 int btrfs_tree_read_lock_atomic(struct extent_buffer *eb)
202 {
203 	if (eb->blocking_writers)
204 		return 0;
205 
206 	read_lock(&eb->lock);
207 	if (eb->blocking_writers) {
208 		read_unlock(&eb->lock);
209 		return 0;
210 	}
211 	btrfs_assert_tree_read_locks_get(eb);
212 	btrfs_assert_spinning_readers_get(eb);
213 	trace_btrfs_tree_read_lock_atomic(eb);
214 	return 1;
215 }
216 
217 /*
218  * returns 1 if we get the read lock and 0 if we don't
219  * this won't wait for blocking writers
220  */
221 int btrfs_try_tree_read_lock(struct extent_buffer *eb)
222 {
223 	if (eb->blocking_writers)
224 		return 0;
225 
226 	if (!read_trylock(&eb->lock))
227 		return 0;
228 
229 	if (eb->blocking_writers) {
230 		read_unlock(&eb->lock);
231 		return 0;
232 	}
233 	btrfs_assert_tree_read_locks_get(eb);
234 	btrfs_assert_spinning_readers_get(eb);
235 	trace_btrfs_try_tree_read_lock(eb);
236 	return 1;
237 }
238 
239 /*
240  * returns 1 if we get the read lock and 0 if we don't
241  * this won't wait for blocking writers or readers
242  */
243 int btrfs_try_tree_write_lock(struct extent_buffer *eb)
244 {
245 	if (eb->blocking_writers || atomic_read(&eb->blocking_readers))
246 		return 0;
247 
248 	write_lock(&eb->lock);
249 	if (eb->blocking_writers || atomic_read(&eb->blocking_readers)) {
250 		write_unlock(&eb->lock);
251 		return 0;
252 	}
253 	btrfs_assert_tree_write_locks_get(eb);
254 	btrfs_assert_spinning_writers_get(eb);
255 	eb->lock_owner = current->pid;
256 	trace_btrfs_try_tree_write_lock(eb);
257 	return 1;
258 }
259 
260 /*
261  * drop a spinning read lock
262  */
263 void btrfs_tree_read_unlock(struct extent_buffer *eb)
264 {
265 	trace_btrfs_tree_read_unlock(eb);
266 	/*
267 	 * if we're nested, we have the write lock.  No new locking
268 	 * is needed as long as we are the lock owner.
269 	 * The write unlock will do a barrier for us, and the lock_nested
270 	 * field only matters to the lock owner.
271 	 */
272 	if (eb->lock_nested && current->pid == eb->lock_owner) {
273 		eb->lock_nested = false;
274 		return;
275 	}
276 	btrfs_assert_tree_read_locked(eb);
277 	btrfs_assert_spinning_readers_put(eb);
278 	btrfs_assert_tree_read_locks_put(eb);
279 	read_unlock(&eb->lock);
280 }
281 
282 /*
283  * drop a blocking read lock
284  */
285 void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb)
286 {
287 	trace_btrfs_tree_read_unlock_blocking(eb);
288 	/*
289 	 * if we're nested, we have the write lock.  No new locking
290 	 * is needed as long as we are the lock owner.
291 	 * The write unlock will do a barrier for us, and the lock_nested
292 	 * field only matters to the lock owner.
293 	 */
294 	if (eb->lock_nested && current->pid == eb->lock_owner) {
295 		eb->lock_nested = false;
296 		return;
297 	}
298 	btrfs_assert_tree_read_locked(eb);
299 	WARN_ON(atomic_read(&eb->blocking_readers) == 0);
300 	/* atomic_dec_and_test implies a barrier */
301 	if (atomic_dec_and_test(&eb->blocking_readers))
302 		cond_wake_up_nomb(&eb->read_lock_wq);
303 	btrfs_assert_tree_read_locks_put(eb);
304 }
305 
306 /*
307  * take a spinning write lock.  This will wait for both
308  * blocking readers or writers
309  */
310 void btrfs_tree_lock(struct extent_buffer *eb)
311 {
312 	u64 start_ns = 0;
313 
314 	if (trace_btrfs_tree_lock_enabled())
315 		start_ns = ktime_get_ns();
316 
317 	WARN_ON(eb->lock_owner == current->pid);
318 again:
319 	wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0);
320 	wait_event(eb->write_lock_wq, eb->blocking_writers == 0);
321 	write_lock(&eb->lock);
322 	if (atomic_read(&eb->blocking_readers) || eb->blocking_writers) {
323 		write_unlock(&eb->lock);
324 		goto again;
325 	}
326 	btrfs_assert_spinning_writers_get(eb);
327 	btrfs_assert_tree_write_locks_get(eb);
328 	eb->lock_owner = current->pid;
329 	trace_btrfs_tree_lock(eb, start_ns);
330 }
331 
332 /*
333  * drop a spinning or a blocking write lock.
334  */
335 void btrfs_tree_unlock(struct extent_buffer *eb)
336 {
337 	int blockers = eb->blocking_writers;
338 
339 	BUG_ON(blockers > 1);
340 
341 	btrfs_assert_tree_locked(eb);
342 	trace_btrfs_tree_unlock(eb);
343 	eb->lock_owner = 0;
344 	btrfs_assert_tree_write_locks_put(eb);
345 
346 	if (blockers) {
347 		btrfs_assert_no_spinning_writers(eb);
348 		eb->blocking_writers--;
349 		/*
350 		 * We need to order modifying blocking_writers above with
351 		 * actually waking up the sleepers to ensure they see the
352 		 * updated value of blocking_writers
353 		 */
354 		cond_wake_up(&eb->write_lock_wq);
355 	} else {
356 		btrfs_assert_spinning_writers_put(eb);
357 		write_unlock(&eb->lock);
358 	}
359 }
360