xref: /openbmc/linux/fs/xfs/xfs_buf.c (revision 0edbfea5)
1 /*
2  * Copyright (c) 2000-2006 Silicon Graphics, Inc.
3  * All Rights Reserved.
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License as
7  * published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it would be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write the Free Software Foundation,
16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17  */
18 #include "xfs.h"
19 #include <linux/stddef.h>
20 #include <linux/errno.h>
21 #include <linux/gfp.h>
22 #include <linux/pagemap.h>
23 #include <linux/init.h>
24 #include <linux/vmalloc.h>
25 #include <linux/bio.h>
26 #include <linux/sysctl.h>
27 #include <linux/proc_fs.h>
28 #include <linux/workqueue.h>
29 #include <linux/percpu.h>
30 #include <linux/blkdev.h>
31 #include <linux/hash.h>
32 #include <linux/kthread.h>
33 #include <linux/migrate.h>
34 #include <linux/backing-dev.h>
35 #include <linux/freezer.h>
36 
37 #include "xfs_format.h"
38 #include "xfs_log_format.h"
39 #include "xfs_trans_resv.h"
40 #include "xfs_sb.h"
41 #include "xfs_mount.h"
42 #include "xfs_trace.h"
43 #include "xfs_log.h"
44 
45 static kmem_zone_t *xfs_buf_zone;
46 
47 #ifdef XFS_BUF_LOCK_TRACKING
48 # define XB_SET_OWNER(bp)	((bp)->b_last_holder = current->pid)
49 # define XB_CLEAR_OWNER(bp)	((bp)->b_last_holder = -1)
50 # define XB_GET_OWNER(bp)	((bp)->b_last_holder)
51 #else
52 # define XB_SET_OWNER(bp)	do { } while (0)
53 # define XB_CLEAR_OWNER(bp)	do { } while (0)
54 # define XB_GET_OWNER(bp)	do { } while (0)
55 #endif
56 
57 #define xb_to_gfp(flags) \
58 	((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : GFP_NOFS) | __GFP_NOWARN)
59 
60 
61 static inline int
62 xfs_buf_is_vmapped(
63 	struct xfs_buf	*bp)
64 {
65 	/*
66 	 * Return true if the buffer is vmapped.
67 	 *
68 	 * b_addr is null if the buffer is not mapped, but the code is clever
69 	 * enough to know it doesn't have to map a single page, so the check has
70 	 * to be both for b_addr and bp->b_page_count > 1.
71 	 */
72 	return bp->b_addr && bp->b_page_count > 1;
73 }
74 
75 static inline int
76 xfs_buf_vmap_len(
77 	struct xfs_buf	*bp)
78 {
79 	return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
80 }
81 
82 /*
83  * When we mark a buffer stale, we remove the buffer from the LRU and clear the
84  * b_lru_ref count so that the buffer is freed immediately when the buffer
85  * reference count falls to zero. If the buffer is already on the LRU, we need
86  * to remove the reference that LRU holds on the buffer.
87  *
88  * This prevents build-up of stale buffers on the LRU.
89  */
90 void
91 xfs_buf_stale(
92 	struct xfs_buf	*bp)
93 {
94 	ASSERT(xfs_buf_islocked(bp));
95 
96 	bp->b_flags |= XBF_STALE;
97 
98 	/*
99 	 * Clear the delwri status so that a delwri queue walker will not
100 	 * flush this buffer to disk now that it is stale. The delwri queue has
101 	 * a reference to the buffer, so this is safe to do.
102 	 */
103 	bp->b_flags &= ~_XBF_DELWRI_Q;
104 
105 	spin_lock(&bp->b_lock);
106 	atomic_set(&bp->b_lru_ref, 0);
107 	if (!(bp->b_state & XFS_BSTATE_DISPOSE) &&
108 	    (list_lru_del(&bp->b_target->bt_lru, &bp->b_lru)))
109 		atomic_dec(&bp->b_hold);
110 
111 	ASSERT(atomic_read(&bp->b_hold) >= 1);
112 	spin_unlock(&bp->b_lock);
113 }
114 
115 static int
116 xfs_buf_get_maps(
117 	struct xfs_buf		*bp,
118 	int			map_count)
119 {
120 	ASSERT(bp->b_maps == NULL);
121 	bp->b_map_count = map_count;
122 
123 	if (map_count == 1) {
124 		bp->b_maps = &bp->__b_map;
125 		return 0;
126 	}
127 
128 	bp->b_maps = kmem_zalloc(map_count * sizeof(struct xfs_buf_map),
129 				KM_NOFS);
130 	if (!bp->b_maps)
131 		return -ENOMEM;
132 	return 0;
133 }
134 
135 /*
136  *	Frees b_pages if it was allocated.
137  */
138 static void
139 xfs_buf_free_maps(
140 	struct xfs_buf	*bp)
141 {
142 	if (bp->b_maps != &bp->__b_map) {
143 		kmem_free(bp->b_maps);
144 		bp->b_maps = NULL;
145 	}
146 }
147 
148 struct xfs_buf *
149 _xfs_buf_alloc(
150 	struct xfs_buftarg	*target,
151 	struct xfs_buf_map	*map,
152 	int			nmaps,
153 	xfs_buf_flags_t		flags)
154 {
155 	struct xfs_buf		*bp;
156 	int			error;
157 	int			i;
158 
159 	bp = kmem_zone_zalloc(xfs_buf_zone, KM_NOFS);
160 	if (unlikely(!bp))
161 		return NULL;
162 
163 	/*
164 	 * We don't want certain flags to appear in b_flags unless they are
165 	 * specifically set by later operations on the buffer.
166 	 */
167 	flags &= ~(XBF_UNMAPPED | XBF_TRYLOCK | XBF_ASYNC | XBF_READ_AHEAD);
168 
169 	atomic_set(&bp->b_hold, 1);
170 	atomic_set(&bp->b_lru_ref, 1);
171 	init_completion(&bp->b_iowait);
172 	INIT_LIST_HEAD(&bp->b_lru);
173 	INIT_LIST_HEAD(&bp->b_list);
174 	RB_CLEAR_NODE(&bp->b_rbnode);
175 	sema_init(&bp->b_sema, 0); /* held, no waiters */
176 	spin_lock_init(&bp->b_lock);
177 	XB_SET_OWNER(bp);
178 	bp->b_target = target;
179 	bp->b_flags = flags;
180 
181 	/*
182 	 * Set length and io_length to the same value initially.
183 	 * I/O routines should use io_length, which will be the same in
184 	 * most cases but may be reset (e.g. XFS recovery).
185 	 */
186 	error = xfs_buf_get_maps(bp, nmaps);
187 	if (error)  {
188 		kmem_zone_free(xfs_buf_zone, bp);
189 		return NULL;
190 	}
191 
192 	bp->b_bn = map[0].bm_bn;
193 	bp->b_length = 0;
194 	for (i = 0; i < nmaps; i++) {
195 		bp->b_maps[i].bm_bn = map[i].bm_bn;
196 		bp->b_maps[i].bm_len = map[i].bm_len;
197 		bp->b_length += map[i].bm_len;
198 	}
199 	bp->b_io_length = bp->b_length;
200 
201 	atomic_set(&bp->b_pin_count, 0);
202 	init_waitqueue_head(&bp->b_waiters);
203 
204 	XFS_STATS_INC(target->bt_mount, xb_create);
205 	trace_xfs_buf_init(bp, _RET_IP_);
206 
207 	return bp;
208 }
209 
210 /*
211  *	Allocate a page array capable of holding a specified number
212  *	of pages, and point the page buf at it.
213  */
214 STATIC int
215 _xfs_buf_get_pages(
216 	xfs_buf_t		*bp,
217 	int			page_count)
218 {
219 	/* Make sure that we have a page list */
220 	if (bp->b_pages == NULL) {
221 		bp->b_page_count = page_count;
222 		if (page_count <= XB_PAGES) {
223 			bp->b_pages = bp->b_page_array;
224 		} else {
225 			bp->b_pages = kmem_alloc(sizeof(struct page *) *
226 						 page_count, KM_NOFS);
227 			if (bp->b_pages == NULL)
228 				return -ENOMEM;
229 		}
230 		memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
231 	}
232 	return 0;
233 }
234 
235 /*
236  *	Frees b_pages if it was allocated.
237  */
238 STATIC void
239 _xfs_buf_free_pages(
240 	xfs_buf_t	*bp)
241 {
242 	if (bp->b_pages != bp->b_page_array) {
243 		kmem_free(bp->b_pages);
244 		bp->b_pages = NULL;
245 	}
246 }
247 
248 /*
249  *	Releases the specified buffer.
250  *
251  * 	The modification state of any associated pages is left unchanged.
252  * 	The buffer must not be on any hash - use xfs_buf_rele instead for
253  * 	hashed and refcounted buffers
254  */
255 void
256 xfs_buf_free(
257 	xfs_buf_t		*bp)
258 {
259 	trace_xfs_buf_free(bp, _RET_IP_);
260 
261 	ASSERT(list_empty(&bp->b_lru));
262 
263 	if (bp->b_flags & _XBF_PAGES) {
264 		uint		i;
265 
266 		if (xfs_buf_is_vmapped(bp))
267 			vm_unmap_ram(bp->b_addr - bp->b_offset,
268 					bp->b_page_count);
269 
270 		for (i = 0; i < bp->b_page_count; i++) {
271 			struct page	*page = bp->b_pages[i];
272 
273 			__free_page(page);
274 		}
275 	} else if (bp->b_flags & _XBF_KMEM)
276 		kmem_free(bp->b_addr);
277 	_xfs_buf_free_pages(bp);
278 	xfs_buf_free_maps(bp);
279 	kmem_zone_free(xfs_buf_zone, bp);
280 }
281 
282 /*
283  * Allocates all the pages for buffer in question and builds it's page list.
284  */
285 STATIC int
286 xfs_buf_allocate_memory(
287 	xfs_buf_t		*bp,
288 	uint			flags)
289 {
290 	size_t			size;
291 	size_t			nbytes, offset;
292 	gfp_t			gfp_mask = xb_to_gfp(flags);
293 	unsigned short		page_count, i;
294 	xfs_off_t		start, end;
295 	int			error;
296 
297 	/*
298 	 * for buffers that are contained within a single page, just allocate
299 	 * the memory from the heap - there's no need for the complexity of
300 	 * page arrays to keep allocation down to order 0.
301 	 */
302 	size = BBTOB(bp->b_length);
303 	if (size < PAGE_SIZE) {
304 		bp->b_addr = kmem_alloc(size, KM_NOFS);
305 		if (!bp->b_addr) {
306 			/* low memory - use alloc_page loop instead */
307 			goto use_alloc_page;
308 		}
309 
310 		if (((unsigned long)(bp->b_addr + size - 1) & PAGE_MASK) !=
311 		    ((unsigned long)bp->b_addr & PAGE_MASK)) {
312 			/* b_addr spans two pages - use alloc_page instead */
313 			kmem_free(bp->b_addr);
314 			bp->b_addr = NULL;
315 			goto use_alloc_page;
316 		}
317 		bp->b_offset = offset_in_page(bp->b_addr);
318 		bp->b_pages = bp->b_page_array;
319 		bp->b_pages[0] = virt_to_page(bp->b_addr);
320 		bp->b_page_count = 1;
321 		bp->b_flags |= _XBF_KMEM;
322 		return 0;
323 	}
324 
325 use_alloc_page:
326 	start = BBTOB(bp->b_maps[0].bm_bn) >> PAGE_SHIFT;
327 	end = (BBTOB(bp->b_maps[0].bm_bn + bp->b_length) + PAGE_SIZE - 1)
328 								>> PAGE_SHIFT;
329 	page_count = end - start;
330 	error = _xfs_buf_get_pages(bp, page_count);
331 	if (unlikely(error))
332 		return error;
333 
334 	offset = bp->b_offset;
335 	bp->b_flags |= _XBF_PAGES;
336 
337 	for (i = 0; i < bp->b_page_count; i++) {
338 		struct page	*page;
339 		uint		retries = 0;
340 retry:
341 		page = alloc_page(gfp_mask);
342 		if (unlikely(page == NULL)) {
343 			if (flags & XBF_READ_AHEAD) {
344 				bp->b_page_count = i;
345 				error = -ENOMEM;
346 				goto out_free_pages;
347 			}
348 
349 			/*
350 			 * This could deadlock.
351 			 *
352 			 * But until all the XFS lowlevel code is revamped to
353 			 * handle buffer allocation failures we can't do much.
354 			 */
355 			if (!(++retries % 100))
356 				xfs_err(NULL,
357 		"%s(%u) possible memory allocation deadlock in %s (mode:0x%x)",
358 					current->comm, current->pid,
359 					__func__, gfp_mask);
360 
361 			XFS_STATS_INC(bp->b_target->bt_mount, xb_page_retries);
362 			congestion_wait(BLK_RW_ASYNC, HZ/50);
363 			goto retry;
364 		}
365 
366 		XFS_STATS_INC(bp->b_target->bt_mount, xb_page_found);
367 
368 		nbytes = min_t(size_t, size, PAGE_SIZE - offset);
369 		size -= nbytes;
370 		bp->b_pages[i] = page;
371 		offset = 0;
372 	}
373 	return 0;
374 
375 out_free_pages:
376 	for (i = 0; i < bp->b_page_count; i++)
377 		__free_page(bp->b_pages[i]);
378 	return error;
379 }
380 
381 /*
382  *	Map buffer into kernel address-space if necessary.
383  */
384 STATIC int
385 _xfs_buf_map_pages(
386 	xfs_buf_t		*bp,
387 	uint			flags)
388 {
389 	ASSERT(bp->b_flags & _XBF_PAGES);
390 	if (bp->b_page_count == 1) {
391 		/* A single page buffer is always mappable */
392 		bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
393 	} else if (flags & XBF_UNMAPPED) {
394 		bp->b_addr = NULL;
395 	} else {
396 		int retried = 0;
397 		unsigned noio_flag;
398 
399 		/*
400 		 * vm_map_ram() will allocate auxillary structures (e.g.
401 		 * pagetables) with GFP_KERNEL, yet we are likely to be under
402 		 * GFP_NOFS context here. Hence we need to tell memory reclaim
403 		 * that we are in such a context via PF_MEMALLOC_NOIO to prevent
404 		 * memory reclaim re-entering the filesystem here and
405 		 * potentially deadlocking.
406 		 */
407 		noio_flag = memalloc_noio_save();
408 		do {
409 			bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
410 						-1, PAGE_KERNEL);
411 			if (bp->b_addr)
412 				break;
413 			vm_unmap_aliases();
414 		} while (retried++ <= 1);
415 		memalloc_noio_restore(noio_flag);
416 
417 		if (!bp->b_addr)
418 			return -ENOMEM;
419 		bp->b_addr += bp->b_offset;
420 	}
421 
422 	return 0;
423 }
424 
425 /*
426  *	Finding and Reading Buffers
427  */
428 
429 /*
430  *	Look up, and creates if absent, a lockable buffer for
431  *	a given range of an inode.  The buffer is returned
432  *	locked.	No I/O is implied by this call.
433  */
434 xfs_buf_t *
435 _xfs_buf_find(
436 	struct xfs_buftarg	*btp,
437 	struct xfs_buf_map	*map,
438 	int			nmaps,
439 	xfs_buf_flags_t		flags,
440 	xfs_buf_t		*new_bp)
441 {
442 	struct xfs_perag	*pag;
443 	struct rb_node		**rbp;
444 	struct rb_node		*parent;
445 	xfs_buf_t		*bp;
446 	xfs_daddr_t		blkno = map[0].bm_bn;
447 	xfs_daddr_t		eofs;
448 	int			numblks = 0;
449 	int			i;
450 
451 	for (i = 0; i < nmaps; i++)
452 		numblks += map[i].bm_len;
453 
454 	/* Check for IOs smaller than the sector size / not sector aligned */
455 	ASSERT(!(BBTOB(numblks) < btp->bt_meta_sectorsize));
456 	ASSERT(!(BBTOB(blkno) & (xfs_off_t)btp->bt_meta_sectormask));
457 
458 	/*
459 	 * Corrupted block numbers can get through to here, unfortunately, so we
460 	 * have to check that the buffer falls within the filesystem bounds.
461 	 */
462 	eofs = XFS_FSB_TO_BB(btp->bt_mount, btp->bt_mount->m_sb.sb_dblocks);
463 	if (blkno < 0 || blkno >= eofs) {
464 		/*
465 		 * XXX (dgc): we should really be returning -EFSCORRUPTED here,
466 		 * but none of the higher level infrastructure supports
467 		 * returning a specific error on buffer lookup failures.
468 		 */
469 		xfs_alert(btp->bt_mount,
470 			  "%s: Block out of range: block 0x%llx, EOFS 0x%llx ",
471 			  __func__, blkno, eofs);
472 		WARN_ON(1);
473 		return NULL;
474 	}
475 
476 	/* get tree root */
477 	pag = xfs_perag_get(btp->bt_mount,
478 				xfs_daddr_to_agno(btp->bt_mount, blkno));
479 
480 	/* walk tree */
481 	spin_lock(&pag->pag_buf_lock);
482 	rbp = &pag->pag_buf_tree.rb_node;
483 	parent = NULL;
484 	bp = NULL;
485 	while (*rbp) {
486 		parent = *rbp;
487 		bp = rb_entry(parent, struct xfs_buf, b_rbnode);
488 
489 		if (blkno < bp->b_bn)
490 			rbp = &(*rbp)->rb_left;
491 		else if (blkno > bp->b_bn)
492 			rbp = &(*rbp)->rb_right;
493 		else {
494 			/*
495 			 * found a block number match. If the range doesn't
496 			 * match, the only way this is allowed is if the buffer
497 			 * in the cache is stale and the transaction that made
498 			 * it stale has not yet committed. i.e. we are
499 			 * reallocating a busy extent. Skip this buffer and
500 			 * continue searching to the right for an exact match.
501 			 */
502 			if (bp->b_length != numblks) {
503 				ASSERT(bp->b_flags & XBF_STALE);
504 				rbp = &(*rbp)->rb_right;
505 				continue;
506 			}
507 			atomic_inc(&bp->b_hold);
508 			goto found;
509 		}
510 	}
511 
512 	/* No match found */
513 	if (new_bp) {
514 		rb_link_node(&new_bp->b_rbnode, parent, rbp);
515 		rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
516 		/* the buffer keeps the perag reference until it is freed */
517 		new_bp->b_pag = pag;
518 		spin_unlock(&pag->pag_buf_lock);
519 	} else {
520 		XFS_STATS_INC(btp->bt_mount, xb_miss_locked);
521 		spin_unlock(&pag->pag_buf_lock);
522 		xfs_perag_put(pag);
523 	}
524 	return new_bp;
525 
526 found:
527 	spin_unlock(&pag->pag_buf_lock);
528 	xfs_perag_put(pag);
529 
530 	if (!xfs_buf_trylock(bp)) {
531 		if (flags & XBF_TRYLOCK) {
532 			xfs_buf_rele(bp);
533 			XFS_STATS_INC(btp->bt_mount, xb_busy_locked);
534 			return NULL;
535 		}
536 		xfs_buf_lock(bp);
537 		XFS_STATS_INC(btp->bt_mount, xb_get_locked_waited);
538 	}
539 
540 	/*
541 	 * if the buffer is stale, clear all the external state associated with
542 	 * it. We need to keep flags such as how we allocated the buffer memory
543 	 * intact here.
544 	 */
545 	if (bp->b_flags & XBF_STALE) {
546 		ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
547 		ASSERT(bp->b_iodone == NULL);
548 		bp->b_flags &= _XBF_KMEM | _XBF_PAGES;
549 		bp->b_ops = NULL;
550 	}
551 
552 	trace_xfs_buf_find(bp, flags, _RET_IP_);
553 	XFS_STATS_INC(btp->bt_mount, xb_get_locked);
554 	return bp;
555 }
556 
557 /*
558  * Assembles a buffer covering the specified range. The code is optimised for
559  * cache hits, as metadata intensive workloads will see 3 orders of magnitude
560  * more hits than misses.
561  */
562 struct xfs_buf *
563 xfs_buf_get_map(
564 	struct xfs_buftarg	*target,
565 	struct xfs_buf_map	*map,
566 	int			nmaps,
567 	xfs_buf_flags_t		flags)
568 {
569 	struct xfs_buf		*bp;
570 	struct xfs_buf		*new_bp;
571 	int			error = 0;
572 
573 	bp = _xfs_buf_find(target, map, nmaps, flags, NULL);
574 	if (likely(bp))
575 		goto found;
576 
577 	new_bp = _xfs_buf_alloc(target, map, nmaps, flags);
578 	if (unlikely(!new_bp))
579 		return NULL;
580 
581 	error = xfs_buf_allocate_memory(new_bp, flags);
582 	if (error) {
583 		xfs_buf_free(new_bp);
584 		return NULL;
585 	}
586 
587 	bp = _xfs_buf_find(target, map, nmaps, flags, new_bp);
588 	if (!bp) {
589 		xfs_buf_free(new_bp);
590 		return NULL;
591 	}
592 
593 	if (bp != new_bp)
594 		xfs_buf_free(new_bp);
595 
596 found:
597 	if (!bp->b_addr) {
598 		error = _xfs_buf_map_pages(bp, flags);
599 		if (unlikely(error)) {
600 			xfs_warn(target->bt_mount,
601 				"%s: failed to map pagesn", __func__);
602 			xfs_buf_relse(bp);
603 			return NULL;
604 		}
605 	}
606 
607 	/*
608 	 * Clear b_error if this is a lookup from a caller that doesn't expect
609 	 * valid data to be found in the buffer.
610 	 */
611 	if (!(flags & XBF_READ))
612 		xfs_buf_ioerror(bp, 0);
613 
614 	XFS_STATS_INC(target->bt_mount, xb_get);
615 	trace_xfs_buf_get(bp, flags, _RET_IP_);
616 	return bp;
617 }
618 
619 STATIC int
620 _xfs_buf_read(
621 	xfs_buf_t		*bp,
622 	xfs_buf_flags_t		flags)
623 {
624 	ASSERT(!(flags & XBF_WRITE));
625 	ASSERT(bp->b_maps[0].bm_bn != XFS_BUF_DADDR_NULL);
626 
627 	bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_READ_AHEAD);
628 	bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | XBF_READ_AHEAD);
629 
630 	if (flags & XBF_ASYNC) {
631 		xfs_buf_submit(bp);
632 		return 0;
633 	}
634 	return xfs_buf_submit_wait(bp);
635 }
636 
637 xfs_buf_t *
638 xfs_buf_read_map(
639 	struct xfs_buftarg	*target,
640 	struct xfs_buf_map	*map,
641 	int			nmaps,
642 	xfs_buf_flags_t		flags,
643 	const struct xfs_buf_ops *ops)
644 {
645 	struct xfs_buf		*bp;
646 
647 	flags |= XBF_READ;
648 
649 	bp = xfs_buf_get_map(target, map, nmaps, flags);
650 	if (bp) {
651 		trace_xfs_buf_read(bp, flags, _RET_IP_);
652 
653 		if (!(bp->b_flags & XBF_DONE)) {
654 			XFS_STATS_INC(target->bt_mount, xb_get_read);
655 			bp->b_ops = ops;
656 			_xfs_buf_read(bp, flags);
657 		} else if (flags & XBF_ASYNC) {
658 			/*
659 			 * Read ahead call which is already satisfied,
660 			 * drop the buffer
661 			 */
662 			xfs_buf_relse(bp);
663 			return NULL;
664 		} else {
665 			/* We do not want read in the flags */
666 			bp->b_flags &= ~XBF_READ;
667 		}
668 	}
669 
670 	return bp;
671 }
672 
673 /*
674  *	If we are not low on memory then do the readahead in a deadlock
675  *	safe manner.
676  */
677 void
678 xfs_buf_readahead_map(
679 	struct xfs_buftarg	*target,
680 	struct xfs_buf_map	*map,
681 	int			nmaps,
682 	const struct xfs_buf_ops *ops)
683 {
684 	if (bdi_read_congested(target->bt_bdi))
685 		return;
686 
687 	xfs_buf_read_map(target, map, nmaps,
688 		     XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD, ops);
689 }
690 
691 /*
692  * Read an uncached buffer from disk. Allocates and returns a locked
693  * buffer containing the disk contents or nothing.
694  */
695 int
696 xfs_buf_read_uncached(
697 	struct xfs_buftarg	*target,
698 	xfs_daddr_t		daddr,
699 	size_t			numblks,
700 	int			flags,
701 	struct xfs_buf		**bpp,
702 	const struct xfs_buf_ops *ops)
703 {
704 	struct xfs_buf		*bp;
705 
706 	*bpp = NULL;
707 
708 	bp = xfs_buf_get_uncached(target, numblks, flags);
709 	if (!bp)
710 		return -ENOMEM;
711 
712 	/* set up the buffer for a read IO */
713 	ASSERT(bp->b_map_count == 1);
714 	bp->b_bn = XFS_BUF_DADDR_NULL;  /* always null for uncached buffers */
715 	bp->b_maps[0].bm_bn = daddr;
716 	bp->b_flags |= XBF_READ;
717 	bp->b_ops = ops;
718 
719 	xfs_buf_submit_wait(bp);
720 	if (bp->b_error) {
721 		int	error = bp->b_error;
722 		xfs_buf_relse(bp);
723 		return error;
724 	}
725 
726 	*bpp = bp;
727 	return 0;
728 }
729 
730 /*
731  * Return a buffer allocated as an empty buffer and associated to external
732  * memory via xfs_buf_associate_memory() back to it's empty state.
733  */
734 void
735 xfs_buf_set_empty(
736 	struct xfs_buf		*bp,
737 	size_t			numblks)
738 {
739 	if (bp->b_pages)
740 		_xfs_buf_free_pages(bp);
741 
742 	bp->b_pages = NULL;
743 	bp->b_page_count = 0;
744 	bp->b_addr = NULL;
745 	bp->b_length = numblks;
746 	bp->b_io_length = numblks;
747 
748 	ASSERT(bp->b_map_count == 1);
749 	bp->b_bn = XFS_BUF_DADDR_NULL;
750 	bp->b_maps[0].bm_bn = XFS_BUF_DADDR_NULL;
751 	bp->b_maps[0].bm_len = bp->b_length;
752 }
753 
754 static inline struct page *
755 mem_to_page(
756 	void			*addr)
757 {
758 	if ((!is_vmalloc_addr(addr))) {
759 		return virt_to_page(addr);
760 	} else {
761 		return vmalloc_to_page(addr);
762 	}
763 }
764 
765 int
766 xfs_buf_associate_memory(
767 	xfs_buf_t		*bp,
768 	void			*mem,
769 	size_t			len)
770 {
771 	int			rval;
772 	int			i = 0;
773 	unsigned long		pageaddr;
774 	unsigned long		offset;
775 	size_t			buflen;
776 	int			page_count;
777 
778 	pageaddr = (unsigned long)mem & PAGE_MASK;
779 	offset = (unsigned long)mem - pageaddr;
780 	buflen = PAGE_ALIGN(len + offset);
781 	page_count = buflen >> PAGE_SHIFT;
782 
783 	/* Free any previous set of page pointers */
784 	if (bp->b_pages)
785 		_xfs_buf_free_pages(bp);
786 
787 	bp->b_pages = NULL;
788 	bp->b_addr = mem;
789 
790 	rval = _xfs_buf_get_pages(bp, page_count);
791 	if (rval)
792 		return rval;
793 
794 	bp->b_offset = offset;
795 
796 	for (i = 0; i < bp->b_page_count; i++) {
797 		bp->b_pages[i] = mem_to_page((void *)pageaddr);
798 		pageaddr += PAGE_SIZE;
799 	}
800 
801 	bp->b_io_length = BTOBB(len);
802 	bp->b_length = BTOBB(buflen);
803 
804 	return 0;
805 }
806 
807 xfs_buf_t *
808 xfs_buf_get_uncached(
809 	struct xfs_buftarg	*target,
810 	size_t			numblks,
811 	int			flags)
812 {
813 	unsigned long		page_count;
814 	int			error, i;
815 	struct xfs_buf		*bp;
816 	DEFINE_SINGLE_BUF_MAP(map, XFS_BUF_DADDR_NULL, numblks);
817 
818 	bp = _xfs_buf_alloc(target, &map, 1, 0);
819 	if (unlikely(bp == NULL))
820 		goto fail;
821 
822 	page_count = PAGE_ALIGN(numblks << BBSHIFT) >> PAGE_SHIFT;
823 	error = _xfs_buf_get_pages(bp, page_count);
824 	if (error)
825 		goto fail_free_buf;
826 
827 	for (i = 0; i < page_count; i++) {
828 		bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
829 		if (!bp->b_pages[i])
830 			goto fail_free_mem;
831 	}
832 	bp->b_flags |= _XBF_PAGES;
833 
834 	error = _xfs_buf_map_pages(bp, 0);
835 	if (unlikely(error)) {
836 		xfs_warn(target->bt_mount,
837 			"%s: failed to map pages", __func__);
838 		goto fail_free_mem;
839 	}
840 
841 	trace_xfs_buf_get_uncached(bp, _RET_IP_);
842 	return bp;
843 
844  fail_free_mem:
845 	while (--i >= 0)
846 		__free_page(bp->b_pages[i]);
847 	_xfs_buf_free_pages(bp);
848  fail_free_buf:
849 	xfs_buf_free_maps(bp);
850 	kmem_zone_free(xfs_buf_zone, bp);
851  fail:
852 	return NULL;
853 }
854 
855 /*
856  *	Increment reference count on buffer, to hold the buffer concurrently
857  *	with another thread which may release (free) the buffer asynchronously.
858  *	Must hold the buffer already to call this function.
859  */
860 void
861 xfs_buf_hold(
862 	xfs_buf_t		*bp)
863 {
864 	trace_xfs_buf_hold(bp, _RET_IP_);
865 	atomic_inc(&bp->b_hold);
866 }
867 
868 /*
869  *	Releases a hold on the specified buffer.  If the
870  *	the hold count is 1, calls xfs_buf_free.
871  */
872 void
873 xfs_buf_rele(
874 	xfs_buf_t		*bp)
875 {
876 	struct xfs_perag	*pag = bp->b_pag;
877 
878 	trace_xfs_buf_rele(bp, _RET_IP_);
879 
880 	if (!pag) {
881 		ASSERT(list_empty(&bp->b_lru));
882 		ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
883 		if (atomic_dec_and_test(&bp->b_hold))
884 			xfs_buf_free(bp);
885 		return;
886 	}
887 
888 	ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
889 
890 	ASSERT(atomic_read(&bp->b_hold) > 0);
891 	if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
892 		spin_lock(&bp->b_lock);
893 		if (!(bp->b_flags & XBF_STALE) && atomic_read(&bp->b_lru_ref)) {
894 			/*
895 			 * If the buffer is added to the LRU take a new
896 			 * reference to the buffer for the LRU and clear the
897 			 * (now stale) dispose list state flag
898 			 */
899 			if (list_lru_add(&bp->b_target->bt_lru, &bp->b_lru)) {
900 				bp->b_state &= ~XFS_BSTATE_DISPOSE;
901 				atomic_inc(&bp->b_hold);
902 			}
903 			spin_unlock(&bp->b_lock);
904 			spin_unlock(&pag->pag_buf_lock);
905 		} else {
906 			/*
907 			 * most of the time buffers will already be removed from
908 			 * the LRU, so optimise that case by checking for the
909 			 * XFS_BSTATE_DISPOSE flag indicating the last list the
910 			 * buffer was on was the disposal list
911 			 */
912 			if (!(bp->b_state & XFS_BSTATE_DISPOSE)) {
913 				list_lru_del(&bp->b_target->bt_lru, &bp->b_lru);
914 			} else {
915 				ASSERT(list_empty(&bp->b_lru));
916 			}
917 			spin_unlock(&bp->b_lock);
918 
919 			ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
920 			rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
921 			spin_unlock(&pag->pag_buf_lock);
922 			xfs_perag_put(pag);
923 			xfs_buf_free(bp);
924 		}
925 	}
926 }
927 
928 
929 /*
930  *	Lock a buffer object, if it is not already locked.
931  *
932  *	If we come across a stale, pinned, locked buffer, we know that we are
933  *	being asked to lock a buffer that has been reallocated. Because it is
934  *	pinned, we know that the log has not been pushed to disk and hence it
935  *	will still be locked.  Rather than continuing to have trylock attempts
936  *	fail until someone else pushes the log, push it ourselves before
937  *	returning.  This means that the xfsaild will not get stuck trying
938  *	to push on stale inode buffers.
939  */
940 int
941 xfs_buf_trylock(
942 	struct xfs_buf		*bp)
943 {
944 	int			locked;
945 
946 	locked = down_trylock(&bp->b_sema) == 0;
947 	if (locked)
948 		XB_SET_OWNER(bp);
949 
950 	trace_xfs_buf_trylock(bp, _RET_IP_);
951 	return locked;
952 }
953 
954 /*
955  *	Lock a buffer object.
956  *
957  *	If we come across a stale, pinned, locked buffer, we know that we
958  *	are being asked to lock a buffer that has been reallocated. Because
959  *	it is pinned, we know that the log has not been pushed to disk and
960  *	hence it will still be locked. Rather than sleeping until someone
961  *	else pushes the log, push it ourselves before trying to get the lock.
962  */
963 void
964 xfs_buf_lock(
965 	struct xfs_buf		*bp)
966 {
967 	trace_xfs_buf_lock(bp, _RET_IP_);
968 
969 	if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
970 		xfs_log_force(bp->b_target->bt_mount, 0);
971 	down(&bp->b_sema);
972 	XB_SET_OWNER(bp);
973 
974 	trace_xfs_buf_lock_done(bp, _RET_IP_);
975 }
976 
977 void
978 xfs_buf_unlock(
979 	struct xfs_buf		*bp)
980 {
981 	XB_CLEAR_OWNER(bp);
982 	up(&bp->b_sema);
983 
984 	trace_xfs_buf_unlock(bp, _RET_IP_);
985 }
986 
987 STATIC void
988 xfs_buf_wait_unpin(
989 	xfs_buf_t		*bp)
990 {
991 	DECLARE_WAITQUEUE	(wait, current);
992 
993 	if (atomic_read(&bp->b_pin_count) == 0)
994 		return;
995 
996 	add_wait_queue(&bp->b_waiters, &wait);
997 	for (;;) {
998 		set_current_state(TASK_UNINTERRUPTIBLE);
999 		if (atomic_read(&bp->b_pin_count) == 0)
1000 			break;
1001 		io_schedule();
1002 	}
1003 	remove_wait_queue(&bp->b_waiters, &wait);
1004 	set_current_state(TASK_RUNNING);
1005 }
1006 
1007 /*
1008  *	Buffer Utility Routines
1009  */
1010 
1011 void
1012 xfs_buf_ioend(
1013 	struct xfs_buf	*bp)
1014 {
1015 	bool		read = bp->b_flags & XBF_READ;
1016 
1017 	trace_xfs_buf_iodone(bp, _RET_IP_);
1018 
1019 	bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
1020 
1021 	/*
1022 	 * Pull in IO completion errors now. We are guaranteed to be running
1023 	 * single threaded, so we don't need the lock to read b_io_error.
1024 	 */
1025 	if (!bp->b_error && bp->b_io_error)
1026 		xfs_buf_ioerror(bp, bp->b_io_error);
1027 
1028 	/* Only validate buffers that were read without errors */
1029 	if (read && !bp->b_error && bp->b_ops) {
1030 		ASSERT(!bp->b_iodone);
1031 		bp->b_ops->verify_read(bp);
1032 	}
1033 
1034 	if (!bp->b_error)
1035 		bp->b_flags |= XBF_DONE;
1036 
1037 	if (bp->b_iodone)
1038 		(*(bp->b_iodone))(bp);
1039 	else if (bp->b_flags & XBF_ASYNC)
1040 		xfs_buf_relse(bp);
1041 	else
1042 		complete(&bp->b_iowait);
1043 }
1044 
1045 static void
1046 xfs_buf_ioend_work(
1047 	struct work_struct	*work)
1048 {
1049 	struct xfs_buf		*bp =
1050 		container_of(work, xfs_buf_t, b_ioend_work);
1051 
1052 	xfs_buf_ioend(bp);
1053 }
1054 
1055 static void
1056 xfs_buf_ioend_async(
1057 	struct xfs_buf	*bp)
1058 {
1059 	INIT_WORK(&bp->b_ioend_work, xfs_buf_ioend_work);
1060 	queue_work(bp->b_ioend_wq, &bp->b_ioend_work);
1061 }
1062 
1063 void
1064 xfs_buf_ioerror(
1065 	xfs_buf_t		*bp,
1066 	int			error)
1067 {
1068 	ASSERT(error <= 0 && error >= -1000);
1069 	bp->b_error = error;
1070 	trace_xfs_buf_ioerror(bp, error, _RET_IP_);
1071 }
1072 
1073 void
1074 xfs_buf_ioerror_alert(
1075 	struct xfs_buf		*bp,
1076 	const char		*func)
1077 {
1078 	xfs_alert(bp->b_target->bt_mount,
1079 "metadata I/O error: block 0x%llx (\"%s\") error %d numblks %d",
1080 		(__uint64_t)XFS_BUF_ADDR(bp), func, -bp->b_error, bp->b_length);
1081 }
1082 
1083 int
1084 xfs_bwrite(
1085 	struct xfs_buf		*bp)
1086 {
1087 	int			error;
1088 
1089 	ASSERT(xfs_buf_islocked(bp));
1090 
1091 	bp->b_flags |= XBF_WRITE;
1092 	bp->b_flags &= ~(XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q |
1093 			 XBF_WRITE_FAIL | XBF_DONE);
1094 
1095 	error = xfs_buf_submit_wait(bp);
1096 	if (error) {
1097 		xfs_force_shutdown(bp->b_target->bt_mount,
1098 				   SHUTDOWN_META_IO_ERROR);
1099 	}
1100 	return error;
1101 }
1102 
1103 static void
1104 xfs_buf_bio_end_io(
1105 	struct bio		*bio)
1106 {
1107 	struct xfs_buf		*bp = (struct xfs_buf *)bio->bi_private;
1108 
1109 	/*
1110 	 * don't overwrite existing errors - otherwise we can lose errors on
1111 	 * buffers that require multiple bios to complete.
1112 	 */
1113 	if (bio->bi_error)
1114 		cmpxchg(&bp->b_io_error, 0, bio->bi_error);
1115 
1116 	if (!bp->b_error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
1117 		invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1118 
1119 	if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
1120 		xfs_buf_ioend_async(bp);
1121 	bio_put(bio);
1122 }
1123 
1124 static void
1125 xfs_buf_ioapply_map(
1126 	struct xfs_buf	*bp,
1127 	int		map,
1128 	int		*buf_offset,
1129 	int		*count,
1130 	int		rw)
1131 {
1132 	int		page_index;
1133 	int		total_nr_pages = bp->b_page_count;
1134 	int		nr_pages;
1135 	struct bio	*bio;
1136 	sector_t	sector =  bp->b_maps[map].bm_bn;
1137 	int		size;
1138 	int		offset;
1139 
1140 	total_nr_pages = bp->b_page_count;
1141 
1142 	/* skip the pages in the buffer before the start offset */
1143 	page_index = 0;
1144 	offset = *buf_offset;
1145 	while (offset >= PAGE_SIZE) {
1146 		page_index++;
1147 		offset -= PAGE_SIZE;
1148 	}
1149 
1150 	/*
1151 	 * Limit the IO size to the length of the current vector, and update the
1152 	 * remaining IO count for the next time around.
1153 	 */
1154 	size = min_t(int, BBTOB(bp->b_maps[map].bm_len), *count);
1155 	*count -= size;
1156 	*buf_offset += size;
1157 
1158 next_chunk:
1159 	atomic_inc(&bp->b_io_remaining);
1160 	nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1161 	if (nr_pages > total_nr_pages)
1162 		nr_pages = total_nr_pages;
1163 
1164 	bio = bio_alloc(GFP_NOIO, nr_pages);
1165 	bio->bi_bdev = bp->b_target->bt_bdev;
1166 	bio->bi_iter.bi_sector = sector;
1167 	bio->bi_end_io = xfs_buf_bio_end_io;
1168 	bio->bi_private = bp;
1169 
1170 
1171 	for (; size && nr_pages; nr_pages--, page_index++) {
1172 		int	rbytes, nbytes = PAGE_SIZE - offset;
1173 
1174 		if (nbytes > size)
1175 			nbytes = size;
1176 
1177 		rbytes = bio_add_page(bio, bp->b_pages[page_index], nbytes,
1178 				      offset);
1179 		if (rbytes < nbytes)
1180 			break;
1181 
1182 		offset = 0;
1183 		sector += BTOBB(nbytes);
1184 		size -= nbytes;
1185 		total_nr_pages--;
1186 	}
1187 
1188 	if (likely(bio->bi_iter.bi_size)) {
1189 		if (xfs_buf_is_vmapped(bp)) {
1190 			flush_kernel_vmap_range(bp->b_addr,
1191 						xfs_buf_vmap_len(bp));
1192 		}
1193 		submit_bio(rw, bio);
1194 		if (size)
1195 			goto next_chunk;
1196 	} else {
1197 		/*
1198 		 * This is guaranteed not to be the last io reference count
1199 		 * because the caller (xfs_buf_submit) holds a count itself.
1200 		 */
1201 		atomic_dec(&bp->b_io_remaining);
1202 		xfs_buf_ioerror(bp, -EIO);
1203 		bio_put(bio);
1204 	}
1205 
1206 }
1207 
1208 STATIC void
1209 _xfs_buf_ioapply(
1210 	struct xfs_buf	*bp)
1211 {
1212 	struct blk_plug	plug;
1213 	int		rw;
1214 	int		offset;
1215 	int		size;
1216 	int		i;
1217 
1218 	/*
1219 	 * Make sure we capture only current IO errors rather than stale errors
1220 	 * left over from previous use of the buffer (e.g. failed readahead).
1221 	 */
1222 	bp->b_error = 0;
1223 
1224 	/*
1225 	 * Initialize the I/O completion workqueue if we haven't yet or the
1226 	 * submitter has not opted to specify a custom one.
1227 	 */
1228 	if (!bp->b_ioend_wq)
1229 		bp->b_ioend_wq = bp->b_target->bt_mount->m_buf_workqueue;
1230 
1231 	if (bp->b_flags & XBF_WRITE) {
1232 		if (bp->b_flags & XBF_SYNCIO)
1233 			rw = WRITE_SYNC;
1234 		else
1235 			rw = WRITE;
1236 		if (bp->b_flags & XBF_FUA)
1237 			rw |= REQ_FUA;
1238 		if (bp->b_flags & XBF_FLUSH)
1239 			rw |= REQ_FLUSH;
1240 
1241 		/*
1242 		 * Run the write verifier callback function if it exists. If
1243 		 * this function fails it will mark the buffer with an error and
1244 		 * the IO should not be dispatched.
1245 		 */
1246 		if (bp->b_ops) {
1247 			bp->b_ops->verify_write(bp);
1248 			if (bp->b_error) {
1249 				xfs_force_shutdown(bp->b_target->bt_mount,
1250 						   SHUTDOWN_CORRUPT_INCORE);
1251 				return;
1252 			}
1253 		} else if (bp->b_bn != XFS_BUF_DADDR_NULL) {
1254 			struct xfs_mount *mp = bp->b_target->bt_mount;
1255 
1256 			/*
1257 			 * non-crc filesystems don't attach verifiers during
1258 			 * log recovery, so don't warn for such filesystems.
1259 			 */
1260 			if (xfs_sb_version_hascrc(&mp->m_sb)) {
1261 				xfs_warn(mp,
1262 					"%s: no ops on block 0x%llx/0x%x",
1263 					__func__, bp->b_bn, bp->b_length);
1264 				xfs_hex_dump(bp->b_addr, 64);
1265 				dump_stack();
1266 			}
1267 		}
1268 	} else if (bp->b_flags & XBF_READ_AHEAD) {
1269 		rw = READA;
1270 	} else {
1271 		rw = READ;
1272 	}
1273 
1274 	/* we only use the buffer cache for meta-data */
1275 	rw |= REQ_META;
1276 
1277 	/*
1278 	 * Walk all the vectors issuing IO on them. Set up the initial offset
1279 	 * into the buffer and the desired IO size before we start -
1280 	 * _xfs_buf_ioapply_vec() will modify them appropriately for each
1281 	 * subsequent call.
1282 	 */
1283 	offset = bp->b_offset;
1284 	size = BBTOB(bp->b_io_length);
1285 	blk_start_plug(&plug);
1286 	for (i = 0; i < bp->b_map_count; i++) {
1287 		xfs_buf_ioapply_map(bp, i, &offset, &size, rw);
1288 		if (bp->b_error)
1289 			break;
1290 		if (size <= 0)
1291 			break;	/* all done */
1292 	}
1293 	blk_finish_plug(&plug);
1294 }
1295 
1296 /*
1297  * Asynchronous IO submission path. This transfers the buffer lock ownership and
1298  * the current reference to the IO. It is not safe to reference the buffer after
1299  * a call to this function unless the caller holds an additional reference
1300  * itself.
1301  */
1302 void
1303 xfs_buf_submit(
1304 	struct xfs_buf	*bp)
1305 {
1306 	trace_xfs_buf_submit(bp, _RET_IP_);
1307 
1308 	ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
1309 	ASSERT(bp->b_flags & XBF_ASYNC);
1310 
1311 	/* on shutdown we stale and complete the buffer immediately */
1312 	if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
1313 		xfs_buf_ioerror(bp, -EIO);
1314 		bp->b_flags &= ~XBF_DONE;
1315 		xfs_buf_stale(bp);
1316 		xfs_buf_ioend(bp);
1317 		return;
1318 	}
1319 
1320 	if (bp->b_flags & XBF_WRITE)
1321 		xfs_buf_wait_unpin(bp);
1322 
1323 	/* clear the internal error state to avoid spurious errors */
1324 	bp->b_io_error = 0;
1325 
1326 	/*
1327 	 * The caller's reference is released during I/O completion.
1328 	 * This occurs some time after the last b_io_remaining reference is
1329 	 * released, so after we drop our Io reference we have to have some
1330 	 * other reference to ensure the buffer doesn't go away from underneath
1331 	 * us. Take a direct reference to ensure we have safe access to the
1332 	 * buffer until we are finished with it.
1333 	 */
1334 	xfs_buf_hold(bp);
1335 
1336 	/*
1337 	 * Set the count to 1 initially, this will stop an I/O completion
1338 	 * callout which happens before we have started all the I/O from calling
1339 	 * xfs_buf_ioend too early.
1340 	 */
1341 	atomic_set(&bp->b_io_remaining, 1);
1342 	_xfs_buf_ioapply(bp);
1343 
1344 	/*
1345 	 * If _xfs_buf_ioapply failed, we can get back here with only the IO
1346 	 * reference we took above. If we drop it to zero, run completion so
1347 	 * that we don't return to the caller with completion still pending.
1348 	 */
1349 	if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1350 		if (bp->b_error)
1351 			xfs_buf_ioend(bp);
1352 		else
1353 			xfs_buf_ioend_async(bp);
1354 	}
1355 
1356 	xfs_buf_rele(bp);
1357 	/* Note: it is not safe to reference bp now we've dropped our ref */
1358 }
1359 
1360 /*
1361  * Synchronous buffer IO submission path, read or write.
1362  */
1363 int
1364 xfs_buf_submit_wait(
1365 	struct xfs_buf	*bp)
1366 {
1367 	int		error;
1368 
1369 	trace_xfs_buf_submit_wait(bp, _RET_IP_);
1370 
1371 	ASSERT(!(bp->b_flags & (_XBF_DELWRI_Q | XBF_ASYNC)));
1372 
1373 	if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
1374 		xfs_buf_ioerror(bp, -EIO);
1375 		xfs_buf_stale(bp);
1376 		bp->b_flags &= ~XBF_DONE;
1377 		return -EIO;
1378 	}
1379 
1380 	if (bp->b_flags & XBF_WRITE)
1381 		xfs_buf_wait_unpin(bp);
1382 
1383 	/* clear the internal error state to avoid spurious errors */
1384 	bp->b_io_error = 0;
1385 
1386 	/*
1387 	 * For synchronous IO, the IO does not inherit the submitters reference
1388 	 * count, nor the buffer lock. Hence we cannot release the reference we
1389 	 * are about to take until we've waited for all IO completion to occur,
1390 	 * including any xfs_buf_ioend_async() work that may be pending.
1391 	 */
1392 	xfs_buf_hold(bp);
1393 
1394 	/*
1395 	 * Set the count to 1 initially, this will stop an I/O completion
1396 	 * callout which happens before we have started all the I/O from calling
1397 	 * xfs_buf_ioend too early.
1398 	 */
1399 	atomic_set(&bp->b_io_remaining, 1);
1400 	_xfs_buf_ioapply(bp);
1401 
1402 	/*
1403 	 * make sure we run completion synchronously if it raced with us and is
1404 	 * already complete.
1405 	 */
1406 	if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
1407 		xfs_buf_ioend(bp);
1408 
1409 	/* wait for completion before gathering the error from the buffer */
1410 	trace_xfs_buf_iowait(bp, _RET_IP_);
1411 	wait_for_completion(&bp->b_iowait);
1412 	trace_xfs_buf_iowait_done(bp, _RET_IP_);
1413 	error = bp->b_error;
1414 
1415 	/*
1416 	 * all done now, we can release the hold that keeps the buffer
1417 	 * referenced for the entire IO.
1418 	 */
1419 	xfs_buf_rele(bp);
1420 	return error;
1421 }
1422 
1423 void *
1424 xfs_buf_offset(
1425 	struct xfs_buf		*bp,
1426 	size_t			offset)
1427 {
1428 	struct page		*page;
1429 
1430 	if (bp->b_addr)
1431 		return bp->b_addr + offset;
1432 
1433 	offset += bp->b_offset;
1434 	page = bp->b_pages[offset >> PAGE_SHIFT];
1435 	return page_address(page) + (offset & (PAGE_SIZE-1));
1436 }
1437 
1438 /*
1439  *	Move data into or out of a buffer.
1440  */
1441 void
1442 xfs_buf_iomove(
1443 	xfs_buf_t		*bp,	/* buffer to process		*/
1444 	size_t			boff,	/* starting buffer offset	*/
1445 	size_t			bsize,	/* length to copy		*/
1446 	void			*data,	/* data address			*/
1447 	xfs_buf_rw_t		mode)	/* read/write/zero flag		*/
1448 {
1449 	size_t			bend;
1450 
1451 	bend = boff + bsize;
1452 	while (boff < bend) {
1453 		struct page	*page;
1454 		int		page_index, page_offset, csize;
1455 
1456 		page_index = (boff + bp->b_offset) >> PAGE_SHIFT;
1457 		page_offset = (boff + bp->b_offset) & ~PAGE_MASK;
1458 		page = bp->b_pages[page_index];
1459 		csize = min_t(size_t, PAGE_SIZE - page_offset,
1460 				      BBTOB(bp->b_io_length) - boff);
1461 
1462 		ASSERT((csize + page_offset) <= PAGE_SIZE);
1463 
1464 		switch (mode) {
1465 		case XBRW_ZERO:
1466 			memset(page_address(page) + page_offset, 0, csize);
1467 			break;
1468 		case XBRW_READ:
1469 			memcpy(data, page_address(page) + page_offset, csize);
1470 			break;
1471 		case XBRW_WRITE:
1472 			memcpy(page_address(page) + page_offset, data, csize);
1473 		}
1474 
1475 		boff += csize;
1476 		data += csize;
1477 	}
1478 }
1479 
1480 /*
1481  *	Handling of buffer targets (buftargs).
1482  */
1483 
1484 /*
1485  * Wait for any bufs with callbacks that have been submitted but have not yet
1486  * returned. These buffers will have an elevated hold count, so wait on those
1487  * while freeing all the buffers only held by the LRU.
1488  */
1489 static enum lru_status
1490 xfs_buftarg_wait_rele(
1491 	struct list_head	*item,
1492 	struct list_lru_one	*lru,
1493 	spinlock_t		*lru_lock,
1494 	void			*arg)
1495 
1496 {
1497 	struct xfs_buf		*bp = container_of(item, struct xfs_buf, b_lru);
1498 	struct list_head	*dispose = arg;
1499 
1500 	if (atomic_read(&bp->b_hold) > 1) {
1501 		/* need to wait, so skip it this pass */
1502 		trace_xfs_buf_wait_buftarg(bp, _RET_IP_);
1503 		return LRU_SKIP;
1504 	}
1505 	if (!spin_trylock(&bp->b_lock))
1506 		return LRU_SKIP;
1507 
1508 	/*
1509 	 * clear the LRU reference count so the buffer doesn't get
1510 	 * ignored in xfs_buf_rele().
1511 	 */
1512 	atomic_set(&bp->b_lru_ref, 0);
1513 	bp->b_state |= XFS_BSTATE_DISPOSE;
1514 	list_lru_isolate_move(lru, item, dispose);
1515 	spin_unlock(&bp->b_lock);
1516 	return LRU_REMOVED;
1517 }
1518 
1519 void
1520 xfs_wait_buftarg(
1521 	struct xfs_buftarg	*btp)
1522 {
1523 	LIST_HEAD(dispose);
1524 	int loop = 0;
1525 
1526 	/*
1527 	 * We need to flush the buffer workqueue to ensure that all IO
1528 	 * completion processing is 100% done. Just waiting on buffer locks is
1529 	 * not sufficient for async IO as the reference count held over IO is
1530 	 * not released until after the buffer lock is dropped. Hence we need to
1531 	 * ensure here that all reference counts have been dropped before we
1532 	 * start walking the LRU list.
1533 	 */
1534 	drain_workqueue(btp->bt_mount->m_buf_workqueue);
1535 
1536 	/* loop until there is nothing left on the lru list. */
1537 	while (list_lru_count(&btp->bt_lru)) {
1538 		list_lru_walk(&btp->bt_lru, xfs_buftarg_wait_rele,
1539 			      &dispose, LONG_MAX);
1540 
1541 		while (!list_empty(&dispose)) {
1542 			struct xfs_buf *bp;
1543 			bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1544 			list_del_init(&bp->b_lru);
1545 			if (bp->b_flags & XBF_WRITE_FAIL) {
1546 				xfs_alert(btp->bt_mount,
1547 "Corruption Alert: Buffer at block 0x%llx had permanent write failures!",
1548 					(long long)bp->b_bn);
1549 				xfs_alert(btp->bt_mount,
1550 "Please run xfs_repair to determine the extent of the problem.");
1551 			}
1552 			xfs_buf_rele(bp);
1553 		}
1554 		if (loop++ != 0)
1555 			delay(100);
1556 	}
1557 }
1558 
1559 static enum lru_status
1560 xfs_buftarg_isolate(
1561 	struct list_head	*item,
1562 	struct list_lru_one	*lru,
1563 	spinlock_t		*lru_lock,
1564 	void			*arg)
1565 {
1566 	struct xfs_buf		*bp = container_of(item, struct xfs_buf, b_lru);
1567 	struct list_head	*dispose = arg;
1568 
1569 	/*
1570 	 * we are inverting the lru lock/bp->b_lock here, so use a trylock.
1571 	 * If we fail to get the lock, just skip it.
1572 	 */
1573 	if (!spin_trylock(&bp->b_lock))
1574 		return LRU_SKIP;
1575 	/*
1576 	 * Decrement the b_lru_ref count unless the value is already
1577 	 * zero. If the value is already zero, we need to reclaim the
1578 	 * buffer, otherwise it gets another trip through the LRU.
1579 	 */
1580 	if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
1581 		spin_unlock(&bp->b_lock);
1582 		return LRU_ROTATE;
1583 	}
1584 
1585 	bp->b_state |= XFS_BSTATE_DISPOSE;
1586 	list_lru_isolate_move(lru, item, dispose);
1587 	spin_unlock(&bp->b_lock);
1588 	return LRU_REMOVED;
1589 }
1590 
1591 static unsigned long
1592 xfs_buftarg_shrink_scan(
1593 	struct shrinker		*shrink,
1594 	struct shrink_control	*sc)
1595 {
1596 	struct xfs_buftarg	*btp = container_of(shrink,
1597 					struct xfs_buftarg, bt_shrinker);
1598 	LIST_HEAD(dispose);
1599 	unsigned long		freed;
1600 
1601 	freed = list_lru_shrink_walk(&btp->bt_lru, sc,
1602 				     xfs_buftarg_isolate, &dispose);
1603 
1604 	while (!list_empty(&dispose)) {
1605 		struct xfs_buf *bp;
1606 		bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1607 		list_del_init(&bp->b_lru);
1608 		xfs_buf_rele(bp);
1609 	}
1610 
1611 	return freed;
1612 }
1613 
1614 static unsigned long
1615 xfs_buftarg_shrink_count(
1616 	struct shrinker		*shrink,
1617 	struct shrink_control	*sc)
1618 {
1619 	struct xfs_buftarg	*btp = container_of(shrink,
1620 					struct xfs_buftarg, bt_shrinker);
1621 	return list_lru_shrink_count(&btp->bt_lru, sc);
1622 }
1623 
1624 void
1625 xfs_free_buftarg(
1626 	struct xfs_mount	*mp,
1627 	struct xfs_buftarg	*btp)
1628 {
1629 	unregister_shrinker(&btp->bt_shrinker);
1630 	list_lru_destroy(&btp->bt_lru);
1631 
1632 	if (mp->m_flags & XFS_MOUNT_BARRIER)
1633 		xfs_blkdev_issue_flush(btp);
1634 
1635 	kmem_free(btp);
1636 }
1637 
1638 int
1639 xfs_setsize_buftarg(
1640 	xfs_buftarg_t		*btp,
1641 	unsigned int		sectorsize)
1642 {
1643 	/* Set up metadata sector size info */
1644 	btp->bt_meta_sectorsize = sectorsize;
1645 	btp->bt_meta_sectormask = sectorsize - 1;
1646 
1647 	if (set_blocksize(btp->bt_bdev, sectorsize)) {
1648 		xfs_warn(btp->bt_mount,
1649 			"Cannot set_blocksize to %u on device %pg",
1650 			sectorsize, btp->bt_bdev);
1651 		return -EINVAL;
1652 	}
1653 
1654 	/* Set up device logical sector size mask */
1655 	btp->bt_logical_sectorsize = bdev_logical_block_size(btp->bt_bdev);
1656 	btp->bt_logical_sectormask = bdev_logical_block_size(btp->bt_bdev) - 1;
1657 
1658 	return 0;
1659 }
1660 
1661 /*
1662  * When allocating the initial buffer target we have not yet
1663  * read in the superblock, so don't know what sized sectors
1664  * are being used at this early stage.  Play safe.
1665  */
1666 STATIC int
1667 xfs_setsize_buftarg_early(
1668 	xfs_buftarg_t		*btp,
1669 	struct block_device	*bdev)
1670 {
1671 	return xfs_setsize_buftarg(btp, bdev_logical_block_size(bdev));
1672 }
1673 
1674 xfs_buftarg_t *
1675 xfs_alloc_buftarg(
1676 	struct xfs_mount	*mp,
1677 	struct block_device	*bdev)
1678 {
1679 	xfs_buftarg_t		*btp;
1680 
1681 	btp = kmem_zalloc(sizeof(*btp), KM_SLEEP | KM_NOFS);
1682 
1683 	btp->bt_mount = mp;
1684 	btp->bt_dev =  bdev->bd_dev;
1685 	btp->bt_bdev = bdev;
1686 	btp->bt_bdi = blk_get_backing_dev_info(bdev);
1687 
1688 	if (xfs_setsize_buftarg_early(btp, bdev))
1689 		goto error;
1690 
1691 	if (list_lru_init(&btp->bt_lru))
1692 		goto error;
1693 
1694 	btp->bt_shrinker.count_objects = xfs_buftarg_shrink_count;
1695 	btp->bt_shrinker.scan_objects = xfs_buftarg_shrink_scan;
1696 	btp->bt_shrinker.seeks = DEFAULT_SEEKS;
1697 	btp->bt_shrinker.flags = SHRINKER_NUMA_AWARE;
1698 	register_shrinker(&btp->bt_shrinker);
1699 	return btp;
1700 
1701 error:
1702 	kmem_free(btp);
1703 	return NULL;
1704 }
1705 
1706 /*
1707  * Add a buffer to the delayed write list.
1708  *
1709  * This queues a buffer for writeout if it hasn't already been.  Note that
1710  * neither this routine nor the buffer list submission functions perform
1711  * any internal synchronization.  It is expected that the lists are thread-local
1712  * to the callers.
1713  *
1714  * Returns true if we queued up the buffer, or false if it already had
1715  * been on the buffer list.
1716  */
1717 bool
1718 xfs_buf_delwri_queue(
1719 	struct xfs_buf		*bp,
1720 	struct list_head	*list)
1721 {
1722 	ASSERT(xfs_buf_islocked(bp));
1723 	ASSERT(!(bp->b_flags & XBF_READ));
1724 
1725 	/*
1726 	 * If the buffer is already marked delwri it already is queued up
1727 	 * by someone else for imediate writeout.  Just ignore it in that
1728 	 * case.
1729 	 */
1730 	if (bp->b_flags & _XBF_DELWRI_Q) {
1731 		trace_xfs_buf_delwri_queued(bp, _RET_IP_);
1732 		return false;
1733 	}
1734 
1735 	trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1736 
1737 	/*
1738 	 * If a buffer gets written out synchronously or marked stale while it
1739 	 * is on a delwri list we lazily remove it. To do this, the other party
1740 	 * clears the  _XBF_DELWRI_Q flag but otherwise leaves the buffer alone.
1741 	 * It remains referenced and on the list.  In a rare corner case it
1742 	 * might get readded to a delwri list after the synchronous writeout, in
1743 	 * which case we need just need to re-add the flag here.
1744 	 */
1745 	bp->b_flags |= _XBF_DELWRI_Q;
1746 	if (list_empty(&bp->b_list)) {
1747 		atomic_inc(&bp->b_hold);
1748 		list_add_tail(&bp->b_list, list);
1749 	}
1750 
1751 	return true;
1752 }
1753 
1754 /*
1755  * Compare function is more complex than it needs to be because
1756  * the return value is only 32 bits and we are doing comparisons
1757  * on 64 bit values
1758  */
1759 static int
1760 xfs_buf_cmp(
1761 	void		*priv,
1762 	struct list_head *a,
1763 	struct list_head *b)
1764 {
1765 	struct xfs_buf	*ap = container_of(a, struct xfs_buf, b_list);
1766 	struct xfs_buf	*bp = container_of(b, struct xfs_buf, b_list);
1767 	xfs_daddr_t		diff;
1768 
1769 	diff = ap->b_maps[0].bm_bn - bp->b_maps[0].bm_bn;
1770 	if (diff < 0)
1771 		return -1;
1772 	if (diff > 0)
1773 		return 1;
1774 	return 0;
1775 }
1776 
1777 static int
1778 __xfs_buf_delwri_submit(
1779 	struct list_head	*buffer_list,
1780 	struct list_head	*io_list,
1781 	bool			wait)
1782 {
1783 	struct blk_plug		plug;
1784 	struct xfs_buf		*bp, *n;
1785 	int			pinned = 0;
1786 
1787 	list_for_each_entry_safe(bp, n, buffer_list, b_list) {
1788 		if (!wait) {
1789 			if (xfs_buf_ispinned(bp)) {
1790 				pinned++;
1791 				continue;
1792 			}
1793 			if (!xfs_buf_trylock(bp))
1794 				continue;
1795 		} else {
1796 			xfs_buf_lock(bp);
1797 		}
1798 
1799 		/*
1800 		 * Someone else might have written the buffer synchronously or
1801 		 * marked it stale in the meantime.  In that case only the
1802 		 * _XBF_DELWRI_Q flag got cleared, and we have to drop the
1803 		 * reference and remove it from the list here.
1804 		 */
1805 		if (!(bp->b_flags & _XBF_DELWRI_Q)) {
1806 			list_del_init(&bp->b_list);
1807 			xfs_buf_relse(bp);
1808 			continue;
1809 		}
1810 
1811 		list_move_tail(&bp->b_list, io_list);
1812 		trace_xfs_buf_delwri_split(bp, _RET_IP_);
1813 	}
1814 
1815 	list_sort(NULL, io_list, xfs_buf_cmp);
1816 
1817 	blk_start_plug(&plug);
1818 	list_for_each_entry_safe(bp, n, io_list, b_list) {
1819 		bp->b_flags &= ~(_XBF_DELWRI_Q | XBF_ASYNC | XBF_WRITE_FAIL);
1820 		bp->b_flags |= XBF_WRITE | XBF_ASYNC;
1821 
1822 		/*
1823 		 * we do all Io submission async. This means if we need to wait
1824 		 * for IO completion we need to take an extra reference so the
1825 		 * buffer is still valid on the other side.
1826 		 */
1827 		if (wait)
1828 			xfs_buf_hold(bp);
1829 		else
1830 			list_del_init(&bp->b_list);
1831 
1832 		xfs_buf_submit(bp);
1833 	}
1834 	blk_finish_plug(&plug);
1835 
1836 	return pinned;
1837 }
1838 
1839 /*
1840  * Write out a buffer list asynchronously.
1841  *
1842  * This will take the @buffer_list, write all non-locked and non-pinned buffers
1843  * out and not wait for I/O completion on any of the buffers.  This interface
1844  * is only safely useable for callers that can track I/O completion by higher
1845  * level means, e.g. AIL pushing as the @buffer_list is consumed in this
1846  * function.
1847  */
1848 int
1849 xfs_buf_delwri_submit_nowait(
1850 	struct list_head	*buffer_list)
1851 {
1852 	LIST_HEAD		(io_list);
1853 	return __xfs_buf_delwri_submit(buffer_list, &io_list, false);
1854 }
1855 
1856 /*
1857  * Write out a buffer list synchronously.
1858  *
1859  * This will take the @buffer_list, write all buffers out and wait for I/O
1860  * completion on all of the buffers. @buffer_list is consumed by the function,
1861  * so callers must have some other way of tracking buffers if they require such
1862  * functionality.
1863  */
1864 int
1865 xfs_buf_delwri_submit(
1866 	struct list_head	*buffer_list)
1867 {
1868 	LIST_HEAD		(io_list);
1869 	int			error = 0, error2;
1870 	struct xfs_buf		*bp;
1871 
1872 	__xfs_buf_delwri_submit(buffer_list, &io_list, true);
1873 
1874 	/* Wait for IO to complete. */
1875 	while (!list_empty(&io_list)) {
1876 		bp = list_first_entry(&io_list, struct xfs_buf, b_list);
1877 
1878 		list_del_init(&bp->b_list);
1879 
1880 		/* locking the buffer will wait for async IO completion. */
1881 		xfs_buf_lock(bp);
1882 		error2 = bp->b_error;
1883 		xfs_buf_relse(bp);
1884 		if (!error)
1885 			error = error2;
1886 	}
1887 
1888 	return error;
1889 }
1890 
1891 int __init
1892 xfs_buf_init(void)
1893 {
1894 	xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1895 						KM_ZONE_HWALIGN, NULL);
1896 	if (!xfs_buf_zone)
1897 		goto out;
1898 
1899 	return 0;
1900 
1901  out:
1902 	return -ENOMEM;
1903 }
1904 
1905 void
1906 xfs_buf_terminate(void)
1907 {
1908 	kmem_zone_destroy(xfs_buf_zone);
1909 }
1910