xref: /openbmc/linux/fs/xfs/xfs_buf.c (revision b34e08d5)
1 /*
2  * Copyright (c) 2000-2006 Silicon Graphics, Inc.
3  * All Rights Reserved.
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License as
7  * published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it would be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write the Free Software Foundation,
16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17  */
18 #include "xfs.h"
19 #include <linux/stddef.h>
20 #include <linux/errno.h>
21 #include <linux/gfp.h>
22 #include <linux/pagemap.h>
23 #include <linux/init.h>
24 #include <linux/vmalloc.h>
25 #include <linux/bio.h>
26 #include <linux/sysctl.h>
27 #include <linux/proc_fs.h>
28 #include <linux/workqueue.h>
29 #include <linux/percpu.h>
30 #include <linux/blkdev.h>
31 #include <linux/hash.h>
32 #include <linux/kthread.h>
33 #include <linux/migrate.h>
34 #include <linux/backing-dev.h>
35 #include <linux/freezer.h>
36 
37 #include "xfs_log_format.h"
38 #include "xfs_trans_resv.h"
39 #include "xfs_sb.h"
40 #include "xfs_ag.h"
41 #include "xfs_mount.h"
42 #include "xfs_trace.h"
43 #include "xfs_log.h"
44 
45 static kmem_zone_t *xfs_buf_zone;
46 
47 static struct workqueue_struct *xfslogd_workqueue;
48 
49 #ifdef XFS_BUF_LOCK_TRACKING
50 # define XB_SET_OWNER(bp)	((bp)->b_last_holder = current->pid)
51 # define XB_CLEAR_OWNER(bp)	((bp)->b_last_holder = -1)
52 # define XB_GET_OWNER(bp)	((bp)->b_last_holder)
53 #else
54 # define XB_SET_OWNER(bp)	do { } while (0)
55 # define XB_CLEAR_OWNER(bp)	do { } while (0)
56 # define XB_GET_OWNER(bp)	do { } while (0)
57 #endif
58 
59 #define xb_to_gfp(flags) \
60 	((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : GFP_NOFS) | __GFP_NOWARN)
61 
62 
63 static inline int
64 xfs_buf_is_vmapped(
65 	struct xfs_buf	*bp)
66 {
67 	/*
68 	 * Return true if the buffer is vmapped.
69 	 *
70 	 * b_addr is null if the buffer is not mapped, but the code is clever
71 	 * enough to know it doesn't have to map a single page, so the check has
72 	 * to be both for b_addr and bp->b_page_count > 1.
73 	 */
74 	return bp->b_addr && bp->b_page_count > 1;
75 }
76 
77 static inline int
78 xfs_buf_vmap_len(
79 	struct xfs_buf	*bp)
80 {
81 	return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
82 }
83 
84 /*
85  * When we mark a buffer stale, we remove the buffer from the LRU and clear the
86  * b_lru_ref count so that the buffer is freed immediately when the buffer
87  * reference count falls to zero. If the buffer is already on the LRU, we need
88  * to remove the reference that LRU holds on the buffer.
89  *
90  * This prevents build-up of stale buffers on the LRU.
91  */
92 void
93 xfs_buf_stale(
94 	struct xfs_buf	*bp)
95 {
96 	ASSERT(xfs_buf_islocked(bp));
97 
98 	bp->b_flags |= XBF_STALE;
99 
100 	/*
101 	 * Clear the delwri status so that a delwri queue walker will not
102 	 * flush this buffer to disk now that it is stale. The delwri queue has
103 	 * a reference to the buffer, so this is safe to do.
104 	 */
105 	bp->b_flags &= ~_XBF_DELWRI_Q;
106 
107 	spin_lock(&bp->b_lock);
108 	atomic_set(&bp->b_lru_ref, 0);
109 	if (!(bp->b_state & XFS_BSTATE_DISPOSE) &&
110 	    (list_lru_del(&bp->b_target->bt_lru, &bp->b_lru)))
111 		atomic_dec(&bp->b_hold);
112 
113 	ASSERT(atomic_read(&bp->b_hold) >= 1);
114 	spin_unlock(&bp->b_lock);
115 }
116 
117 static int
118 xfs_buf_get_maps(
119 	struct xfs_buf		*bp,
120 	int			map_count)
121 {
122 	ASSERT(bp->b_maps == NULL);
123 	bp->b_map_count = map_count;
124 
125 	if (map_count == 1) {
126 		bp->b_maps = &bp->__b_map;
127 		return 0;
128 	}
129 
130 	bp->b_maps = kmem_zalloc(map_count * sizeof(struct xfs_buf_map),
131 				KM_NOFS);
132 	if (!bp->b_maps)
133 		return ENOMEM;
134 	return 0;
135 }
136 
137 /*
138  *	Frees b_pages if it was allocated.
139  */
140 static void
141 xfs_buf_free_maps(
142 	struct xfs_buf	*bp)
143 {
144 	if (bp->b_maps != &bp->__b_map) {
145 		kmem_free(bp->b_maps);
146 		bp->b_maps = NULL;
147 	}
148 }
149 
150 struct xfs_buf *
151 _xfs_buf_alloc(
152 	struct xfs_buftarg	*target,
153 	struct xfs_buf_map	*map,
154 	int			nmaps,
155 	xfs_buf_flags_t		flags)
156 {
157 	struct xfs_buf		*bp;
158 	int			error;
159 	int			i;
160 
161 	bp = kmem_zone_zalloc(xfs_buf_zone, KM_NOFS);
162 	if (unlikely(!bp))
163 		return NULL;
164 
165 	/*
166 	 * We don't want certain flags to appear in b_flags unless they are
167 	 * specifically set by later operations on the buffer.
168 	 */
169 	flags &= ~(XBF_UNMAPPED | XBF_TRYLOCK | XBF_ASYNC | XBF_READ_AHEAD);
170 
171 	atomic_set(&bp->b_hold, 1);
172 	atomic_set(&bp->b_lru_ref, 1);
173 	init_completion(&bp->b_iowait);
174 	INIT_LIST_HEAD(&bp->b_lru);
175 	INIT_LIST_HEAD(&bp->b_list);
176 	RB_CLEAR_NODE(&bp->b_rbnode);
177 	sema_init(&bp->b_sema, 0); /* held, no waiters */
178 	spin_lock_init(&bp->b_lock);
179 	XB_SET_OWNER(bp);
180 	bp->b_target = target;
181 	bp->b_flags = flags;
182 
183 	/*
184 	 * Set length and io_length to the same value initially.
185 	 * I/O routines should use io_length, which will be the same in
186 	 * most cases but may be reset (e.g. XFS recovery).
187 	 */
188 	error = xfs_buf_get_maps(bp, nmaps);
189 	if (error)  {
190 		kmem_zone_free(xfs_buf_zone, bp);
191 		return NULL;
192 	}
193 
194 	bp->b_bn = map[0].bm_bn;
195 	bp->b_length = 0;
196 	for (i = 0; i < nmaps; i++) {
197 		bp->b_maps[i].bm_bn = map[i].bm_bn;
198 		bp->b_maps[i].bm_len = map[i].bm_len;
199 		bp->b_length += map[i].bm_len;
200 	}
201 	bp->b_io_length = bp->b_length;
202 
203 	atomic_set(&bp->b_pin_count, 0);
204 	init_waitqueue_head(&bp->b_waiters);
205 
206 	XFS_STATS_INC(xb_create);
207 	trace_xfs_buf_init(bp, _RET_IP_);
208 
209 	return bp;
210 }
211 
212 /*
213  *	Allocate a page array capable of holding a specified number
214  *	of pages, and point the page buf at it.
215  */
216 STATIC int
217 _xfs_buf_get_pages(
218 	xfs_buf_t		*bp,
219 	int			page_count,
220 	xfs_buf_flags_t		flags)
221 {
222 	/* Make sure that we have a page list */
223 	if (bp->b_pages == NULL) {
224 		bp->b_page_count = page_count;
225 		if (page_count <= XB_PAGES) {
226 			bp->b_pages = bp->b_page_array;
227 		} else {
228 			bp->b_pages = kmem_alloc(sizeof(struct page *) *
229 						 page_count, KM_NOFS);
230 			if (bp->b_pages == NULL)
231 				return -ENOMEM;
232 		}
233 		memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
234 	}
235 	return 0;
236 }
237 
238 /*
239  *	Frees b_pages if it was allocated.
240  */
241 STATIC void
242 _xfs_buf_free_pages(
243 	xfs_buf_t	*bp)
244 {
245 	if (bp->b_pages != bp->b_page_array) {
246 		kmem_free(bp->b_pages);
247 		bp->b_pages = NULL;
248 	}
249 }
250 
251 /*
252  *	Releases the specified buffer.
253  *
254  * 	The modification state of any associated pages is left unchanged.
255  * 	The buffer must not be on any hash - use xfs_buf_rele instead for
256  * 	hashed and refcounted buffers
257  */
258 void
259 xfs_buf_free(
260 	xfs_buf_t		*bp)
261 {
262 	trace_xfs_buf_free(bp, _RET_IP_);
263 
264 	ASSERT(list_empty(&bp->b_lru));
265 
266 	if (bp->b_flags & _XBF_PAGES) {
267 		uint		i;
268 
269 		if (xfs_buf_is_vmapped(bp))
270 			vm_unmap_ram(bp->b_addr - bp->b_offset,
271 					bp->b_page_count);
272 
273 		for (i = 0; i < bp->b_page_count; i++) {
274 			struct page	*page = bp->b_pages[i];
275 
276 			__free_page(page);
277 		}
278 	} else if (bp->b_flags & _XBF_KMEM)
279 		kmem_free(bp->b_addr);
280 	_xfs_buf_free_pages(bp);
281 	xfs_buf_free_maps(bp);
282 	kmem_zone_free(xfs_buf_zone, bp);
283 }
284 
285 /*
286  * Allocates all the pages for buffer in question and builds it's page list.
287  */
288 STATIC int
289 xfs_buf_allocate_memory(
290 	xfs_buf_t		*bp,
291 	uint			flags)
292 {
293 	size_t			size;
294 	size_t			nbytes, offset;
295 	gfp_t			gfp_mask = xb_to_gfp(flags);
296 	unsigned short		page_count, i;
297 	xfs_off_t		start, end;
298 	int			error;
299 
300 	/*
301 	 * for buffers that are contained within a single page, just allocate
302 	 * the memory from the heap - there's no need for the complexity of
303 	 * page arrays to keep allocation down to order 0.
304 	 */
305 	size = BBTOB(bp->b_length);
306 	if (size < PAGE_SIZE) {
307 		bp->b_addr = kmem_alloc(size, KM_NOFS);
308 		if (!bp->b_addr) {
309 			/* low memory - use alloc_page loop instead */
310 			goto use_alloc_page;
311 		}
312 
313 		if (((unsigned long)(bp->b_addr + size - 1) & PAGE_MASK) !=
314 		    ((unsigned long)bp->b_addr & PAGE_MASK)) {
315 			/* b_addr spans two pages - use alloc_page instead */
316 			kmem_free(bp->b_addr);
317 			bp->b_addr = NULL;
318 			goto use_alloc_page;
319 		}
320 		bp->b_offset = offset_in_page(bp->b_addr);
321 		bp->b_pages = bp->b_page_array;
322 		bp->b_pages[0] = virt_to_page(bp->b_addr);
323 		bp->b_page_count = 1;
324 		bp->b_flags |= _XBF_KMEM;
325 		return 0;
326 	}
327 
328 use_alloc_page:
329 	start = BBTOB(bp->b_maps[0].bm_bn) >> PAGE_SHIFT;
330 	end = (BBTOB(bp->b_maps[0].bm_bn + bp->b_length) + PAGE_SIZE - 1)
331 								>> PAGE_SHIFT;
332 	page_count = end - start;
333 	error = _xfs_buf_get_pages(bp, page_count, flags);
334 	if (unlikely(error))
335 		return error;
336 
337 	offset = bp->b_offset;
338 	bp->b_flags |= _XBF_PAGES;
339 
340 	for (i = 0; i < bp->b_page_count; i++) {
341 		struct page	*page;
342 		uint		retries = 0;
343 retry:
344 		page = alloc_page(gfp_mask);
345 		if (unlikely(page == NULL)) {
346 			if (flags & XBF_READ_AHEAD) {
347 				bp->b_page_count = i;
348 				error = ENOMEM;
349 				goto out_free_pages;
350 			}
351 
352 			/*
353 			 * This could deadlock.
354 			 *
355 			 * But until all the XFS lowlevel code is revamped to
356 			 * handle buffer allocation failures we can't do much.
357 			 */
358 			if (!(++retries % 100))
359 				xfs_err(NULL,
360 		"possible memory allocation deadlock in %s (mode:0x%x)",
361 					__func__, gfp_mask);
362 
363 			XFS_STATS_INC(xb_page_retries);
364 			congestion_wait(BLK_RW_ASYNC, HZ/50);
365 			goto retry;
366 		}
367 
368 		XFS_STATS_INC(xb_page_found);
369 
370 		nbytes = min_t(size_t, size, PAGE_SIZE - offset);
371 		size -= nbytes;
372 		bp->b_pages[i] = page;
373 		offset = 0;
374 	}
375 	return 0;
376 
377 out_free_pages:
378 	for (i = 0; i < bp->b_page_count; i++)
379 		__free_page(bp->b_pages[i]);
380 	return error;
381 }
382 
383 /*
384  *	Map buffer into kernel address-space if necessary.
385  */
386 STATIC int
387 _xfs_buf_map_pages(
388 	xfs_buf_t		*bp,
389 	uint			flags)
390 {
391 	ASSERT(bp->b_flags & _XBF_PAGES);
392 	if (bp->b_page_count == 1) {
393 		/* A single page buffer is always mappable */
394 		bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
395 	} else if (flags & XBF_UNMAPPED) {
396 		bp->b_addr = NULL;
397 	} else {
398 		int retried = 0;
399 		unsigned noio_flag;
400 
401 		/*
402 		 * vm_map_ram() will allocate auxillary structures (e.g.
403 		 * pagetables) with GFP_KERNEL, yet we are likely to be under
404 		 * GFP_NOFS context here. Hence we need to tell memory reclaim
405 		 * that we are in such a context via PF_MEMALLOC_NOIO to prevent
406 		 * memory reclaim re-entering the filesystem here and
407 		 * potentially deadlocking.
408 		 */
409 		noio_flag = memalloc_noio_save();
410 		do {
411 			bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
412 						-1, PAGE_KERNEL);
413 			if (bp->b_addr)
414 				break;
415 			vm_unmap_aliases();
416 		} while (retried++ <= 1);
417 		memalloc_noio_restore(noio_flag);
418 
419 		if (!bp->b_addr)
420 			return -ENOMEM;
421 		bp->b_addr += bp->b_offset;
422 	}
423 
424 	return 0;
425 }
426 
427 /*
428  *	Finding and Reading Buffers
429  */
430 
431 /*
432  *	Look up, and creates if absent, a lockable buffer for
433  *	a given range of an inode.  The buffer is returned
434  *	locked.	No I/O is implied by this call.
435  */
436 xfs_buf_t *
437 _xfs_buf_find(
438 	struct xfs_buftarg	*btp,
439 	struct xfs_buf_map	*map,
440 	int			nmaps,
441 	xfs_buf_flags_t		flags,
442 	xfs_buf_t		*new_bp)
443 {
444 	size_t			numbytes;
445 	struct xfs_perag	*pag;
446 	struct rb_node		**rbp;
447 	struct rb_node		*parent;
448 	xfs_buf_t		*bp;
449 	xfs_daddr_t		blkno = map[0].bm_bn;
450 	xfs_daddr_t		eofs;
451 	int			numblks = 0;
452 	int			i;
453 
454 	for (i = 0; i < nmaps; i++)
455 		numblks += map[i].bm_len;
456 	numbytes = BBTOB(numblks);
457 
458 	/* Check for IOs smaller than the sector size / not sector aligned */
459 	ASSERT(!(numbytes < btp->bt_meta_sectorsize));
460 	ASSERT(!(BBTOB(blkno) & (xfs_off_t)btp->bt_meta_sectormask));
461 
462 	/*
463 	 * Corrupted block numbers can get through to here, unfortunately, so we
464 	 * have to check that the buffer falls within the filesystem bounds.
465 	 */
466 	eofs = XFS_FSB_TO_BB(btp->bt_mount, btp->bt_mount->m_sb.sb_dblocks);
467 	if (blkno >= eofs) {
468 		/*
469 		 * XXX (dgc): we should really be returning EFSCORRUPTED here,
470 		 * but none of the higher level infrastructure supports
471 		 * returning a specific error on buffer lookup failures.
472 		 */
473 		xfs_alert(btp->bt_mount,
474 			  "%s: Block out of range: block 0x%llx, EOFS 0x%llx ",
475 			  __func__, blkno, eofs);
476 		WARN_ON(1);
477 		return NULL;
478 	}
479 
480 	/* get tree root */
481 	pag = xfs_perag_get(btp->bt_mount,
482 				xfs_daddr_to_agno(btp->bt_mount, blkno));
483 
484 	/* walk tree */
485 	spin_lock(&pag->pag_buf_lock);
486 	rbp = &pag->pag_buf_tree.rb_node;
487 	parent = NULL;
488 	bp = NULL;
489 	while (*rbp) {
490 		parent = *rbp;
491 		bp = rb_entry(parent, struct xfs_buf, b_rbnode);
492 
493 		if (blkno < bp->b_bn)
494 			rbp = &(*rbp)->rb_left;
495 		else if (blkno > bp->b_bn)
496 			rbp = &(*rbp)->rb_right;
497 		else {
498 			/*
499 			 * found a block number match. If the range doesn't
500 			 * match, the only way this is allowed is if the buffer
501 			 * in the cache is stale and the transaction that made
502 			 * it stale has not yet committed. i.e. we are
503 			 * reallocating a busy extent. Skip this buffer and
504 			 * continue searching to the right for an exact match.
505 			 */
506 			if (bp->b_length != numblks) {
507 				ASSERT(bp->b_flags & XBF_STALE);
508 				rbp = &(*rbp)->rb_right;
509 				continue;
510 			}
511 			atomic_inc(&bp->b_hold);
512 			goto found;
513 		}
514 	}
515 
516 	/* No match found */
517 	if (new_bp) {
518 		rb_link_node(&new_bp->b_rbnode, parent, rbp);
519 		rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
520 		/* the buffer keeps the perag reference until it is freed */
521 		new_bp->b_pag = pag;
522 		spin_unlock(&pag->pag_buf_lock);
523 	} else {
524 		XFS_STATS_INC(xb_miss_locked);
525 		spin_unlock(&pag->pag_buf_lock);
526 		xfs_perag_put(pag);
527 	}
528 	return new_bp;
529 
530 found:
531 	spin_unlock(&pag->pag_buf_lock);
532 	xfs_perag_put(pag);
533 
534 	if (!xfs_buf_trylock(bp)) {
535 		if (flags & XBF_TRYLOCK) {
536 			xfs_buf_rele(bp);
537 			XFS_STATS_INC(xb_busy_locked);
538 			return NULL;
539 		}
540 		xfs_buf_lock(bp);
541 		XFS_STATS_INC(xb_get_locked_waited);
542 	}
543 
544 	/*
545 	 * if the buffer is stale, clear all the external state associated with
546 	 * it. We need to keep flags such as how we allocated the buffer memory
547 	 * intact here.
548 	 */
549 	if (bp->b_flags & XBF_STALE) {
550 		ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
551 		ASSERT(bp->b_iodone == NULL);
552 		bp->b_flags &= _XBF_KMEM | _XBF_PAGES;
553 		bp->b_ops = NULL;
554 	}
555 
556 	trace_xfs_buf_find(bp, flags, _RET_IP_);
557 	XFS_STATS_INC(xb_get_locked);
558 	return bp;
559 }
560 
561 /*
562  * Assembles a buffer covering the specified range. The code is optimised for
563  * cache hits, as metadata intensive workloads will see 3 orders of magnitude
564  * more hits than misses.
565  */
566 struct xfs_buf *
567 xfs_buf_get_map(
568 	struct xfs_buftarg	*target,
569 	struct xfs_buf_map	*map,
570 	int			nmaps,
571 	xfs_buf_flags_t		flags)
572 {
573 	struct xfs_buf		*bp;
574 	struct xfs_buf		*new_bp;
575 	int			error = 0;
576 
577 	bp = _xfs_buf_find(target, map, nmaps, flags, NULL);
578 	if (likely(bp))
579 		goto found;
580 
581 	new_bp = _xfs_buf_alloc(target, map, nmaps, flags);
582 	if (unlikely(!new_bp))
583 		return NULL;
584 
585 	error = xfs_buf_allocate_memory(new_bp, flags);
586 	if (error) {
587 		xfs_buf_free(new_bp);
588 		return NULL;
589 	}
590 
591 	bp = _xfs_buf_find(target, map, nmaps, flags, new_bp);
592 	if (!bp) {
593 		xfs_buf_free(new_bp);
594 		return NULL;
595 	}
596 
597 	if (bp != new_bp)
598 		xfs_buf_free(new_bp);
599 
600 found:
601 	if (!bp->b_addr) {
602 		error = _xfs_buf_map_pages(bp, flags);
603 		if (unlikely(error)) {
604 			xfs_warn(target->bt_mount,
605 				"%s: failed to map pagesn", __func__);
606 			xfs_buf_relse(bp);
607 			return NULL;
608 		}
609 	}
610 
611 	XFS_STATS_INC(xb_get);
612 	trace_xfs_buf_get(bp, flags, _RET_IP_);
613 	return bp;
614 }
615 
616 STATIC int
617 _xfs_buf_read(
618 	xfs_buf_t		*bp,
619 	xfs_buf_flags_t		flags)
620 {
621 	ASSERT(!(flags & XBF_WRITE));
622 	ASSERT(bp->b_maps[0].bm_bn != XFS_BUF_DADDR_NULL);
623 
624 	bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_READ_AHEAD);
625 	bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | XBF_READ_AHEAD);
626 
627 	xfs_buf_iorequest(bp);
628 	if (flags & XBF_ASYNC)
629 		return 0;
630 	return xfs_buf_iowait(bp);
631 }
632 
633 xfs_buf_t *
634 xfs_buf_read_map(
635 	struct xfs_buftarg	*target,
636 	struct xfs_buf_map	*map,
637 	int			nmaps,
638 	xfs_buf_flags_t		flags,
639 	const struct xfs_buf_ops *ops)
640 {
641 	struct xfs_buf		*bp;
642 
643 	flags |= XBF_READ;
644 
645 	bp = xfs_buf_get_map(target, map, nmaps, flags);
646 	if (bp) {
647 		trace_xfs_buf_read(bp, flags, _RET_IP_);
648 
649 		if (!XFS_BUF_ISDONE(bp)) {
650 			XFS_STATS_INC(xb_get_read);
651 			bp->b_ops = ops;
652 			_xfs_buf_read(bp, flags);
653 		} else if (flags & XBF_ASYNC) {
654 			/*
655 			 * Read ahead call which is already satisfied,
656 			 * drop the buffer
657 			 */
658 			xfs_buf_relse(bp);
659 			return NULL;
660 		} else {
661 			/* We do not want read in the flags */
662 			bp->b_flags &= ~XBF_READ;
663 		}
664 	}
665 
666 	return bp;
667 }
668 
669 /*
670  *	If we are not low on memory then do the readahead in a deadlock
671  *	safe manner.
672  */
673 void
674 xfs_buf_readahead_map(
675 	struct xfs_buftarg	*target,
676 	struct xfs_buf_map	*map,
677 	int			nmaps,
678 	const struct xfs_buf_ops *ops)
679 {
680 	if (bdi_read_congested(target->bt_bdi))
681 		return;
682 
683 	xfs_buf_read_map(target, map, nmaps,
684 		     XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD, ops);
685 }
686 
687 /*
688  * Read an uncached buffer from disk. Allocates and returns a locked
689  * buffer containing the disk contents or nothing.
690  */
691 struct xfs_buf *
692 xfs_buf_read_uncached(
693 	struct xfs_buftarg	*target,
694 	xfs_daddr_t		daddr,
695 	size_t			numblks,
696 	int			flags,
697 	const struct xfs_buf_ops *ops)
698 {
699 	struct xfs_buf		*bp;
700 
701 	bp = xfs_buf_get_uncached(target, numblks, flags);
702 	if (!bp)
703 		return NULL;
704 
705 	/* set up the buffer for a read IO */
706 	ASSERT(bp->b_map_count == 1);
707 	bp->b_bn = daddr;
708 	bp->b_maps[0].bm_bn = daddr;
709 	bp->b_flags |= XBF_READ;
710 	bp->b_ops = ops;
711 
712 	if (XFS_FORCED_SHUTDOWN(target->bt_mount)) {
713 		xfs_buf_relse(bp);
714 		return NULL;
715 	}
716 	xfs_buf_iorequest(bp);
717 	xfs_buf_iowait(bp);
718 	return bp;
719 }
720 
721 /*
722  * Return a buffer allocated as an empty buffer and associated to external
723  * memory via xfs_buf_associate_memory() back to it's empty state.
724  */
725 void
726 xfs_buf_set_empty(
727 	struct xfs_buf		*bp,
728 	size_t			numblks)
729 {
730 	if (bp->b_pages)
731 		_xfs_buf_free_pages(bp);
732 
733 	bp->b_pages = NULL;
734 	bp->b_page_count = 0;
735 	bp->b_addr = NULL;
736 	bp->b_length = numblks;
737 	bp->b_io_length = numblks;
738 
739 	ASSERT(bp->b_map_count == 1);
740 	bp->b_bn = XFS_BUF_DADDR_NULL;
741 	bp->b_maps[0].bm_bn = XFS_BUF_DADDR_NULL;
742 	bp->b_maps[0].bm_len = bp->b_length;
743 }
744 
745 static inline struct page *
746 mem_to_page(
747 	void			*addr)
748 {
749 	if ((!is_vmalloc_addr(addr))) {
750 		return virt_to_page(addr);
751 	} else {
752 		return vmalloc_to_page(addr);
753 	}
754 }
755 
756 int
757 xfs_buf_associate_memory(
758 	xfs_buf_t		*bp,
759 	void			*mem,
760 	size_t			len)
761 {
762 	int			rval;
763 	int			i = 0;
764 	unsigned long		pageaddr;
765 	unsigned long		offset;
766 	size_t			buflen;
767 	int			page_count;
768 
769 	pageaddr = (unsigned long)mem & PAGE_MASK;
770 	offset = (unsigned long)mem - pageaddr;
771 	buflen = PAGE_ALIGN(len + offset);
772 	page_count = buflen >> PAGE_SHIFT;
773 
774 	/* Free any previous set of page pointers */
775 	if (bp->b_pages)
776 		_xfs_buf_free_pages(bp);
777 
778 	bp->b_pages = NULL;
779 	bp->b_addr = mem;
780 
781 	rval = _xfs_buf_get_pages(bp, page_count, 0);
782 	if (rval)
783 		return rval;
784 
785 	bp->b_offset = offset;
786 
787 	for (i = 0; i < bp->b_page_count; i++) {
788 		bp->b_pages[i] = mem_to_page((void *)pageaddr);
789 		pageaddr += PAGE_SIZE;
790 	}
791 
792 	bp->b_io_length = BTOBB(len);
793 	bp->b_length = BTOBB(buflen);
794 
795 	return 0;
796 }
797 
798 xfs_buf_t *
799 xfs_buf_get_uncached(
800 	struct xfs_buftarg	*target,
801 	size_t			numblks,
802 	int			flags)
803 {
804 	unsigned long		page_count;
805 	int			error, i;
806 	struct xfs_buf		*bp;
807 	DEFINE_SINGLE_BUF_MAP(map, XFS_BUF_DADDR_NULL, numblks);
808 
809 	bp = _xfs_buf_alloc(target, &map, 1, 0);
810 	if (unlikely(bp == NULL))
811 		goto fail;
812 
813 	page_count = PAGE_ALIGN(numblks << BBSHIFT) >> PAGE_SHIFT;
814 	error = _xfs_buf_get_pages(bp, page_count, 0);
815 	if (error)
816 		goto fail_free_buf;
817 
818 	for (i = 0; i < page_count; i++) {
819 		bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
820 		if (!bp->b_pages[i])
821 			goto fail_free_mem;
822 	}
823 	bp->b_flags |= _XBF_PAGES;
824 
825 	error = _xfs_buf_map_pages(bp, 0);
826 	if (unlikely(error)) {
827 		xfs_warn(target->bt_mount,
828 			"%s: failed to map pages", __func__);
829 		goto fail_free_mem;
830 	}
831 
832 	trace_xfs_buf_get_uncached(bp, _RET_IP_);
833 	return bp;
834 
835  fail_free_mem:
836 	while (--i >= 0)
837 		__free_page(bp->b_pages[i]);
838 	_xfs_buf_free_pages(bp);
839  fail_free_buf:
840 	xfs_buf_free_maps(bp);
841 	kmem_zone_free(xfs_buf_zone, bp);
842  fail:
843 	return NULL;
844 }
845 
846 /*
847  *	Increment reference count on buffer, to hold the buffer concurrently
848  *	with another thread which may release (free) the buffer asynchronously.
849  *	Must hold the buffer already to call this function.
850  */
851 void
852 xfs_buf_hold(
853 	xfs_buf_t		*bp)
854 {
855 	trace_xfs_buf_hold(bp, _RET_IP_);
856 	atomic_inc(&bp->b_hold);
857 }
858 
859 /*
860  *	Releases a hold on the specified buffer.  If the
861  *	the hold count is 1, calls xfs_buf_free.
862  */
863 void
864 xfs_buf_rele(
865 	xfs_buf_t		*bp)
866 {
867 	struct xfs_perag	*pag = bp->b_pag;
868 
869 	trace_xfs_buf_rele(bp, _RET_IP_);
870 
871 	if (!pag) {
872 		ASSERT(list_empty(&bp->b_lru));
873 		ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
874 		if (atomic_dec_and_test(&bp->b_hold))
875 			xfs_buf_free(bp);
876 		return;
877 	}
878 
879 	ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
880 
881 	ASSERT(atomic_read(&bp->b_hold) > 0);
882 	if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
883 		spin_lock(&bp->b_lock);
884 		if (!(bp->b_flags & XBF_STALE) && atomic_read(&bp->b_lru_ref)) {
885 			/*
886 			 * If the buffer is added to the LRU take a new
887 			 * reference to the buffer for the LRU and clear the
888 			 * (now stale) dispose list state flag
889 			 */
890 			if (list_lru_add(&bp->b_target->bt_lru, &bp->b_lru)) {
891 				bp->b_state &= ~XFS_BSTATE_DISPOSE;
892 				atomic_inc(&bp->b_hold);
893 			}
894 			spin_unlock(&bp->b_lock);
895 			spin_unlock(&pag->pag_buf_lock);
896 		} else {
897 			/*
898 			 * most of the time buffers will already be removed from
899 			 * the LRU, so optimise that case by checking for the
900 			 * XFS_BSTATE_DISPOSE flag indicating the last list the
901 			 * buffer was on was the disposal list
902 			 */
903 			if (!(bp->b_state & XFS_BSTATE_DISPOSE)) {
904 				list_lru_del(&bp->b_target->bt_lru, &bp->b_lru);
905 			} else {
906 				ASSERT(list_empty(&bp->b_lru));
907 			}
908 			spin_unlock(&bp->b_lock);
909 
910 			ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
911 			rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
912 			spin_unlock(&pag->pag_buf_lock);
913 			xfs_perag_put(pag);
914 			xfs_buf_free(bp);
915 		}
916 	}
917 }
918 
919 
920 /*
921  *	Lock a buffer object, if it is not already locked.
922  *
923  *	If we come across a stale, pinned, locked buffer, we know that we are
924  *	being asked to lock a buffer that has been reallocated. Because it is
925  *	pinned, we know that the log has not been pushed to disk and hence it
926  *	will still be locked.  Rather than continuing to have trylock attempts
927  *	fail until someone else pushes the log, push it ourselves before
928  *	returning.  This means that the xfsaild will not get stuck trying
929  *	to push on stale inode buffers.
930  */
931 int
932 xfs_buf_trylock(
933 	struct xfs_buf		*bp)
934 {
935 	int			locked;
936 
937 	locked = down_trylock(&bp->b_sema) == 0;
938 	if (locked)
939 		XB_SET_OWNER(bp);
940 
941 	trace_xfs_buf_trylock(bp, _RET_IP_);
942 	return locked;
943 }
944 
945 /*
946  *	Lock a buffer object.
947  *
948  *	If we come across a stale, pinned, locked buffer, we know that we
949  *	are being asked to lock a buffer that has been reallocated. Because
950  *	it is pinned, we know that the log has not been pushed to disk and
951  *	hence it will still be locked. Rather than sleeping until someone
952  *	else pushes the log, push it ourselves before trying to get the lock.
953  */
954 void
955 xfs_buf_lock(
956 	struct xfs_buf		*bp)
957 {
958 	trace_xfs_buf_lock(bp, _RET_IP_);
959 
960 	if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
961 		xfs_log_force(bp->b_target->bt_mount, 0);
962 	down(&bp->b_sema);
963 	XB_SET_OWNER(bp);
964 
965 	trace_xfs_buf_lock_done(bp, _RET_IP_);
966 }
967 
968 void
969 xfs_buf_unlock(
970 	struct xfs_buf		*bp)
971 {
972 	XB_CLEAR_OWNER(bp);
973 	up(&bp->b_sema);
974 
975 	trace_xfs_buf_unlock(bp, _RET_IP_);
976 }
977 
978 STATIC void
979 xfs_buf_wait_unpin(
980 	xfs_buf_t		*bp)
981 {
982 	DECLARE_WAITQUEUE	(wait, current);
983 
984 	if (atomic_read(&bp->b_pin_count) == 0)
985 		return;
986 
987 	add_wait_queue(&bp->b_waiters, &wait);
988 	for (;;) {
989 		set_current_state(TASK_UNINTERRUPTIBLE);
990 		if (atomic_read(&bp->b_pin_count) == 0)
991 			break;
992 		io_schedule();
993 	}
994 	remove_wait_queue(&bp->b_waiters, &wait);
995 	set_current_state(TASK_RUNNING);
996 }
997 
998 /*
999  *	Buffer Utility Routines
1000  */
1001 
1002 STATIC void
1003 xfs_buf_iodone_work(
1004 	struct work_struct	*work)
1005 {
1006 	struct xfs_buf		*bp =
1007 		container_of(work, xfs_buf_t, b_iodone_work);
1008 	bool			read = !!(bp->b_flags & XBF_READ);
1009 
1010 	bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
1011 
1012 	/* only validate buffers that were read without errors */
1013 	if (read && bp->b_ops && !bp->b_error && (bp->b_flags & XBF_DONE))
1014 		bp->b_ops->verify_read(bp);
1015 
1016 	if (bp->b_iodone)
1017 		(*(bp->b_iodone))(bp);
1018 	else if (bp->b_flags & XBF_ASYNC)
1019 		xfs_buf_relse(bp);
1020 	else {
1021 		ASSERT(read && bp->b_ops);
1022 		complete(&bp->b_iowait);
1023 	}
1024 }
1025 
1026 void
1027 xfs_buf_ioend(
1028 	struct xfs_buf	*bp,
1029 	int		schedule)
1030 {
1031 	bool		read = !!(bp->b_flags & XBF_READ);
1032 
1033 	trace_xfs_buf_iodone(bp, _RET_IP_);
1034 
1035 	if (bp->b_error == 0)
1036 		bp->b_flags |= XBF_DONE;
1037 
1038 	if (bp->b_iodone || (read && bp->b_ops) || (bp->b_flags & XBF_ASYNC)) {
1039 		if (schedule) {
1040 			INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
1041 			queue_work(xfslogd_workqueue, &bp->b_iodone_work);
1042 		} else {
1043 			xfs_buf_iodone_work(&bp->b_iodone_work);
1044 		}
1045 	} else {
1046 		bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
1047 		complete(&bp->b_iowait);
1048 	}
1049 }
1050 
1051 void
1052 xfs_buf_ioerror(
1053 	xfs_buf_t		*bp,
1054 	int			error)
1055 {
1056 	ASSERT(error >= 0 && error <= 0xffff);
1057 	bp->b_error = (unsigned short)error;
1058 	trace_xfs_buf_ioerror(bp, error, _RET_IP_);
1059 }
1060 
1061 void
1062 xfs_buf_ioerror_alert(
1063 	struct xfs_buf		*bp,
1064 	const char		*func)
1065 {
1066 	xfs_alert(bp->b_target->bt_mount,
1067 "metadata I/O error: block 0x%llx (\"%s\") error %d numblks %d",
1068 		(__uint64_t)XFS_BUF_ADDR(bp), func, bp->b_error, bp->b_length);
1069 }
1070 
1071 /*
1072  * Called when we want to stop a buffer from getting written or read.
1073  * We attach the EIO error, muck with its flags, and call xfs_buf_ioend
1074  * so that the proper iodone callbacks get called.
1075  */
1076 STATIC int
1077 xfs_bioerror(
1078 	xfs_buf_t *bp)
1079 {
1080 #ifdef XFSERRORDEBUG
1081 	ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1082 #endif
1083 
1084 	/*
1085 	 * No need to wait until the buffer is unpinned, we aren't flushing it.
1086 	 */
1087 	xfs_buf_ioerror(bp, EIO);
1088 
1089 	/*
1090 	 * We're calling xfs_buf_ioend, so delete XBF_DONE flag.
1091 	 */
1092 	XFS_BUF_UNREAD(bp);
1093 	XFS_BUF_UNDONE(bp);
1094 	xfs_buf_stale(bp);
1095 
1096 	xfs_buf_ioend(bp, 0);
1097 
1098 	return EIO;
1099 }
1100 
1101 /*
1102  * Same as xfs_bioerror, except that we are releasing the buffer
1103  * here ourselves, and avoiding the xfs_buf_ioend call.
1104  * This is meant for userdata errors; metadata bufs come with
1105  * iodone functions attached, so that we can track down errors.
1106  */
1107 int
1108 xfs_bioerror_relse(
1109 	struct xfs_buf	*bp)
1110 {
1111 	int64_t		fl = bp->b_flags;
1112 	/*
1113 	 * No need to wait until the buffer is unpinned.
1114 	 * We aren't flushing it.
1115 	 *
1116 	 * chunkhold expects B_DONE to be set, whether
1117 	 * we actually finish the I/O or not. We don't want to
1118 	 * change that interface.
1119 	 */
1120 	XFS_BUF_UNREAD(bp);
1121 	XFS_BUF_DONE(bp);
1122 	xfs_buf_stale(bp);
1123 	bp->b_iodone = NULL;
1124 	if (!(fl & XBF_ASYNC)) {
1125 		/*
1126 		 * Mark b_error and B_ERROR _both_.
1127 		 * Lot's of chunkcache code assumes that.
1128 		 * There's no reason to mark error for
1129 		 * ASYNC buffers.
1130 		 */
1131 		xfs_buf_ioerror(bp, EIO);
1132 		complete(&bp->b_iowait);
1133 	} else {
1134 		xfs_buf_relse(bp);
1135 	}
1136 
1137 	return EIO;
1138 }
1139 
1140 STATIC int
1141 xfs_bdstrat_cb(
1142 	struct xfs_buf	*bp)
1143 {
1144 	if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
1145 		trace_xfs_bdstrat_shut(bp, _RET_IP_);
1146 		/*
1147 		 * Metadata write that didn't get logged but
1148 		 * written delayed anyway. These aren't associated
1149 		 * with a transaction, and can be ignored.
1150 		 */
1151 		if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1152 			return xfs_bioerror_relse(bp);
1153 		else
1154 			return xfs_bioerror(bp);
1155 	}
1156 
1157 	xfs_buf_iorequest(bp);
1158 	return 0;
1159 }
1160 
1161 int
1162 xfs_bwrite(
1163 	struct xfs_buf		*bp)
1164 {
1165 	int			error;
1166 
1167 	ASSERT(xfs_buf_islocked(bp));
1168 
1169 	bp->b_flags |= XBF_WRITE;
1170 	bp->b_flags &= ~(XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q | XBF_WRITE_FAIL);
1171 
1172 	xfs_bdstrat_cb(bp);
1173 
1174 	error = xfs_buf_iowait(bp);
1175 	if (error) {
1176 		xfs_force_shutdown(bp->b_target->bt_mount,
1177 				   SHUTDOWN_META_IO_ERROR);
1178 	}
1179 	return error;
1180 }
1181 
1182 STATIC void
1183 _xfs_buf_ioend(
1184 	xfs_buf_t		*bp,
1185 	int			schedule)
1186 {
1187 	if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
1188 		xfs_buf_ioend(bp, schedule);
1189 }
1190 
1191 STATIC void
1192 xfs_buf_bio_end_io(
1193 	struct bio		*bio,
1194 	int			error)
1195 {
1196 	xfs_buf_t		*bp = (xfs_buf_t *)bio->bi_private;
1197 
1198 	/*
1199 	 * don't overwrite existing errors - otherwise we can lose errors on
1200 	 * buffers that require multiple bios to complete.
1201 	 */
1202 	if (!bp->b_error)
1203 		xfs_buf_ioerror(bp, -error);
1204 
1205 	if (!bp->b_error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
1206 		invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1207 
1208 	_xfs_buf_ioend(bp, 1);
1209 	bio_put(bio);
1210 }
1211 
1212 static void
1213 xfs_buf_ioapply_map(
1214 	struct xfs_buf	*bp,
1215 	int		map,
1216 	int		*buf_offset,
1217 	int		*count,
1218 	int		rw)
1219 {
1220 	int		page_index;
1221 	int		total_nr_pages = bp->b_page_count;
1222 	int		nr_pages;
1223 	struct bio	*bio;
1224 	sector_t	sector =  bp->b_maps[map].bm_bn;
1225 	int		size;
1226 	int		offset;
1227 
1228 	total_nr_pages = bp->b_page_count;
1229 
1230 	/* skip the pages in the buffer before the start offset */
1231 	page_index = 0;
1232 	offset = *buf_offset;
1233 	while (offset >= PAGE_SIZE) {
1234 		page_index++;
1235 		offset -= PAGE_SIZE;
1236 	}
1237 
1238 	/*
1239 	 * Limit the IO size to the length of the current vector, and update the
1240 	 * remaining IO count for the next time around.
1241 	 */
1242 	size = min_t(int, BBTOB(bp->b_maps[map].bm_len), *count);
1243 	*count -= size;
1244 	*buf_offset += size;
1245 
1246 next_chunk:
1247 	atomic_inc(&bp->b_io_remaining);
1248 	nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1249 	if (nr_pages > total_nr_pages)
1250 		nr_pages = total_nr_pages;
1251 
1252 	bio = bio_alloc(GFP_NOIO, nr_pages);
1253 	bio->bi_bdev = bp->b_target->bt_bdev;
1254 	bio->bi_iter.bi_sector = sector;
1255 	bio->bi_end_io = xfs_buf_bio_end_io;
1256 	bio->bi_private = bp;
1257 
1258 
1259 	for (; size && nr_pages; nr_pages--, page_index++) {
1260 		int	rbytes, nbytes = PAGE_SIZE - offset;
1261 
1262 		if (nbytes > size)
1263 			nbytes = size;
1264 
1265 		rbytes = bio_add_page(bio, bp->b_pages[page_index], nbytes,
1266 				      offset);
1267 		if (rbytes < nbytes)
1268 			break;
1269 
1270 		offset = 0;
1271 		sector += BTOBB(nbytes);
1272 		size -= nbytes;
1273 		total_nr_pages--;
1274 	}
1275 
1276 	if (likely(bio->bi_iter.bi_size)) {
1277 		if (xfs_buf_is_vmapped(bp)) {
1278 			flush_kernel_vmap_range(bp->b_addr,
1279 						xfs_buf_vmap_len(bp));
1280 		}
1281 		submit_bio(rw, bio);
1282 		if (size)
1283 			goto next_chunk;
1284 	} else {
1285 		/*
1286 		 * This is guaranteed not to be the last io reference count
1287 		 * because the caller (xfs_buf_iorequest) holds a count itself.
1288 		 */
1289 		atomic_dec(&bp->b_io_remaining);
1290 		xfs_buf_ioerror(bp, EIO);
1291 		bio_put(bio);
1292 	}
1293 
1294 }
1295 
1296 STATIC void
1297 _xfs_buf_ioapply(
1298 	struct xfs_buf	*bp)
1299 {
1300 	struct blk_plug	plug;
1301 	int		rw;
1302 	int		offset;
1303 	int		size;
1304 	int		i;
1305 
1306 	/*
1307 	 * Make sure we capture only current IO errors rather than stale errors
1308 	 * left over from previous use of the buffer (e.g. failed readahead).
1309 	 */
1310 	bp->b_error = 0;
1311 
1312 	if (bp->b_flags & XBF_WRITE) {
1313 		if (bp->b_flags & XBF_SYNCIO)
1314 			rw = WRITE_SYNC;
1315 		else
1316 			rw = WRITE;
1317 		if (bp->b_flags & XBF_FUA)
1318 			rw |= REQ_FUA;
1319 		if (bp->b_flags & XBF_FLUSH)
1320 			rw |= REQ_FLUSH;
1321 
1322 		/*
1323 		 * Run the write verifier callback function if it exists. If
1324 		 * this function fails it will mark the buffer with an error and
1325 		 * the IO should not be dispatched.
1326 		 */
1327 		if (bp->b_ops) {
1328 			bp->b_ops->verify_write(bp);
1329 			if (bp->b_error) {
1330 				xfs_force_shutdown(bp->b_target->bt_mount,
1331 						   SHUTDOWN_CORRUPT_INCORE);
1332 				return;
1333 			}
1334 		}
1335 	} else if (bp->b_flags & XBF_READ_AHEAD) {
1336 		rw = READA;
1337 	} else {
1338 		rw = READ;
1339 	}
1340 
1341 	/* we only use the buffer cache for meta-data */
1342 	rw |= REQ_META;
1343 
1344 	/*
1345 	 * Walk all the vectors issuing IO on them. Set up the initial offset
1346 	 * into the buffer and the desired IO size before we start -
1347 	 * _xfs_buf_ioapply_vec() will modify them appropriately for each
1348 	 * subsequent call.
1349 	 */
1350 	offset = bp->b_offset;
1351 	size = BBTOB(bp->b_io_length);
1352 	blk_start_plug(&plug);
1353 	for (i = 0; i < bp->b_map_count; i++) {
1354 		xfs_buf_ioapply_map(bp, i, &offset, &size, rw);
1355 		if (bp->b_error)
1356 			break;
1357 		if (size <= 0)
1358 			break;	/* all done */
1359 	}
1360 	blk_finish_plug(&plug);
1361 }
1362 
1363 void
1364 xfs_buf_iorequest(
1365 	xfs_buf_t		*bp)
1366 {
1367 	trace_xfs_buf_iorequest(bp, _RET_IP_);
1368 
1369 	ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
1370 
1371 	if (bp->b_flags & XBF_WRITE)
1372 		xfs_buf_wait_unpin(bp);
1373 	xfs_buf_hold(bp);
1374 
1375 	/* Set the count to 1 initially, this will stop an I/O
1376 	 * completion callout which happens before we have started
1377 	 * all the I/O from calling xfs_buf_ioend too early.
1378 	 */
1379 	atomic_set(&bp->b_io_remaining, 1);
1380 	_xfs_buf_ioapply(bp);
1381 	_xfs_buf_ioend(bp, 1);
1382 
1383 	xfs_buf_rele(bp);
1384 }
1385 
1386 /*
1387  * Waits for I/O to complete on the buffer supplied.  It returns immediately if
1388  * no I/O is pending or there is already a pending error on the buffer.  It
1389  * returns the I/O error code, if any, or 0 if there was no error.
1390  */
1391 int
1392 xfs_buf_iowait(
1393 	xfs_buf_t		*bp)
1394 {
1395 	trace_xfs_buf_iowait(bp, _RET_IP_);
1396 
1397 	if (!bp->b_error)
1398 		wait_for_completion(&bp->b_iowait);
1399 
1400 	trace_xfs_buf_iowait_done(bp, _RET_IP_);
1401 	return bp->b_error;
1402 }
1403 
1404 xfs_caddr_t
1405 xfs_buf_offset(
1406 	xfs_buf_t		*bp,
1407 	size_t			offset)
1408 {
1409 	struct page		*page;
1410 
1411 	if (bp->b_addr)
1412 		return bp->b_addr + offset;
1413 
1414 	offset += bp->b_offset;
1415 	page = bp->b_pages[offset >> PAGE_SHIFT];
1416 	return (xfs_caddr_t)page_address(page) + (offset & (PAGE_SIZE-1));
1417 }
1418 
1419 /*
1420  *	Move data into or out of a buffer.
1421  */
1422 void
1423 xfs_buf_iomove(
1424 	xfs_buf_t		*bp,	/* buffer to process		*/
1425 	size_t			boff,	/* starting buffer offset	*/
1426 	size_t			bsize,	/* length to copy		*/
1427 	void			*data,	/* data address			*/
1428 	xfs_buf_rw_t		mode)	/* read/write/zero flag		*/
1429 {
1430 	size_t			bend;
1431 
1432 	bend = boff + bsize;
1433 	while (boff < bend) {
1434 		struct page	*page;
1435 		int		page_index, page_offset, csize;
1436 
1437 		page_index = (boff + bp->b_offset) >> PAGE_SHIFT;
1438 		page_offset = (boff + bp->b_offset) & ~PAGE_MASK;
1439 		page = bp->b_pages[page_index];
1440 		csize = min_t(size_t, PAGE_SIZE - page_offset,
1441 				      BBTOB(bp->b_io_length) - boff);
1442 
1443 		ASSERT((csize + page_offset) <= PAGE_SIZE);
1444 
1445 		switch (mode) {
1446 		case XBRW_ZERO:
1447 			memset(page_address(page) + page_offset, 0, csize);
1448 			break;
1449 		case XBRW_READ:
1450 			memcpy(data, page_address(page) + page_offset, csize);
1451 			break;
1452 		case XBRW_WRITE:
1453 			memcpy(page_address(page) + page_offset, data, csize);
1454 		}
1455 
1456 		boff += csize;
1457 		data += csize;
1458 	}
1459 }
1460 
1461 /*
1462  *	Handling of buffer targets (buftargs).
1463  */
1464 
1465 /*
1466  * Wait for any bufs with callbacks that have been submitted but have not yet
1467  * returned. These buffers will have an elevated hold count, so wait on those
1468  * while freeing all the buffers only held by the LRU.
1469  */
1470 static enum lru_status
1471 xfs_buftarg_wait_rele(
1472 	struct list_head	*item,
1473 	spinlock_t		*lru_lock,
1474 	void			*arg)
1475 
1476 {
1477 	struct xfs_buf		*bp = container_of(item, struct xfs_buf, b_lru);
1478 	struct list_head	*dispose = arg;
1479 
1480 	if (atomic_read(&bp->b_hold) > 1) {
1481 		/* need to wait, so skip it this pass */
1482 		trace_xfs_buf_wait_buftarg(bp, _RET_IP_);
1483 		return LRU_SKIP;
1484 	}
1485 	if (!spin_trylock(&bp->b_lock))
1486 		return LRU_SKIP;
1487 
1488 	/*
1489 	 * clear the LRU reference count so the buffer doesn't get
1490 	 * ignored in xfs_buf_rele().
1491 	 */
1492 	atomic_set(&bp->b_lru_ref, 0);
1493 	bp->b_state |= XFS_BSTATE_DISPOSE;
1494 	list_move(item, dispose);
1495 	spin_unlock(&bp->b_lock);
1496 	return LRU_REMOVED;
1497 }
1498 
1499 void
1500 xfs_wait_buftarg(
1501 	struct xfs_buftarg	*btp)
1502 {
1503 	LIST_HEAD(dispose);
1504 	int loop = 0;
1505 
1506 	/* loop until there is nothing left on the lru list. */
1507 	while (list_lru_count(&btp->bt_lru)) {
1508 		list_lru_walk(&btp->bt_lru, xfs_buftarg_wait_rele,
1509 			      &dispose, LONG_MAX);
1510 
1511 		while (!list_empty(&dispose)) {
1512 			struct xfs_buf *bp;
1513 			bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1514 			list_del_init(&bp->b_lru);
1515 			if (bp->b_flags & XBF_WRITE_FAIL) {
1516 				xfs_alert(btp->bt_mount,
1517 "Corruption Alert: Buffer at block 0x%llx had permanent write failures!\n"
1518 "Please run xfs_repair to determine the extent of the problem.",
1519 					(long long)bp->b_bn);
1520 			}
1521 			xfs_buf_rele(bp);
1522 		}
1523 		if (loop++ != 0)
1524 			delay(100);
1525 	}
1526 }
1527 
1528 static enum lru_status
1529 xfs_buftarg_isolate(
1530 	struct list_head	*item,
1531 	spinlock_t		*lru_lock,
1532 	void			*arg)
1533 {
1534 	struct xfs_buf		*bp = container_of(item, struct xfs_buf, b_lru);
1535 	struct list_head	*dispose = arg;
1536 
1537 	/*
1538 	 * we are inverting the lru lock/bp->b_lock here, so use a trylock.
1539 	 * If we fail to get the lock, just skip it.
1540 	 */
1541 	if (!spin_trylock(&bp->b_lock))
1542 		return LRU_SKIP;
1543 	/*
1544 	 * Decrement the b_lru_ref count unless the value is already
1545 	 * zero. If the value is already zero, we need to reclaim the
1546 	 * buffer, otherwise it gets another trip through the LRU.
1547 	 */
1548 	if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
1549 		spin_unlock(&bp->b_lock);
1550 		return LRU_ROTATE;
1551 	}
1552 
1553 	bp->b_state |= XFS_BSTATE_DISPOSE;
1554 	list_move(item, dispose);
1555 	spin_unlock(&bp->b_lock);
1556 	return LRU_REMOVED;
1557 }
1558 
1559 static unsigned long
1560 xfs_buftarg_shrink_scan(
1561 	struct shrinker		*shrink,
1562 	struct shrink_control	*sc)
1563 {
1564 	struct xfs_buftarg	*btp = container_of(shrink,
1565 					struct xfs_buftarg, bt_shrinker);
1566 	LIST_HEAD(dispose);
1567 	unsigned long		freed;
1568 	unsigned long		nr_to_scan = sc->nr_to_scan;
1569 
1570 	freed = list_lru_walk_node(&btp->bt_lru, sc->nid, xfs_buftarg_isolate,
1571 				       &dispose, &nr_to_scan);
1572 
1573 	while (!list_empty(&dispose)) {
1574 		struct xfs_buf *bp;
1575 		bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1576 		list_del_init(&bp->b_lru);
1577 		xfs_buf_rele(bp);
1578 	}
1579 
1580 	return freed;
1581 }
1582 
1583 static unsigned long
1584 xfs_buftarg_shrink_count(
1585 	struct shrinker		*shrink,
1586 	struct shrink_control	*sc)
1587 {
1588 	struct xfs_buftarg	*btp = container_of(shrink,
1589 					struct xfs_buftarg, bt_shrinker);
1590 	return list_lru_count_node(&btp->bt_lru, sc->nid);
1591 }
1592 
1593 void
1594 xfs_free_buftarg(
1595 	struct xfs_mount	*mp,
1596 	struct xfs_buftarg	*btp)
1597 {
1598 	unregister_shrinker(&btp->bt_shrinker);
1599 	list_lru_destroy(&btp->bt_lru);
1600 
1601 	if (mp->m_flags & XFS_MOUNT_BARRIER)
1602 		xfs_blkdev_issue_flush(btp);
1603 
1604 	kmem_free(btp);
1605 }
1606 
1607 int
1608 xfs_setsize_buftarg(
1609 	xfs_buftarg_t		*btp,
1610 	unsigned int		blocksize,
1611 	unsigned int		sectorsize)
1612 {
1613 	/* Set up metadata sector size info */
1614 	btp->bt_meta_sectorsize = sectorsize;
1615 	btp->bt_meta_sectormask = sectorsize - 1;
1616 
1617 	if (set_blocksize(btp->bt_bdev, sectorsize)) {
1618 		char name[BDEVNAME_SIZE];
1619 
1620 		bdevname(btp->bt_bdev, name);
1621 
1622 		xfs_warn(btp->bt_mount,
1623 			"Cannot set_blocksize to %u on device %s",
1624 			sectorsize, name);
1625 		return EINVAL;
1626 	}
1627 
1628 	/* Set up device logical sector size mask */
1629 	btp->bt_logical_sectorsize = bdev_logical_block_size(btp->bt_bdev);
1630 	btp->bt_logical_sectormask = bdev_logical_block_size(btp->bt_bdev) - 1;
1631 
1632 	return 0;
1633 }
1634 
1635 /*
1636  * When allocating the initial buffer target we have not yet
1637  * read in the superblock, so don't know what sized sectors
1638  * are being used at this early stage.  Play safe.
1639  */
1640 STATIC int
1641 xfs_setsize_buftarg_early(
1642 	xfs_buftarg_t		*btp,
1643 	struct block_device	*bdev)
1644 {
1645 	return xfs_setsize_buftarg(btp, PAGE_SIZE,
1646 				   bdev_logical_block_size(bdev));
1647 }
1648 
1649 xfs_buftarg_t *
1650 xfs_alloc_buftarg(
1651 	struct xfs_mount	*mp,
1652 	struct block_device	*bdev,
1653 	int			external,
1654 	const char		*fsname)
1655 {
1656 	xfs_buftarg_t		*btp;
1657 
1658 	btp = kmem_zalloc(sizeof(*btp), KM_SLEEP | KM_NOFS);
1659 
1660 	btp->bt_mount = mp;
1661 	btp->bt_dev =  bdev->bd_dev;
1662 	btp->bt_bdev = bdev;
1663 	btp->bt_bdi = blk_get_backing_dev_info(bdev);
1664 	if (!btp->bt_bdi)
1665 		goto error;
1666 
1667 	if (xfs_setsize_buftarg_early(btp, bdev))
1668 		goto error;
1669 
1670 	if (list_lru_init(&btp->bt_lru))
1671 		goto error;
1672 
1673 	btp->bt_shrinker.count_objects = xfs_buftarg_shrink_count;
1674 	btp->bt_shrinker.scan_objects = xfs_buftarg_shrink_scan;
1675 	btp->bt_shrinker.seeks = DEFAULT_SEEKS;
1676 	btp->bt_shrinker.flags = SHRINKER_NUMA_AWARE;
1677 	register_shrinker(&btp->bt_shrinker);
1678 	return btp;
1679 
1680 error:
1681 	kmem_free(btp);
1682 	return NULL;
1683 }
1684 
1685 /*
1686  * Add a buffer to the delayed write list.
1687  *
1688  * This queues a buffer for writeout if it hasn't already been.  Note that
1689  * neither this routine nor the buffer list submission functions perform
1690  * any internal synchronization.  It is expected that the lists are thread-local
1691  * to the callers.
1692  *
1693  * Returns true if we queued up the buffer, or false if it already had
1694  * been on the buffer list.
1695  */
1696 bool
1697 xfs_buf_delwri_queue(
1698 	struct xfs_buf		*bp,
1699 	struct list_head	*list)
1700 {
1701 	ASSERT(xfs_buf_islocked(bp));
1702 	ASSERT(!(bp->b_flags & XBF_READ));
1703 
1704 	/*
1705 	 * If the buffer is already marked delwri it already is queued up
1706 	 * by someone else for imediate writeout.  Just ignore it in that
1707 	 * case.
1708 	 */
1709 	if (bp->b_flags & _XBF_DELWRI_Q) {
1710 		trace_xfs_buf_delwri_queued(bp, _RET_IP_);
1711 		return false;
1712 	}
1713 
1714 	trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1715 
1716 	/*
1717 	 * If a buffer gets written out synchronously or marked stale while it
1718 	 * is on a delwri list we lazily remove it. To do this, the other party
1719 	 * clears the  _XBF_DELWRI_Q flag but otherwise leaves the buffer alone.
1720 	 * It remains referenced and on the list.  In a rare corner case it
1721 	 * might get readded to a delwri list after the synchronous writeout, in
1722 	 * which case we need just need to re-add the flag here.
1723 	 */
1724 	bp->b_flags |= _XBF_DELWRI_Q;
1725 	if (list_empty(&bp->b_list)) {
1726 		atomic_inc(&bp->b_hold);
1727 		list_add_tail(&bp->b_list, list);
1728 	}
1729 
1730 	return true;
1731 }
1732 
1733 /*
1734  * Compare function is more complex than it needs to be because
1735  * the return value is only 32 bits and we are doing comparisons
1736  * on 64 bit values
1737  */
1738 static int
1739 xfs_buf_cmp(
1740 	void		*priv,
1741 	struct list_head *a,
1742 	struct list_head *b)
1743 {
1744 	struct xfs_buf	*ap = container_of(a, struct xfs_buf, b_list);
1745 	struct xfs_buf	*bp = container_of(b, struct xfs_buf, b_list);
1746 	xfs_daddr_t		diff;
1747 
1748 	diff = ap->b_maps[0].bm_bn - bp->b_maps[0].bm_bn;
1749 	if (diff < 0)
1750 		return -1;
1751 	if (diff > 0)
1752 		return 1;
1753 	return 0;
1754 }
1755 
1756 static int
1757 __xfs_buf_delwri_submit(
1758 	struct list_head	*buffer_list,
1759 	struct list_head	*io_list,
1760 	bool			wait)
1761 {
1762 	struct blk_plug		plug;
1763 	struct xfs_buf		*bp, *n;
1764 	int			pinned = 0;
1765 
1766 	list_for_each_entry_safe(bp, n, buffer_list, b_list) {
1767 		if (!wait) {
1768 			if (xfs_buf_ispinned(bp)) {
1769 				pinned++;
1770 				continue;
1771 			}
1772 			if (!xfs_buf_trylock(bp))
1773 				continue;
1774 		} else {
1775 			xfs_buf_lock(bp);
1776 		}
1777 
1778 		/*
1779 		 * Someone else might have written the buffer synchronously or
1780 		 * marked it stale in the meantime.  In that case only the
1781 		 * _XBF_DELWRI_Q flag got cleared, and we have to drop the
1782 		 * reference and remove it from the list here.
1783 		 */
1784 		if (!(bp->b_flags & _XBF_DELWRI_Q)) {
1785 			list_del_init(&bp->b_list);
1786 			xfs_buf_relse(bp);
1787 			continue;
1788 		}
1789 
1790 		list_move_tail(&bp->b_list, io_list);
1791 		trace_xfs_buf_delwri_split(bp, _RET_IP_);
1792 	}
1793 
1794 	list_sort(NULL, io_list, xfs_buf_cmp);
1795 
1796 	blk_start_plug(&plug);
1797 	list_for_each_entry_safe(bp, n, io_list, b_list) {
1798 		bp->b_flags &= ~(_XBF_DELWRI_Q | XBF_ASYNC | XBF_WRITE_FAIL);
1799 		bp->b_flags |= XBF_WRITE;
1800 
1801 		if (!wait) {
1802 			bp->b_flags |= XBF_ASYNC;
1803 			list_del_init(&bp->b_list);
1804 		}
1805 		xfs_bdstrat_cb(bp);
1806 	}
1807 	blk_finish_plug(&plug);
1808 
1809 	return pinned;
1810 }
1811 
1812 /*
1813  * Write out a buffer list asynchronously.
1814  *
1815  * This will take the @buffer_list, write all non-locked and non-pinned buffers
1816  * out and not wait for I/O completion on any of the buffers.  This interface
1817  * is only safely useable for callers that can track I/O completion by higher
1818  * level means, e.g. AIL pushing as the @buffer_list is consumed in this
1819  * function.
1820  */
1821 int
1822 xfs_buf_delwri_submit_nowait(
1823 	struct list_head	*buffer_list)
1824 {
1825 	LIST_HEAD		(io_list);
1826 	return __xfs_buf_delwri_submit(buffer_list, &io_list, false);
1827 }
1828 
1829 /*
1830  * Write out a buffer list synchronously.
1831  *
1832  * This will take the @buffer_list, write all buffers out and wait for I/O
1833  * completion on all of the buffers. @buffer_list is consumed by the function,
1834  * so callers must have some other way of tracking buffers if they require such
1835  * functionality.
1836  */
1837 int
1838 xfs_buf_delwri_submit(
1839 	struct list_head	*buffer_list)
1840 {
1841 	LIST_HEAD		(io_list);
1842 	int			error = 0, error2;
1843 	struct xfs_buf		*bp;
1844 
1845 	__xfs_buf_delwri_submit(buffer_list, &io_list, true);
1846 
1847 	/* Wait for IO to complete. */
1848 	while (!list_empty(&io_list)) {
1849 		bp = list_first_entry(&io_list, struct xfs_buf, b_list);
1850 
1851 		list_del_init(&bp->b_list);
1852 		error2 = xfs_buf_iowait(bp);
1853 		xfs_buf_relse(bp);
1854 		if (!error)
1855 			error = error2;
1856 	}
1857 
1858 	return error;
1859 }
1860 
1861 int __init
1862 xfs_buf_init(void)
1863 {
1864 	xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1865 						KM_ZONE_HWALIGN, NULL);
1866 	if (!xfs_buf_zone)
1867 		goto out;
1868 
1869 	xfslogd_workqueue = alloc_workqueue("xfslogd",
1870 					WQ_MEM_RECLAIM | WQ_HIGHPRI, 1);
1871 	if (!xfslogd_workqueue)
1872 		goto out_free_buf_zone;
1873 
1874 	return 0;
1875 
1876  out_free_buf_zone:
1877 	kmem_zone_destroy(xfs_buf_zone);
1878  out:
1879 	return -ENOMEM;
1880 }
1881 
1882 void
1883 xfs_buf_terminate(void)
1884 {
1885 	destroy_workqueue(xfslogd_workqueue);
1886 	kmem_zone_destroy(xfs_buf_zone);
1887 }
1888