xref: /openbmc/linux/fs/xfs/xfs_aops.c (revision d2999e1b)
1 /*
2  * Copyright (c) 2000-2005 Silicon Graphics, Inc.
3  * All Rights Reserved.
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License as
7  * published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it would be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write the Free Software Foundation,
16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17  */
18 #include "xfs.h"
19 #include "xfs_shared.h"
20 #include "xfs_format.h"
21 #include "xfs_log_format.h"
22 #include "xfs_trans_resv.h"
23 #include "xfs_sb.h"
24 #include "xfs_ag.h"
25 #include "xfs_mount.h"
26 #include "xfs_inode.h"
27 #include "xfs_trans.h"
28 #include "xfs_inode_item.h"
29 #include "xfs_alloc.h"
30 #include "xfs_error.h"
31 #include "xfs_iomap.h"
32 #include "xfs_trace.h"
33 #include "xfs_bmap.h"
34 #include "xfs_bmap_util.h"
35 #include "xfs_bmap_btree.h"
36 #include "xfs_dinode.h"
37 #include <linux/aio.h>
38 #include <linux/gfp.h>
39 #include <linux/mpage.h>
40 #include <linux/pagevec.h>
41 #include <linux/writeback.h>
42 
43 void
44 xfs_count_page_state(
45 	struct page		*page,
46 	int			*delalloc,
47 	int			*unwritten)
48 {
49 	struct buffer_head	*bh, *head;
50 
51 	*delalloc = *unwritten = 0;
52 
53 	bh = head = page_buffers(page);
54 	do {
55 		if (buffer_unwritten(bh))
56 			(*unwritten) = 1;
57 		else if (buffer_delay(bh))
58 			(*delalloc) = 1;
59 	} while ((bh = bh->b_this_page) != head);
60 }
61 
62 STATIC struct block_device *
63 xfs_find_bdev_for_inode(
64 	struct inode		*inode)
65 {
66 	struct xfs_inode	*ip = XFS_I(inode);
67 	struct xfs_mount	*mp = ip->i_mount;
68 
69 	if (XFS_IS_REALTIME_INODE(ip))
70 		return mp->m_rtdev_targp->bt_bdev;
71 	else
72 		return mp->m_ddev_targp->bt_bdev;
73 }
74 
75 /*
76  * We're now finished for good with this ioend structure.
77  * Update the page state via the associated buffer_heads,
78  * release holds on the inode and bio, and finally free
79  * up memory.  Do not use the ioend after this.
80  */
81 STATIC void
82 xfs_destroy_ioend(
83 	xfs_ioend_t		*ioend)
84 {
85 	struct buffer_head	*bh, *next;
86 
87 	for (bh = ioend->io_buffer_head; bh; bh = next) {
88 		next = bh->b_private;
89 		bh->b_end_io(bh, !ioend->io_error);
90 	}
91 
92 	mempool_free(ioend, xfs_ioend_pool);
93 }
94 
95 /*
96  * Fast and loose check if this write could update the on-disk inode size.
97  */
98 static inline bool xfs_ioend_is_append(struct xfs_ioend *ioend)
99 {
100 	return ioend->io_offset + ioend->io_size >
101 		XFS_I(ioend->io_inode)->i_d.di_size;
102 }
103 
104 STATIC int
105 xfs_setfilesize_trans_alloc(
106 	struct xfs_ioend	*ioend)
107 {
108 	struct xfs_mount	*mp = XFS_I(ioend->io_inode)->i_mount;
109 	struct xfs_trans	*tp;
110 	int			error;
111 
112 	tp = xfs_trans_alloc(mp, XFS_TRANS_FSYNC_TS);
113 
114 	error = xfs_trans_reserve(tp, &M_RES(mp)->tr_fsyncts, 0, 0);
115 	if (error) {
116 		xfs_trans_cancel(tp, 0);
117 		return error;
118 	}
119 
120 	ioend->io_append_trans = tp;
121 
122 	/*
123 	 * We may pass freeze protection with a transaction.  So tell lockdep
124 	 * we released it.
125 	 */
126 	rwsem_release(&ioend->io_inode->i_sb->s_writers.lock_map[SB_FREEZE_FS-1],
127 		      1, _THIS_IP_);
128 	/*
129 	 * We hand off the transaction to the completion thread now, so
130 	 * clear the flag here.
131 	 */
132 	current_restore_flags_nested(&tp->t_pflags, PF_FSTRANS);
133 	return 0;
134 }
135 
136 /*
137  * Update on-disk file size now that data has been written to disk.
138  */
139 STATIC int
140 xfs_setfilesize(
141 	struct xfs_ioend	*ioend)
142 {
143 	struct xfs_inode	*ip = XFS_I(ioend->io_inode);
144 	struct xfs_trans	*tp = ioend->io_append_trans;
145 	xfs_fsize_t		isize;
146 
147 	/*
148 	 * The transaction may have been allocated in the I/O submission thread,
149 	 * thus we need to mark ourselves as beeing in a transaction manually.
150 	 * Similarly for freeze protection.
151 	 */
152 	current_set_flags_nested(&tp->t_pflags, PF_FSTRANS);
153 	rwsem_acquire_read(&VFS_I(ip)->i_sb->s_writers.lock_map[SB_FREEZE_FS-1],
154 			   0, 1, _THIS_IP_);
155 
156 	xfs_ilock(ip, XFS_ILOCK_EXCL);
157 	isize = xfs_new_eof(ip, ioend->io_offset + ioend->io_size);
158 	if (!isize) {
159 		xfs_iunlock(ip, XFS_ILOCK_EXCL);
160 		xfs_trans_cancel(tp, 0);
161 		return 0;
162 	}
163 
164 	trace_xfs_setfilesize(ip, ioend->io_offset, ioend->io_size);
165 
166 	ip->i_d.di_size = isize;
167 	xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
168 	xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
169 
170 	return xfs_trans_commit(tp, 0);
171 }
172 
173 /*
174  * Schedule IO completion handling on the final put of an ioend.
175  *
176  * If there is no work to do we might as well call it a day and free the
177  * ioend right now.
178  */
179 STATIC void
180 xfs_finish_ioend(
181 	struct xfs_ioend	*ioend)
182 {
183 	if (atomic_dec_and_test(&ioend->io_remaining)) {
184 		struct xfs_mount	*mp = XFS_I(ioend->io_inode)->i_mount;
185 
186 		if (ioend->io_type == XFS_IO_UNWRITTEN)
187 			queue_work(mp->m_unwritten_workqueue, &ioend->io_work);
188 		else if (ioend->io_append_trans ||
189 			 (ioend->io_isdirect && xfs_ioend_is_append(ioend)))
190 			queue_work(mp->m_data_workqueue, &ioend->io_work);
191 		else
192 			xfs_destroy_ioend(ioend);
193 	}
194 }
195 
196 /*
197  * IO write completion.
198  */
199 STATIC void
200 xfs_end_io(
201 	struct work_struct *work)
202 {
203 	xfs_ioend_t	*ioend = container_of(work, xfs_ioend_t, io_work);
204 	struct xfs_inode *ip = XFS_I(ioend->io_inode);
205 	int		error = 0;
206 
207 	if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
208 		ioend->io_error = -EIO;
209 		goto done;
210 	}
211 	if (ioend->io_error)
212 		goto done;
213 
214 	/*
215 	 * For unwritten extents we need to issue transactions to convert a
216 	 * range to normal written extens after the data I/O has finished.
217 	 */
218 	if (ioend->io_type == XFS_IO_UNWRITTEN) {
219 		error = xfs_iomap_write_unwritten(ip, ioend->io_offset,
220 						  ioend->io_size);
221 	} else if (ioend->io_isdirect && xfs_ioend_is_append(ioend)) {
222 		/*
223 		 * For direct I/O we do not know if we need to allocate blocks
224 		 * or not so we can't preallocate an append transaction as that
225 		 * results in nested reservations and log space deadlocks. Hence
226 		 * allocate the transaction here. While this is sub-optimal and
227 		 * can block IO completion for some time, we're stuck with doing
228 		 * it this way until we can pass the ioend to the direct IO
229 		 * allocation callbacks and avoid nesting that way.
230 		 */
231 		error = xfs_setfilesize_trans_alloc(ioend);
232 		if (error)
233 			goto done;
234 		error = xfs_setfilesize(ioend);
235 	} else if (ioend->io_append_trans) {
236 		error = xfs_setfilesize(ioend);
237 	} else {
238 		ASSERT(!xfs_ioend_is_append(ioend));
239 	}
240 
241 done:
242 	if (error)
243 		ioend->io_error = -error;
244 	xfs_destroy_ioend(ioend);
245 }
246 
247 /*
248  * Call IO completion handling in caller context on the final put of an ioend.
249  */
250 STATIC void
251 xfs_finish_ioend_sync(
252 	struct xfs_ioend	*ioend)
253 {
254 	if (atomic_dec_and_test(&ioend->io_remaining))
255 		xfs_end_io(&ioend->io_work);
256 }
257 
258 /*
259  * Allocate and initialise an IO completion structure.
260  * We need to track unwritten extent write completion here initially.
261  * We'll need to extend this for updating the ondisk inode size later
262  * (vs. incore size).
263  */
264 STATIC xfs_ioend_t *
265 xfs_alloc_ioend(
266 	struct inode		*inode,
267 	unsigned int		type)
268 {
269 	xfs_ioend_t		*ioend;
270 
271 	ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS);
272 
273 	/*
274 	 * Set the count to 1 initially, which will prevent an I/O
275 	 * completion callback from happening before we have started
276 	 * all the I/O from calling the completion routine too early.
277 	 */
278 	atomic_set(&ioend->io_remaining, 1);
279 	ioend->io_isdirect = 0;
280 	ioend->io_error = 0;
281 	ioend->io_list = NULL;
282 	ioend->io_type = type;
283 	ioend->io_inode = inode;
284 	ioend->io_buffer_head = NULL;
285 	ioend->io_buffer_tail = NULL;
286 	ioend->io_offset = 0;
287 	ioend->io_size = 0;
288 	ioend->io_append_trans = NULL;
289 
290 	INIT_WORK(&ioend->io_work, xfs_end_io);
291 	return ioend;
292 }
293 
294 STATIC int
295 xfs_map_blocks(
296 	struct inode		*inode,
297 	loff_t			offset,
298 	struct xfs_bmbt_irec	*imap,
299 	int			type,
300 	int			nonblocking)
301 {
302 	struct xfs_inode	*ip = XFS_I(inode);
303 	struct xfs_mount	*mp = ip->i_mount;
304 	ssize_t			count = 1 << inode->i_blkbits;
305 	xfs_fileoff_t		offset_fsb, end_fsb;
306 	int			error = 0;
307 	int			bmapi_flags = XFS_BMAPI_ENTIRE;
308 	int			nimaps = 1;
309 
310 	if (XFS_FORCED_SHUTDOWN(mp))
311 		return -XFS_ERROR(EIO);
312 
313 	if (type == XFS_IO_UNWRITTEN)
314 		bmapi_flags |= XFS_BMAPI_IGSTATE;
315 
316 	if (!xfs_ilock_nowait(ip, XFS_ILOCK_SHARED)) {
317 		if (nonblocking)
318 			return -XFS_ERROR(EAGAIN);
319 		xfs_ilock(ip, XFS_ILOCK_SHARED);
320 	}
321 
322 	ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE ||
323 	       (ip->i_df.if_flags & XFS_IFEXTENTS));
324 	ASSERT(offset <= mp->m_super->s_maxbytes);
325 
326 	if (offset + count > mp->m_super->s_maxbytes)
327 		count = mp->m_super->s_maxbytes - offset;
328 	end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count);
329 	offset_fsb = XFS_B_TO_FSBT(mp, offset);
330 	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb,
331 				imap, &nimaps, bmapi_flags);
332 	xfs_iunlock(ip, XFS_ILOCK_SHARED);
333 
334 	if (error)
335 		return -XFS_ERROR(error);
336 
337 	if (type == XFS_IO_DELALLOC &&
338 	    (!nimaps || isnullstartblock(imap->br_startblock))) {
339 		error = xfs_iomap_write_allocate(ip, offset, imap);
340 		if (!error)
341 			trace_xfs_map_blocks_alloc(ip, offset, count, type, imap);
342 		return -XFS_ERROR(error);
343 	}
344 
345 #ifdef DEBUG
346 	if (type == XFS_IO_UNWRITTEN) {
347 		ASSERT(nimaps);
348 		ASSERT(imap->br_startblock != HOLESTARTBLOCK);
349 		ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
350 	}
351 #endif
352 	if (nimaps)
353 		trace_xfs_map_blocks_found(ip, offset, count, type, imap);
354 	return 0;
355 }
356 
357 STATIC int
358 xfs_imap_valid(
359 	struct inode		*inode,
360 	struct xfs_bmbt_irec	*imap,
361 	xfs_off_t		offset)
362 {
363 	offset >>= inode->i_blkbits;
364 
365 	return offset >= imap->br_startoff &&
366 		offset < imap->br_startoff + imap->br_blockcount;
367 }
368 
369 /*
370  * BIO completion handler for buffered IO.
371  */
372 STATIC void
373 xfs_end_bio(
374 	struct bio		*bio,
375 	int			error)
376 {
377 	xfs_ioend_t		*ioend = bio->bi_private;
378 
379 	ASSERT(atomic_read(&bio->bi_cnt) >= 1);
380 	ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error;
381 
382 	/* Toss bio and pass work off to an xfsdatad thread */
383 	bio->bi_private = NULL;
384 	bio->bi_end_io = NULL;
385 	bio_put(bio);
386 
387 	xfs_finish_ioend(ioend);
388 }
389 
390 STATIC void
391 xfs_submit_ioend_bio(
392 	struct writeback_control *wbc,
393 	xfs_ioend_t		*ioend,
394 	struct bio		*bio)
395 {
396 	atomic_inc(&ioend->io_remaining);
397 	bio->bi_private = ioend;
398 	bio->bi_end_io = xfs_end_bio;
399 	submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE, bio);
400 }
401 
402 STATIC struct bio *
403 xfs_alloc_ioend_bio(
404 	struct buffer_head	*bh)
405 {
406 	int			nvecs = bio_get_nr_vecs(bh->b_bdev);
407 	struct bio		*bio = bio_alloc(GFP_NOIO, nvecs);
408 
409 	ASSERT(bio->bi_private == NULL);
410 	bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
411 	bio->bi_bdev = bh->b_bdev;
412 	return bio;
413 }
414 
415 STATIC void
416 xfs_start_buffer_writeback(
417 	struct buffer_head	*bh)
418 {
419 	ASSERT(buffer_mapped(bh));
420 	ASSERT(buffer_locked(bh));
421 	ASSERT(!buffer_delay(bh));
422 	ASSERT(!buffer_unwritten(bh));
423 
424 	mark_buffer_async_write(bh);
425 	set_buffer_uptodate(bh);
426 	clear_buffer_dirty(bh);
427 }
428 
429 STATIC void
430 xfs_start_page_writeback(
431 	struct page		*page,
432 	int			clear_dirty,
433 	int			buffers)
434 {
435 	ASSERT(PageLocked(page));
436 	ASSERT(!PageWriteback(page));
437 	if (clear_dirty)
438 		clear_page_dirty_for_io(page);
439 	set_page_writeback(page);
440 	unlock_page(page);
441 	/* If no buffers on the page are to be written, finish it here */
442 	if (!buffers)
443 		end_page_writeback(page);
444 }
445 
446 static inline int xfs_bio_add_buffer(struct bio *bio, struct buffer_head *bh)
447 {
448 	return bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
449 }
450 
451 /*
452  * Submit all of the bios for all of the ioends we have saved up, covering the
453  * initial writepage page and also any probed pages.
454  *
455  * Because we may have multiple ioends spanning a page, we need to start
456  * writeback on all the buffers before we submit them for I/O. If we mark the
457  * buffers as we got, then we can end up with a page that only has buffers
458  * marked async write and I/O complete on can occur before we mark the other
459  * buffers async write.
460  *
461  * The end result of this is that we trip a bug in end_page_writeback() because
462  * we call it twice for the one page as the code in end_buffer_async_write()
463  * assumes that all buffers on the page are started at the same time.
464  *
465  * The fix is two passes across the ioend list - one to start writeback on the
466  * buffer_heads, and then submit them for I/O on the second pass.
467  *
468  * If @fail is non-zero, it means that we have a situation where some part of
469  * the submission process has failed after we have marked paged for writeback
470  * and unlocked them. In this situation, we need to fail the ioend chain rather
471  * than submit it to IO. This typically only happens on a filesystem shutdown.
472  */
473 STATIC void
474 xfs_submit_ioend(
475 	struct writeback_control *wbc,
476 	xfs_ioend_t		*ioend,
477 	int			fail)
478 {
479 	xfs_ioend_t		*head = ioend;
480 	xfs_ioend_t		*next;
481 	struct buffer_head	*bh;
482 	struct bio		*bio;
483 	sector_t		lastblock = 0;
484 
485 	/* Pass 1 - start writeback */
486 	do {
487 		next = ioend->io_list;
488 		for (bh = ioend->io_buffer_head; bh; bh = bh->b_private)
489 			xfs_start_buffer_writeback(bh);
490 	} while ((ioend = next) != NULL);
491 
492 	/* Pass 2 - submit I/O */
493 	ioend = head;
494 	do {
495 		next = ioend->io_list;
496 		bio = NULL;
497 
498 		/*
499 		 * If we are failing the IO now, just mark the ioend with an
500 		 * error and finish it. This will run IO completion immediately
501 		 * as there is only one reference to the ioend at this point in
502 		 * time.
503 		 */
504 		if (fail) {
505 			ioend->io_error = -fail;
506 			xfs_finish_ioend(ioend);
507 			continue;
508 		}
509 
510 		for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
511 
512 			if (!bio) {
513  retry:
514 				bio = xfs_alloc_ioend_bio(bh);
515 			} else if (bh->b_blocknr != lastblock + 1) {
516 				xfs_submit_ioend_bio(wbc, ioend, bio);
517 				goto retry;
518 			}
519 
520 			if (xfs_bio_add_buffer(bio, bh) != bh->b_size) {
521 				xfs_submit_ioend_bio(wbc, ioend, bio);
522 				goto retry;
523 			}
524 
525 			lastblock = bh->b_blocknr;
526 		}
527 		if (bio)
528 			xfs_submit_ioend_bio(wbc, ioend, bio);
529 		xfs_finish_ioend(ioend);
530 	} while ((ioend = next) != NULL);
531 }
532 
533 /*
534  * Cancel submission of all buffer_heads so far in this endio.
535  * Toss the endio too.  Only ever called for the initial page
536  * in a writepage request, so only ever one page.
537  */
538 STATIC void
539 xfs_cancel_ioend(
540 	xfs_ioend_t		*ioend)
541 {
542 	xfs_ioend_t		*next;
543 	struct buffer_head	*bh, *next_bh;
544 
545 	do {
546 		next = ioend->io_list;
547 		bh = ioend->io_buffer_head;
548 		do {
549 			next_bh = bh->b_private;
550 			clear_buffer_async_write(bh);
551 			unlock_buffer(bh);
552 		} while ((bh = next_bh) != NULL);
553 
554 		mempool_free(ioend, xfs_ioend_pool);
555 	} while ((ioend = next) != NULL);
556 }
557 
558 /*
559  * Test to see if we've been building up a completion structure for
560  * earlier buffers -- if so, we try to append to this ioend if we
561  * can, otherwise we finish off any current ioend and start another.
562  * Return true if we've finished the given ioend.
563  */
564 STATIC void
565 xfs_add_to_ioend(
566 	struct inode		*inode,
567 	struct buffer_head	*bh,
568 	xfs_off_t		offset,
569 	unsigned int		type,
570 	xfs_ioend_t		**result,
571 	int			need_ioend)
572 {
573 	xfs_ioend_t		*ioend = *result;
574 
575 	if (!ioend || need_ioend || type != ioend->io_type) {
576 		xfs_ioend_t	*previous = *result;
577 
578 		ioend = xfs_alloc_ioend(inode, type);
579 		ioend->io_offset = offset;
580 		ioend->io_buffer_head = bh;
581 		ioend->io_buffer_tail = bh;
582 		if (previous)
583 			previous->io_list = ioend;
584 		*result = ioend;
585 	} else {
586 		ioend->io_buffer_tail->b_private = bh;
587 		ioend->io_buffer_tail = bh;
588 	}
589 
590 	bh->b_private = NULL;
591 	ioend->io_size += bh->b_size;
592 }
593 
594 STATIC void
595 xfs_map_buffer(
596 	struct inode		*inode,
597 	struct buffer_head	*bh,
598 	struct xfs_bmbt_irec	*imap,
599 	xfs_off_t		offset)
600 {
601 	sector_t		bn;
602 	struct xfs_mount	*m = XFS_I(inode)->i_mount;
603 	xfs_off_t		iomap_offset = XFS_FSB_TO_B(m, imap->br_startoff);
604 	xfs_daddr_t		iomap_bn = xfs_fsb_to_db(XFS_I(inode), imap->br_startblock);
605 
606 	ASSERT(imap->br_startblock != HOLESTARTBLOCK);
607 	ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
608 
609 	bn = (iomap_bn >> (inode->i_blkbits - BBSHIFT)) +
610 	      ((offset - iomap_offset) >> inode->i_blkbits);
611 
612 	ASSERT(bn || XFS_IS_REALTIME_INODE(XFS_I(inode)));
613 
614 	bh->b_blocknr = bn;
615 	set_buffer_mapped(bh);
616 }
617 
618 STATIC void
619 xfs_map_at_offset(
620 	struct inode		*inode,
621 	struct buffer_head	*bh,
622 	struct xfs_bmbt_irec	*imap,
623 	xfs_off_t		offset)
624 {
625 	ASSERT(imap->br_startblock != HOLESTARTBLOCK);
626 	ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
627 
628 	xfs_map_buffer(inode, bh, imap, offset);
629 	set_buffer_mapped(bh);
630 	clear_buffer_delay(bh);
631 	clear_buffer_unwritten(bh);
632 }
633 
634 /*
635  * Test if a given page contains at least one buffer of a given @type.
636  * If @check_all_buffers is true, then we walk all the buffers in the page to
637  * try to find one of the type passed in. If it is not set, then the caller only
638  * needs to check the first buffer on the page for a match.
639  */
640 STATIC bool
641 xfs_check_page_type(
642 	struct page		*page,
643 	unsigned int		type,
644 	bool			check_all_buffers)
645 {
646 	struct buffer_head	*bh;
647 	struct buffer_head	*head;
648 
649 	if (PageWriteback(page))
650 		return false;
651 	if (!page->mapping)
652 		return false;
653 	if (!page_has_buffers(page))
654 		return false;
655 
656 	bh = head = page_buffers(page);
657 	do {
658 		if (buffer_unwritten(bh)) {
659 			if (type == XFS_IO_UNWRITTEN)
660 				return true;
661 		} else if (buffer_delay(bh)) {
662 			if (type == XFS_IO_DELALLOC)
663 				return true;
664 		} else if (buffer_dirty(bh) && buffer_mapped(bh)) {
665 			if (type == XFS_IO_OVERWRITE)
666 				return true;
667 		}
668 
669 		/* If we are only checking the first buffer, we are done now. */
670 		if (!check_all_buffers)
671 			break;
672 	} while ((bh = bh->b_this_page) != head);
673 
674 	return false;
675 }
676 
677 /*
678  * Allocate & map buffers for page given the extent map. Write it out.
679  * except for the original page of a writepage, this is called on
680  * delalloc/unwritten pages only, for the original page it is possible
681  * that the page has no mapping at all.
682  */
683 STATIC int
684 xfs_convert_page(
685 	struct inode		*inode,
686 	struct page		*page,
687 	loff_t			tindex,
688 	struct xfs_bmbt_irec	*imap,
689 	xfs_ioend_t		**ioendp,
690 	struct writeback_control *wbc)
691 {
692 	struct buffer_head	*bh, *head;
693 	xfs_off_t		end_offset;
694 	unsigned long		p_offset;
695 	unsigned int		type;
696 	int			len, page_dirty;
697 	int			count = 0, done = 0, uptodate = 1;
698  	xfs_off_t		offset = page_offset(page);
699 
700 	if (page->index != tindex)
701 		goto fail;
702 	if (!trylock_page(page))
703 		goto fail;
704 	if (PageWriteback(page))
705 		goto fail_unlock_page;
706 	if (page->mapping != inode->i_mapping)
707 		goto fail_unlock_page;
708 	if (!xfs_check_page_type(page, (*ioendp)->io_type, false))
709 		goto fail_unlock_page;
710 
711 	/*
712 	 * page_dirty is initially a count of buffers on the page before
713 	 * EOF and is decremented as we move each into a cleanable state.
714 	 *
715 	 * Derivation:
716 	 *
717 	 * End offset is the highest offset that this page should represent.
718 	 * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1))
719 	 * will evaluate non-zero and be less than PAGE_CACHE_SIZE and
720 	 * hence give us the correct page_dirty count. On any other page,
721 	 * it will be zero and in that case we need page_dirty to be the
722 	 * count of buffers on the page.
723 	 */
724 	end_offset = min_t(unsigned long long,
725 			(xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
726 			i_size_read(inode));
727 
728 	/*
729 	 * If the current map does not span the entire page we are about to try
730 	 * to write, then give up. The only way we can write a page that spans
731 	 * multiple mappings in a single writeback iteration is via the
732 	 * xfs_vm_writepage() function. Data integrity writeback requires the
733 	 * entire page to be written in a single attempt, otherwise the part of
734 	 * the page we don't write here doesn't get written as part of the data
735 	 * integrity sync.
736 	 *
737 	 * For normal writeback, we also don't attempt to write partial pages
738 	 * here as it simply means that write_cache_pages() will see it under
739 	 * writeback and ignore the page until some point in the future, at
740 	 * which time this will be the only page in the file that needs
741 	 * writeback.  Hence for more optimal IO patterns, we should always
742 	 * avoid partial page writeback due to multiple mappings on a page here.
743 	 */
744 	if (!xfs_imap_valid(inode, imap, end_offset))
745 		goto fail_unlock_page;
746 
747 	len = 1 << inode->i_blkbits;
748 	p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1),
749 					PAGE_CACHE_SIZE);
750 	p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE;
751 	page_dirty = p_offset / len;
752 
753 	/*
754 	 * The moment we find a buffer that doesn't match our current type
755 	 * specification or can't be written, abort the loop and start
756 	 * writeback. As per the above xfs_imap_valid() check, only
757 	 * xfs_vm_writepage() can handle partial page writeback fully - we are
758 	 * limited here to the buffers that are contiguous with the current
759 	 * ioend, and hence a buffer we can't write breaks that contiguity and
760 	 * we have to defer the rest of the IO to xfs_vm_writepage().
761 	 */
762 	bh = head = page_buffers(page);
763 	do {
764 		if (offset >= end_offset)
765 			break;
766 		if (!buffer_uptodate(bh))
767 			uptodate = 0;
768 		if (!(PageUptodate(page) || buffer_uptodate(bh))) {
769 			done = 1;
770 			break;
771 		}
772 
773 		if (buffer_unwritten(bh) || buffer_delay(bh) ||
774 		    buffer_mapped(bh)) {
775 			if (buffer_unwritten(bh))
776 				type = XFS_IO_UNWRITTEN;
777 			else if (buffer_delay(bh))
778 				type = XFS_IO_DELALLOC;
779 			else
780 				type = XFS_IO_OVERWRITE;
781 
782 			/*
783 			 * imap should always be valid because of the above
784 			 * partial page end_offset check on the imap.
785 			 */
786 			ASSERT(xfs_imap_valid(inode, imap, offset));
787 
788 			lock_buffer(bh);
789 			if (type != XFS_IO_OVERWRITE)
790 				xfs_map_at_offset(inode, bh, imap, offset);
791 			xfs_add_to_ioend(inode, bh, offset, type,
792 					 ioendp, done);
793 
794 			page_dirty--;
795 			count++;
796 		} else {
797 			done = 1;
798 			break;
799 		}
800 	} while (offset += len, (bh = bh->b_this_page) != head);
801 
802 	if (uptodate && bh == head)
803 		SetPageUptodate(page);
804 
805 	if (count) {
806 		if (--wbc->nr_to_write <= 0 &&
807 		    wbc->sync_mode == WB_SYNC_NONE)
808 			done = 1;
809 	}
810 	xfs_start_page_writeback(page, !page_dirty, count);
811 
812 	return done;
813  fail_unlock_page:
814 	unlock_page(page);
815  fail:
816 	return 1;
817 }
818 
819 /*
820  * Convert & write out a cluster of pages in the same extent as defined
821  * by mp and following the start page.
822  */
823 STATIC void
824 xfs_cluster_write(
825 	struct inode		*inode,
826 	pgoff_t			tindex,
827 	struct xfs_bmbt_irec	*imap,
828 	xfs_ioend_t		**ioendp,
829 	struct writeback_control *wbc,
830 	pgoff_t			tlast)
831 {
832 	struct pagevec		pvec;
833 	int			done = 0, i;
834 
835 	pagevec_init(&pvec, 0);
836 	while (!done && tindex <= tlast) {
837 		unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
838 
839 		if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
840 			break;
841 
842 		for (i = 0; i < pagevec_count(&pvec); i++) {
843 			done = xfs_convert_page(inode, pvec.pages[i], tindex++,
844 					imap, ioendp, wbc);
845 			if (done)
846 				break;
847 		}
848 
849 		pagevec_release(&pvec);
850 		cond_resched();
851 	}
852 }
853 
854 STATIC void
855 xfs_vm_invalidatepage(
856 	struct page		*page,
857 	unsigned int		offset,
858 	unsigned int		length)
859 {
860 	trace_xfs_invalidatepage(page->mapping->host, page, offset,
861 				 length);
862 	block_invalidatepage(page, offset, length);
863 }
864 
865 /*
866  * If the page has delalloc buffers on it, we need to punch them out before we
867  * invalidate the page. If we don't, we leave a stale delalloc mapping on the
868  * inode that can trip a BUG() in xfs_get_blocks() later on if a direct IO read
869  * is done on that same region - the delalloc extent is returned when none is
870  * supposed to be there.
871  *
872  * We prevent this by truncating away the delalloc regions on the page before
873  * invalidating it. Because they are delalloc, we can do this without needing a
874  * transaction. Indeed - if we get ENOSPC errors, we have to be able to do this
875  * truncation without a transaction as there is no space left for block
876  * reservation (typically why we see a ENOSPC in writeback).
877  *
878  * This is not a performance critical path, so for now just do the punching a
879  * buffer head at a time.
880  */
881 STATIC void
882 xfs_aops_discard_page(
883 	struct page		*page)
884 {
885 	struct inode		*inode = page->mapping->host;
886 	struct xfs_inode	*ip = XFS_I(inode);
887 	struct buffer_head	*bh, *head;
888 	loff_t			offset = page_offset(page);
889 
890 	if (!xfs_check_page_type(page, XFS_IO_DELALLOC, true))
891 		goto out_invalidate;
892 
893 	if (XFS_FORCED_SHUTDOWN(ip->i_mount))
894 		goto out_invalidate;
895 
896 	xfs_alert(ip->i_mount,
897 		"page discard on page %p, inode 0x%llx, offset %llu.",
898 			page, ip->i_ino, offset);
899 
900 	xfs_ilock(ip, XFS_ILOCK_EXCL);
901 	bh = head = page_buffers(page);
902 	do {
903 		int		error;
904 		xfs_fileoff_t	start_fsb;
905 
906 		if (!buffer_delay(bh))
907 			goto next_buffer;
908 
909 		start_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
910 		error = xfs_bmap_punch_delalloc_range(ip, start_fsb, 1);
911 		if (error) {
912 			/* something screwed, just bail */
913 			if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
914 				xfs_alert(ip->i_mount,
915 			"page discard unable to remove delalloc mapping.");
916 			}
917 			break;
918 		}
919 next_buffer:
920 		offset += 1 << inode->i_blkbits;
921 
922 	} while ((bh = bh->b_this_page) != head);
923 
924 	xfs_iunlock(ip, XFS_ILOCK_EXCL);
925 out_invalidate:
926 	xfs_vm_invalidatepage(page, 0, PAGE_CACHE_SIZE);
927 	return;
928 }
929 
930 /*
931  * Write out a dirty page.
932  *
933  * For delalloc space on the page we need to allocate space and flush it.
934  * For unwritten space on the page we need to start the conversion to
935  * regular allocated space.
936  * For any other dirty buffer heads on the page we should flush them.
937  */
938 STATIC int
939 xfs_vm_writepage(
940 	struct page		*page,
941 	struct writeback_control *wbc)
942 {
943 	struct inode		*inode = page->mapping->host;
944 	struct buffer_head	*bh, *head;
945 	struct xfs_bmbt_irec	imap;
946 	xfs_ioend_t		*ioend = NULL, *iohead = NULL;
947 	loff_t			offset;
948 	unsigned int		type;
949 	__uint64_t              end_offset;
950 	pgoff_t                 end_index, last_index;
951 	ssize_t			len;
952 	int			err, imap_valid = 0, uptodate = 1;
953 	int			count = 0;
954 	int			nonblocking = 0;
955 
956 	trace_xfs_writepage(inode, page, 0, 0);
957 
958 	ASSERT(page_has_buffers(page));
959 
960 	/*
961 	 * Refuse to write the page out if we are called from reclaim context.
962 	 *
963 	 * This avoids stack overflows when called from deeply used stacks in
964 	 * random callers for direct reclaim or memcg reclaim.  We explicitly
965 	 * allow reclaim from kswapd as the stack usage there is relatively low.
966 	 *
967 	 * This should never happen except in the case of a VM regression so
968 	 * warn about it.
969 	 */
970 	if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) ==
971 			PF_MEMALLOC))
972 		goto redirty;
973 
974 	/*
975 	 * Given that we do not allow direct reclaim to call us, we should
976 	 * never be called while in a filesystem transaction.
977 	 */
978 	if (WARN_ON_ONCE(current->flags & PF_FSTRANS))
979 		goto redirty;
980 
981 	/* Is this page beyond the end of the file? */
982 	offset = i_size_read(inode);
983 	end_index = offset >> PAGE_CACHE_SHIFT;
984 	last_index = (offset - 1) >> PAGE_CACHE_SHIFT;
985 
986 	/*
987 	 * The page index is less than the end_index, adjust the end_offset
988 	 * to the highest offset that this page should represent.
989 	 * -----------------------------------------------------
990 	 * |			file mapping	       | <EOF> |
991 	 * -----------------------------------------------------
992 	 * | Page ... | Page N-2 | Page N-1 |  Page N  |       |
993 	 * ^--------------------------------^----------|--------
994 	 * |     desired writeback range    |      see else    |
995 	 * ---------------------------------^------------------|
996 	 */
997 	if (page->index < end_index)
998 		end_offset = (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT;
999 	else {
1000 		/*
1001 		 * Check whether the page to write out is beyond or straddles
1002 		 * i_size or not.
1003 		 * -------------------------------------------------------
1004 		 * |		file mapping		        | <EOF>  |
1005 		 * -------------------------------------------------------
1006 		 * | Page ... | Page N-2 | Page N-1 |  Page N   | Beyond |
1007 		 * ^--------------------------------^-----------|---------
1008 		 * |				    |      Straddles     |
1009 		 * ---------------------------------^-----------|--------|
1010 		 */
1011 		unsigned offset_into_page = offset & (PAGE_CACHE_SIZE - 1);
1012 
1013 		/*
1014 		 * Skip the page if it is fully outside i_size, e.g. due to a
1015 		 * truncate operation that is in progress. We must redirty the
1016 		 * page so that reclaim stops reclaiming it. Otherwise
1017 		 * xfs_vm_releasepage() is called on it and gets confused.
1018 		 *
1019 		 * Note that the end_index is unsigned long, it would overflow
1020 		 * if the given offset is greater than 16TB on 32-bit system
1021 		 * and if we do check the page is fully outside i_size or not
1022 		 * via "if (page->index >= end_index + 1)" as "end_index + 1"
1023 		 * will be evaluated to 0.  Hence this page will be redirtied
1024 		 * and be written out repeatedly which would result in an
1025 		 * infinite loop, the user program that perform this operation
1026 		 * will hang.  Instead, we can verify this situation by checking
1027 		 * if the page to write is totally beyond the i_size or if it's
1028 		 * offset is just equal to the EOF.
1029 		 */
1030 		if (page->index > end_index ||
1031 		    (page->index == end_index && offset_into_page == 0))
1032 			goto redirty;
1033 
1034 		/*
1035 		 * The page straddles i_size.  It must be zeroed out on each
1036 		 * and every writepage invocation because it may be mmapped.
1037 		 * "A file is mapped in multiples of the page size.  For a file
1038 		 * that is not a multiple of the page size, the remaining
1039 		 * memory is zeroed when mapped, and writes to that region are
1040 		 * not written out to the file."
1041 		 */
1042 		zero_user_segment(page, offset_into_page, PAGE_CACHE_SIZE);
1043 
1044 		/* Adjust the end_offset to the end of file */
1045 		end_offset = offset;
1046 	}
1047 
1048 	len = 1 << inode->i_blkbits;
1049 
1050 	bh = head = page_buffers(page);
1051 	offset = page_offset(page);
1052 	type = XFS_IO_OVERWRITE;
1053 
1054 	if (wbc->sync_mode == WB_SYNC_NONE)
1055 		nonblocking = 1;
1056 
1057 	do {
1058 		int new_ioend = 0;
1059 
1060 		if (offset >= end_offset)
1061 			break;
1062 		if (!buffer_uptodate(bh))
1063 			uptodate = 0;
1064 
1065 		/*
1066 		 * set_page_dirty dirties all buffers in a page, independent
1067 		 * of their state.  The dirty state however is entirely
1068 		 * meaningless for holes (!mapped && uptodate), so skip
1069 		 * buffers covering holes here.
1070 		 */
1071 		if (!buffer_mapped(bh) && buffer_uptodate(bh)) {
1072 			imap_valid = 0;
1073 			continue;
1074 		}
1075 
1076 		if (buffer_unwritten(bh)) {
1077 			if (type != XFS_IO_UNWRITTEN) {
1078 				type = XFS_IO_UNWRITTEN;
1079 				imap_valid = 0;
1080 			}
1081 		} else if (buffer_delay(bh)) {
1082 			if (type != XFS_IO_DELALLOC) {
1083 				type = XFS_IO_DELALLOC;
1084 				imap_valid = 0;
1085 			}
1086 		} else if (buffer_uptodate(bh)) {
1087 			if (type != XFS_IO_OVERWRITE) {
1088 				type = XFS_IO_OVERWRITE;
1089 				imap_valid = 0;
1090 			}
1091 		} else {
1092 			if (PageUptodate(page))
1093 				ASSERT(buffer_mapped(bh));
1094 			/*
1095 			 * This buffer is not uptodate and will not be
1096 			 * written to disk.  Ensure that we will put any
1097 			 * subsequent writeable buffers into a new
1098 			 * ioend.
1099 			 */
1100 			imap_valid = 0;
1101 			continue;
1102 		}
1103 
1104 		if (imap_valid)
1105 			imap_valid = xfs_imap_valid(inode, &imap, offset);
1106 		if (!imap_valid) {
1107 			/*
1108 			 * If we didn't have a valid mapping then we need to
1109 			 * put the new mapping into a separate ioend structure.
1110 			 * This ensures non-contiguous extents always have
1111 			 * separate ioends, which is particularly important
1112 			 * for unwritten extent conversion at I/O completion
1113 			 * time.
1114 			 */
1115 			new_ioend = 1;
1116 			err = xfs_map_blocks(inode, offset, &imap, type,
1117 					     nonblocking);
1118 			if (err)
1119 				goto error;
1120 			imap_valid = xfs_imap_valid(inode, &imap, offset);
1121 		}
1122 		if (imap_valid) {
1123 			lock_buffer(bh);
1124 			if (type != XFS_IO_OVERWRITE)
1125 				xfs_map_at_offset(inode, bh, &imap, offset);
1126 			xfs_add_to_ioend(inode, bh, offset, type, &ioend,
1127 					 new_ioend);
1128 			count++;
1129 		}
1130 
1131 		if (!iohead)
1132 			iohead = ioend;
1133 
1134 	} while (offset += len, ((bh = bh->b_this_page) != head));
1135 
1136 	if (uptodate && bh == head)
1137 		SetPageUptodate(page);
1138 
1139 	xfs_start_page_writeback(page, 1, count);
1140 
1141 	/* if there is no IO to be submitted for this page, we are done */
1142 	if (!ioend)
1143 		return 0;
1144 
1145 	ASSERT(iohead);
1146 
1147 	/*
1148 	 * Any errors from this point onwards need tobe reported through the IO
1149 	 * completion path as we have marked the initial page as under writeback
1150 	 * and unlocked it.
1151 	 */
1152 	if (imap_valid) {
1153 		xfs_off_t		end_index;
1154 
1155 		end_index = imap.br_startoff + imap.br_blockcount;
1156 
1157 		/* to bytes */
1158 		end_index <<= inode->i_blkbits;
1159 
1160 		/* to pages */
1161 		end_index = (end_index - 1) >> PAGE_CACHE_SHIFT;
1162 
1163 		/* check against file size */
1164 		if (end_index > last_index)
1165 			end_index = last_index;
1166 
1167 		xfs_cluster_write(inode, page->index + 1, &imap, &ioend,
1168 				  wbc, end_index);
1169 	}
1170 
1171 
1172 	/*
1173 	 * Reserve log space if we might write beyond the on-disk inode size.
1174 	 */
1175 	err = 0;
1176 	if (ioend->io_type != XFS_IO_UNWRITTEN && xfs_ioend_is_append(ioend))
1177 		err = xfs_setfilesize_trans_alloc(ioend);
1178 
1179 	xfs_submit_ioend(wbc, iohead, err);
1180 
1181 	return 0;
1182 
1183 error:
1184 	if (iohead)
1185 		xfs_cancel_ioend(iohead);
1186 
1187 	if (err == -EAGAIN)
1188 		goto redirty;
1189 
1190 	xfs_aops_discard_page(page);
1191 	ClearPageUptodate(page);
1192 	unlock_page(page);
1193 	return err;
1194 
1195 redirty:
1196 	redirty_page_for_writepage(wbc, page);
1197 	unlock_page(page);
1198 	return 0;
1199 }
1200 
1201 STATIC int
1202 xfs_vm_writepages(
1203 	struct address_space	*mapping,
1204 	struct writeback_control *wbc)
1205 {
1206 	xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
1207 	return generic_writepages(mapping, wbc);
1208 }
1209 
1210 /*
1211  * Called to move a page into cleanable state - and from there
1212  * to be released. The page should already be clean. We always
1213  * have buffer heads in this call.
1214  *
1215  * Returns 1 if the page is ok to release, 0 otherwise.
1216  */
1217 STATIC int
1218 xfs_vm_releasepage(
1219 	struct page		*page,
1220 	gfp_t			gfp_mask)
1221 {
1222 	int			delalloc, unwritten;
1223 
1224 	trace_xfs_releasepage(page->mapping->host, page, 0, 0);
1225 
1226 	xfs_count_page_state(page, &delalloc, &unwritten);
1227 
1228 	if (WARN_ON_ONCE(delalloc))
1229 		return 0;
1230 	if (WARN_ON_ONCE(unwritten))
1231 		return 0;
1232 
1233 	return try_to_free_buffers(page);
1234 }
1235 
1236 STATIC int
1237 __xfs_get_blocks(
1238 	struct inode		*inode,
1239 	sector_t		iblock,
1240 	struct buffer_head	*bh_result,
1241 	int			create,
1242 	int			direct)
1243 {
1244 	struct xfs_inode	*ip = XFS_I(inode);
1245 	struct xfs_mount	*mp = ip->i_mount;
1246 	xfs_fileoff_t		offset_fsb, end_fsb;
1247 	int			error = 0;
1248 	int			lockmode = 0;
1249 	struct xfs_bmbt_irec	imap;
1250 	int			nimaps = 1;
1251 	xfs_off_t		offset;
1252 	ssize_t			size;
1253 	int			new = 0;
1254 
1255 	if (XFS_FORCED_SHUTDOWN(mp))
1256 		return -XFS_ERROR(EIO);
1257 
1258 	offset = (xfs_off_t)iblock << inode->i_blkbits;
1259 	ASSERT(bh_result->b_size >= (1 << inode->i_blkbits));
1260 	size = bh_result->b_size;
1261 
1262 	if (!create && direct && offset >= i_size_read(inode))
1263 		return 0;
1264 
1265 	/*
1266 	 * Direct I/O is usually done on preallocated files, so try getting
1267 	 * a block mapping without an exclusive lock first.  For buffered
1268 	 * writes we already have the exclusive iolock anyway, so avoiding
1269 	 * a lock roundtrip here by taking the ilock exclusive from the
1270 	 * beginning is a useful micro optimization.
1271 	 */
1272 	if (create && !direct) {
1273 		lockmode = XFS_ILOCK_EXCL;
1274 		xfs_ilock(ip, lockmode);
1275 	} else {
1276 		lockmode = xfs_ilock_data_map_shared(ip);
1277 	}
1278 
1279 	ASSERT(offset <= mp->m_super->s_maxbytes);
1280 	if (offset + size > mp->m_super->s_maxbytes)
1281 		size = mp->m_super->s_maxbytes - offset;
1282 	end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + size);
1283 	offset_fsb = XFS_B_TO_FSBT(mp, offset);
1284 
1285 	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb,
1286 				&imap, &nimaps, XFS_BMAPI_ENTIRE);
1287 	if (error)
1288 		goto out_unlock;
1289 
1290 	if (create &&
1291 	    (!nimaps ||
1292 	     (imap.br_startblock == HOLESTARTBLOCK ||
1293 	      imap.br_startblock == DELAYSTARTBLOCK))) {
1294 		if (direct || xfs_get_extsz_hint(ip)) {
1295 			/*
1296 			 * Drop the ilock in preparation for starting the block
1297 			 * allocation transaction.  It will be retaken
1298 			 * exclusively inside xfs_iomap_write_direct for the
1299 			 * actual allocation.
1300 			 */
1301 			xfs_iunlock(ip, lockmode);
1302 			error = xfs_iomap_write_direct(ip, offset, size,
1303 						       &imap, nimaps);
1304 			if (error)
1305 				return -error;
1306 			new = 1;
1307 		} else {
1308 			/*
1309 			 * Delalloc reservations do not require a transaction,
1310 			 * we can go on without dropping the lock here. If we
1311 			 * are allocating a new delalloc block, make sure that
1312 			 * we set the new flag so that we mark the buffer new so
1313 			 * that we know that it is newly allocated if the write
1314 			 * fails.
1315 			 */
1316 			if (nimaps && imap.br_startblock == HOLESTARTBLOCK)
1317 				new = 1;
1318 			error = xfs_iomap_write_delay(ip, offset, size, &imap);
1319 			if (error)
1320 				goto out_unlock;
1321 
1322 			xfs_iunlock(ip, lockmode);
1323 		}
1324 
1325 		trace_xfs_get_blocks_alloc(ip, offset, size, 0, &imap);
1326 	} else if (nimaps) {
1327 		trace_xfs_get_blocks_found(ip, offset, size, 0, &imap);
1328 		xfs_iunlock(ip, lockmode);
1329 	} else {
1330 		trace_xfs_get_blocks_notfound(ip, offset, size);
1331 		goto out_unlock;
1332 	}
1333 
1334 	if (imap.br_startblock != HOLESTARTBLOCK &&
1335 	    imap.br_startblock != DELAYSTARTBLOCK) {
1336 		/*
1337 		 * For unwritten extents do not report a disk address on
1338 		 * the read case (treat as if we're reading into a hole).
1339 		 */
1340 		if (create || !ISUNWRITTEN(&imap))
1341 			xfs_map_buffer(inode, bh_result, &imap, offset);
1342 		if (create && ISUNWRITTEN(&imap)) {
1343 			if (direct) {
1344 				bh_result->b_private = inode;
1345 				set_buffer_defer_completion(bh_result);
1346 			}
1347 			set_buffer_unwritten(bh_result);
1348 		}
1349 	}
1350 
1351 	/*
1352 	 * If this is a realtime file, data may be on a different device.
1353 	 * to that pointed to from the buffer_head b_bdev currently.
1354 	 */
1355 	bh_result->b_bdev = xfs_find_bdev_for_inode(inode);
1356 
1357 	/*
1358 	 * If we previously allocated a block out beyond eof and we are now
1359 	 * coming back to use it then we will need to flag it as new even if it
1360 	 * has a disk address.
1361 	 *
1362 	 * With sub-block writes into unwritten extents we also need to mark
1363 	 * the buffer as new so that the unwritten parts of the buffer gets
1364 	 * correctly zeroed.
1365 	 */
1366 	if (create &&
1367 	    ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) ||
1368 	     (offset >= i_size_read(inode)) ||
1369 	     (new || ISUNWRITTEN(&imap))))
1370 		set_buffer_new(bh_result);
1371 
1372 	if (imap.br_startblock == DELAYSTARTBLOCK) {
1373 		BUG_ON(direct);
1374 		if (create) {
1375 			set_buffer_uptodate(bh_result);
1376 			set_buffer_mapped(bh_result);
1377 			set_buffer_delay(bh_result);
1378 		}
1379 	}
1380 
1381 	/*
1382 	 * If this is O_DIRECT or the mpage code calling tell them how large
1383 	 * the mapping is, so that we can avoid repeated get_blocks calls.
1384 	 *
1385 	 * If the mapping spans EOF, then we have to break the mapping up as the
1386 	 * mapping for blocks beyond EOF must be marked new so that sub block
1387 	 * regions can be correctly zeroed. We can't do this for mappings within
1388 	 * EOF unless the mapping was just allocated or is unwritten, otherwise
1389 	 * the callers would overwrite existing data with zeros. Hence we have
1390 	 * to split the mapping into a range up to and including EOF, and a
1391 	 * second mapping for beyond EOF.
1392 	 */
1393 	if (direct || size > (1 << inode->i_blkbits)) {
1394 		xfs_off_t		mapping_size;
1395 
1396 		mapping_size = imap.br_startoff + imap.br_blockcount - iblock;
1397 		mapping_size <<= inode->i_blkbits;
1398 
1399 		ASSERT(mapping_size > 0);
1400 		if (mapping_size > size)
1401 			mapping_size = size;
1402 		if (offset < i_size_read(inode) &&
1403 		    offset + mapping_size >= i_size_read(inode)) {
1404 			/* limit mapping to block that spans EOF */
1405 			mapping_size = roundup_64(i_size_read(inode) - offset,
1406 						  1 << inode->i_blkbits);
1407 		}
1408 		if (mapping_size > LONG_MAX)
1409 			mapping_size = LONG_MAX;
1410 
1411 		bh_result->b_size = mapping_size;
1412 	}
1413 
1414 	return 0;
1415 
1416 out_unlock:
1417 	xfs_iunlock(ip, lockmode);
1418 	return -error;
1419 }
1420 
1421 int
1422 xfs_get_blocks(
1423 	struct inode		*inode,
1424 	sector_t		iblock,
1425 	struct buffer_head	*bh_result,
1426 	int			create)
1427 {
1428 	return __xfs_get_blocks(inode, iblock, bh_result, create, 0);
1429 }
1430 
1431 STATIC int
1432 xfs_get_blocks_direct(
1433 	struct inode		*inode,
1434 	sector_t		iblock,
1435 	struct buffer_head	*bh_result,
1436 	int			create)
1437 {
1438 	return __xfs_get_blocks(inode, iblock, bh_result, create, 1);
1439 }
1440 
1441 /*
1442  * Complete a direct I/O write request.
1443  *
1444  * If the private argument is non-NULL __xfs_get_blocks signals us that we
1445  * need to issue a transaction to convert the range from unwritten to written
1446  * extents.  In case this is regular synchronous I/O we just call xfs_end_io
1447  * to do this and we are done.  But in case this was a successful AIO
1448  * request this handler is called from interrupt context, from which we
1449  * can't start transactions.  In that case offload the I/O completion to
1450  * the workqueues we also use for buffered I/O completion.
1451  */
1452 STATIC void
1453 xfs_end_io_direct_write(
1454 	struct kiocb		*iocb,
1455 	loff_t			offset,
1456 	ssize_t			size,
1457 	void			*private)
1458 {
1459 	struct xfs_ioend	*ioend = iocb->private;
1460 
1461 	/*
1462 	 * While the generic direct I/O code updates the inode size, it does
1463 	 * so only after the end_io handler is called, which means our
1464 	 * end_io handler thinks the on-disk size is outside the in-core
1465 	 * size.  To prevent this just update it a little bit earlier here.
1466 	 */
1467 	if (offset + size > i_size_read(ioend->io_inode))
1468 		i_size_write(ioend->io_inode, offset + size);
1469 
1470 	/*
1471 	 * blockdev_direct_IO can return an error even after the I/O
1472 	 * completion handler was called.  Thus we need to protect
1473 	 * against double-freeing.
1474 	 */
1475 	iocb->private = NULL;
1476 
1477 	ioend->io_offset = offset;
1478 	ioend->io_size = size;
1479 	if (private && size > 0)
1480 		ioend->io_type = XFS_IO_UNWRITTEN;
1481 
1482 	xfs_finish_ioend_sync(ioend);
1483 }
1484 
1485 STATIC ssize_t
1486 xfs_vm_direct_IO(
1487 	int			rw,
1488 	struct kiocb		*iocb,
1489 	struct iov_iter		*iter,
1490 	loff_t			offset)
1491 {
1492 	struct inode		*inode = iocb->ki_filp->f_mapping->host;
1493 	struct block_device	*bdev = xfs_find_bdev_for_inode(inode);
1494 	struct xfs_ioend	*ioend = NULL;
1495 	ssize_t			ret;
1496 
1497 	if (rw & WRITE) {
1498 		size_t size = iov_iter_count(iter);
1499 
1500 		/*
1501 		 * We cannot preallocate a size update transaction here as we
1502 		 * don't know whether allocation is necessary or not. Hence we
1503 		 * can only tell IO completion that one is necessary if we are
1504 		 * not doing unwritten extent conversion.
1505 		 */
1506 		iocb->private = ioend = xfs_alloc_ioend(inode, XFS_IO_DIRECT);
1507 		if (offset + size > XFS_I(inode)->i_d.di_size)
1508 			ioend->io_isdirect = 1;
1509 
1510 		ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iter,
1511 					    offset, xfs_get_blocks_direct,
1512 					    xfs_end_io_direct_write, NULL,
1513 					    DIO_ASYNC_EXTEND);
1514 		if (ret != -EIOCBQUEUED && iocb->private)
1515 			goto out_destroy_ioend;
1516 	} else {
1517 		ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iter,
1518 					    offset, xfs_get_blocks_direct,
1519 					    NULL, NULL, 0);
1520 	}
1521 
1522 	return ret;
1523 
1524 out_destroy_ioend:
1525 	xfs_destroy_ioend(ioend);
1526 	return ret;
1527 }
1528 
1529 /*
1530  * Punch out the delalloc blocks we have already allocated.
1531  *
1532  * Don't bother with xfs_setattr given that nothing can have made it to disk yet
1533  * as the page is still locked at this point.
1534  */
1535 STATIC void
1536 xfs_vm_kill_delalloc_range(
1537 	struct inode		*inode,
1538 	loff_t			start,
1539 	loff_t			end)
1540 {
1541 	struct xfs_inode	*ip = XFS_I(inode);
1542 	xfs_fileoff_t		start_fsb;
1543 	xfs_fileoff_t		end_fsb;
1544 	int			error;
1545 
1546 	start_fsb = XFS_B_TO_FSB(ip->i_mount, start);
1547 	end_fsb = XFS_B_TO_FSB(ip->i_mount, end);
1548 	if (end_fsb <= start_fsb)
1549 		return;
1550 
1551 	xfs_ilock(ip, XFS_ILOCK_EXCL);
1552 	error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
1553 						end_fsb - start_fsb);
1554 	if (error) {
1555 		/* something screwed, just bail */
1556 		if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
1557 			xfs_alert(ip->i_mount,
1558 		"xfs_vm_write_failed: unable to clean up ino %lld",
1559 					ip->i_ino);
1560 		}
1561 	}
1562 	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1563 }
1564 
1565 STATIC void
1566 xfs_vm_write_failed(
1567 	struct inode		*inode,
1568 	struct page		*page,
1569 	loff_t			pos,
1570 	unsigned		len)
1571 {
1572 	loff_t			block_offset;
1573 	loff_t			block_start;
1574 	loff_t			block_end;
1575 	loff_t			from = pos & (PAGE_CACHE_SIZE - 1);
1576 	loff_t			to = from + len;
1577 	struct buffer_head	*bh, *head;
1578 
1579 	/*
1580 	 * The request pos offset might be 32 or 64 bit, this is all fine
1581 	 * on 64-bit platform.  However, for 64-bit pos request on 32-bit
1582 	 * platform, the high 32-bit will be masked off if we evaluate the
1583 	 * block_offset via (pos & PAGE_MASK) because the PAGE_MASK is
1584 	 * 0xfffff000 as an unsigned long, hence the result is incorrect
1585 	 * which could cause the following ASSERT failed in most cases.
1586 	 * In order to avoid this, we can evaluate the block_offset of the
1587 	 * start of the page by using shifts rather than masks the mismatch
1588 	 * problem.
1589 	 */
1590 	block_offset = (pos >> PAGE_CACHE_SHIFT) << PAGE_CACHE_SHIFT;
1591 
1592 	ASSERT(block_offset + from == pos);
1593 
1594 	head = page_buffers(page);
1595 	block_start = 0;
1596 	for (bh = head; bh != head || !block_start;
1597 	     bh = bh->b_this_page, block_start = block_end,
1598 				   block_offset += bh->b_size) {
1599 		block_end = block_start + bh->b_size;
1600 
1601 		/* skip buffers before the write */
1602 		if (block_end <= from)
1603 			continue;
1604 
1605 		/* if the buffer is after the write, we're done */
1606 		if (block_start >= to)
1607 			break;
1608 
1609 		if (!buffer_delay(bh))
1610 			continue;
1611 
1612 		if (!buffer_new(bh) && block_offset < i_size_read(inode))
1613 			continue;
1614 
1615 		xfs_vm_kill_delalloc_range(inode, block_offset,
1616 					   block_offset + bh->b_size);
1617 
1618 		/*
1619 		 * This buffer does not contain data anymore. make sure anyone
1620 		 * who finds it knows that for certain.
1621 		 */
1622 		clear_buffer_delay(bh);
1623 		clear_buffer_uptodate(bh);
1624 		clear_buffer_mapped(bh);
1625 		clear_buffer_new(bh);
1626 		clear_buffer_dirty(bh);
1627 	}
1628 
1629 }
1630 
1631 /*
1632  * This used to call block_write_begin(), but it unlocks and releases the page
1633  * on error, and we need that page to be able to punch stale delalloc blocks out
1634  * on failure. hence we copy-n-waste it here and call xfs_vm_write_failed() at
1635  * the appropriate point.
1636  */
1637 STATIC int
1638 xfs_vm_write_begin(
1639 	struct file		*file,
1640 	struct address_space	*mapping,
1641 	loff_t			pos,
1642 	unsigned		len,
1643 	unsigned		flags,
1644 	struct page		**pagep,
1645 	void			**fsdata)
1646 {
1647 	pgoff_t			index = pos >> PAGE_CACHE_SHIFT;
1648 	struct page		*page;
1649 	int			status;
1650 
1651 	ASSERT(len <= PAGE_CACHE_SIZE);
1652 
1653 	page = grab_cache_page_write_begin(mapping, index, flags);
1654 	if (!page)
1655 		return -ENOMEM;
1656 
1657 	status = __block_write_begin(page, pos, len, xfs_get_blocks);
1658 	if (unlikely(status)) {
1659 		struct inode	*inode = mapping->host;
1660 		size_t		isize = i_size_read(inode);
1661 
1662 		xfs_vm_write_failed(inode, page, pos, len);
1663 		unlock_page(page);
1664 
1665 		/*
1666 		 * If the write is beyond EOF, we only want to kill blocks
1667 		 * allocated in this write, not blocks that were previously
1668 		 * written successfully.
1669 		 */
1670 		if (pos + len > isize) {
1671 			ssize_t start = max_t(ssize_t, pos, isize);
1672 
1673 			truncate_pagecache_range(inode, start, pos + len);
1674 		}
1675 
1676 		page_cache_release(page);
1677 		page = NULL;
1678 	}
1679 
1680 	*pagep = page;
1681 	return status;
1682 }
1683 
1684 /*
1685  * On failure, we only need to kill delalloc blocks beyond EOF in the range of
1686  * this specific write because they will never be written. Previous writes
1687  * beyond EOF where block allocation succeeded do not need to be trashed, so
1688  * only new blocks from this write should be trashed. For blocks within
1689  * EOF, generic_write_end() zeros them so they are safe to leave alone and be
1690  * written with all the other valid data.
1691  */
1692 STATIC int
1693 xfs_vm_write_end(
1694 	struct file		*file,
1695 	struct address_space	*mapping,
1696 	loff_t			pos,
1697 	unsigned		len,
1698 	unsigned		copied,
1699 	struct page		*page,
1700 	void			*fsdata)
1701 {
1702 	int			ret;
1703 
1704 	ASSERT(len <= PAGE_CACHE_SIZE);
1705 
1706 	ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata);
1707 	if (unlikely(ret < len)) {
1708 		struct inode	*inode = mapping->host;
1709 		size_t		isize = i_size_read(inode);
1710 		loff_t		to = pos + len;
1711 
1712 		if (to > isize) {
1713 			/* only kill blocks in this write beyond EOF */
1714 			if (pos > isize)
1715 				isize = pos;
1716 			xfs_vm_kill_delalloc_range(inode, isize, to);
1717 			truncate_pagecache_range(inode, isize, to);
1718 		}
1719 	}
1720 	return ret;
1721 }
1722 
1723 STATIC sector_t
1724 xfs_vm_bmap(
1725 	struct address_space	*mapping,
1726 	sector_t		block)
1727 {
1728 	struct inode		*inode = (struct inode *)mapping->host;
1729 	struct xfs_inode	*ip = XFS_I(inode);
1730 
1731 	trace_xfs_vm_bmap(XFS_I(inode));
1732 	xfs_ilock(ip, XFS_IOLOCK_SHARED);
1733 	filemap_write_and_wait(mapping);
1734 	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
1735 	return generic_block_bmap(mapping, block, xfs_get_blocks);
1736 }
1737 
1738 STATIC int
1739 xfs_vm_readpage(
1740 	struct file		*unused,
1741 	struct page		*page)
1742 {
1743 	return mpage_readpage(page, xfs_get_blocks);
1744 }
1745 
1746 STATIC int
1747 xfs_vm_readpages(
1748 	struct file		*unused,
1749 	struct address_space	*mapping,
1750 	struct list_head	*pages,
1751 	unsigned		nr_pages)
1752 {
1753 	return mpage_readpages(mapping, pages, nr_pages, xfs_get_blocks);
1754 }
1755 
1756 const struct address_space_operations xfs_address_space_operations = {
1757 	.readpage		= xfs_vm_readpage,
1758 	.readpages		= xfs_vm_readpages,
1759 	.writepage		= xfs_vm_writepage,
1760 	.writepages		= xfs_vm_writepages,
1761 	.releasepage		= xfs_vm_releasepage,
1762 	.invalidatepage		= xfs_vm_invalidatepage,
1763 	.write_begin		= xfs_vm_write_begin,
1764 	.write_end		= xfs_vm_write_end,
1765 	.bmap			= xfs_vm_bmap,
1766 	.direct_IO		= xfs_vm_direct_IO,
1767 	.migratepage		= buffer_migrate_page,
1768 	.is_partially_uptodate  = block_is_partially_uptodate,
1769 	.error_remove_page	= generic_error_remove_page,
1770 };
1771