1 /* 2 * Copyright (c) 2000-2005 Silicon Graphics, Inc. 3 * All Rights Reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it would be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17 */ 18 #include "xfs.h" 19 #include "xfs_shared.h" 20 #include "xfs_format.h" 21 #include "xfs_log_format.h" 22 #include "xfs_trans_resv.h" 23 #include "xfs_mount.h" 24 #include "xfs_inode.h" 25 #include "xfs_trans.h" 26 #include "xfs_inode_item.h" 27 #include "xfs_alloc.h" 28 #include "xfs_error.h" 29 #include "xfs_iomap.h" 30 #include "xfs_trace.h" 31 #include "xfs_bmap.h" 32 #include "xfs_bmap_util.h" 33 #include "xfs_bmap_btree.h" 34 #include "xfs_reflink.h" 35 #include <linux/gfp.h> 36 #include <linux/mpage.h> 37 #include <linux/pagevec.h> 38 #include <linux/writeback.h> 39 40 /* flags for direct write completions */ 41 #define XFS_DIO_FLAG_UNWRITTEN (1 << 0) 42 #define XFS_DIO_FLAG_APPEND (1 << 1) 43 #define XFS_DIO_FLAG_COW (1 << 2) 44 45 /* 46 * structure owned by writepages passed to individual writepage calls 47 */ 48 struct xfs_writepage_ctx { 49 struct xfs_bmbt_irec imap; 50 bool imap_valid; 51 unsigned int io_type; 52 struct xfs_ioend *ioend; 53 sector_t last_block; 54 }; 55 56 void 57 xfs_count_page_state( 58 struct page *page, 59 int *delalloc, 60 int *unwritten) 61 { 62 struct buffer_head *bh, *head; 63 64 *delalloc = *unwritten = 0; 65 66 bh = head = page_buffers(page); 67 do { 68 if (buffer_unwritten(bh)) 69 (*unwritten) = 1; 70 else if (buffer_delay(bh)) 71 (*delalloc) = 1; 72 } while ((bh = bh->b_this_page) != head); 73 } 74 75 struct block_device * 76 xfs_find_bdev_for_inode( 77 struct inode *inode) 78 { 79 struct xfs_inode *ip = XFS_I(inode); 80 struct xfs_mount *mp = ip->i_mount; 81 82 if (XFS_IS_REALTIME_INODE(ip)) 83 return mp->m_rtdev_targp->bt_bdev; 84 else 85 return mp->m_ddev_targp->bt_bdev; 86 } 87 88 /* 89 * We're now finished for good with this page. Update the page state via the 90 * associated buffer_heads, paying attention to the start and end offsets that 91 * we need to process on the page. 92 * 93 * Landmine Warning: bh->b_end_io() will call end_page_writeback() on the last 94 * buffer in the IO. Once it does this, it is unsafe to access the bufferhead or 95 * the page at all, as we may be racing with memory reclaim and it can free both 96 * the bufferhead chain and the page as it will see the page as clean and 97 * unused. 98 */ 99 static void 100 xfs_finish_page_writeback( 101 struct inode *inode, 102 struct bio_vec *bvec, 103 int error) 104 { 105 unsigned int end = bvec->bv_offset + bvec->bv_len - 1; 106 struct buffer_head *head, *bh, *next; 107 unsigned int off = 0; 108 unsigned int bsize; 109 110 ASSERT(bvec->bv_offset < PAGE_SIZE); 111 ASSERT((bvec->bv_offset & ((1 << inode->i_blkbits) - 1)) == 0); 112 ASSERT(end < PAGE_SIZE); 113 ASSERT((bvec->bv_len & ((1 << inode->i_blkbits) - 1)) == 0); 114 115 bh = head = page_buffers(bvec->bv_page); 116 117 bsize = bh->b_size; 118 do { 119 next = bh->b_this_page; 120 if (off < bvec->bv_offset) 121 goto next_bh; 122 if (off > end) 123 break; 124 bh->b_end_io(bh, !error); 125 next_bh: 126 off += bsize; 127 } while ((bh = next) != head); 128 } 129 130 /* 131 * We're now finished for good with this ioend structure. Update the page 132 * state, release holds on bios, and finally free up memory. Do not use the 133 * ioend after this. 134 */ 135 STATIC void 136 xfs_destroy_ioend( 137 struct xfs_ioend *ioend, 138 int error) 139 { 140 struct inode *inode = ioend->io_inode; 141 struct bio *last = ioend->io_bio; 142 struct bio *bio, *next; 143 144 for (bio = &ioend->io_inline_bio; bio; bio = next) { 145 struct bio_vec *bvec; 146 int i; 147 148 /* 149 * For the last bio, bi_private points to the ioend, so we 150 * need to explicitly end the iteration here. 151 */ 152 if (bio == last) 153 next = NULL; 154 else 155 next = bio->bi_private; 156 157 /* walk each page on bio, ending page IO on them */ 158 bio_for_each_segment_all(bvec, bio, i) 159 xfs_finish_page_writeback(inode, bvec, error); 160 161 bio_put(bio); 162 } 163 } 164 165 /* 166 * Fast and loose check if this write could update the on-disk inode size. 167 */ 168 static inline bool xfs_ioend_is_append(struct xfs_ioend *ioend) 169 { 170 return ioend->io_offset + ioend->io_size > 171 XFS_I(ioend->io_inode)->i_d.di_size; 172 } 173 174 STATIC int 175 xfs_setfilesize_trans_alloc( 176 struct xfs_ioend *ioend) 177 { 178 struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount; 179 struct xfs_trans *tp; 180 int error; 181 182 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp); 183 if (error) 184 return error; 185 186 ioend->io_append_trans = tp; 187 188 /* 189 * We may pass freeze protection with a transaction. So tell lockdep 190 * we released it. 191 */ 192 __sb_writers_release(ioend->io_inode->i_sb, SB_FREEZE_FS); 193 /* 194 * We hand off the transaction to the completion thread now, so 195 * clear the flag here. 196 */ 197 current_restore_flags_nested(&tp->t_pflags, PF_FSTRANS); 198 return 0; 199 } 200 201 /* 202 * Update on-disk file size now that data has been written to disk. 203 */ 204 STATIC int 205 __xfs_setfilesize( 206 struct xfs_inode *ip, 207 struct xfs_trans *tp, 208 xfs_off_t offset, 209 size_t size) 210 { 211 xfs_fsize_t isize; 212 213 xfs_ilock(ip, XFS_ILOCK_EXCL); 214 isize = xfs_new_eof(ip, offset + size); 215 if (!isize) { 216 xfs_iunlock(ip, XFS_ILOCK_EXCL); 217 xfs_trans_cancel(tp); 218 return 0; 219 } 220 221 trace_xfs_setfilesize(ip, offset, size); 222 223 ip->i_d.di_size = isize; 224 xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); 225 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 226 227 return xfs_trans_commit(tp); 228 } 229 230 int 231 xfs_setfilesize( 232 struct xfs_inode *ip, 233 xfs_off_t offset, 234 size_t size) 235 { 236 struct xfs_mount *mp = ip->i_mount; 237 struct xfs_trans *tp; 238 int error; 239 240 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp); 241 if (error) 242 return error; 243 244 return __xfs_setfilesize(ip, tp, offset, size); 245 } 246 247 STATIC int 248 xfs_setfilesize_ioend( 249 struct xfs_ioend *ioend, 250 int error) 251 { 252 struct xfs_inode *ip = XFS_I(ioend->io_inode); 253 struct xfs_trans *tp = ioend->io_append_trans; 254 255 /* 256 * The transaction may have been allocated in the I/O submission thread, 257 * thus we need to mark ourselves as being in a transaction manually. 258 * Similarly for freeze protection. 259 */ 260 current_set_flags_nested(&tp->t_pflags, PF_FSTRANS); 261 __sb_writers_acquired(VFS_I(ip)->i_sb, SB_FREEZE_FS); 262 263 /* we abort the update if there was an IO error */ 264 if (error) { 265 xfs_trans_cancel(tp); 266 return error; 267 } 268 269 return __xfs_setfilesize(ip, tp, ioend->io_offset, ioend->io_size); 270 } 271 272 /* 273 * IO write completion. 274 */ 275 STATIC void 276 xfs_end_io( 277 struct work_struct *work) 278 { 279 struct xfs_ioend *ioend = 280 container_of(work, struct xfs_ioend, io_work); 281 struct xfs_inode *ip = XFS_I(ioend->io_inode); 282 int error = ioend->io_bio->bi_error; 283 284 /* 285 * Set an error if the mount has shut down and proceed with end I/O 286 * processing so it can perform whatever cleanups are necessary. 287 */ 288 if (XFS_FORCED_SHUTDOWN(ip->i_mount)) 289 error = -EIO; 290 291 /* 292 * For a CoW extent, we need to move the mapping from the CoW fork 293 * to the data fork. If instead an error happened, just dump the 294 * new blocks. 295 */ 296 if (ioend->io_type == XFS_IO_COW) { 297 if (error) 298 goto done; 299 if (ioend->io_bio->bi_error) { 300 error = xfs_reflink_cancel_cow_range(ip, 301 ioend->io_offset, ioend->io_size); 302 goto done; 303 } 304 error = xfs_reflink_end_cow(ip, ioend->io_offset, 305 ioend->io_size); 306 if (error) 307 goto done; 308 } 309 310 /* 311 * For unwritten extents we need to issue transactions to convert a 312 * range to normal written extens after the data I/O has finished. 313 * Detecting and handling completion IO errors is done individually 314 * for each case as different cleanup operations need to be performed 315 * on error. 316 */ 317 if (ioend->io_type == XFS_IO_UNWRITTEN) { 318 if (error) 319 goto done; 320 error = xfs_iomap_write_unwritten(ip, ioend->io_offset, 321 ioend->io_size); 322 } else if (ioend->io_append_trans) { 323 error = xfs_setfilesize_ioend(ioend, error); 324 } else { 325 ASSERT(!xfs_ioend_is_append(ioend) || 326 ioend->io_type == XFS_IO_COW); 327 } 328 329 done: 330 xfs_destroy_ioend(ioend, error); 331 } 332 333 STATIC void 334 xfs_end_bio( 335 struct bio *bio) 336 { 337 struct xfs_ioend *ioend = bio->bi_private; 338 struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount; 339 340 if (ioend->io_type == XFS_IO_UNWRITTEN || ioend->io_type == XFS_IO_COW) 341 queue_work(mp->m_unwritten_workqueue, &ioend->io_work); 342 else if (ioend->io_append_trans) 343 queue_work(mp->m_data_workqueue, &ioend->io_work); 344 else 345 xfs_destroy_ioend(ioend, bio->bi_error); 346 } 347 348 STATIC int 349 xfs_map_blocks( 350 struct inode *inode, 351 loff_t offset, 352 struct xfs_bmbt_irec *imap, 353 int type) 354 { 355 struct xfs_inode *ip = XFS_I(inode); 356 struct xfs_mount *mp = ip->i_mount; 357 ssize_t count = 1 << inode->i_blkbits; 358 xfs_fileoff_t offset_fsb, end_fsb; 359 int error = 0; 360 int bmapi_flags = XFS_BMAPI_ENTIRE; 361 int nimaps = 1; 362 363 if (XFS_FORCED_SHUTDOWN(mp)) 364 return -EIO; 365 366 ASSERT(type != XFS_IO_COW); 367 if (type == XFS_IO_UNWRITTEN) 368 bmapi_flags |= XFS_BMAPI_IGSTATE; 369 370 xfs_ilock(ip, XFS_ILOCK_SHARED); 371 ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE || 372 (ip->i_df.if_flags & XFS_IFEXTENTS)); 373 ASSERT(offset <= mp->m_super->s_maxbytes); 374 375 if (offset + count > mp->m_super->s_maxbytes) 376 count = mp->m_super->s_maxbytes - offset; 377 end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count); 378 offset_fsb = XFS_B_TO_FSBT(mp, offset); 379 error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, 380 imap, &nimaps, bmapi_flags); 381 /* 382 * Truncate an overwrite extent if there's a pending CoW 383 * reservation before the end of this extent. This forces us 384 * to come back to writepage to take care of the CoW. 385 */ 386 if (nimaps && type == XFS_IO_OVERWRITE) 387 xfs_reflink_trim_irec_to_next_cow(ip, offset_fsb, imap); 388 xfs_iunlock(ip, XFS_ILOCK_SHARED); 389 390 if (error) 391 return error; 392 393 if (type == XFS_IO_DELALLOC && 394 (!nimaps || isnullstartblock(imap->br_startblock))) { 395 error = xfs_iomap_write_allocate(ip, XFS_DATA_FORK, offset, 396 imap); 397 if (!error) 398 trace_xfs_map_blocks_alloc(ip, offset, count, type, imap); 399 return error; 400 } 401 402 #ifdef DEBUG 403 if (type == XFS_IO_UNWRITTEN) { 404 ASSERT(nimaps); 405 ASSERT(imap->br_startblock != HOLESTARTBLOCK); 406 ASSERT(imap->br_startblock != DELAYSTARTBLOCK); 407 } 408 #endif 409 if (nimaps) 410 trace_xfs_map_blocks_found(ip, offset, count, type, imap); 411 return 0; 412 } 413 414 STATIC bool 415 xfs_imap_valid( 416 struct inode *inode, 417 struct xfs_bmbt_irec *imap, 418 xfs_off_t offset) 419 { 420 offset >>= inode->i_blkbits; 421 422 return offset >= imap->br_startoff && 423 offset < imap->br_startoff + imap->br_blockcount; 424 } 425 426 STATIC void 427 xfs_start_buffer_writeback( 428 struct buffer_head *bh) 429 { 430 ASSERT(buffer_mapped(bh)); 431 ASSERT(buffer_locked(bh)); 432 ASSERT(!buffer_delay(bh)); 433 ASSERT(!buffer_unwritten(bh)); 434 435 mark_buffer_async_write(bh); 436 set_buffer_uptodate(bh); 437 clear_buffer_dirty(bh); 438 } 439 440 STATIC void 441 xfs_start_page_writeback( 442 struct page *page, 443 int clear_dirty) 444 { 445 ASSERT(PageLocked(page)); 446 ASSERT(!PageWriteback(page)); 447 448 /* 449 * if the page was not fully cleaned, we need to ensure that the higher 450 * layers come back to it correctly. That means we need to keep the page 451 * dirty, and for WB_SYNC_ALL writeback we need to ensure the 452 * PAGECACHE_TAG_TOWRITE index mark is not removed so another attempt to 453 * write this page in this writeback sweep will be made. 454 */ 455 if (clear_dirty) { 456 clear_page_dirty_for_io(page); 457 set_page_writeback(page); 458 } else 459 set_page_writeback_keepwrite(page); 460 461 unlock_page(page); 462 } 463 464 static inline int xfs_bio_add_buffer(struct bio *bio, struct buffer_head *bh) 465 { 466 return bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh)); 467 } 468 469 /* 470 * Submit the bio for an ioend. We are passed an ioend with a bio attached to 471 * it, and we submit that bio. The ioend may be used for multiple bio 472 * submissions, so we only want to allocate an append transaction for the ioend 473 * once. In the case of multiple bio submission, each bio will take an IO 474 * reference to the ioend to ensure that the ioend completion is only done once 475 * all bios have been submitted and the ioend is really done. 476 * 477 * If @fail is non-zero, it means that we have a situation where some part of 478 * the submission process has failed after we have marked paged for writeback 479 * and unlocked them. In this situation, we need to fail the bio and ioend 480 * rather than submit it to IO. This typically only happens on a filesystem 481 * shutdown. 482 */ 483 STATIC int 484 xfs_submit_ioend( 485 struct writeback_control *wbc, 486 struct xfs_ioend *ioend, 487 int status) 488 { 489 /* Reserve log space if we might write beyond the on-disk inode size. */ 490 if (!status && 491 ioend->io_type != XFS_IO_UNWRITTEN && 492 xfs_ioend_is_append(ioend) && 493 !ioend->io_append_trans) 494 status = xfs_setfilesize_trans_alloc(ioend); 495 496 ioend->io_bio->bi_private = ioend; 497 ioend->io_bio->bi_end_io = xfs_end_bio; 498 bio_set_op_attrs(ioend->io_bio, REQ_OP_WRITE, 499 (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : 0); 500 /* 501 * If we are failing the IO now, just mark the ioend with an 502 * error and finish it. This will run IO completion immediately 503 * as there is only one reference to the ioend at this point in 504 * time. 505 */ 506 if (status) { 507 ioend->io_bio->bi_error = status; 508 bio_endio(ioend->io_bio); 509 return status; 510 } 511 512 submit_bio(ioend->io_bio); 513 return 0; 514 } 515 516 static void 517 xfs_init_bio_from_bh( 518 struct bio *bio, 519 struct buffer_head *bh) 520 { 521 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); 522 bio->bi_bdev = bh->b_bdev; 523 } 524 525 static struct xfs_ioend * 526 xfs_alloc_ioend( 527 struct inode *inode, 528 unsigned int type, 529 xfs_off_t offset, 530 struct buffer_head *bh) 531 { 532 struct xfs_ioend *ioend; 533 struct bio *bio; 534 535 bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, xfs_ioend_bioset); 536 xfs_init_bio_from_bh(bio, bh); 537 538 ioend = container_of(bio, struct xfs_ioend, io_inline_bio); 539 INIT_LIST_HEAD(&ioend->io_list); 540 ioend->io_type = type; 541 ioend->io_inode = inode; 542 ioend->io_size = 0; 543 ioend->io_offset = offset; 544 INIT_WORK(&ioend->io_work, xfs_end_io); 545 ioend->io_append_trans = NULL; 546 ioend->io_bio = bio; 547 return ioend; 548 } 549 550 /* 551 * Allocate a new bio, and chain the old bio to the new one. 552 * 553 * Note that we have to do perform the chaining in this unintuitive order 554 * so that the bi_private linkage is set up in the right direction for the 555 * traversal in xfs_destroy_ioend(). 556 */ 557 static void 558 xfs_chain_bio( 559 struct xfs_ioend *ioend, 560 struct writeback_control *wbc, 561 struct buffer_head *bh) 562 { 563 struct bio *new; 564 565 new = bio_alloc(GFP_NOFS, BIO_MAX_PAGES); 566 xfs_init_bio_from_bh(new, bh); 567 568 bio_chain(ioend->io_bio, new); 569 bio_get(ioend->io_bio); /* for xfs_destroy_ioend */ 570 bio_set_op_attrs(ioend->io_bio, REQ_OP_WRITE, 571 (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : 0); 572 submit_bio(ioend->io_bio); 573 ioend->io_bio = new; 574 } 575 576 /* 577 * Test to see if we've been building up a completion structure for 578 * earlier buffers -- if so, we try to append to this ioend if we 579 * can, otherwise we finish off any current ioend and start another. 580 * Return the ioend we finished off so that the caller can submit it 581 * once it has finished processing the dirty page. 582 */ 583 STATIC void 584 xfs_add_to_ioend( 585 struct inode *inode, 586 struct buffer_head *bh, 587 xfs_off_t offset, 588 struct xfs_writepage_ctx *wpc, 589 struct writeback_control *wbc, 590 struct list_head *iolist) 591 { 592 if (!wpc->ioend || wpc->io_type != wpc->ioend->io_type || 593 bh->b_blocknr != wpc->last_block + 1 || 594 offset != wpc->ioend->io_offset + wpc->ioend->io_size) { 595 if (wpc->ioend) 596 list_add(&wpc->ioend->io_list, iolist); 597 wpc->ioend = xfs_alloc_ioend(inode, wpc->io_type, offset, bh); 598 } 599 600 /* 601 * If the buffer doesn't fit into the bio we need to allocate a new 602 * one. This shouldn't happen more than once for a given buffer. 603 */ 604 while (xfs_bio_add_buffer(wpc->ioend->io_bio, bh) != bh->b_size) 605 xfs_chain_bio(wpc->ioend, wbc, bh); 606 607 wpc->ioend->io_size += bh->b_size; 608 wpc->last_block = bh->b_blocknr; 609 xfs_start_buffer_writeback(bh); 610 } 611 612 STATIC void 613 xfs_map_buffer( 614 struct inode *inode, 615 struct buffer_head *bh, 616 struct xfs_bmbt_irec *imap, 617 xfs_off_t offset) 618 { 619 sector_t bn; 620 struct xfs_mount *m = XFS_I(inode)->i_mount; 621 xfs_off_t iomap_offset = XFS_FSB_TO_B(m, imap->br_startoff); 622 xfs_daddr_t iomap_bn = xfs_fsb_to_db(XFS_I(inode), imap->br_startblock); 623 624 ASSERT(imap->br_startblock != HOLESTARTBLOCK); 625 ASSERT(imap->br_startblock != DELAYSTARTBLOCK); 626 627 bn = (iomap_bn >> (inode->i_blkbits - BBSHIFT)) + 628 ((offset - iomap_offset) >> inode->i_blkbits); 629 630 ASSERT(bn || XFS_IS_REALTIME_INODE(XFS_I(inode))); 631 632 bh->b_blocknr = bn; 633 set_buffer_mapped(bh); 634 } 635 636 STATIC void 637 xfs_map_at_offset( 638 struct inode *inode, 639 struct buffer_head *bh, 640 struct xfs_bmbt_irec *imap, 641 xfs_off_t offset) 642 { 643 ASSERT(imap->br_startblock != HOLESTARTBLOCK); 644 ASSERT(imap->br_startblock != DELAYSTARTBLOCK); 645 646 xfs_map_buffer(inode, bh, imap, offset); 647 set_buffer_mapped(bh); 648 clear_buffer_delay(bh); 649 clear_buffer_unwritten(bh); 650 } 651 652 /* 653 * Test if a given page contains at least one buffer of a given @type. 654 * If @check_all_buffers is true, then we walk all the buffers in the page to 655 * try to find one of the type passed in. If it is not set, then the caller only 656 * needs to check the first buffer on the page for a match. 657 */ 658 STATIC bool 659 xfs_check_page_type( 660 struct page *page, 661 unsigned int type, 662 bool check_all_buffers) 663 { 664 struct buffer_head *bh; 665 struct buffer_head *head; 666 667 if (PageWriteback(page)) 668 return false; 669 if (!page->mapping) 670 return false; 671 if (!page_has_buffers(page)) 672 return false; 673 674 bh = head = page_buffers(page); 675 do { 676 if (buffer_unwritten(bh)) { 677 if (type == XFS_IO_UNWRITTEN) 678 return true; 679 } else if (buffer_delay(bh)) { 680 if (type == XFS_IO_DELALLOC) 681 return true; 682 } else if (buffer_dirty(bh) && buffer_mapped(bh)) { 683 if (type == XFS_IO_OVERWRITE) 684 return true; 685 } 686 687 /* If we are only checking the first buffer, we are done now. */ 688 if (!check_all_buffers) 689 break; 690 } while ((bh = bh->b_this_page) != head); 691 692 return false; 693 } 694 695 STATIC void 696 xfs_vm_invalidatepage( 697 struct page *page, 698 unsigned int offset, 699 unsigned int length) 700 { 701 trace_xfs_invalidatepage(page->mapping->host, page, offset, 702 length); 703 block_invalidatepage(page, offset, length); 704 } 705 706 /* 707 * If the page has delalloc buffers on it, we need to punch them out before we 708 * invalidate the page. If we don't, we leave a stale delalloc mapping on the 709 * inode that can trip a BUG() in xfs_get_blocks() later on if a direct IO read 710 * is done on that same region - the delalloc extent is returned when none is 711 * supposed to be there. 712 * 713 * We prevent this by truncating away the delalloc regions on the page before 714 * invalidating it. Because they are delalloc, we can do this without needing a 715 * transaction. Indeed - if we get ENOSPC errors, we have to be able to do this 716 * truncation without a transaction as there is no space left for block 717 * reservation (typically why we see a ENOSPC in writeback). 718 * 719 * This is not a performance critical path, so for now just do the punching a 720 * buffer head at a time. 721 */ 722 STATIC void 723 xfs_aops_discard_page( 724 struct page *page) 725 { 726 struct inode *inode = page->mapping->host; 727 struct xfs_inode *ip = XFS_I(inode); 728 struct buffer_head *bh, *head; 729 loff_t offset = page_offset(page); 730 731 if (!xfs_check_page_type(page, XFS_IO_DELALLOC, true)) 732 goto out_invalidate; 733 734 if (XFS_FORCED_SHUTDOWN(ip->i_mount)) 735 goto out_invalidate; 736 737 xfs_alert(ip->i_mount, 738 "page discard on page %p, inode 0x%llx, offset %llu.", 739 page, ip->i_ino, offset); 740 741 xfs_ilock(ip, XFS_ILOCK_EXCL); 742 bh = head = page_buffers(page); 743 do { 744 int error; 745 xfs_fileoff_t start_fsb; 746 747 if (!buffer_delay(bh)) 748 goto next_buffer; 749 750 start_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 751 error = xfs_bmap_punch_delalloc_range(ip, start_fsb, 1); 752 if (error) { 753 /* something screwed, just bail */ 754 if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) { 755 xfs_alert(ip->i_mount, 756 "page discard unable to remove delalloc mapping."); 757 } 758 break; 759 } 760 next_buffer: 761 offset += 1 << inode->i_blkbits; 762 763 } while ((bh = bh->b_this_page) != head); 764 765 xfs_iunlock(ip, XFS_ILOCK_EXCL); 766 out_invalidate: 767 xfs_vm_invalidatepage(page, 0, PAGE_SIZE); 768 return; 769 } 770 771 static int 772 xfs_map_cow( 773 struct xfs_writepage_ctx *wpc, 774 struct inode *inode, 775 loff_t offset, 776 unsigned int *new_type) 777 { 778 struct xfs_inode *ip = XFS_I(inode); 779 struct xfs_bmbt_irec imap; 780 bool is_cow = false, need_alloc = false; 781 int error; 782 783 /* 784 * If we already have a valid COW mapping keep using it. 785 */ 786 if (wpc->io_type == XFS_IO_COW) { 787 wpc->imap_valid = xfs_imap_valid(inode, &wpc->imap, offset); 788 if (wpc->imap_valid) { 789 *new_type = XFS_IO_COW; 790 return 0; 791 } 792 } 793 794 /* 795 * Else we need to check if there is a COW mapping at this offset. 796 */ 797 xfs_ilock(ip, XFS_ILOCK_SHARED); 798 is_cow = xfs_reflink_find_cow_mapping(ip, offset, &imap, &need_alloc); 799 xfs_iunlock(ip, XFS_ILOCK_SHARED); 800 801 if (!is_cow) 802 return 0; 803 804 /* 805 * And if the COW mapping has a delayed extent here we need to 806 * allocate real space for it now. 807 */ 808 if (need_alloc) { 809 error = xfs_iomap_write_allocate(ip, XFS_COW_FORK, offset, 810 &imap); 811 if (error) 812 return error; 813 } 814 815 wpc->io_type = *new_type = XFS_IO_COW; 816 wpc->imap_valid = true; 817 wpc->imap = imap; 818 return 0; 819 } 820 821 /* 822 * We implement an immediate ioend submission policy here to avoid needing to 823 * chain multiple ioends and hence nest mempool allocations which can violate 824 * forward progress guarantees we need to provide. The current ioend we are 825 * adding buffers to is cached on the writepage context, and if the new buffer 826 * does not append to the cached ioend it will create a new ioend and cache that 827 * instead. 828 * 829 * If a new ioend is created and cached, the old ioend is returned and queued 830 * locally for submission once the entire page is processed or an error has been 831 * detected. While ioends are submitted immediately after they are completed, 832 * batching optimisations are provided by higher level block plugging. 833 * 834 * At the end of a writeback pass, there will be a cached ioend remaining on the 835 * writepage context that the caller will need to submit. 836 */ 837 static int 838 xfs_writepage_map( 839 struct xfs_writepage_ctx *wpc, 840 struct writeback_control *wbc, 841 struct inode *inode, 842 struct page *page, 843 loff_t offset, 844 __uint64_t end_offset) 845 { 846 LIST_HEAD(submit_list); 847 struct xfs_ioend *ioend, *next; 848 struct buffer_head *bh, *head; 849 ssize_t len = 1 << inode->i_blkbits; 850 int error = 0; 851 int count = 0; 852 int uptodate = 1; 853 unsigned int new_type; 854 855 bh = head = page_buffers(page); 856 offset = page_offset(page); 857 do { 858 if (offset >= end_offset) 859 break; 860 if (!buffer_uptodate(bh)) 861 uptodate = 0; 862 863 /* 864 * set_page_dirty dirties all buffers in a page, independent 865 * of their state. The dirty state however is entirely 866 * meaningless for holes (!mapped && uptodate), so skip 867 * buffers covering holes here. 868 */ 869 if (!buffer_mapped(bh) && buffer_uptodate(bh)) { 870 wpc->imap_valid = false; 871 continue; 872 } 873 874 if (buffer_unwritten(bh)) 875 new_type = XFS_IO_UNWRITTEN; 876 else if (buffer_delay(bh)) 877 new_type = XFS_IO_DELALLOC; 878 else if (buffer_uptodate(bh)) 879 new_type = XFS_IO_OVERWRITE; 880 else { 881 if (PageUptodate(page)) 882 ASSERT(buffer_mapped(bh)); 883 /* 884 * This buffer is not uptodate and will not be 885 * written to disk. Ensure that we will put any 886 * subsequent writeable buffers into a new 887 * ioend. 888 */ 889 wpc->imap_valid = false; 890 continue; 891 } 892 893 if (xfs_is_reflink_inode(XFS_I(inode))) { 894 error = xfs_map_cow(wpc, inode, offset, &new_type); 895 if (error) 896 goto out; 897 } 898 899 if (wpc->io_type != new_type) { 900 wpc->io_type = new_type; 901 wpc->imap_valid = false; 902 } 903 904 if (wpc->imap_valid) 905 wpc->imap_valid = xfs_imap_valid(inode, &wpc->imap, 906 offset); 907 if (!wpc->imap_valid) { 908 error = xfs_map_blocks(inode, offset, &wpc->imap, 909 wpc->io_type); 910 if (error) 911 goto out; 912 wpc->imap_valid = xfs_imap_valid(inode, &wpc->imap, 913 offset); 914 } 915 if (wpc->imap_valid) { 916 lock_buffer(bh); 917 if (wpc->io_type != XFS_IO_OVERWRITE) 918 xfs_map_at_offset(inode, bh, &wpc->imap, offset); 919 xfs_add_to_ioend(inode, bh, offset, wpc, wbc, &submit_list); 920 count++; 921 } 922 923 } while (offset += len, ((bh = bh->b_this_page) != head)); 924 925 if (uptodate && bh == head) 926 SetPageUptodate(page); 927 928 ASSERT(wpc->ioend || list_empty(&submit_list)); 929 930 out: 931 /* 932 * On error, we have to fail the ioend here because we have locked 933 * buffers in the ioend. If we don't do this, we'll deadlock 934 * invalidating the page as that tries to lock the buffers on the page. 935 * Also, because we may have set pages under writeback, we have to make 936 * sure we run IO completion to mark the error state of the IO 937 * appropriately, so we can't cancel the ioend directly here. That means 938 * we have to mark this page as under writeback if we included any 939 * buffers from it in the ioend chain so that completion treats it 940 * correctly. 941 * 942 * If we didn't include the page in the ioend, the on error we can 943 * simply discard and unlock it as there are no other users of the page 944 * or it's buffers right now. The caller will still need to trigger 945 * submission of outstanding ioends on the writepage context so they are 946 * treated correctly on error. 947 */ 948 if (count) { 949 xfs_start_page_writeback(page, !error); 950 951 /* 952 * Preserve the original error if there was one, otherwise catch 953 * submission errors here and propagate into subsequent ioend 954 * submissions. 955 */ 956 list_for_each_entry_safe(ioend, next, &submit_list, io_list) { 957 int error2; 958 959 list_del_init(&ioend->io_list); 960 error2 = xfs_submit_ioend(wbc, ioend, error); 961 if (error2 && !error) 962 error = error2; 963 } 964 } else if (error) { 965 xfs_aops_discard_page(page); 966 ClearPageUptodate(page); 967 unlock_page(page); 968 } else { 969 /* 970 * We can end up here with no error and nothing to write if we 971 * race with a partial page truncate on a sub-page block sized 972 * filesystem. In that case we need to mark the page clean. 973 */ 974 xfs_start_page_writeback(page, 1); 975 end_page_writeback(page); 976 } 977 978 mapping_set_error(page->mapping, error); 979 return error; 980 } 981 982 /* 983 * Write out a dirty page. 984 * 985 * For delalloc space on the page we need to allocate space and flush it. 986 * For unwritten space on the page we need to start the conversion to 987 * regular allocated space. 988 * For any other dirty buffer heads on the page we should flush them. 989 */ 990 STATIC int 991 xfs_do_writepage( 992 struct page *page, 993 struct writeback_control *wbc, 994 void *data) 995 { 996 struct xfs_writepage_ctx *wpc = data; 997 struct inode *inode = page->mapping->host; 998 loff_t offset; 999 __uint64_t end_offset; 1000 pgoff_t end_index; 1001 1002 trace_xfs_writepage(inode, page, 0, 0); 1003 1004 ASSERT(page_has_buffers(page)); 1005 1006 /* 1007 * Refuse to write the page out if we are called from reclaim context. 1008 * 1009 * This avoids stack overflows when called from deeply used stacks in 1010 * random callers for direct reclaim or memcg reclaim. We explicitly 1011 * allow reclaim from kswapd as the stack usage there is relatively low. 1012 * 1013 * This should never happen except in the case of a VM regression so 1014 * warn about it. 1015 */ 1016 if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) == 1017 PF_MEMALLOC)) 1018 goto redirty; 1019 1020 /* 1021 * Given that we do not allow direct reclaim to call us, we should 1022 * never be called while in a filesystem transaction. 1023 */ 1024 if (WARN_ON_ONCE(current->flags & PF_FSTRANS)) 1025 goto redirty; 1026 1027 /* 1028 * Is this page beyond the end of the file? 1029 * 1030 * The page index is less than the end_index, adjust the end_offset 1031 * to the highest offset that this page should represent. 1032 * ----------------------------------------------------- 1033 * | file mapping | <EOF> | 1034 * ----------------------------------------------------- 1035 * | Page ... | Page N-2 | Page N-1 | Page N | | 1036 * ^--------------------------------^----------|-------- 1037 * | desired writeback range | see else | 1038 * ---------------------------------^------------------| 1039 */ 1040 offset = i_size_read(inode); 1041 end_index = offset >> PAGE_SHIFT; 1042 if (page->index < end_index) 1043 end_offset = (xfs_off_t)(page->index + 1) << PAGE_SHIFT; 1044 else { 1045 /* 1046 * Check whether the page to write out is beyond or straddles 1047 * i_size or not. 1048 * ------------------------------------------------------- 1049 * | file mapping | <EOF> | 1050 * ------------------------------------------------------- 1051 * | Page ... | Page N-2 | Page N-1 | Page N | Beyond | 1052 * ^--------------------------------^-----------|--------- 1053 * | | Straddles | 1054 * ---------------------------------^-----------|--------| 1055 */ 1056 unsigned offset_into_page = offset & (PAGE_SIZE - 1); 1057 1058 /* 1059 * Skip the page if it is fully outside i_size, e.g. due to a 1060 * truncate operation that is in progress. We must redirty the 1061 * page so that reclaim stops reclaiming it. Otherwise 1062 * xfs_vm_releasepage() is called on it and gets confused. 1063 * 1064 * Note that the end_index is unsigned long, it would overflow 1065 * if the given offset is greater than 16TB on 32-bit system 1066 * and if we do check the page is fully outside i_size or not 1067 * via "if (page->index >= end_index + 1)" as "end_index + 1" 1068 * will be evaluated to 0. Hence this page will be redirtied 1069 * and be written out repeatedly which would result in an 1070 * infinite loop, the user program that perform this operation 1071 * will hang. Instead, we can verify this situation by checking 1072 * if the page to write is totally beyond the i_size or if it's 1073 * offset is just equal to the EOF. 1074 */ 1075 if (page->index > end_index || 1076 (page->index == end_index && offset_into_page == 0)) 1077 goto redirty; 1078 1079 /* 1080 * The page straddles i_size. It must be zeroed out on each 1081 * and every writepage invocation because it may be mmapped. 1082 * "A file is mapped in multiples of the page size. For a file 1083 * that is not a multiple of the page size, the remaining 1084 * memory is zeroed when mapped, and writes to that region are 1085 * not written out to the file." 1086 */ 1087 zero_user_segment(page, offset_into_page, PAGE_SIZE); 1088 1089 /* Adjust the end_offset to the end of file */ 1090 end_offset = offset; 1091 } 1092 1093 return xfs_writepage_map(wpc, wbc, inode, page, offset, end_offset); 1094 1095 redirty: 1096 redirty_page_for_writepage(wbc, page); 1097 unlock_page(page); 1098 return 0; 1099 } 1100 1101 STATIC int 1102 xfs_vm_writepage( 1103 struct page *page, 1104 struct writeback_control *wbc) 1105 { 1106 struct xfs_writepage_ctx wpc = { 1107 .io_type = XFS_IO_INVALID, 1108 }; 1109 int ret; 1110 1111 ret = xfs_do_writepage(page, wbc, &wpc); 1112 if (wpc.ioend) 1113 ret = xfs_submit_ioend(wbc, wpc.ioend, ret); 1114 return ret; 1115 } 1116 1117 STATIC int 1118 xfs_vm_writepages( 1119 struct address_space *mapping, 1120 struct writeback_control *wbc) 1121 { 1122 struct xfs_writepage_ctx wpc = { 1123 .io_type = XFS_IO_INVALID, 1124 }; 1125 int ret; 1126 1127 xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED); 1128 if (dax_mapping(mapping)) 1129 return dax_writeback_mapping_range(mapping, 1130 xfs_find_bdev_for_inode(mapping->host), wbc); 1131 1132 ret = write_cache_pages(mapping, wbc, xfs_do_writepage, &wpc); 1133 if (wpc.ioend) 1134 ret = xfs_submit_ioend(wbc, wpc.ioend, ret); 1135 return ret; 1136 } 1137 1138 /* 1139 * Called to move a page into cleanable state - and from there 1140 * to be released. The page should already be clean. We always 1141 * have buffer heads in this call. 1142 * 1143 * Returns 1 if the page is ok to release, 0 otherwise. 1144 */ 1145 STATIC int 1146 xfs_vm_releasepage( 1147 struct page *page, 1148 gfp_t gfp_mask) 1149 { 1150 int delalloc, unwritten; 1151 1152 trace_xfs_releasepage(page->mapping->host, page, 0, 0); 1153 1154 /* 1155 * mm accommodates an old ext3 case where clean pages might not have had 1156 * the dirty bit cleared. Thus, it can send actual dirty pages to 1157 * ->releasepage() via shrink_active_list(). Conversely, 1158 * block_invalidatepage() can send pages that are still marked dirty 1159 * but otherwise have invalidated buffers. 1160 * 1161 * We've historically freed buffers on the latter. Instead, quietly 1162 * filter out all dirty pages to avoid spurious buffer state warnings. 1163 * This can likely be removed once shrink_active_list() is fixed. 1164 */ 1165 if (PageDirty(page)) 1166 return 0; 1167 1168 xfs_count_page_state(page, &delalloc, &unwritten); 1169 1170 if (WARN_ON_ONCE(delalloc)) 1171 return 0; 1172 if (WARN_ON_ONCE(unwritten)) 1173 return 0; 1174 1175 return try_to_free_buffers(page); 1176 } 1177 1178 /* 1179 * When we map a DIO buffer, we may need to pass flags to 1180 * xfs_end_io_direct_write to tell it what kind of write IO we are doing. 1181 * 1182 * Note that for DIO, an IO to the highest supported file block offset (i.e. 1183 * 2^63 - 1FSB bytes) will result in the offset + count overflowing a signed 64 1184 * bit variable. Hence if we see this overflow, we have to assume that the IO is 1185 * extending the file size. We won't know for sure until IO completion is run 1186 * and the actual max write offset is communicated to the IO completion 1187 * routine. 1188 */ 1189 static void 1190 xfs_map_direct( 1191 struct inode *inode, 1192 struct buffer_head *bh_result, 1193 struct xfs_bmbt_irec *imap, 1194 xfs_off_t offset, 1195 bool is_cow) 1196 { 1197 uintptr_t *flags = (uintptr_t *)&bh_result->b_private; 1198 xfs_off_t size = bh_result->b_size; 1199 1200 trace_xfs_get_blocks_map_direct(XFS_I(inode), offset, size, 1201 ISUNWRITTEN(imap) ? XFS_IO_UNWRITTEN : is_cow ? XFS_IO_COW : 1202 XFS_IO_OVERWRITE, imap); 1203 1204 if (ISUNWRITTEN(imap)) { 1205 *flags |= XFS_DIO_FLAG_UNWRITTEN; 1206 set_buffer_defer_completion(bh_result); 1207 } else if (is_cow) { 1208 *flags |= XFS_DIO_FLAG_COW; 1209 set_buffer_defer_completion(bh_result); 1210 } 1211 if (offset + size > i_size_read(inode) || offset + size < 0) { 1212 *flags |= XFS_DIO_FLAG_APPEND; 1213 set_buffer_defer_completion(bh_result); 1214 } 1215 } 1216 1217 /* 1218 * If this is O_DIRECT or the mpage code calling tell them how large the mapping 1219 * is, so that we can avoid repeated get_blocks calls. 1220 * 1221 * If the mapping spans EOF, then we have to break the mapping up as the mapping 1222 * for blocks beyond EOF must be marked new so that sub block regions can be 1223 * correctly zeroed. We can't do this for mappings within EOF unless the mapping 1224 * was just allocated or is unwritten, otherwise the callers would overwrite 1225 * existing data with zeros. Hence we have to split the mapping into a range up 1226 * to and including EOF, and a second mapping for beyond EOF. 1227 */ 1228 static void 1229 xfs_map_trim_size( 1230 struct inode *inode, 1231 sector_t iblock, 1232 struct buffer_head *bh_result, 1233 struct xfs_bmbt_irec *imap, 1234 xfs_off_t offset, 1235 ssize_t size) 1236 { 1237 xfs_off_t mapping_size; 1238 1239 mapping_size = imap->br_startoff + imap->br_blockcount - iblock; 1240 mapping_size <<= inode->i_blkbits; 1241 1242 ASSERT(mapping_size > 0); 1243 if (mapping_size > size) 1244 mapping_size = size; 1245 if (offset < i_size_read(inode) && 1246 offset + mapping_size >= i_size_read(inode)) { 1247 /* limit mapping to block that spans EOF */ 1248 mapping_size = roundup_64(i_size_read(inode) - offset, 1249 1 << inode->i_blkbits); 1250 } 1251 if (mapping_size > LONG_MAX) 1252 mapping_size = LONG_MAX; 1253 1254 bh_result->b_size = mapping_size; 1255 } 1256 1257 /* Bounce unaligned directio writes to the page cache. */ 1258 static int 1259 xfs_bounce_unaligned_dio_write( 1260 struct xfs_inode *ip, 1261 xfs_fileoff_t offset_fsb, 1262 struct xfs_bmbt_irec *imap) 1263 { 1264 struct xfs_bmbt_irec irec; 1265 xfs_fileoff_t delta; 1266 bool shared; 1267 bool x; 1268 int error; 1269 1270 irec = *imap; 1271 if (offset_fsb > irec.br_startoff) { 1272 delta = offset_fsb - irec.br_startoff; 1273 irec.br_blockcount -= delta; 1274 irec.br_startblock += delta; 1275 irec.br_startoff = offset_fsb; 1276 } 1277 error = xfs_reflink_trim_around_shared(ip, &irec, &shared, &x); 1278 if (error) 1279 return error; 1280 1281 /* 1282 * We're here because we're trying to do a directio write to a 1283 * region that isn't aligned to a filesystem block. If any part 1284 * of the extent is shared, fall back to buffered mode to handle 1285 * the RMW. This is done by returning -EREMCHG ("remote addr 1286 * changed"), which is caught further up the call stack. 1287 */ 1288 if (shared) { 1289 trace_xfs_reflink_bounce_dio_write(ip, imap); 1290 return -EREMCHG; 1291 } 1292 return 0; 1293 } 1294 1295 STATIC int 1296 __xfs_get_blocks( 1297 struct inode *inode, 1298 sector_t iblock, 1299 struct buffer_head *bh_result, 1300 int create, 1301 bool direct, 1302 bool dax_fault) 1303 { 1304 struct xfs_inode *ip = XFS_I(inode); 1305 struct xfs_mount *mp = ip->i_mount; 1306 xfs_fileoff_t offset_fsb, end_fsb; 1307 int error = 0; 1308 int lockmode = 0; 1309 struct xfs_bmbt_irec imap; 1310 int nimaps = 1; 1311 xfs_off_t offset; 1312 ssize_t size; 1313 int new = 0; 1314 bool is_cow = false; 1315 bool need_alloc = false; 1316 1317 BUG_ON(create && !direct); 1318 1319 if (XFS_FORCED_SHUTDOWN(mp)) 1320 return -EIO; 1321 1322 offset = (xfs_off_t)iblock << inode->i_blkbits; 1323 ASSERT(bh_result->b_size >= (1 << inode->i_blkbits)); 1324 size = bh_result->b_size; 1325 1326 if (!create && offset >= i_size_read(inode)) 1327 return 0; 1328 1329 /* 1330 * Direct I/O is usually done on preallocated files, so try getting 1331 * a block mapping without an exclusive lock first. 1332 */ 1333 lockmode = xfs_ilock_data_map_shared(ip); 1334 1335 ASSERT(offset <= mp->m_super->s_maxbytes); 1336 if (offset + size > mp->m_super->s_maxbytes) 1337 size = mp->m_super->s_maxbytes - offset; 1338 end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + size); 1339 offset_fsb = XFS_B_TO_FSBT(mp, offset); 1340 1341 if (create && direct && xfs_is_reflink_inode(ip)) 1342 is_cow = xfs_reflink_find_cow_mapping(ip, offset, &imap, 1343 &need_alloc); 1344 if (!is_cow) { 1345 error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, 1346 &imap, &nimaps, XFS_BMAPI_ENTIRE); 1347 /* 1348 * Truncate an overwrite extent if there's a pending CoW 1349 * reservation before the end of this extent. This 1350 * forces us to come back to get_blocks to take care of 1351 * the CoW. 1352 */ 1353 if (create && direct && nimaps && 1354 imap.br_startblock != HOLESTARTBLOCK && 1355 imap.br_startblock != DELAYSTARTBLOCK && 1356 !ISUNWRITTEN(&imap)) 1357 xfs_reflink_trim_irec_to_next_cow(ip, offset_fsb, 1358 &imap); 1359 } 1360 ASSERT(!need_alloc); 1361 if (error) 1362 goto out_unlock; 1363 1364 /* for DAX, we convert unwritten extents directly */ 1365 if (create && 1366 (!nimaps || 1367 (imap.br_startblock == HOLESTARTBLOCK || 1368 imap.br_startblock == DELAYSTARTBLOCK) || 1369 (IS_DAX(inode) && ISUNWRITTEN(&imap)))) { 1370 /* 1371 * xfs_iomap_write_direct() expects the shared lock. It 1372 * is unlocked on return. 1373 */ 1374 if (lockmode == XFS_ILOCK_EXCL) 1375 xfs_ilock_demote(ip, lockmode); 1376 1377 error = xfs_iomap_write_direct(ip, offset, size, 1378 &imap, nimaps); 1379 if (error) 1380 return error; 1381 new = 1; 1382 1383 trace_xfs_get_blocks_alloc(ip, offset, size, 1384 ISUNWRITTEN(&imap) ? XFS_IO_UNWRITTEN 1385 : XFS_IO_DELALLOC, &imap); 1386 } else if (nimaps) { 1387 trace_xfs_get_blocks_found(ip, offset, size, 1388 ISUNWRITTEN(&imap) ? XFS_IO_UNWRITTEN 1389 : XFS_IO_OVERWRITE, &imap); 1390 xfs_iunlock(ip, lockmode); 1391 } else { 1392 trace_xfs_get_blocks_notfound(ip, offset, size); 1393 goto out_unlock; 1394 } 1395 1396 if (IS_DAX(inode) && create) { 1397 ASSERT(!ISUNWRITTEN(&imap)); 1398 /* zeroing is not needed at a higher layer */ 1399 new = 0; 1400 } 1401 1402 /* trim mapping down to size requested */ 1403 xfs_map_trim_size(inode, iblock, bh_result, &imap, offset, size); 1404 1405 /* 1406 * For unwritten extents do not report a disk address in the buffered 1407 * read case (treat as if we're reading into a hole). 1408 */ 1409 if (imap.br_startblock != HOLESTARTBLOCK && 1410 imap.br_startblock != DELAYSTARTBLOCK && 1411 (create || !ISUNWRITTEN(&imap))) { 1412 if (create && direct && !is_cow) { 1413 error = xfs_bounce_unaligned_dio_write(ip, offset_fsb, 1414 &imap); 1415 if (error) 1416 return error; 1417 } 1418 1419 xfs_map_buffer(inode, bh_result, &imap, offset); 1420 if (ISUNWRITTEN(&imap)) 1421 set_buffer_unwritten(bh_result); 1422 /* direct IO needs special help */ 1423 if (create) { 1424 if (dax_fault) 1425 ASSERT(!ISUNWRITTEN(&imap)); 1426 else 1427 xfs_map_direct(inode, bh_result, &imap, offset, 1428 is_cow); 1429 } 1430 } 1431 1432 /* 1433 * If this is a realtime file, data may be on a different device. 1434 * to that pointed to from the buffer_head b_bdev currently. 1435 */ 1436 bh_result->b_bdev = xfs_find_bdev_for_inode(inode); 1437 1438 /* 1439 * If we previously allocated a block out beyond eof and we are now 1440 * coming back to use it then we will need to flag it as new even if it 1441 * has a disk address. 1442 * 1443 * With sub-block writes into unwritten extents we also need to mark 1444 * the buffer as new so that the unwritten parts of the buffer gets 1445 * correctly zeroed. 1446 */ 1447 if (create && 1448 ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) || 1449 (offset >= i_size_read(inode)) || 1450 (new || ISUNWRITTEN(&imap)))) 1451 set_buffer_new(bh_result); 1452 1453 BUG_ON(direct && imap.br_startblock == DELAYSTARTBLOCK); 1454 1455 return 0; 1456 1457 out_unlock: 1458 xfs_iunlock(ip, lockmode); 1459 return error; 1460 } 1461 1462 int 1463 xfs_get_blocks( 1464 struct inode *inode, 1465 sector_t iblock, 1466 struct buffer_head *bh_result, 1467 int create) 1468 { 1469 return __xfs_get_blocks(inode, iblock, bh_result, create, false, false); 1470 } 1471 1472 int 1473 xfs_get_blocks_direct( 1474 struct inode *inode, 1475 sector_t iblock, 1476 struct buffer_head *bh_result, 1477 int create) 1478 { 1479 return __xfs_get_blocks(inode, iblock, bh_result, create, true, false); 1480 } 1481 1482 int 1483 xfs_get_blocks_dax_fault( 1484 struct inode *inode, 1485 sector_t iblock, 1486 struct buffer_head *bh_result, 1487 int create) 1488 { 1489 return __xfs_get_blocks(inode, iblock, bh_result, create, true, true); 1490 } 1491 1492 /* 1493 * Complete a direct I/O write request. 1494 * 1495 * xfs_map_direct passes us some flags in the private data to tell us what to 1496 * do. If no flags are set, then the write IO is an overwrite wholly within 1497 * the existing allocated file size and so there is nothing for us to do. 1498 * 1499 * Note that in this case the completion can be called in interrupt context, 1500 * whereas if we have flags set we will always be called in task context 1501 * (i.e. from a workqueue). 1502 */ 1503 int 1504 xfs_end_io_direct_write( 1505 struct kiocb *iocb, 1506 loff_t offset, 1507 ssize_t size, 1508 void *private) 1509 { 1510 struct inode *inode = file_inode(iocb->ki_filp); 1511 struct xfs_inode *ip = XFS_I(inode); 1512 uintptr_t flags = (uintptr_t)private; 1513 int error = 0; 1514 1515 trace_xfs_end_io_direct_write(ip, offset, size); 1516 1517 if (XFS_FORCED_SHUTDOWN(ip->i_mount)) 1518 return -EIO; 1519 1520 if (size <= 0) 1521 return size; 1522 1523 /* 1524 * The flags tell us whether we are doing unwritten extent conversions 1525 * or an append transaction that updates the on-disk file size. These 1526 * cases are the only cases where we should *potentially* be needing 1527 * to update the VFS inode size. 1528 */ 1529 if (flags == 0) { 1530 ASSERT(offset + size <= i_size_read(inode)); 1531 return 0; 1532 } 1533 1534 /* 1535 * We need to update the in-core inode size here so that we don't end up 1536 * with the on-disk inode size being outside the in-core inode size. We 1537 * have no other method of updating EOF for AIO, so always do it here 1538 * if necessary. 1539 * 1540 * We need to lock the test/set EOF update as we can be racing with 1541 * other IO completions here to update the EOF. Failing to serialise 1542 * here can result in EOF moving backwards and Bad Things Happen when 1543 * that occurs. 1544 */ 1545 spin_lock(&ip->i_flags_lock); 1546 if (offset + size > i_size_read(inode)) 1547 i_size_write(inode, offset + size); 1548 spin_unlock(&ip->i_flags_lock); 1549 1550 if (flags & XFS_DIO_FLAG_COW) 1551 error = xfs_reflink_end_cow(ip, offset, size); 1552 if (flags & XFS_DIO_FLAG_UNWRITTEN) { 1553 trace_xfs_end_io_direct_write_unwritten(ip, offset, size); 1554 1555 error = xfs_iomap_write_unwritten(ip, offset, size); 1556 } 1557 if (flags & XFS_DIO_FLAG_APPEND) { 1558 trace_xfs_end_io_direct_write_append(ip, offset, size); 1559 1560 error = xfs_setfilesize(ip, offset, size); 1561 } 1562 1563 return error; 1564 } 1565 1566 STATIC ssize_t 1567 xfs_vm_direct_IO( 1568 struct kiocb *iocb, 1569 struct iov_iter *iter) 1570 { 1571 /* 1572 * We just need the method present so that open/fcntl allow direct I/O. 1573 */ 1574 return -EINVAL; 1575 } 1576 1577 STATIC sector_t 1578 xfs_vm_bmap( 1579 struct address_space *mapping, 1580 sector_t block) 1581 { 1582 struct inode *inode = (struct inode *)mapping->host; 1583 struct xfs_inode *ip = XFS_I(inode); 1584 1585 trace_xfs_vm_bmap(XFS_I(inode)); 1586 xfs_ilock(ip, XFS_IOLOCK_SHARED); 1587 1588 /* 1589 * The swap code (ab-)uses ->bmap to get a block mapping and then 1590 * bypasseѕ the file system for actual I/O. We really can't allow 1591 * that on reflinks inodes, so we have to skip out here. And yes, 1592 * 0 is the magic code for a bmap error.. 1593 */ 1594 if (xfs_is_reflink_inode(ip)) { 1595 xfs_iunlock(ip, XFS_IOLOCK_SHARED); 1596 return 0; 1597 } 1598 filemap_write_and_wait(mapping); 1599 xfs_iunlock(ip, XFS_IOLOCK_SHARED); 1600 return generic_block_bmap(mapping, block, xfs_get_blocks); 1601 } 1602 1603 STATIC int 1604 xfs_vm_readpage( 1605 struct file *unused, 1606 struct page *page) 1607 { 1608 trace_xfs_vm_readpage(page->mapping->host, 1); 1609 return mpage_readpage(page, xfs_get_blocks); 1610 } 1611 1612 STATIC int 1613 xfs_vm_readpages( 1614 struct file *unused, 1615 struct address_space *mapping, 1616 struct list_head *pages, 1617 unsigned nr_pages) 1618 { 1619 trace_xfs_vm_readpages(mapping->host, nr_pages); 1620 return mpage_readpages(mapping, pages, nr_pages, xfs_get_blocks); 1621 } 1622 1623 /* 1624 * This is basically a copy of __set_page_dirty_buffers() with one 1625 * small tweak: buffers beyond EOF do not get marked dirty. If we mark them 1626 * dirty, we'll never be able to clean them because we don't write buffers 1627 * beyond EOF, and that means we can't invalidate pages that span EOF 1628 * that have been marked dirty. Further, the dirty state can leak into 1629 * the file interior if the file is extended, resulting in all sorts of 1630 * bad things happening as the state does not match the underlying data. 1631 * 1632 * XXX: this really indicates that bufferheads in XFS need to die. Warts like 1633 * this only exist because of bufferheads and how the generic code manages them. 1634 */ 1635 STATIC int 1636 xfs_vm_set_page_dirty( 1637 struct page *page) 1638 { 1639 struct address_space *mapping = page->mapping; 1640 struct inode *inode = mapping->host; 1641 loff_t end_offset; 1642 loff_t offset; 1643 int newly_dirty; 1644 1645 if (unlikely(!mapping)) 1646 return !TestSetPageDirty(page); 1647 1648 end_offset = i_size_read(inode); 1649 offset = page_offset(page); 1650 1651 spin_lock(&mapping->private_lock); 1652 if (page_has_buffers(page)) { 1653 struct buffer_head *head = page_buffers(page); 1654 struct buffer_head *bh = head; 1655 1656 do { 1657 if (offset < end_offset) 1658 set_buffer_dirty(bh); 1659 bh = bh->b_this_page; 1660 offset += 1 << inode->i_blkbits; 1661 } while (bh != head); 1662 } 1663 /* 1664 * Lock out page->mem_cgroup migration to keep PageDirty 1665 * synchronized with per-memcg dirty page counters. 1666 */ 1667 lock_page_memcg(page); 1668 newly_dirty = !TestSetPageDirty(page); 1669 spin_unlock(&mapping->private_lock); 1670 1671 if (newly_dirty) { 1672 /* sigh - __set_page_dirty() is static, so copy it here, too */ 1673 unsigned long flags; 1674 1675 spin_lock_irqsave(&mapping->tree_lock, flags); 1676 if (page->mapping) { /* Race with truncate? */ 1677 WARN_ON_ONCE(!PageUptodate(page)); 1678 account_page_dirtied(page, mapping); 1679 radix_tree_tag_set(&mapping->page_tree, 1680 page_index(page), PAGECACHE_TAG_DIRTY); 1681 } 1682 spin_unlock_irqrestore(&mapping->tree_lock, flags); 1683 } 1684 unlock_page_memcg(page); 1685 if (newly_dirty) 1686 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 1687 return newly_dirty; 1688 } 1689 1690 const struct address_space_operations xfs_address_space_operations = { 1691 .readpage = xfs_vm_readpage, 1692 .readpages = xfs_vm_readpages, 1693 .writepage = xfs_vm_writepage, 1694 .writepages = xfs_vm_writepages, 1695 .set_page_dirty = xfs_vm_set_page_dirty, 1696 .releasepage = xfs_vm_releasepage, 1697 .invalidatepage = xfs_vm_invalidatepage, 1698 .bmap = xfs_vm_bmap, 1699 .direct_IO = xfs_vm_direct_IO, 1700 .migratepage = buffer_migrate_page, 1701 .is_partially_uptodate = block_is_partially_uptodate, 1702 .error_remove_page = generic_error_remove_page, 1703 }; 1704