1 /* 2 * Copyright (c) 2000-2005 Silicon Graphics, Inc. 3 * All Rights Reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it would be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17 */ 18 #include "xfs.h" 19 #include "xfs_shared.h" 20 #include "xfs_format.h" 21 #include "xfs_log_format.h" 22 #include "xfs_trans_resv.h" 23 #include "xfs_sb.h" 24 #include "xfs_ag.h" 25 #include "xfs_mount.h" 26 #include "xfs_inode.h" 27 #include "xfs_trans.h" 28 #include "xfs_inode_item.h" 29 #include "xfs_alloc.h" 30 #include "xfs_error.h" 31 #include "xfs_iomap.h" 32 #include "xfs_trace.h" 33 #include "xfs_bmap.h" 34 #include "xfs_bmap_util.h" 35 #include "xfs_bmap_btree.h" 36 #include "xfs_dinode.h" 37 #include <linux/aio.h> 38 #include <linux/gfp.h> 39 #include <linux/mpage.h> 40 #include <linux/pagevec.h> 41 #include <linux/writeback.h> 42 43 void 44 xfs_count_page_state( 45 struct page *page, 46 int *delalloc, 47 int *unwritten) 48 { 49 struct buffer_head *bh, *head; 50 51 *delalloc = *unwritten = 0; 52 53 bh = head = page_buffers(page); 54 do { 55 if (buffer_unwritten(bh)) 56 (*unwritten) = 1; 57 else if (buffer_delay(bh)) 58 (*delalloc) = 1; 59 } while ((bh = bh->b_this_page) != head); 60 } 61 62 STATIC struct block_device * 63 xfs_find_bdev_for_inode( 64 struct inode *inode) 65 { 66 struct xfs_inode *ip = XFS_I(inode); 67 struct xfs_mount *mp = ip->i_mount; 68 69 if (XFS_IS_REALTIME_INODE(ip)) 70 return mp->m_rtdev_targp->bt_bdev; 71 else 72 return mp->m_ddev_targp->bt_bdev; 73 } 74 75 /* 76 * We're now finished for good with this ioend structure. 77 * Update the page state via the associated buffer_heads, 78 * release holds on the inode and bio, and finally free 79 * up memory. Do not use the ioend after this. 80 */ 81 STATIC void 82 xfs_destroy_ioend( 83 xfs_ioend_t *ioend) 84 { 85 struct buffer_head *bh, *next; 86 87 for (bh = ioend->io_buffer_head; bh; bh = next) { 88 next = bh->b_private; 89 bh->b_end_io(bh, !ioend->io_error); 90 } 91 92 mempool_free(ioend, xfs_ioend_pool); 93 } 94 95 /* 96 * Fast and loose check if this write could update the on-disk inode size. 97 */ 98 static inline bool xfs_ioend_is_append(struct xfs_ioend *ioend) 99 { 100 return ioend->io_offset + ioend->io_size > 101 XFS_I(ioend->io_inode)->i_d.di_size; 102 } 103 104 STATIC int 105 xfs_setfilesize_trans_alloc( 106 struct xfs_ioend *ioend) 107 { 108 struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount; 109 struct xfs_trans *tp; 110 int error; 111 112 tp = xfs_trans_alloc(mp, XFS_TRANS_FSYNC_TS); 113 114 error = xfs_trans_reserve(tp, &M_RES(mp)->tr_fsyncts, 0, 0); 115 if (error) { 116 xfs_trans_cancel(tp, 0); 117 return error; 118 } 119 120 ioend->io_append_trans = tp; 121 122 /* 123 * We may pass freeze protection with a transaction. So tell lockdep 124 * we released it. 125 */ 126 rwsem_release(&ioend->io_inode->i_sb->s_writers.lock_map[SB_FREEZE_FS-1], 127 1, _THIS_IP_); 128 /* 129 * We hand off the transaction to the completion thread now, so 130 * clear the flag here. 131 */ 132 current_restore_flags_nested(&tp->t_pflags, PF_FSTRANS); 133 return 0; 134 } 135 136 /* 137 * Update on-disk file size now that data has been written to disk. 138 */ 139 STATIC int 140 xfs_setfilesize( 141 struct xfs_ioend *ioend) 142 { 143 struct xfs_inode *ip = XFS_I(ioend->io_inode); 144 struct xfs_trans *tp = ioend->io_append_trans; 145 xfs_fsize_t isize; 146 147 /* 148 * The transaction may have been allocated in the I/O submission thread, 149 * thus we need to mark ourselves as beeing in a transaction manually. 150 * Similarly for freeze protection. 151 */ 152 current_set_flags_nested(&tp->t_pflags, PF_FSTRANS); 153 rwsem_acquire_read(&VFS_I(ip)->i_sb->s_writers.lock_map[SB_FREEZE_FS-1], 154 0, 1, _THIS_IP_); 155 156 xfs_ilock(ip, XFS_ILOCK_EXCL); 157 isize = xfs_new_eof(ip, ioend->io_offset + ioend->io_size); 158 if (!isize) { 159 xfs_iunlock(ip, XFS_ILOCK_EXCL); 160 xfs_trans_cancel(tp, 0); 161 return 0; 162 } 163 164 trace_xfs_setfilesize(ip, ioend->io_offset, ioend->io_size); 165 166 ip->i_d.di_size = isize; 167 xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); 168 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 169 170 return xfs_trans_commit(tp, 0); 171 } 172 173 /* 174 * Schedule IO completion handling on the final put of an ioend. 175 * 176 * If there is no work to do we might as well call it a day and free the 177 * ioend right now. 178 */ 179 STATIC void 180 xfs_finish_ioend( 181 struct xfs_ioend *ioend) 182 { 183 if (atomic_dec_and_test(&ioend->io_remaining)) { 184 struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount; 185 186 if (ioend->io_type == XFS_IO_UNWRITTEN) 187 queue_work(mp->m_unwritten_workqueue, &ioend->io_work); 188 else if (ioend->io_append_trans || 189 (ioend->io_isdirect && xfs_ioend_is_append(ioend))) 190 queue_work(mp->m_data_workqueue, &ioend->io_work); 191 else 192 xfs_destroy_ioend(ioend); 193 } 194 } 195 196 /* 197 * IO write completion. 198 */ 199 STATIC void 200 xfs_end_io( 201 struct work_struct *work) 202 { 203 xfs_ioend_t *ioend = container_of(work, xfs_ioend_t, io_work); 204 struct xfs_inode *ip = XFS_I(ioend->io_inode); 205 int error = 0; 206 207 if (XFS_FORCED_SHUTDOWN(ip->i_mount)) { 208 ioend->io_error = -EIO; 209 goto done; 210 } 211 if (ioend->io_error) 212 goto done; 213 214 /* 215 * For unwritten extents we need to issue transactions to convert a 216 * range to normal written extens after the data I/O has finished. 217 */ 218 if (ioend->io_type == XFS_IO_UNWRITTEN) { 219 error = xfs_iomap_write_unwritten(ip, ioend->io_offset, 220 ioend->io_size); 221 } else if (ioend->io_isdirect && xfs_ioend_is_append(ioend)) { 222 /* 223 * For direct I/O we do not know if we need to allocate blocks 224 * or not so we can't preallocate an append transaction as that 225 * results in nested reservations and log space deadlocks. Hence 226 * allocate the transaction here. While this is sub-optimal and 227 * can block IO completion for some time, we're stuck with doing 228 * it this way until we can pass the ioend to the direct IO 229 * allocation callbacks and avoid nesting that way. 230 */ 231 error = xfs_setfilesize_trans_alloc(ioend); 232 if (error) 233 goto done; 234 error = xfs_setfilesize(ioend); 235 } else if (ioend->io_append_trans) { 236 error = xfs_setfilesize(ioend); 237 } else { 238 ASSERT(!xfs_ioend_is_append(ioend)); 239 } 240 241 done: 242 if (error) 243 ioend->io_error = -error; 244 xfs_destroy_ioend(ioend); 245 } 246 247 /* 248 * Call IO completion handling in caller context on the final put of an ioend. 249 */ 250 STATIC void 251 xfs_finish_ioend_sync( 252 struct xfs_ioend *ioend) 253 { 254 if (atomic_dec_and_test(&ioend->io_remaining)) 255 xfs_end_io(&ioend->io_work); 256 } 257 258 /* 259 * Allocate and initialise an IO completion structure. 260 * We need to track unwritten extent write completion here initially. 261 * We'll need to extend this for updating the ondisk inode size later 262 * (vs. incore size). 263 */ 264 STATIC xfs_ioend_t * 265 xfs_alloc_ioend( 266 struct inode *inode, 267 unsigned int type) 268 { 269 xfs_ioend_t *ioend; 270 271 ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS); 272 273 /* 274 * Set the count to 1 initially, which will prevent an I/O 275 * completion callback from happening before we have started 276 * all the I/O from calling the completion routine too early. 277 */ 278 atomic_set(&ioend->io_remaining, 1); 279 ioend->io_isdirect = 0; 280 ioend->io_error = 0; 281 ioend->io_list = NULL; 282 ioend->io_type = type; 283 ioend->io_inode = inode; 284 ioend->io_buffer_head = NULL; 285 ioend->io_buffer_tail = NULL; 286 ioend->io_offset = 0; 287 ioend->io_size = 0; 288 ioend->io_append_trans = NULL; 289 290 INIT_WORK(&ioend->io_work, xfs_end_io); 291 return ioend; 292 } 293 294 STATIC int 295 xfs_map_blocks( 296 struct inode *inode, 297 loff_t offset, 298 struct xfs_bmbt_irec *imap, 299 int type, 300 int nonblocking) 301 { 302 struct xfs_inode *ip = XFS_I(inode); 303 struct xfs_mount *mp = ip->i_mount; 304 ssize_t count = 1 << inode->i_blkbits; 305 xfs_fileoff_t offset_fsb, end_fsb; 306 int error = 0; 307 int bmapi_flags = XFS_BMAPI_ENTIRE; 308 int nimaps = 1; 309 310 if (XFS_FORCED_SHUTDOWN(mp)) 311 return -XFS_ERROR(EIO); 312 313 if (type == XFS_IO_UNWRITTEN) 314 bmapi_flags |= XFS_BMAPI_IGSTATE; 315 316 if (!xfs_ilock_nowait(ip, XFS_ILOCK_SHARED)) { 317 if (nonblocking) 318 return -XFS_ERROR(EAGAIN); 319 xfs_ilock(ip, XFS_ILOCK_SHARED); 320 } 321 322 ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE || 323 (ip->i_df.if_flags & XFS_IFEXTENTS)); 324 ASSERT(offset <= mp->m_super->s_maxbytes); 325 326 if (offset + count > mp->m_super->s_maxbytes) 327 count = mp->m_super->s_maxbytes - offset; 328 end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count); 329 offset_fsb = XFS_B_TO_FSBT(mp, offset); 330 error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, 331 imap, &nimaps, bmapi_flags); 332 xfs_iunlock(ip, XFS_ILOCK_SHARED); 333 334 if (error) 335 return -XFS_ERROR(error); 336 337 if (type == XFS_IO_DELALLOC && 338 (!nimaps || isnullstartblock(imap->br_startblock))) { 339 error = xfs_iomap_write_allocate(ip, offset, imap); 340 if (!error) 341 trace_xfs_map_blocks_alloc(ip, offset, count, type, imap); 342 return -XFS_ERROR(error); 343 } 344 345 #ifdef DEBUG 346 if (type == XFS_IO_UNWRITTEN) { 347 ASSERT(nimaps); 348 ASSERT(imap->br_startblock != HOLESTARTBLOCK); 349 ASSERT(imap->br_startblock != DELAYSTARTBLOCK); 350 } 351 #endif 352 if (nimaps) 353 trace_xfs_map_blocks_found(ip, offset, count, type, imap); 354 return 0; 355 } 356 357 STATIC int 358 xfs_imap_valid( 359 struct inode *inode, 360 struct xfs_bmbt_irec *imap, 361 xfs_off_t offset) 362 { 363 offset >>= inode->i_blkbits; 364 365 return offset >= imap->br_startoff && 366 offset < imap->br_startoff + imap->br_blockcount; 367 } 368 369 /* 370 * BIO completion handler for buffered IO. 371 */ 372 STATIC void 373 xfs_end_bio( 374 struct bio *bio, 375 int error) 376 { 377 xfs_ioend_t *ioend = bio->bi_private; 378 379 ASSERT(atomic_read(&bio->bi_cnt) >= 1); 380 ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error; 381 382 /* Toss bio and pass work off to an xfsdatad thread */ 383 bio->bi_private = NULL; 384 bio->bi_end_io = NULL; 385 bio_put(bio); 386 387 xfs_finish_ioend(ioend); 388 } 389 390 STATIC void 391 xfs_submit_ioend_bio( 392 struct writeback_control *wbc, 393 xfs_ioend_t *ioend, 394 struct bio *bio) 395 { 396 atomic_inc(&ioend->io_remaining); 397 bio->bi_private = ioend; 398 bio->bi_end_io = xfs_end_bio; 399 submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE, bio); 400 } 401 402 STATIC struct bio * 403 xfs_alloc_ioend_bio( 404 struct buffer_head *bh) 405 { 406 int nvecs = bio_get_nr_vecs(bh->b_bdev); 407 struct bio *bio = bio_alloc(GFP_NOIO, nvecs); 408 409 ASSERT(bio->bi_private == NULL); 410 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); 411 bio->bi_bdev = bh->b_bdev; 412 return bio; 413 } 414 415 STATIC void 416 xfs_start_buffer_writeback( 417 struct buffer_head *bh) 418 { 419 ASSERT(buffer_mapped(bh)); 420 ASSERT(buffer_locked(bh)); 421 ASSERT(!buffer_delay(bh)); 422 ASSERT(!buffer_unwritten(bh)); 423 424 mark_buffer_async_write(bh); 425 set_buffer_uptodate(bh); 426 clear_buffer_dirty(bh); 427 } 428 429 STATIC void 430 xfs_start_page_writeback( 431 struct page *page, 432 int clear_dirty, 433 int buffers) 434 { 435 ASSERT(PageLocked(page)); 436 ASSERT(!PageWriteback(page)); 437 if (clear_dirty) 438 clear_page_dirty_for_io(page); 439 set_page_writeback(page); 440 unlock_page(page); 441 /* If no buffers on the page are to be written, finish it here */ 442 if (!buffers) 443 end_page_writeback(page); 444 } 445 446 static inline int xfs_bio_add_buffer(struct bio *bio, struct buffer_head *bh) 447 { 448 return bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh)); 449 } 450 451 /* 452 * Submit all of the bios for all of the ioends we have saved up, covering the 453 * initial writepage page and also any probed pages. 454 * 455 * Because we may have multiple ioends spanning a page, we need to start 456 * writeback on all the buffers before we submit them for I/O. If we mark the 457 * buffers as we got, then we can end up with a page that only has buffers 458 * marked async write and I/O complete on can occur before we mark the other 459 * buffers async write. 460 * 461 * The end result of this is that we trip a bug in end_page_writeback() because 462 * we call it twice for the one page as the code in end_buffer_async_write() 463 * assumes that all buffers on the page are started at the same time. 464 * 465 * The fix is two passes across the ioend list - one to start writeback on the 466 * buffer_heads, and then submit them for I/O on the second pass. 467 * 468 * If @fail is non-zero, it means that we have a situation where some part of 469 * the submission process has failed after we have marked paged for writeback 470 * and unlocked them. In this situation, we need to fail the ioend chain rather 471 * than submit it to IO. This typically only happens on a filesystem shutdown. 472 */ 473 STATIC void 474 xfs_submit_ioend( 475 struct writeback_control *wbc, 476 xfs_ioend_t *ioend, 477 int fail) 478 { 479 xfs_ioend_t *head = ioend; 480 xfs_ioend_t *next; 481 struct buffer_head *bh; 482 struct bio *bio; 483 sector_t lastblock = 0; 484 485 /* Pass 1 - start writeback */ 486 do { 487 next = ioend->io_list; 488 for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) 489 xfs_start_buffer_writeback(bh); 490 } while ((ioend = next) != NULL); 491 492 /* Pass 2 - submit I/O */ 493 ioend = head; 494 do { 495 next = ioend->io_list; 496 bio = NULL; 497 498 /* 499 * If we are failing the IO now, just mark the ioend with an 500 * error and finish it. This will run IO completion immediately 501 * as there is only one reference to the ioend at this point in 502 * time. 503 */ 504 if (fail) { 505 ioend->io_error = -fail; 506 xfs_finish_ioend(ioend); 507 continue; 508 } 509 510 for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) { 511 512 if (!bio) { 513 retry: 514 bio = xfs_alloc_ioend_bio(bh); 515 } else if (bh->b_blocknr != lastblock + 1) { 516 xfs_submit_ioend_bio(wbc, ioend, bio); 517 goto retry; 518 } 519 520 if (xfs_bio_add_buffer(bio, bh) != bh->b_size) { 521 xfs_submit_ioend_bio(wbc, ioend, bio); 522 goto retry; 523 } 524 525 lastblock = bh->b_blocknr; 526 } 527 if (bio) 528 xfs_submit_ioend_bio(wbc, ioend, bio); 529 xfs_finish_ioend(ioend); 530 } while ((ioend = next) != NULL); 531 } 532 533 /* 534 * Cancel submission of all buffer_heads so far in this endio. 535 * Toss the endio too. Only ever called for the initial page 536 * in a writepage request, so only ever one page. 537 */ 538 STATIC void 539 xfs_cancel_ioend( 540 xfs_ioend_t *ioend) 541 { 542 xfs_ioend_t *next; 543 struct buffer_head *bh, *next_bh; 544 545 do { 546 next = ioend->io_list; 547 bh = ioend->io_buffer_head; 548 do { 549 next_bh = bh->b_private; 550 clear_buffer_async_write(bh); 551 unlock_buffer(bh); 552 } while ((bh = next_bh) != NULL); 553 554 mempool_free(ioend, xfs_ioend_pool); 555 } while ((ioend = next) != NULL); 556 } 557 558 /* 559 * Test to see if we've been building up a completion structure for 560 * earlier buffers -- if so, we try to append to this ioend if we 561 * can, otherwise we finish off any current ioend and start another. 562 * Return true if we've finished the given ioend. 563 */ 564 STATIC void 565 xfs_add_to_ioend( 566 struct inode *inode, 567 struct buffer_head *bh, 568 xfs_off_t offset, 569 unsigned int type, 570 xfs_ioend_t **result, 571 int need_ioend) 572 { 573 xfs_ioend_t *ioend = *result; 574 575 if (!ioend || need_ioend || type != ioend->io_type) { 576 xfs_ioend_t *previous = *result; 577 578 ioend = xfs_alloc_ioend(inode, type); 579 ioend->io_offset = offset; 580 ioend->io_buffer_head = bh; 581 ioend->io_buffer_tail = bh; 582 if (previous) 583 previous->io_list = ioend; 584 *result = ioend; 585 } else { 586 ioend->io_buffer_tail->b_private = bh; 587 ioend->io_buffer_tail = bh; 588 } 589 590 bh->b_private = NULL; 591 ioend->io_size += bh->b_size; 592 } 593 594 STATIC void 595 xfs_map_buffer( 596 struct inode *inode, 597 struct buffer_head *bh, 598 struct xfs_bmbt_irec *imap, 599 xfs_off_t offset) 600 { 601 sector_t bn; 602 struct xfs_mount *m = XFS_I(inode)->i_mount; 603 xfs_off_t iomap_offset = XFS_FSB_TO_B(m, imap->br_startoff); 604 xfs_daddr_t iomap_bn = xfs_fsb_to_db(XFS_I(inode), imap->br_startblock); 605 606 ASSERT(imap->br_startblock != HOLESTARTBLOCK); 607 ASSERT(imap->br_startblock != DELAYSTARTBLOCK); 608 609 bn = (iomap_bn >> (inode->i_blkbits - BBSHIFT)) + 610 ((offset - iomap_offset) >> inode->i_blkbits); 611 612 ASSERT(bn || XFS_IS_REALTIME_INODE(XFS_I(inode))); 613 614 bh->b_blocknr = bn; 615 set_buffer_mapped(bh); 616 } 617 618 STATIC void 619 xfs_map_at_offset( 620 struct inode *inode, 621 struct buffer_head *bh, 622 struct xfs_bmbt_irec *imap, 623 xfs_off_t offset) 624 { 625 ASSERT(imap->br_startblock != HOLESTARTBLOCK); 626 ASSERT(imap->br_startblock != DELAYSTARTBLOCK); 627 628 xfs_map_buffer(inode, bh, imap, offset); 629 set_buffer_mapped(bh); 630 clear_buffer_delay(bh); 631 clear_buffer_unwritten(bh); 632 } 633 634 /* 635 * Test if a given page contains at least one buffer of a given @type. 636 * If @check_all_buffers is true, then we walk all the buffers in the page to 637 * try to find one of the type passed in. If it is not set, then the caller only 638 * needs to check the first buffer on the page for a match. 639 */ 640 STATIC bool 641 xfs_check_page_type( 642 struct page *page, 643 unsigned int type, 644 bool check_all_buffers) 645 { 646 struct buffer_head *bh; 647 struct buffer_head *head; 648 649 if (PageWriteback(page)) 650 return false; 651 if (!page->mapping) 652 return false; 653 if (!page_has_buffers(page)) 654 return false; 655 656 bh = head = page_buffers(page); 657 do { 658 if (buffer_unwritten(bh)) { 659 if (type == XFS_IO_UNWRITTEN) 660 return true; 661 } else if (buffer_delay(bh)) { 662 if (type == XFS_IO_DELALLOC) 663 return true; 664 } else if (buffer_dirty(bh) && buffer_mapped(bh)) { 665 if (type == XFS_IO_OVERWRITE) 666 return true; 667 } 668 669 /* If we are only checking the first buffer, we are done now. */ 670 if (!check_all_buffers) 671 break; 672 } while ((bh = bh->b_this_page) != head); 673 674 return false; 675 } 676 677 /* 678 * Allocate & map buffers for page given the extent map. Write it out. 679 * except for the original page of a writepage, this is called on 680 * delalloc/unwritten pages only, for the original page it is possible 681 * that the page has no mapping at all. 682 */ 683 STATIC int 684 xfs_convert_page( 685 struct inode *inode, 686 struct page *page, 687 loff_t tindex, 688 struct xfs_bmbt_irec *imap, 689 xfs_ioend_t **ioendp, 690 struct writeback_control *wbc) 691 { 692 struct buffer_head *bh, *head; 693 xfs_off_t end_offset; 694 unsigned long p_offset; 695 unsigned int type; 696 int len, page_dirty; 697 int count = 0, done = 0, uptodate = 1; 698 xfs_off_t offset = page_offset(page); 699 700 if (page->index != tindex) 701 goto fail; 702 if (!trylock_page(page)) 703 goto fail; 704 if (PageWriteback(page)) 705 goto fail_unlock_page; 706 if (page->mapping != inode->i_mapping) 707 goto fail_unlock_page; 708 if (!xfs_check_page_type(page, (*ioendp)->io_type, false)) 709 goto fail_unlock_page; 710 711 /* 712 * page_dirty is initially a count of buffers on the page before 713 * EOF and is decremented as we move each into a cleanable state. 714 * 715 * Derivation: 716 * 717 * End offset is the highest offset that this page should represent. 718 * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1)) 719 * will evaluate non-zero and be less than PAGE_CACHE_SIZE and 720 * hence give us the correct page_dirty count. On any other page, 721 * it will be zero and in that case we need page_dirty to be the 722 * count of buffers on the page. 723 */ 724 end_offset = min_t(unsigned long long, 725 (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT, 726 i_size_read(inode)); 727 728 /* 729 * If the current map does not span the entire page we are about to try 730 * to write, then give up. The only way we can write a page that spans 731 * multiple mappings in a single writeback iteration is via the 732 * xfs_vm_writepage() function. Data integrity writeback requires the 733 * entire page to be written in a single attempt, otherwise the part of 734 * the page we don't write here doesn't get written as part of the data 735 * integrity sync. 736 * 737 * For normal writeback, we also don't attempt to write partial pages 738 * here as it simply means that write_cache_pages() will see it under 739 * writeback and ignore the page until some point in the future, at 740 * which time this will be the only page in the file that needs 741 * writeback. Hence for more optimal IO patterns, we should always 742 * avoid partial page writeback due to multiple mappings on a page here. 743 */ 744 if (!xfs_imap_valid(inode, imap, end_offset)) 745 goto fail_unlock_page; 746 747 len = 1 << inode->i_blkbits; 748 p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1), 749 PAGE_CACHE_SIZE); 750 p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE; 751 page_dirty = p_offset / len; 752 753 /* 754 * The moment we find a buffer that doesn't match our current type 755 * specification or can't be written, abort the loop and start 756 * writeback. As per the above xfs_imap_valid() check, only 757 * xfs_vm_writepage() can handle partial page writeback fully - we are 758 * limited here to the buffers that are contiguous with the current 759 * ioend, and hence a buffer we can't write breaks that contiguity and 760 * we have to defer the rest of the IO to xfs_vm_writepage(). 761 */ 762 bh = head = page_buffers(page); 763 do { 764 if (offset >= end_offset) 765 break; 766 if (!buffer_uptodate(bh)) 767 uptodate = 0; 768 if (!(PageUptodate(page) || buffer_uptodate(bh))) { 769 done = 1; 770 break; 771 } 772 773 if (buffer_unwritten(bh) || buffer_delay(bh) || 774 buffer_mapped(bh)) { 775 if (buffer_unwritten(bh)) 776 type = XFS_IO_UNWRITTEN; 777 else if (buffer_delay(bh)) 778 type = XFS_IO_DELALLOC; 779 else 780 type = XFS_IO_OVERWRITE; 781 782 /* 783 * imap should always be valid because of the above 784 * partial page end_offset check on the imap. 785 */ 786 ASSERT(xfs_imap_valid(inode, imap, offset)); 787 788 lock_buffer(bh); 789 if (type != XFS_IO_OVERWRITE) 790 xfs_map_at_offset(inode, bh, imap, offset); 791 xfs_add_to_ioend(inode, bh, offset, type, 792 ioendp, done); 793 794 page_dirty--; 795 count++; 796 } else { 797 done = 1; 798 break; 799 } 800 } while (offset += len, (bh = bh->b_this_page) != head); 801 802 if (uptodate && bh == head) 803 SetPageUptodate(page); 804 805 if (count) { 806 if (--wbc->nr_to_write <= 0 && 807 wbc->sync_mode == WB_SYNC_NONE) 808 done = 1; 809 } 810 xfs_start_page_writeback(page, !page_dirty, count); 811 812 return done; 813 fail_unlock_page: 814 unlock_page(page); 815 fail: 816 return 1; 817 } 818 819 /* 820 * Convert & write out a cluster of pages in the same extent as defined 821 * by mp and following the start page. 822 */ 823 STATIC void 824 xfs_cluster_write( 825 struct inode *inode, 826 pgoff_t tindex, 827 struct xfs_bmbt_irec *imap, 828 xfs_ioend_t **ioendp, 829 struct writeback_control *wbc, 830 pgoff_t tlast) 831 { 832 struct pagevec pvec; 833 int done = 0, i; 834 835 pagevec_init(&pvec, 0); 836 while (!done && tindex <= tlast) { 837 unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1); 838 839 if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len)) 840 break; 841 842 for (i = 0; i < pagevec_count(&pvec); i++) { 843 done = xfs_convert_page(inode, pvec.pages[i], tindex++, 844 imap, ioendp, wbc); 845 if (done) 846 break; 847 } 848 849 pagevec_release(&pvec); 850 cond_resched(); 851 } 852 } 853 854 STATIC void 855 xfs_vm_invalidatepage( 856 struct page *page, 857 unsigned int offset, 858 unsigned int length) 859 { 860 trace_xfs_invalidatepage(page->mapping->host, page, offset, 861 length); 862 block_invalidatepage(page, offset, length); 863 } 864 865 /* 866 * If the page has delalloc buffers on it, we need to punch them out before we 867 * invalidate the page. If we don't, we leave a stale delalloc mapping on the 868 * inode that can trip a BUG() in xfs_get_blocks() later on if a direct IO read 869 * is done on that same region - the delalloc extent is returned when none is 870 * supposed to be there. 871 * 872 * We prevent this by truncating away the delalloc regions on the page before 873 * invalidating it. Because they are delalloc, we can do this without needing a 874 * transaction. Indeed - if we get ENOSPC errors, we have to be able to do this 875 * truncation without a transaction as there is no space left for block 876 * reservation (typically why we see a ENOSPC in writeback). 877 * 878 * This is not a performance critical path, so for now just do the punching a 879 * buffer head at a time. 880 */ 881 STATIC void 882 xfs_aops_discard_page( 883 struct page *page) 884 { 885 struct inode *inode = page->mapping->host; 886 struct xfs_inode *ip = XFS_I(inode); 887 struct buffer_head *bh, *head; 888 loff_t offset = page_offset(page); 889 890 if (!xfs_check_page_type(page, XFS_IO_DELALLOC, true)) 891 goto out_invalidate; 892 893 if (XFS_FORCED_SHUTDOWN(ip->i_mount)) 894 goto out_invalidate; 895 896 xfs_alert(ip->i_mount, 897 "page discard on page %p, inode 0x%llx, offset %llu.", 898 page, ip->i_ino, offset); 899 900 xfs_ilock(ip, XFS_ILOCK_EXCL); 901 bh = head = page_buffers(page); 902 do { 903 int error; 904 xfs_fileoff_t start_fsb; 905 906 if (!buffer_delay(bh)) 907 goto next_buffer; 908 909 start_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); 910 error = xfs_bmap_punch_delalloc_range(ip, start_fsb, 1); 911 if (error) { 912 /* something screwed, just bail */ 913 if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) { 914 xfs_alert(ip->i_mount, 915 "page discard unable to remove delalloc mapping."); 916 } 917 break; 918 } 919 next_buffer: 920 offset += 1 << inode->i_blkbits; 921 922 } while ((bh = bh->b_this_page) != head); 923 924 xfs_iunlock(ip, XFS_ILOCK_EXCL); 925 out_invalidate: 926 xfs_vm_invalidatepage(page, 0, PAGE_CACHE_SIZE); 927 return; 928 } 929 930 /* 931 * Write out a dirty page. 932 * 933 * For delalloc space on the page we need to allocate space and flush it. 934 * For unwritten space on the page we need to start the conversion to 935 * regular allocated space. 936 * For any other dirty buffer heads on the page we should flush them. 937 */ 938 STATIC int 939 xfs_vm_writepage( 940 struct page *page, 941 struct writeback_control *wbc) 942 { 943 struct inode *inode = page->mapping->host; 944 struct buffer_head *bh, *head; 945 struct xfs_bmbt_irec imap; 946 xfs_ioend_t *ioend = NULL, *iohead = NULL; 947 loff_t offset; 948 unsigned int type; 949 __uint64_t end_offset; 950 pgoff_t end_index, last_index; 951 ssize_t len; 952 int err, imap_valid = 0, uptodate = 1; 953 int count = 0; 954 int nonblocking = 0; 955 956 trace_xfs_writepage(inode, page, 0, 0); 957 958 ASSERT(page_has_buffers(page)); 959 960 /* 961 * Refuse to write the page out if we are called from reclaim context. 962 * 963 * This avoids stack overflows when called from deeply used stacks in 964 * random callers for direct reclaim or memcg reclaim. We explicitly 965 * allow reclaim from kswapd as the stack usage there is relatively low. 966 * 967 * This should never happen except in the case of a VM regression so 968 * warn about it. 969 */ 970 if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) == 971 PF_MEMALLOC)) 972 goto redirty; 973 974 /* 975 * Given that we do not allow direct reclaim to call us, we should 976 * never be called while in a filesystem transaction. 977 */ 978 if (WARN_ON_ONCE(current->flags & PF_FSTRANS)) 979 goto redirty; 980 981 /* Is this page beyond the end of the file? */ 982 offset = i_size_read(inode); 983 end_index = offset >> PAGE_CACHE_SHIFT; 984 last_index = (offset - 1) >> PAGE_CACHE_SHIFT; 985 986 /* 987 * The page index is less than the end_index, adjust the end_offset 988 * to the highest offset that this page should represent. 989 * ----------------------------------------------------- 990 * | file mapping | <EOF> | 991 * ----------------------------------------------------- 992 * | Page ... | Page N-2 | Page N-1 | Page N | | 993 * ^--------------------------------^----------|-------- 994 * | desired writeback range | see else | 995 * ---------------------------------^------------------| 996 */ 997 if (page->index < end_index) 998 end_offset = (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT; 999 else { 1000 /* 1001 * Check whether the page to write out is beyond or straddles 1002 * i_size or not. 1003 * ------------------------------------------------------- 1004 * | file mapping | <EOF> | 1005 * ------------------------------------------------------- 1006 * | Page ... | Page N-2 | Page N-1 | Page N | Beyond | 1007 * ^--------------------------------^-----------|--------- 1008 * | | Straddles | 1009 * ---------------------------------^-----------|--------| 1010 */ 1011 unsigned offset_into_page = offset & (PAGE_CACHE_SIZE - 1); 1012 1013 /* 1014 * Skip the page if it is fully outside i_size, e.g. due to a 1015 * truncate operation that is in progress. We must redirty the 1016 * page so that reclaim stops reclaiming it. Otherwise 1017 * xfs_vm_releasepage() is called on it and gets confused. 1018 * 1019 * Note that the end_index is unsigned long, it would overflow 1020 * if the given offset is greater than 16TB on 32-bit system 1021 * and if we do check the page is fully outside i_size or not 1022 * via "if (page->index >= end_index + 1)" as "end_index + 1" 1023 * will be evaluated to 0. Hence this page will be redirtied 1024 * and be written out repeatedly which would result in an 1025 * infinite loop, the user program that perform this operation 1026 * will hang. Instead, we can verify this situation by checking 1027 * if the page to write is totally beyond the i_size or if it's 1028 * offset is just equal to the EOF. 1029 */ 1030 if (page->index > end_index || 1031 (page->index == end_index && offset_into_page == 0)) 1032 goto redirty; 1033 1034 /* 1035 * The page straddles i_size. It must be zeroed out on each 1036 * and every writepage invocation because it may be mmapped. 1037 * "A file is mapped in multiples of the page size. For a file 1038 * that is not a multiple of the page size, the remaining 1039 * memory is zeroed when mapped, and writes to that region are 1040 * not written out to the file." 1041 */ 1042 zero_user_segment(page, offset_into_page, PAGE_CACHE_SIZE); 1043 1044 /* Adjust the end_offset to the end of file */ 1045 end_offset = offset; 1046 } 1047 1048 len = 1 << inode->i_blkbits; 1049 1050 bh = head = page_buffers(page); 1051 offset = page_offset(page); 1052 type = XFS_IO_OVERWRITE; 1053 1054 if (wbc->sync_mode == WB_SYNC_NONE) 1055 nonblocking = 1; 1056 1057 do { 1058 int new_ioend = 0; 1059 1060 if (offset >= end_offset) 1061 break; 1062 if (!buffer_uptodate(bh)) 1063 uptodate = 0; 1064 1065 /* 1066 * set_page_dirty dirties all buffers in a page, independent 1067 * of their state. The dirty state however is entirely 1068 * meaningless for holes (!mapped && uptodate), so skip 1069 * buffers covering holes here. 1070 */ 1071 if (!buffer_mapped(bh) && buffer_uptodate(bh)) { 1072 imap_valid = 0; 1073 continue; 1074 } 1075 1076 if (buffer_unwritten(bh)) { 1077 if (type != XFS_IO_UNWRITTEN) { 1078 type = XFS_IO_UNWRITTEN; 1079 imap_valid = 0; 1080 } 1081 } else if (buffer_delay(bh)) { 1082 if (type != XFS_IO_DELALLOC) { 1083 type = XFS_IO_DELALLOC; 1084 imap_valid = 0; 1085 } 1086 } else if (buffer_uptodate(bh)) { 1087 if (type != XFS_IO_OVERWRITE) { 1088 type = XFS_IO_OVERWRITE; 1089 imap_valid = 0; 1090 } 1091 } else { 1092 if (PageUptodate(page)) 1093 ASSERT(buffer_mapped(bh)); 1094 /* 1095 * This buffer is not uptodate and will not be 1096 * written to disk. Ensure that we will put any 1097 * subsequent writeable buffers into a new 1098 * ioend. 1099 */ 1100 imap_valid = 0; 1101 continue; 1102 } 1103 1104 if (imap_valid) 1105 imap_valid = xfs_imap_valid(inode, &imap, offset); 1106 if (!imap_valid) { 1107 /* 1108 * If we didn't have a valid mapping then we need to 1109 * put the new mapping into a separate ioend structure. 1110 * This ensures non-contiguous extents always have 1111 * separate ioends, which is particularly important 1112 * for unwritten extent conversion at I/O completion 1113 * time. 1114 */ 1115 new_ioend = 1; 1116 err = xfs_map_blocks(inode, offset, &imap, type, 1117 nonblocking); 1118 if (err) 1119 goto error; 1120 imap_valid = xfs_imap_valid(inode, &imap, offset); 1121 } 1122 if (imap_valid) { 1123 lock_buffer(bh); 1124 if (type != XFS_IO_OVERWRITE) 1125 xfs_map_at_offset(inode, bh, &imap, offset); 1126 xfs_add_to_ioend(inode, bh, offset, type, &ioend, 1127 new_ioend); 1128 count++; 1129 } 1130 1131 if (!iohead) 1132 iohead = ioend; 1133 1134 } while (offset += len, ((bh = bh->b_this_page) != head)); 1135 1136 if (uptodate && bh == head) 1137 SetPageUptodate(page); 1138 1139 xfs_start_page_writeback(page, 1, count); 1140 1141 /* if there is no IO to be submitted for this page, we are done */ 1142 if (!ioend) 1143 return 0; 1144 1145 ASSERT(iohead); 1146 1147 /* 1148 * Any errors from this point onwards need tobe reported through the IO 1149 * completion path as we have marked the initial page as under writeback 1150 * and unlocked it. 1151 */ 1152 if (imap_valid) { 1153 xfs_off_t end_index; 1154 1155 end_index = imap.br_startoff + imap.br_blockcount; 1156 1157 /* to bytes */ 1158 end_index <<= inode->i_blkbits; 1159 1160 /* to pages */ 1161 end_index = (end_index - 1) >> PAGE_CACHE_SHIFT; 1162 1163 /* check against file size */ 1164 if (end_index > last_index) 1165 end_index = last_index; 1166 1167 xfs_cluster_write(inode, page->index + 1, &imap, &ioend, 1168 wbc, end_index); 1169 } 1170 1171 1172 /* 1173 * Reserve log space if we might write beyond the on-disk inode size. 1174 */ 1175 err = 0; 1176 if (ioend->io_type != XFS_IO_UNWRITTEN && xfs_ioend_is_append(ioend)) 1177 err = xfs_setfilesize_trans_alloc(ioend); 1178 1179 xfs_submit_ioend(wbc, iohead, err); 1180 1181 return 0; 1182 1183 error: 1184 if (iohead) 1185 xfs_cancel_ioend(iohead); 1186 1187 if (err == -EAGAIN) 1188 goto redirty; 1189 1190 xfs_aops_discard_page(page); 1191 ClearPageUptodate(page); 1192 unlock_page(page); 1193 return err; 1194 1195 redirty: 1196 redirty_page_for_writepage(wbc, page); 1197 unlock_page(page); 1198 return 0; 1199 } 1200 1201 STATIC int 1202 xfs_vm_writepages( 1203 struct address_space *mapping, 1204 struct writeback_control *wbc) 1205 { 1206 xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED); 1207 return generic_writepages(mapping, wbc); 1208 } 1209 1210 /* 1211 * Called to move a page into cleanable state - and from there 1212 * to be released. The page should already be clean. We always 1213 * have buffer heads in this call. 1214 * 1215 * Returns 1 if the page is ok to release, 0 otherwise. 1216 */ 1217 STATIC int 1218 xfs_vm_releasepage( 1219 struct page *page, 1220 gfp_t gfp_mask) 1221 { 1222 int delalloc, unwritten; 1223 1224 trace_xfs_releasepage(page->mapping->host, page, 0, 0); 1225 1226 xfs_count_page_state(page, &delalloc, &unwritten); 1227 1228 if (WARN_ON_ONCE(delalloc)) 1229 return 0; 1230 if (WARN_ON_ONCE(unwritten)) 1231 return 0; 1232 1233 return try_to_free_buffers(page); 1234 } 1235 1236 STATIC int 1237 __xfs_get_blocks( 1238 struct inode *inode, 1239 sector_t iblock, 1240 struct buffer_head *bh_result, 1241 int create, 1242 int direct) 1243 { 1244 struct xfs_inode *ip = XFS_I(inode); 1245 struct xfs_mount *mp = ip->i_mount; 1246 xfs_fileoff_t offset_fsb, end_fsb; 1247 int error = 0; 1248 int lockmode = 0; 1249 struct xfs_bmbt_irec imap; 1250 int nimaps = 1; 1251 xfs_off_t offset; 1252 ssize_t size; 1253 int new = 0; 1254 1255 if (XFS_FORCED_SHUTDOWN(mp)) 1256 return -XFS_ERROR(EIO); 1257 1258 offset = (xfs_off_t)iblock << inode->i_blkbits; 1259 ASSERT(bh_result->b_size >= (1 << inode->i_blkbits)); 1260 size = bh_result->b_size; 1261 1262 if (!create && direct && offset >= i_size_read(inode)) 1263 return 0; 1264 1265 /* 1266 * Direct I/O is usually done on preallocated files, so try getting 1267 * a block mapping without an exclusive lock first. For buffered 1268 * writes we already have the exclusive iolock anyway, so avoiding 1269 * a lock roundtrip here by taking the ilock exclusive from the 1270 * beginning is a useful micro optimization. 1271 */ 1272 if (create && !direct) { 1273 lockmode = XFS_ILOCK_EXCL; 1274 xfs_ilock(ip, lockmode); 1275 } else { 1276 lockmode = xfs_ilock_data_map_shared(ip); 1277 } 1278 1279 ASSERT(offset <= mp->m_super->s_maxbytes); 1280 if (offset + size > mp->m_super->s_maxbytes) 1281 size = mp->m_super->s_maxbytes - offset; 1282 end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + size); 1283 offset_fsb = XFS_B_TO_FSBT(mp, offset); 1284 1285 error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, 1286 &imap, &nimaps, XFS_BMAPI_ENTIRE); 1287 if (error) 1288 goto out_unlock; 1289 1290 if (create && 1291 (!nimaps || 1292 (imap.br_startblock == HOLESTARTBLOCK || 1293 imap.br_startblock == DELAYSTARTBLOCK))) { 1294 if (direct || xfs_get_extsz_hint(ip)) { 1295 /* 1296 * Drop the ilock in preparation for starting the block 1297 * allocation transaction. It will be retaken 1298 * exclusively inside xfs_iomap_write_direct for the 1299 * actual allocation. 1300 */ 1301 xfs_iunlock(ip, lockmode); 1302 error = xfs_iomap_write_direct(ip, offset, size, 1303 &imap, nimaps); 1304 if (error) 1305 return -error; 1306 new = 1; 1307 } else { 1308 /* 1309 * Delalloc reservations do not require a transaction, 1310 * we can go on without dropping the lock here. If we 1311 * are allocating a new delalloc block, make sure that 1312 * we set the new flag so that we mark the buffer new so 1313 * that we know that it is newly allocated if the write 1314 * fails. 1315 */ 1316 if (nimaps && imap.br_startblock == HOLESTARTBLOCK) 1317 new = 1; 1318 error = xfs_iomap_write_delay(ip, offset, size, &imap); 1319 if (error) 1320 goto out_unlock; 1321 1322 xfs_iunlock(ip, lockmode); 1323 } 1324 1325 trace_xfs_get_blocks_alloc(ip, offset, size, 0, &imap); 1326 } else if (nimaps) { 1327 trace_xfs_get_blocks_found(ip, offset, size, 0, &imap); 1328 xfs_iunlock(ip, lockmode); 1329 } else { 1330 trace_xfs_get_blocks_notfound(ip, offset, size); 1331 goto out_unlock; 1332 } 1333 1334 if (imap.br_startblock != HOLESTARTBLOCK && 1335 imap.br_startblock != DELAYSTARTBLOCK) { 1336 /* 1337 * For unwritten extents do not report a disk address on 1338 * the read case (treat as if we're reading into a hole). 1339 */ 1340 if (create || !ISUNWRITTEN(&imap)) 1341 xfs_map_buffer(inode, bh_result, &imap, offset); 1342 if (create && ISUNWRITTEN(&imap)) { 1343 if (direct) { 1344 bh_result->b_private = inode; 1345 set_buffer_defer_completion(bh_result); 1346 } 1347 set_buffer_unwritten(bh_result); 1348 } 1349 } 1350 1351 /* 1352 * If this is a realtime file, data may be on a different device. 1353 * to that pointed to from the buffer_head b_bdev currently. 1354 */ 1355 bh_result->b_bdev = xfs_find_bdev_for_inode(inode); 1356 1357 /* 1358 * If we previously allocated a block out beyond eof and we are now 1359 * coming back to use it then we will need to flag it as new even if it 1360 * has a disk address. 1361 * 1362 * With sub-block writes into unwritten extents we also need to mark 1363 * the buffer as new so that the unwritten parts of the buffer gets 1364 * correctly zeroed. 1365 */ 1366 if (create && 1367 ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) || 1368 (offset >= i_size_read(inode)) || 1369 (new || ISUNWRITTEN(&imap)))) 1370 set_buffer_new(bh_result); 1371 1372 if (imap.br_startblock == DELAYSTARTBLOCK) { 1373 BUG_ON(direct); 1374 if (create) { 1375 set_buffer_uptodate(bh_result); 1376 set_buffer_mapped(bh_result); 1377 set_buffer_delay(bh_result); 1378 } 1379 } 1380 1381 /* 1382 * If this is O_DIRECT or the mpage code calling tell them how large 1383 * the mapping is, so that we can avoid repeated get_blocks calls. 1384 * 1385 * If the mapping spans EOF, then we have to break the mapping up as the 1386 * mapping for blocks beyond EOF must be marked new so that sub block 1387 * regions can be correctly zeroed. We can't do this for mappings within 1388 * EOF unless the mapping was just allocated or is unwritten, otherwise 1389 * the callers would overwrite existing data with zeros. Hence we have 1390 * to split the mapping into a range up to and including EOF, and a 1391 * second mapping for beyond EOF. 1392 */ 1393 if (direct || size > (1 << inode->i_blkbits)) { 1394 xfs_off_t mapping_size; 1395 1396 mapping_size = imap.br_startoff + imap.br_blockcount - iblock; 1397 mapping_size <<= inode->i_blkbits; 1398 1399 ASSERT(mapping_size > 0); 1400 if (mapping_size > size) 1401 mapping_size = size; 1402 if (offset < i_size_read(inode) && 1403 offset + mapping_size >= i_size_read(inode)) { 1404 /* limit mapping to block that spans EOF */ 1405 mapping_size = roundup_64(i_size_read(inode) - offset, 1406 1 << inode->i_blkbits); 1407 } 1408 if (mapping_size > LONG_MAX) 1409 mapping_size = LONG_MAX; 1410 1411 bh_result->b_size = mapping_size; 1412 } 1413 1414 return 0; 1415 1416 out_unlock: 1417 xfs_iunlock(ip, lockmode); 1418 return -error; 1419 } 1420 1421 int 1422 xfs_get_blocks( 1423 struct inode *inode, 1424 sector_t iblock, 1425 struct buffer_head *bh_result, 1426 int create) 1427 { 1428 return __xfs_get_blocks(inode, iblock, bh_result, create, 0); 1429 } 1430 1431 STATIC int 1432 xfs_get_blocks_direct( 1433 struct inode *inode, 1434 sector_t iblock, 1435 struct buffer_head *bh_result, 1436 int create) 1437 { 1438 return __xfs_get_blocks(inode, iblock, bh_result, create, 1); 1439 } 1440 1441 /* 1442 * Complete a direct I/O write request. 1443 * 1444 * If the private argument is non-NULL __xfs_get_blocks signals us that we 1445 * need to issue a transaction to convert the range from unwritten to written 1446 * extents. In case this is regular synchronous I/O we just call xfs_end_io 1447 * to do this and we are done. But in case this was a successful AIO 1448 * request this handler is called from interrupt context, from which we 1449 * can't start transactions. In that case offload the I/O completion to 1450 * the workqueues we also use for buffered I/O completion. 1451 */ 1452 STATIC void 1453 xfs_end_io_direct_write( 1454 struct kiocb *iocb, 1455 loff_t offset, 1456 ssize_t size, 1457 void *private) 1458 { 1459 struct xfs_ioend *ioend = iocb->private; 1460 1461 /* 1462 * While the generic direct I/O code updates the inode size, it does 1463 * so only after the end_io handler is called, which means our 1464 * end_io handler thinks the on-disk size is outside the in-core 1465 * size. To prevent this just update it a little bit earlier here. 1466 */ 1467 if (offset + size > i_size_read(ioend->io_inode)) 1468 i_size_write(ioend->io_inode, offset + size); 1469 1470 /* 1471 * blockdev_direct_IO can return an error even after the I/O 1472 * completion handler was called. Thus we need to protect 1473 * against double-freeing. 1474 */ 1475 iocb->private = NULL; 1476 1477 ioend->io_offset = offset; 1478 ioend->io_size = size; 1479 if (private && size > 0) 1480 ioend->io_type = XFS_IO_UNWRITTEN; 1481 1482 xfs_finish_ioend_sync(ioend); 1483 } 1484 1485 STATIC ssize_t 1486 xfs_vm_direct_IO( 1487 int rw, 1488 struct kiocb *iocb, 1489 struct iov_iter *iter, 1490 loff_t offset) 1491 { 1492 struct inode *inode = iocb->ki_filp->f_mapping->host; 1493 struct block_device *bdev = xfs_find_bdev_for_inode(inode); 1494 struct xfs_ioend *ioend = NULL; 1495 ssize_t ret; 1496 1497 if (rw & WRITE) { 1498 size_t size = iov_iter_count(iter); 1499 1500 /* 1501 * We cannot preallocate a size update transaction here as we 1502 * don't know whether allocation is necessary or not. Hence we 1503 * can only tell IO completion that one is necessary if we are 1504 * not doing unwritten extent conversion. 1505 */ 1506 iocb->private = ioend = xfs_alloc_ioend(inode, XFS_IO_DIRECT); 1507 if (offset + size > XFS_I(inode)->i_d.di_size) 1508 ioend->io_isdirect = 1; 1509 1510 ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iter, 1511 offset, xfs_get_blocks_direct, 1512 xfs_end_io_direct_write, NULL, 1513 DIO_ASYNC_EXTEND); 1514 if (ret != -EIOCBQUEUED && iocb->private) 1515 goto out_destroy_ioend; 1516 } else { 1517 ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iter, 1518 offset, xfs_get_blocks_direct, 1519 NULL, NULL, 0); 1520 } 1521 1522 return ret; 1523 1524 out_destroy_ioend: 1525 xfs_destroy_ioend(ioend); 1526 return ret; 1527 } 1528 1529 /* 1530 * Punch out the delalloc blocks we have already allocated. 1531 * 1532 * Don't bother with xfs_setattr given that nothing can have made it to disk yet 1533 * as the page is still locked at this point. 1534 */ 1535 STATIC void 1536 xfs_vm_kill_delalloc_range( 1537 struct inode *inode, 1538 loff_t start, 1539 loff_t end) 1540 { 1541 struct xfs_inode *ip = XFS_I(inode); 1542 xfs_fileoff_t start_fsb; 1543 xfs_fileoff_t end_fsb; 1544 int error; 1545 1546 start_fsb = XFS_B_TO_FSB(ip->i_mount, start); 1547 end_fsb = XFS_B_TO_FSB(ip->i_mount, end); 1548 if (end_fsb <= start_fsb) 1549 return; 1550 1551 xfs_ilock(ip, XFS_ILOCK_EXCL); 1552 error = xfs_bmap_punch_delalloc_range(ip, start_fsb, 1553 end_fsb - start_fsb); 1554 if (error) { 1555 /* something screwed, just bail */ 1556 if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) { 1557 xfs_alert(ip->i_mount, 1558 "xfs_vm_write_failed: unable to clean up ino %lld", 1559 ip->i_ino); 1560 } 1561 } 1562 xfs_iunlock(ip, XFS_ILOCK_EXCL); 1563 } 1564 1565 STATIC void 1566 xfs_vm_write_failed( 1567 struct inode *inode, 1568 struct page *page, 1569 loff_t pos, 1570 unsigned len) 1571 { 1572 loff_t block_offset; 1573 loff_t block_start; 1574 loff_t block_end; 1575 loff_t from = pos & (PAGE_CACHE_SIZE - 1); 1576 loff_t to = from + len; 1577 struct buffer_head *bh, *head; 1578 1579 /* 1580 * The request pos offset might be 32 or 64 bit, this is all fine 1581 * on 64-bit platform. However, for 64-bit pos request on 32-bit 1582 * platform, the high 32-bit will be masked off if we evaluate the 1583 * block_offset via (pos & PAGE_MASK) because the PAGE_MASK is 1584 * 0xfffff000 as an unsigned long, hence the result is incorrect 1585 * which could cause the following ASSERT failed in most cases. 1586 * In order to avoid this, we can evaluate the block_offset of the 1587 * start of the page by using shifts rather than masks the mismatch 1588 * problem. 1589 */ 1590 block_offset = (pos >> PAGE_CACHE_SHIFT) << PAGE_CACHE_SHIFT; 1591 1592 ASSERT(block_offset + from == pos); 1593 1594 head = page_buffers(page); 1595 block_start = 0; 1596 for (bh = head; bh != head || !block_start; 1597 bh = bh->b_this_page, block_start = block_end, 1598 block_offset += bh->b_size) { 1599 block_end = block_start + bh->b_size; 1600 1601 /* skip buffers before the write */ 1602 if (block_end <= from) 1603 continue; 1604 1605 /* if the buffer is after the write, we're done */ 1606 if (block_start >= to) 1607 break; 1608 1609 if (!buffer_delay(bh)) 1610 continue; 1611 1612 if (!buffer_new(bh) && block_offset < i_size_read(inode)) 1613 continue; 1614 1615 xfs_vm_kill_delalloc_range(inode, block_offset, 1616 block_offset + bh->b_size); 1617 1618 /* 1619 * This buffer does not contain data anymore. make sure anyone 1620 * who finds it knows that for certain. 1621 */ 1622 clear_buffer_delay(bh); 1623 clear_buffer_uptodate(bh); 1624 clear_buffer_mapped(bh); 1625 clear_buffer_new(bh); 1626 clear_buffer_dirty(bh); 1627 } 1628 1629 } 1630 1631 /* 1632 * This used to call block_write_begin(), but it unlocks and releases the page 1633 * on error, and we need that page to be able to punch stale delalloc blocks out 1634 * on failure. hence we copy-n-waste it here and call xfs_vm_write_failed() at 1635 * the appropriate point. 1636 */ 1637 STATIC int 1638 xfs_vm_write_begin( 1639 struct file *file, 1640 struct address_space *mapping, 1641 loff_t pos, 1642 unsigned len, 1643 unsigned flags, 1644 struct page **pagep, 1645 void **fsdata) 1646 { 1647 pgoff_t index = pos >> PAGE_CACHE_SHIFT; 1648 struct page *page; 1649 int status; 1650 1651 ASSERT(len <= PAGE_CACHE_SIZE); 1652 1653 page = grab_cache_page_write_begin(mapping, index, flags); 1654 if (!page) 1655 return -ENOMEM; 1656 1657 status = __block_write_begin(page, pos, len, xfs_get_blocks); 1658 if (unlikely(status)) { 1659 struct inode *inode = mapping->host; 1660 size_t isize = i_size_read(inode); 1661 1662 xfs_vm_write_failed(inode, page, pos, len); 1663 unlock_page(page); 1664 1665 /* 1666 * If the write is beyond EOF, we only want to kill blocks 1667 * allocated in this write, not blocks that were previously 1668 * written successfully. 1669 */ 1670 if (pos + len > isize) { 1671 ssize_t start = max_t(ssize_t, pos, isize); 1672 1673 truncate_pagecache_range(inode, start, pos + len); 1674 } 1675 1676 page_cache_release(page); 1677 page = NULL; 1678 } 1679 1680 *pagep = page; 1681 return status; 1682 } 1683 1684 /* 1685 * On failure, we only need to kill delalloc blocks beyond EOF in the range of 1686 * this specific write because they will never be written. Previous writes 1687 * beyond EOF where block allocation succeeded do not need to be trashed, so 1688 * only new blocks from this write should be trashed. For blocks within 1689 * EOF, generic_write_end() zeros them so they are safe to leave alone and be 1690 * written with all the other valid data. 1691 */ 1692 STATIC int 1693 xfs_vm_write_end( 1694 struct file *file, 1695 struct address_space *mapping, 1696 loff_t pos, 1697 unsigned len, 1698 unsigned copied, 1699 struct page *page, 1700 void *fsdata) 1701 { 1702 int ret; 1703 1704 ASSERT(len <= PAGE_CACHE_SIZE); 1705 1706 ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); 1707 if (unlikely(ret < len)) { 1708 struct inode *inode = mapping->host; 1709 size_t isize = i_size_read(inode); 1710 loff_t to = pos + len; 1711 1712 if (to > isize) { 1713 /* only kill blocks in this write beyond EOF */ 1714 if (pos > isize) 1715 isize = pos; 1716 xfs_vm_kill_delalloc_range(inode, isize, to); 1717 truncate_pagecache_range(inode, isize, to); 1718 } 1719 } 1720 return ret; 1721 } 1722 1723 STATIC sector_t 1724 xfs_vm_bmap( 1725 struct address_space *mapping, 1726 sector_t block) 1727 { 1728 struct inode *inode = (struct inode *)mapping->host; 1729 struct xfs_inode *ip = XFS_I(inode); 1730 1731 trace_xfs_vm_bmap(XFS_I(inode)); 1732 xfs_ilock(ip, XFS_IOLOCK_SHARED); 1733 filemap_write_and_wait(mapping); 1734 xfs_iunlock(ip, XFS_IOLOCK_SHARED); 1735 return generic_block_bmap(mapping, block, xfs_get_blocks); 1736 } 1737 1738 STATIC int 1739 xfs_vm_readpage( 1740 struct file *unused, 1741 struct page *page) 1742 { 1743 return mpage_readpage(page, xfs_get_blocks); 1744 } 1745 1746 STATIC int 1747 xfs_vm_readpages( 1748 struct file *unused, 1749 struct address_space *mapping, 1750 struct list_head *pages, 1751 unsigned nr_pages) 1752 { 1753 return mpage_readpages(mapping, pages, nr_pages, xfs_get_blocks); 1754 } 1755 1756 const struct address_space_operations xfs_address_space_operations = { 1757 .readpage = xfs_vm_readpage, 1758 .readpages = xfs_vm_readpages, 1759 .writepage = xfs_vm_writepage, 1760 .writepages = xfs_vm_writepages, 1761 .releasepage = xfs_vm_releasepage, 1762 .invalidatepage = xfs_vm_invalidatepage, 1763 .write_begin = xfs_vm_write_begin, 1764 .write_end = xfs_vm_write_end, 1765 .bmap = xfs_vm_bmap, 1766 .direct_IO = xfs_vm_direct_IO, 1767 .migratepage = buffer_migrate_page, 1768 .is_partially_uptodate = block_is_partially_uptodate, 1769 .error_remove_page = generic_error_remove_page, 1770 }; 1771