1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fs/mpage.c 4 * 5 * Copyright (C) 2002, Linus Torvalds. 6 * 7 * Contains functions related to preparing and submitting BIOs which contain 8 * multiple pagecache pages. 9 * 10 * 15May2002 Andrew Morton 11 * Initial version 12 * 27Jun2002 axboe@suse.de 13 * use bio_add_page() to build bio's just the right size 14 */ 15 16 #include <linux/kernel.h> 17 #include <linux/export.h> 18 #include <linux/mm.h> 19 #include <linux/kdev_t.h> 20 #include <linux/gfp.h> 21 #include <linux/bio.h> 22 #include <linux/fs.h> 23 #include <linux/buffer_head.h> 24 #include <linux/blkdev.h> 25 #include <linux/highmem.h> 26 #include <linux/prefetch.h> 27 #include <linux/mpage.h> 28 #include <linux/mm_inline.h> 29 #include <linux/writeback.h> 30 #include <linux/backing-dev.h> 31 #include <linux/pagevec.h> 32 #include "internal.h" 33 34 /* 35 * I/O completion handler for multipage BIOs. 36 * 37 * The mpage code never puts partial pages into a BIO (except for end-of-file). 38 * If a page does not map to a contiguous run of blocks then it simply falls 39 * back to block_read_full_page(). 40 * 41 * Why is this? If a page's completion depends on a number of different BIOs 42 * which can complete in any order (or at the same time) then determining the 43 * status of that page is hard. See end_buffer_async_read() for the details. 44 * There is no point in duplicating all that complexity. 45 */ 46 static void mpage_end_io(struct bio *bio) 47 { 48 struct bio_vec *bv; 49 struct bvec_iter_all iter_all; 50 51 bio_for_each_segment_all(bv, bio, iter_all) { 52 struct page *page = bv->bv_page; 53 page_endio(page, bio_op(bio), 54 blk_status_to_errno(bio->bi_status)); 55 } 56 57 bio_put(bio); 58 } 59 60 static struct bio *mpage_bio_submit(struct bio *bio) 61 { 62 bio->bi_end_io = mpage_end_io; 63 guard_bio_eod(bio); 64 submit_bio(bio); 65 return NULL; 66 } 67 68 /* 69 * support function for mpage_readahead. The fs supplied get_block might 70 * return an up to date buffer. This is used to map that buffer into 71 * the page, which allows readpage to avoid triggering a duplicate call 72 * to get_block. 73 * 74 * The idea is to avoid adding buffers to pages that don't already have 75 * them. So when the buffer is up to date and the page size == block size, 76 * this marks the page up to date instead of adding new buffers. 77 */ 78 static void 79 map_buffer_to_page(struct page *page, struct buffer_head *bh, int page_block) 80 { 81 struct inode *inode = page->mapping->host; 82 struct buffer_head *page_bh, *head; 83 int block = 0; 84 85 if (!page_has_buffers(page)) { 86 /* 87 * don't make any buffers if there is only one buffer on 88 * the page and the page just needs to be set up to date 89 */ 90 if (inode->i_blkbits == PAGE_SHIFT && 91 buffer_uptodate(bh)) { 92 SetPageUptodate(page); 93 return; 94 } 95 create_empty_buffers(page, i_blocksize(inode), 0); 96 } 97 head = page_buffers(page); 98 page_bh = head; 99 do { 100 if (block == page_block) { 101 page_bh->b_state = bh->b_state; 102 page_bh->b_bdev = bh->b_bdev; 103 page_bh->b_blocknr = bh->b_blocknr; 104 break; 105 } 106 page_bh = page_bh->b_this_page; 107 block++; 108 } while (page_bh != head); 109 } 110 111 struct mpage_readpage_args { 112 struct bio *bio; 113 struct page *page; 114 unsigned int nr_pages; 115 bool is_readahead; 116 sector_t last_block_in_bio; 117 struct buffer_head map_bh; 118 unsigned long first_logical_block; 119 get_block_t *get_block; 120 }; 121 122 /* 123 * This is the worker routine which does all the work of mapping the disk 124 * blocks and constructs largest possible bios, submits them for IO if the 125 * blocks are not contiguous on the disk. 126 * 127 * We pass a buffer_head back and forth and use its buffer_mapped() flag to 128 * represent the validity of its disk mapping and to decide when to do the next 129 * get_block() call. 130 */ 131 static struct bio *do_mpage_readpage(struct mpage_readpage_args *args) 132 { 133 struct page *page = args->page; 134 struct inode *inode = page->mapping->host; 135 const unsigned blkbits = inode->i_blkbits; 136 const unsigned blocks_per_page = PAGE_SIZE >> blkbits; 137 const unsigned blocksize = 1 << blkbits; 138 struct buffer_head *map_bh = &args->map_bh; 139 sector_t block_in_file; 140 sector_t last_block; 141 sector_t last_block_in_file; 142 sector_t blocks[MAX_BUF_PER_PAGE]; 143 unsigned page_block; 144 unsigned first_hole = blocks_per_page; 145 struct block_device *bdev = NULL; 146 int length; 147 int fully_mapped = 1; 148 int op = REQ_OP_READ; 149 unsigned nblocks; 150 unsigned relative_block; 151 gfp_t gfp; 152 153 if (args->is_readahead) { 154 op |= REQ_RAHEAD; 155 gfp = readahead_gfp_mask(page->mapping); 156 } else { 157 gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL); 158 } 159 160 if (page_has_buffers(page)) 161 goto confused; 162 163 block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits); 164 last_block = block_in_file + args->nr_pages * blocks_per_page; 165 last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits; 166 if (last_block > last_block_in_file) 167 last_block = last_block_in_file; 168 page_block = 0; 169 170 /* 171 * Map blocks using the result from the previous get_blocks call first. 172 */ 173 nblocks = map_bh->b_size >> blkbits; 174 if (buffer_mapped(map_bh) && 175 block_in_file > args->first_logical_block && 176 block_in_file < (args->first_logical_block + nblocks)) { 177 unsigned map_offset = block_in_file - args->first_logical_block; 178 unsigned last = nblocks - map_offset; 179 180 for (relative_block = 0; ; relative_block++) { 181 if (relative_block == last) { 182 clear_buffer_mapped(map_bh); 183 break; 184 } 185 if (page_block == blocks_per_page) 186 break; 187 blocks[page_block] = map_bh->b_blocknr + map_offset + 188 relative_block; 189 page_block++; 190 block_in_file++; 191 } 192 bdev = map_bh->b_bdev; 193 } 194 195 /* 196 * Then do more get_blocks calls until we are done with this page. 197 */ 198 map_bh->b_page = page; 199 while (page_block < blocks_per_page) { 200 map_bh->b_state = 0; 201 map_bh->b_size = 0; 202 203 if (block_in_file < last_block) { 204 map_bh->b_size = (last_block-block_in_file) << blkbits; 205 if (args->get_block(inode, block_in_file, map_bh, 0)) 206 goto confused; 207 args->first_logical_block = block_in_file; 208 } 209 210 if (!buffer_mapped(map_bh)) { 211 fully_mapped = 0; 212 if (first_hole == blocks_per_page) 213 first_hole = page_block; 214 page_block++; 215 block_in_file++; 216 continue; 217 } 218 219 /* some filesystems will copy data into the page during 220 * the get_block call, in which case we don't want to 221 * read it again. map_buffer_to_page copies the data 222 * we just collected from get_block into the page's buffers 223 * so readpage doesn't have to repeat the get_block call 224 */ 225 if (buffer_uptodate(map_bh)) { 226 map_buffer_to_page(page, map_bh, page_block); 227 goto confused; 228 } 229 230 if (first_hole != blocks_per_page) 231 goto confused; /* hole -> non-hole */ 232 233 /* Contiguous blocks? */ 234 if (page_block && blocks[page_block-1] != map_bh->b_blocknr-1) 235 goto confused; 236 nblocks = map_bh->b_size >> blkbits; 237 for (relative_block = 0; ; relative_block++) { 238 if (relative_block == nblocks) { 239 clear_buffer_mapped(map_bh); 240 break; 241 } else if (page_block == blocks_per_page) 242 break; 243 blocks[page_block] = map_bh->b_blocknr+relative_block; 244 page_block++; 245 block_in_file++; 246 } 247 bdev = map_bh->b_bdev; 248 } 249 250 if (first_hole != blocks_per_page) { 251 zero_user_segment(page, first_hole << blkbits, PAGE_SIZE); 252 if (first_hole == 0) { 253 SetPageUptodate(page); 254 unlock_page(page); 255 goto out; 256 } 257 } else if (fully_mapped) { 258 SetPageMappedToDisk(page); 259 } 260 261 /* 262 * This page will go to BIO. Do we need to send this BIO off first? 263 */ 264 if (args->bio && (args->last_block_in_bio != blocks[0] - 1)) 265 args->bio = mpage_bio_submit(args->bio); 266 267 alloc_new: 268 if (args->bio == NULL) { 269 if (first_hole == blocks_per_page) { 270 if (!bdev_read_page(bdev, blocks[0] << (blkbits - 9), 271 page)) 272 goto out; 273 } 274 args->bio = bio_alloc(bdev, bio_max_segs(args->nr_pages), op, 275 gfp); 276 if (args->bio == NULL) 277 goto confused; 278 args->bio->bi_iter.bi_sector = blocks[0] << (blkbits - 9); 279 } 280 281 length = first_hole << blkbits; 282 if (bio_add_page(args->bio, page, length, 0) < length) { 283 args->bio = mpage_bio_submit(args->bio); 284 goto alloc_new; 285 } 286 287 relative_block = block_in_file - args->first_logical_block; 288 nblocks = map_bh->b_size >> blkbits; 289 if ((buffer_boundary(map_bh) && relative_block == nblocks) || 290 (first_hole != blocks_per_page)) 291 args->bio = mpage_bio_submit(args->bio); 292 else 293 args->last_block_in_bio = blocks[blocks_per_page - 1]; 294 out: 295 return args->bio; 296 297 confused: 298 if (args->bio) 299 args->bio = mpage_bio_submit(args->bio); 300 if (!PageUptodate(page)) 301 block_read_full_page(page, args->get_block); 302 else 303 unlock_page(page); 304 goto out; 305 } 306 307 /** 308 * mpage_readahead - start reads against pages 309 * @rac: Describes which pages to read. 310 * @get_block: The filesystem's block mapper function. 311 * 312 * This function walks the pages and the blocks within each page, building and 313 * emitting large BIOs. 314 * 315 * If anything unusual happens, such as: 316 * 317 * - encountering a page which has buffers 318 * - encountering a page which has a non-hole after a hole 319 * - encountering a page with non-contiguous blocks 320 * 321 * then this code just gives up and calls the buffer_head-based read function. 322 * It does handle a page which has holes at the end - that is a common case: 323 * the end-of-file on blocksize < PAGE_SIZE setups. 324 * 325 * BH_Boundary explanation: 326 * 327 * There is a problem. The mpage read code assembles several pages, gets all 328 * their disk mappings, and then submits them all. That's fine, but obtaining 329 * the disk mappings may require I/O. Reads of indirect blocks, for example. 330 * 331 * So an mpage read of the first 16 blocks of an ext2 file will cause I/O to be 332 * submitted in the following order: 333 * 334 * 12 0 1 2 3 4 5 6 7 8 9 10 11 13 14 15 16 335 * 336 * because the indirect block has to be read to get the mappings of blocks 337 * 13,14,15,16. Obviously, this impacts performance. 338 * 339 * So what we do it to allow the filesystem's get_block() function to set 340 * BH_Boundary when it maps block 11. BH_Boundary says: mapping of the block 341 * after this one will require I/O against a block which is probably close to 342 * this one. So you should push what I/O you have currently accumulated. 343 * 344 * This all causes the disk requests to be issued in the correct order. 345 */ 346 void mpage_readahead(struct readahead_control *rac, get_block_t get_block) 347 { 348 struct page *page; 349 struct mpage_readpage_args args = { 350 .get_block = get_block, 351 .is_readahead = true, 352 }; 353 354 while ((page = readahead_page(rac))) { 355 prefetchw(&page->flags); 356 args.page = page; 357 args.nr_pages = readahead_count(rac); 358 args.bio = do_mpage_readpage(&args); 359 put_page(page); 360 } 361 if (args.bio) 362 mpage_bio_submit(args.bio); 363 } 364 EXPORT_SYMBOL(mpage_readahead); 365 366 /* 367 * This isn't called much at all 368 */ 369 int mpage_readpage(struct page *page, get_block_t get_block) 370 { 371 struct mpage_readpage_args args = { 372 .page = page, 373 .nr_pages = 1, 374 .get_block = get_block, 375 }; 376 377 args.bio = do_mpage_readpage(&args); 378 if (args.bio) 379 mpage_bio_submit(args.bio); 380 return 0; 381 } 382 EXPORT_SYMBOL(mpage_readpage); 383 384 /* 385 * Writing is not so simple. 386 * 387 * If the page has buffers then they will be used for obtaining the disk 388 * mapping. We only support pages which are fully mapped-and-dirty, with a 389 * special case for pages which are unmapped at the end: end-of-file. 390 * 391 * If the page has no buffers (preferred) then the page is mapped here. 392 * 393 * If all blocks are found to be contiguous then the page can go into the 394 * BIO. Otherwise fall back to the mapping's writepage(). 395 * 396 * FIXME: This code wants an estimate of how many pages are still to be 397 * written, so it can intelligently allocate a suitably-sized BIO. For now, 398 * just allocate full-size (16-page) BIOs. 399 */ 400 401 struct mpage_data { 402 struct bio *bio; 403 sector_t last_block_in_bio; 404 get_block_t *get_block; 405 unsigned use_writepage; 406 }; 407 408 /* 409 * We have our BIO, so we can now mark the buffers clean. Make 410 * sure to only clean buffers which we know we'll be writing. 411 */ 412 static void clean_buffers(struct page *page, unsigned first_unmapped) 413 { 414 unsigned buffer_counter = 0; 415 struct buffer_head *bh, *head; 416 if (!page_has_buffers(page)) 417 return; 418 head = page_buffers(page); 419 bh = head; 420 421 do { 422 if (buffer_counter++ == first_unmapped) 423 break; 424 clear_buffer_dirty(bh); 425 bh = bh->b_this_page; 426 } while (bh != head); 427 428 /* 429 * we cannot drop the bh if the page is not uptodate or a concurrent 430 * readpage would fail to serialize with the bh and it would read from 431 * disk before we reach the platter. 432 */ 433 if (buffer_heads_over_limit && PageUptodate(page)) 434 try_to_free_buffers(page); 435 } 436 437 /* 438 * For situations where we want to clean all buffers attached to a page. 439 * We don't need to calculate how many buffers are attached to the page, 440 * we just need to specify a number larger than the maximum number of buffers. 441 */ 442 void clean_page_buffers(struct page *page) 443 { 444 clean_buffers(page, ~0U); 445 } 446 447 static int __mpage_writepage(struct page *page, struct writeback_control *wbc, 448 void *data) 449 { 450 struct mpage_data *mpd = data; 451 struct bio *bio = mpd->bio; 452 struct address_space *mapping = page->mapping; 453 struct inode *inode = page->mapping->host; 454 const unsigned blkbits = inode->i_blkbits; 455 unsigned long end_index; 456 const unsigned blocks_per_page = PAGE_SIZE >> blkbits; 457 sector_t last_block; 458 sector_t block_in_file; 459 sector_t blocks[MAX_BUF_PER_PAGE]; 460 unsigned page_block; 461 unsigned first_unmapped = blocks_per_page; 462 struct block_device *bdev = NULL; 463 int boundary = 0; 464 sector_t boundary_block = 0; 465 struct block_device *boundary_bdev = NULL; 466 int length; 467 struct buffer_head map_bh; 468 loff_t i_size = i_size_read(inode); 469 int ret = 0; 470 471 if (page_has_buffers(page)) { 472 struct buffer_head *head = page_buffers(page); 473 struct buffer_head *bh = head; 474 475 /* If they're all mapped and dirty, do it */ 476 page_block = 0; 477 do { 478 BUG_ON(buffer_locked(bh)); 479 if (!buffer_mapped(bh)) { 480 /* 481 * unmapped dirty buffers are created by 482 * __set_page_dirty_buffers -> mmapped data 483 */ 484 if (buffer_dirty(bh)) 485 goto confused; 486 if (first_unmapped == blocks_per_page) 487 first_unmapped = page_block; 488 continue; 489 } 490 491 if (first_unmapped != blocks_per_page) 492 goto confused; /* hole -> non-hole */ 493 494 if (!buffer_dirty(bh) || !buffer_uptodate(bh)) 495 goto confused; 496 if (page_block) { 497 if (bh->b_blocknr != blocks[page_block-1] + 1) 498 goto confused; 499 } 500 blocks[page_block++] = bh->b_blocknr; 501 boundary = buffer_boundary(bh); 502 if (boundary) { 503 boundary_block = bh->b_blocknr; 504 boundary_bdev = bh->b_bdev; 505 } 506 bdev = bh->b_bdev; 507 } while ((bh = bh->b_this_page) != head); 508 509 if (first_unmapped) 510 goto page_is_mapped; 511 512 /* 513 * Page has buffers, but they are all unmapped. The page was 514 * created by pagein or read over a hole which was handled by 515 * block_read_full_page(). If this address_space is also 516 * using mpage_readahead then this can rarely happen. 517 */ 518 goto confused; 519 } 520 521 /* 522 * The page has no buffers: map it to disk 523 */ 524 BUG_ON(!PageUptodate(page)); 525 block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits); 526 last_block = (i_size - 1) >> blkbits; 527 map_bh.b_page = page; 528 for (page_block = 0; page_block < blocks_per_page; ) { 529 530 map_bh.b_state = 0; 531 map_bh.b_size = 1 << blkbits; 532 if (mpd->get_block(inode, block_in_file, &map_bh, 1)) 533 goto confused; 534 if (buffer_new(&map_bh)) 535 clean_bdev_bh_alias(&map_bh); 536 if (buffer_boundary(&map_bh)) { 537 boundary_block = map_bh.b_blocknr; 538 boundary_bdev = map_bh.b_bdev; 539 } 540 if (page_block) { 541 if (map_bh.b_blocknr != blocks[page_block-1] + 1) 542 goto confused; 543 } 544 blocks[page_block++] = map_bh.b_blocknr; 545 boundary = buffer_boundary(&map_bh); 546 bdev = map_bh.b_bdev; 547 if (block_in_file == last_block) 548 break; 549 block_in_file++; 550 } 551 BUG_ON(page_block == 0); 552 553 first_unmapped = page_block; 554 555 page_is_mapped: 556 end_index = i_size >> PAGE_SHIFT; 557 if (page->index >= end_index) { 558 /* 559 * The page straddles i_size. It must be zeroed out on each 560 * and every writepage invocation because it may be mmapped. 561 * "A file is mapped in multiples of the page size. For a file 562 * that is not a multiple of the page size, the remaining memory 563 * is zeroed when mapped, and writes to that region are not 564 * written out to the file." 565 */ 566 unsigned offset = i_size & (PAGE_SIZE - 1); 567 568 if (page->index > end_index || !offset) 569 goto confused; 570 zero_user_segment(page, offset, PAGE_SIZE); 571 } 572 573 /* 574 * This page will go to BIO. Do we need to send this BIO off first? 575 */ 576 if (bio && mpd->last_block_in_bio != blocks[0] - 1) 577 bio = mpage_bio_submit(bio); 578 579 alloc_new: 580 if (bio == NULL) { 581 if (first_unmapped == blocks_per_page) { 582 if (!bdev_write_page(bdev, blocks[0] << (blkbits - 9), 583 page, wbc)) 584 goto out; 585 } 586 bio = bio_alloc(bdev, BIO_MAX_VECS, 587 REQ_OP_WRITE | wbc_to_write_flags(wbc), 588 GFP_NOFS); 589 bio->bi_iter.bi_sector = blocks[0] << (blkbits - 9); 590 wbc_init_bio(wbc, bio); 591 bio->bi_write_hint = inode->i_write_hint; 592 } 593 594 /* 595 * Must try to add the page before marking the buffer clean or 596 * the confused fail path above (OOM) will be very confused when 597 * it finds all bh marked clean (i.e. it will not write anything) 598 */ 599 wbc_account_cgroup_owner(wbc, page, PAGE_SIZE); 600 length = first_unmapped << blkbits; 601 if (bio_add_page(bio, page, length, 0) < length) { 602 bio = mpage_bio_submit(bio); 603 goto alloc_new; 604 } 605 606 clean_buffers(page, first_unmapped); 607 608 BUG_ON(PageWriteback(page)); 609 set_page_writeback(page); 610 unlock_page(page); 611 if (boundary || (first_unmapped != blocks_per_page)) { 612 bio = mpage_bio_submit(bio); 613 if (boundary_block) { 614 write_boundary_block(boundary_bdev, 615 boundary_block, 1 << blkbits); 616 } 617 } else { 618 mpd->last_block_in_bio = blocks[blocks_per_page - 1]; 619 } 620 goto out; 621 622 confused: 623 if (bio) 624 bio = mpage_bio_submit(bio); 625 626 if (mpd->use_writepage) { 627 ret = mapping->a_ops->writepage(page, wbc); 628 } else { 629 ret = -EAGAIN; 630 goto out; 631 } 632 /* 633 * The caller has a ref on the inode, so *mapping is stable 634 */ 635 mapping_set_error(mapping, ret); 636 out: 637 mpd->bio = bio; 638 return ret; 639 } 640 641 /** 642 * mpage_writepages - walk the list of dirty pages of the given address space & writepage() all of them 643 * @mapping: address space structure to write 644 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 645 * @get_block: the filesystem's block mapper function. 646 * If this is NULL then use a_ops->writepage. Otherwise, go 647 * direct-to-BIO. 648 * 649 * This is a library function, which implements the writepages() 650 * address_space_operation. 651 * 652 * If a page is already under I/O, generic_writepages() skips it, even 653 * if it's dirty. This is desirable behaviour for memory-cleaning writeback, 654 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync() 655 * and msync() need to guarantee that all the data which was dirty at the time 656 * the call was made get new I/O started against them. If wbc->sync_mode is 657 * WB_SYNC_ALL then we were called for data integrity and we must wait for 658 * existing IO to complete. 659 */ 660 int 661 mpage_writepages(struct address_space *mapping, 662 struct writeback_control *wbc, get_block_t get_block) 663 { 664 struct blk_plug plug; 665 int ret; 666 667 blk_start_plug(&plug); 668 669 if (!get_block) 670 ret = generic_writepages(mapping, wbc); 671 else { 672 struct mpage_data mpd = { 673 .bio = NULL, 674 .last_block_in_bio = 0, 675 .get_block = get_block, 676 .use_writepage = 1, 677 }; 678 679 ret = write_cache_pages(mapping, wbc, __mpage_writepage, &mpd); 680 if (mpd.bio) 681 mpage_bio_submit(mpd.bio); 682 } 683 blk_finish_plug(&plug); 684 return ret; 685 } 686 EXPORT_SYMBOL(mpage_writepages); 687 688 int mpage_writepage(struct page *page, get_block_t get_block, 689 struct writeback_control *wbc) 690 { 691 struct mpage_data mpd = { 692 .bio = NULL, 693 .last_block_in_bio = 0, 694 .get_block = get_block, 695 .use_writepage = 0, 696 }; 697 int ret = __mpage_writepage(page, wbc, &mpd); 698 if (mpd.bio) 699 mpage_bio_submit(mpd.bio); 700 return ret; 701 } 702 EXPORT_SYMBOL(mpage_writepage); 703