1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/fs/buffer.c 4 * 5 * Copyright (C) 1991, 1992, 2002 Linus Torvalds 6 */ 7 8 /* 9 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95 10 * 11 * Removed a lot of unnecessary code and simplified things now that 12 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96 13 * 14 * Speed up hash, lru, and free list operations. Use gfp() for allocating 15 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM 16 * 17 * Added 32k buffer block sizes - these are required older ARM systems. - RMK 18 * 19 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de> 20 */ 21 22 #include <linux/kernel.h> 23 #include <linux/sched/signal.h> 24 #include <linux/syscalls.h> 25 #include <linux/fs.h> 26 #include <linux/iomap.h> 27 #include <linux/mm.h> 28 #include <linux/percpu.h> 29 #include <linux/slab.h> 30 #include <linux/capability.h> 31 #include <linux/blkdev.h> 32 #include <linux/file.h> 33 #include <linux/quotaops.h> 34 #include <linux/highmem.h> 35 #include <linux/export.h> 36 #include <linux/backing-dev.h> 37 #include <linux/writeback.h> 38 #include <linux/hash.h> 39 #include <linux/suspend.h> 40 #include <linux/buffer_head.h> 41 #include <linux/task_io_accounting_ops.h> 42 #include <linux/bio.h> 43 #include <linux/cpu.h> 44 #include <linux/bitops.h> 45 #include <linux/mpage.h> 46 #include <linux/bit_spinlock.h> 47 #include <linux/pagevec.h> 48 #include <linux/sched/mm.h> 49 #include <trace/events/block.h> 50 #include <linux/fscrypt.h> 51 52 #include "internal.h" 53 54 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list); 55 static int submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, 56 struct writeback_control *wbc); 57 58 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) 59 60 inline void touch_buffer(struct buffer_head *bh) 61 { 62 trace_block_touch_buffer(bh); 63 mark_page_accessed(bh->b_page); 64 } 65 EXPORT_SYMBOL(touch_buffer); 66 67 void __lock_buffer(struct buffer_head *bh) 68 { 69 wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE); 70 } 71 EXPORT_SYMBOL(__lock_buffer); 72 73 void unlock_buffer(struct buffer_head *bh) 74 { 75 clear_bit_unlock(BH_Lock, &bh->b_state); 76 smp_mb__after_atomic(); 77 wake_up_bit(&bh->b_state, BH_Lock); 78 } 79 EXPORT_SYMBOL(unlock_buffer); 80 81 /* 82 * Returns if the folio has dirty or writeback buffers. If all the buffers 83 * are unlocked and clean then the folio_test_dirty information is stale. If 84 * any of the buffers are locked, it is assumed they are locked for IO. 85 */ 86 void buffer_check_dirty_writeback(struct folio *folio, 87 bool *dirty, bool *writeback) 88 { 89 struct buffer_head *head, *bh; 90 *dirty = false; 91 *writeback = false; 92 93 BUG_ON(!folio_test_locked(folio)); 94 95 head = folio_buffers(folio); 96 if (!head) 97 return; 98 99 if (folio_test_writeback(folio)) 100 *writeback = true; 101 102 bh = head; 103 do { 104 if (buffer_locked(bh)) 105 *writeback = true; 106 107 if (buffer_dirty(bh)) 108 *dirty = true; 109 110 bh = bh->b_this_page; 111 } while (bh != head); 112 } 113 EXPORT_SYMBOL(buffer_check_dirty_writeback); 114 115 /* 116 * Block until a buffer comes unlocked. This doesn't stop it 117 * from becoming locked again - you have to lock it yourself 118 * if you want to preserve its state. 119 */ 120 void __wait_on_buffer(struct buffer_head * bh) 121 { 122 wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE); 123 } 124 EXPORT_SYMBOL(__wait_on_buffer); 125 126 static void buffer_io_error(struct buffer_head *bh, char *msg) 127 { 128 if (!test_bit(BH_Quiet, &bh->b_state)) 129 printk_ratelimited(KERN_ERR 130 "Buffer I/O error on dev %pg, logical block %llu%s\n", 131 bh->b_bdev, (unsigned long long)bh->b_blocknr, msg); 132 } 133 134 /* 135 * End-of-IO handler helper function which does not touch the bh after 136 * unlocking it. 137 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but 138 * a race there is benign: unlock_buffer() only use the bh's address for 139 * hashing after unlocking the buffer, so it doesn't actually touch the bh 140 * itself. 141 */ 142 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate) 143 { 144 if (uptodate) { 145 set_buffer_uptodate(bh); 146 } else { 147 /* This happens, due to failed read-ahead attempts. */ 148 clear_buffer_uptodate(bh); 149 } 150 unlock_buffer(bh); 151 } 152 153 /* 154 * Default synchronous end-of-IO handler.. Just mark it up-to-date and 155 * unlock the buffer. This is what ll_rw_block uses too. 156 */ 157 void end_buffer_read_sync(struct buffer_head *bh, int uptodate) 158 { 159 __end_buffer_read_notouch(bh, uptodate); 160 put_bh(bh); 161 } 162 EXPORT_SYMBOL(end_buffer_read_sync); 163 164 void end_buffer_write_sync(struct buffer_head *bh, int uptodate) 165 { 166 if (uptodate) { 167 set_buffer_uptodate(bh); 168 } else { 169 buffer_io_error(bh, ", lost sync page write"); 170 mark_buffer_write_io_error(bh); 171 clear_buffer_uptodate(bh); 172 } 173 unlock_buffer(bh); 174 put_bh(bh); 175 } 176 EXPORT_SYMBOL(end_buffer_write_sync); 177 178 /* 179 * Various filesystems appear to want __find_get_block to be non-blocking. 180 * But it's the page lock which protects the buffers. To get around this, 181 * we get exclusion from try_to_free_buffers with the blockdev mapping's 182 * private_lock. 183 * 184 * Hack idea: for the blockdev mapping, private_lock contention 185 * may be quite high. This code could TryLock the page, and if that 186 * succeeds, there is no need to take private_lock. 187 */ 188 static struct buffer_head * 189 __find_get_block_slow(struct block_device *bdev, sector_t block) 190 { 191 struct inode *bd_inode = bdev->bd_inode; 192 struct address_space *bd_mapping = bd_inode->i_mapping; 193 struct buffer_head *ret = NULL; 194 pgoff_t index; 195 struct buffer_head *bh; 196 struct buffer_head *head; 197 struct page *page; 198 int all_mapped = 1; 199 static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1); 200 201 index = block >> (PAGE_SHIFT - bd_inode->i_blkbits); 202 page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED); 203 if (!page) 204 goto out; 205 206 spin_lock(&bd_mapping->private_lock); 207 if (!page_has_buffers(page)) 208 goto out_unlock; 209 head = page_buffers(page); 210 bh = head; 211 do { 212 if (!buffer_mapped(bh)) 213 all_mapped = 0; 214 else if (bh->b_blocknr == block) { 215 ret = bh; 216 get_bh(bh); 217 goto out_unlock; 218 } 219 bh = bh->b_this_page; 220 } while (bh != head); 221 222 /* we might be here because some of the buffers on this page are 223 * not mapped. This is due to various races between 224 * file io on the block device and getblk. It gets dealt with 225 * elsewhere, don't buffer_error if we had some unmapped buffers 226 */ 227 ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE); 228 if (all_mapped && __ratelimit(&last_warned)) { 229 printk("__find_get_block_slow() failed. block=%llu, " 230 "b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, " 231 "device %pg blocksize: %d\n", 232 (unsigned long long)block, 233 (unsigned long long)bh->b_blocknr, 234 bh->b_state, bh->b_size, bdev, 235 1 << bd_inode->i_blkbits); 236 } 237 out_unlock: 238 spin_unlock(&bd_mapping->private_lock); 239 put_page(page); 240 out: 241 return ret; 242 } 243 244 static void end_buffer_async_read(struct buffer_head *bh, int uptodate) 245 { 246 unsigned long flags; 247 struct buffer_head *first; 248 struct buffer_head *tmp; 249 struct page *page; 250 int page_uptodate = 1; 251 252 BUG_ON(!buffer_async_read(bh)); 253 254 page = bh->b_page; 255 if (uptodate) { 256 set_buffer_uptodate(bh); 257 } else { 258 clear_buffer_uptodate(bh); 259 buffer_io_error(bh, ", async page read"); 260 SetPageError(page); 261 } 262 263 /* 264 * Be _very_ careful from here on. Bad things can happen if 265 * two buffer heads end IO at almost the same time and both 266 * decide that the page is now completely done. 267 */ 268 first = page_buffers(page); 269 spin_lock_irqsave(&first->b_uptodate_lock, flags); 270 clear_buffer_async_read(bh); 271 unlock_buffer(bh); 272 tmp = bh; 273 do { 274 if (!buffer_uptodate(tmp)) 275 page_uptodate = 0; 276 if (buffer_async_read(tmp)) { 277 BUG_ON(!buffer_locked(tmp)); 278 goto still_busy; 279 } 280 tmp = tmp->b_this_page; 281 } while (tmp != bh); 282 spin_unlock_irqrestore(&first->b_uptodate_lock, flags); 283 284 /* 285 * If all of the buffers are uptodate then we can set the page 286 * uptodate. 287 */ 288 if (page_uptodate) 289 SetPageUptodate(page); 290 unlock_page(page); 291 return; 292 293 still_busy: 294 spin_unlock_irqrestore(&first->b_uptodate_lock, flags); 295 return; 296 } 297 298 struct decrypt_bh_ctx { 299 struct work_struct work; 300 struct buffer_head *bh; 301 }; 302 303 static void decrypt_bh(struct work_struct *work) 304 { 305 struct decrypt_bh_ctx *ctx = 306 container_of(work, struct decrypt_bh_ctx, work); 307 struct buffer_head *bh = ctx->bh; 308 int err; 309 310 err = fscrypt_decrypt_pagecache_blocks(bh->b_page, bh->b_size, 311 bh_offset(bh)); 312 end_buffer_async_read(bh, err == 0); 313 kfree(ctx); 314 } 315 316 /* 317 * I/O completion handler for block_read_full_folio() - pages 318 * which come unlocked at the end of I/O. 319 */ 320 static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate) 321 { 322 /* Decrypt if needed */ 323 if (uptodate && 324 fscrypt_inode_uses_fs_layer_crypto(bh->b_page->mapping->host)) { 325 struct decrypt_bh_ctx *ctx = kmalloc(sizeof(*ctx), GFP_ATOMIC); 326 327 if (ctx) { 328 INIT_WORK(&ctx->work, decrypt_bh); 329 ctx->bh = bh; 330 fscrypt_enqueue_decrypt_work(&ctx->work); 331 return; 332 } 333 uptodate = 0; 334 } 335 end_buffer_async_read(bh, uptodate); 336 } 337 338 /* 339 * Completion handler for block_write_full_page() - pages which are unlocked 340 * during I/O, and which have PageWriteback cleared upon I/O completion. 341 */ 342 void end_buffer_async_write(struct buffer_head *bh, int uptodate) 343 { 344 unsigned long flags; 345 struct buffer_head *first; 346 struct buffer_head *tmp; 347 struct page *page; 348 349 BUG_ON(!buffer_async_write(bh)); 350 351 page = bh->b_page; 352 if (uptodate) { 353 set_buffer_uptodate(bh); 354 } else { 355 buffer_io_error(bh, ", lost async page write"); 356 mark_buffer_write_io_error(bh); 357 clear_buffer_uptodate(bh); 358 SetPageError(page); 359 } 360 361 first = page_buffers(page); 362 spin_lock_irqsave(&first->b_uptodate_lock, flags); 363 364 clear_buffer_async_write(bh); 365 unlock_buffer(bh); 366 tmp = bh->b_this_page; 367 while (tmp != bh) { 368 if (buffer_async_write(tmp)) { 369 BUG_ON(!buffer_locked(tmp)); 370 goto still_busy; 371 } 372 tmp = tmp->b_this_page; 373 } 374 spin_unlock_irqrestore(&first->b_uptodate_lock, flags); 375 end_page_writeback(page); 376 return; 377 378 still_busy: 379 spin_unlock_irqrestore(&first->b_uptodate_lock, flags); 380 return; 381 } 382 EXPORT_SYMBOL(end_buffer_async_write); 383 384 /* 385 * If a page's buffers are under async readin (end_buffer_async_read 386 * completion) then there is a possibility that another thread of 387 * control could lock one of the buffers after it has completed 388 * but while some of the other buffers have not completed. This 389 * locked buffer would confuse end_buffer_async_read() into not unlocking 390 * the page. So the absence of BH_Async_Read tells end_buffer_async_read() 391 * that this buffer is not under async I/O. 392 * 393 * The page comes unlocked when it has no locked buffer_async buffers 394 * left. 395 * 396 * PageLocked prevents anyone starting new async I/O reads any of 397 * the buffers. 398 * 399 * PageWriteback is used to prevent simultaneous writeout of the same 400 * page. 401 * 402 * PageLocked prevents anyone from starting writeback of a page which is 403 * under read I/O (PageWriteback is only ever set against a locked page). 404 */ 405 static void mark_buffer_async_read(struct buffer_head *bh) 406 { 407 bh->b_end_io = end_buffer_async_read_io; 408 set_buffer_async_read(bh); 409 } 410 411 static void mark_buffer_async_write_endio(struct buffer_head *bh, 412 bh_end_io_t *handler) 413 { 414 bh->b_end_io = handler; 415 set_buffer_async_write(bh); 416 } 417 418 void mark_buffer_async_write(struct buffer_head *bh) 419 { 420 mark_buffer_async_write_endio(bh, end_buffer_async_write); 421 } 422 EXPORT_SYMBOL(mark_buffer_async_write); 423 424 425 /* 426 * fs/buffer.c contains helper functions for buffer-backed address space's 427 * fsync functions. A common requirement for buffer-based filesystems is 428 * that certain data from the backing blockdev needs to be written out for 429 * a successful fsync(). For example, ext2 indirect blocks need to be 430 * written back and waited upon before fsync() returns. 431 * 432 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(), 433 * inode_has_buffers() and invalidate_inode_buffers() are provided for the 434 * management of a list of dependent buffers at ->i_mapping->private_list. 435 * 436 * Locking is a little subtle: try_to_free_buffers() will remove buffers 437 * from their controlling inode's queue when they are being freed. But 438 * try_to_free_buffers() will be operating against the *blockdev* mapping 439 * at the time, not against the S_ISREG file which depends on those buffers. 440 * So the locking for private_list is via the private_lock in the address_space 441 * which backs the buffers. Which is different from the address_space 442 * against which the buffers are listed. So for a particular address_space, 443 * mapping->private_lock does *not* protect mapping->private_list! In fact, 444 * mapping->private_list will always be protected by the backing blockdev's 445 * ->private_lock. 446 * 447 * Which introduces a requirement: all buffers on an address_space's 448 * ->private_list must be from the same address_space: the blockdev's. 449 * 450 * address_spaces which do not place buffers at ->private_list via these 451 * utility functions are free to use private_lock and private_list for 452 * whatever they want. The only requirement is that list_empty(private_list) 453 * be true at clear_inode() time. 454 * 455 * FIXME: clear_inode should not call invalidate_inode_buffers(). The 456 * filesystems should do that. invalidate_inode_buffers() should just go 457 * BUG_ON(!list_empty). 458 * 459 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should 460 * take an address_space, not an inode. And it should be called 461 * mark_buffer_dirty_fsync() to clearly define why those buffers are being 462 * queued up. 463 * 464 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the 465 * list if it is already on a list. Because if the buffer is on a list, 466 * it *must* already be on the right one. If not, the filesystem is being 467 * silly. This will save a ton of locking. But first we have to ensure 468 * that buffers are taken *off* the old inode's list when they are freed 469 * (presumably in truncate). That requires careful auditing of all 470 * filesystems (do it inside bforget()). It could also be done by bringing 471 * b_inode back. 472 */ 473 474 /* 475 * The buffer's backing address_space's private_lock must be held 476 */ 477 static void __remove_assoc_queue(struct buffer_head *bh) 478 { 479 list_del_init(&bh->b_assoc_buffers); 480 WARN_ON(!bh->b_assoc_map); 481 bh->b_assoc_map = NULL; 482 } 483 484 int inode_has_buffers(struct inode *inode) 485 { 486 return !list_empty(&inode->i_data.private_list); 487 } 488 489 /* 490 * osync is designed to support O_SYNC io. It waits synchronously for 491 * all already-submitted IO to complete, but does not queue any new 492 * writes to the disk. 493 * 494 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as 495 * you dirty the buffers, and then use osync_inode_buffers to wait for 496 * completion. Any other dirty buffers which are not yet queued for 497 * write will not be flushed to disk by the osync. 498 */ 499 static int osync_buffers_list(spinlock_t *lock, struct list_head *list) 500 { 501 struct buffer_head *bh; 502 struct list_head *p; 503 int err = 0; 504 505 spin_lock(lock); 506 repeat: 507 list_for_each_prev(p, list) { 508 bh = BH_ENTRY(p); 509 if (buffer_locked(bh)) { 510 get_bh(bh); 511 spin_unlock(lock); 512 wait_on_buffer(bh); 513 if (!buffer_uptodate(bh)) 514 err = -EIO; 515 brelse(bh); 516 spin_lock(lock); 517 goto repeat; 518 } 519 } 520 spin_unlock(lock); 521 return err; 522 } 523 524 void emergency_thaw_bdev(struct super_block *sb) 525 { 526 while (sb->s_bdev && !thaw_bdev(sb->s_bdev)) 527 printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev); 528 } 529 530 /** 531 * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers 532 * @mapping: the mapping which wants those buffers written 533 * 534 * Starts I/O against the buffers at mapping->private_list, and waits upon 535 * that I/O. 536 * 537 * Basically, this is a convenience function for fsync(). 538 * @mapping is a file or directory which needs those buffers to be written for 539 * a successful fsync(). 540 */ 541 int sync_mapping_buffers(struct address_space *mapping) 542 { 543 struct address_space *buffer_mapping = mapping->private_data; 544 545 if (buffer_mapping == NULL || list_empty(&mapping->private_list)) 546 return 0; 547 548 return fsync_buffers_list(&buffer_mapping->private_lock, 549 &mapping->private_list); 550 } 551 EXPORT_SYMBOL(sync_mapping_buffers); 552 553 /* 554 * Called when we've recently written block `bblock', and it is known that 555 * `bblock' was for a buffer_boundary() buffer. This means that the block at 556 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's 557 * dirty, schedule it for IO. So that indirects merge nicely with their data. 558 */ 559 void write_boundary_block(struct block_device *bdev, 560 sector_t bblock, unsigned blocksize) 561 { 562 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize); 563 if (bh) { 564 if (buffer_dirty(bh)) 565 ll_rw_block(REQ_OP_WRITE, 1, &bh); 566 put_bh(bh); 567 } 568 } 569 570 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode) 571 { 572 struct address_space *mapping = inode->i_mapping; 573 struct address_space *buffer_mapping = bh->b_page->mapping; 574 575 mark_buffer_dirty(bh); 576 if (!mapping->private_data) { 577 mapping->private_data = buffer_mapping; 578 } else { 579 BUG_ON(mapping->private_data != buffer_mapping); 580 } 581 if (!bh->b_assoc_map) { 582 spin_lock(&buffer_mapping->private_lock); 583 list_move_tail(&bh->b_assoc_buffers, 584 &mapping->private_list); 585 bh->b_assoc_map = mapping; 586 spin_unlock(&buffer_mapping->private_lock); 587 } 588 } 589 EXPORT_SYMBOL(mark_buffer_dirty_inode); 590 591 /* 592 * Add a page to the dirty page list. 593 * 594 * It is a sad fact of life that this function is called from several places 595 * deeply under spinlocking. It may not sleep. 596 * 597 * If the page has buffers, the uptodate buffers are set dirty, to preserve 598 * dirty-state coherency between the page and the buffers. It the page does 599 * not have buffers then when they are later attached they will all be set 600 * dirty. 601 * 602 * The buffers are dirtied before the page is dirtied. There's a small race 603 * window in which a writepage caller may see the page cleanness but not the 604 * buffer dirtiness. That's fine. If this code were to set the page dirty 605 * before the buffers, a concurrent writepage caller could clear the page dirty 606 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean 607 * page on the dirty page list. 608 * 609 * We use private_lock to lock against try_to_free_buffers while using the 610 * page's buffer list. Also use this to protect against clean buffers being 611 * added to the page after it was set dirty. 612 * 613 * FIXME: may need to call ->reservepage here as well. That's rather up to the 614 * address_space though. 615 */ 616 bool block_dirty_folio(struct address_space *mapping, struct folio *folio) 617 { 618 struct buffer_head *head; 619 bool newly_dirty; 620 621 spin_lock(&mapping->private_lock); 622 head = folio_buffers(folio); 623 if (head) { 624 struct buffer_head *bh = head; 625 626 do { 627 set_buffer_dirty(bh); 628 bh = bh->b_this_page; 629 } while (bh != head); 630 } 631 /* 632 * Lock out page's memcg migration to keep PageDirty 633 * synchronized with per-memcg dirty page counters. 634 */ 635 folio_memcg_lock(folio); 636 newly_dirty = !folio_test_set_dirty(folio); 637 spin_unlock(&mapping->private_lock); 638 639 if (newly_dirty) 640 __folio_mark_dirty(folio, mapping, 1); 641 642 folio_memcg_unlock(folio); 643 644 if (newly_dirty) 645 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 646 647 return newly_dirty; 648 } 649 EXPORT_SYMBOL(block_dirty_folio); 650 651 /* 652 * Write out and wait upon a list of buffers. 653 * 654 * We have conflicting pressures: we want to make sure that all 655 * initially dirty buffers get waited on, but that any subsequently 656 * dirtied buffers don't. After all, we don't want fsync to last 657 * forever if somebody is actively writing to the file. 658 * 659 * Do this in two main stages: first we copy dirty buffers to a 660 * temporary inode list, queueing the writes as we go. Then we clean 661 * up, waiting for those writes to complete. 662 * 663 * During this second stage, any subsequent updates to the file may end 664 * up refiling the buffer on the original inode's dirty list again, so 665 * there is a chance we will end up with a buffer queued for write but 666 * not yet completed on that list. So, as a final cleanup we go through 667 * the osync code to catch these locked, dirty buffers without requeuing 668 * any newly dirty buffers for write. 669 */ 670 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list) 671 { 672 struct buffer_head *bh; 673 struct list_head tmp; 674 struct address_space *mapping; 675 int err = 0, err2; 676 struct blk_plug plug; 677 678 INIT_LIST_HEAD(&tmp); 679 blk_start_plug(&plug); 680 681 spin_lock(lock); 682 while (!list_empty(list)) { 683 bh = BH_ENTRY(list->next); 684 mapping = bh->b_assoc_map; 685 __remove_assoc_queue(bh); 686 /* Avoid race with mark_buffer_dirty_inode() which does 687 * a lockless check and we rely on seeing the dirty bit */ 688 smp_mb(); 689 if (buffer_dirty(bh) || buffer_locked(bh)) { 690 list_add(&bh->b_assoc_buffers, &tmp); 691 bh->b_assoc_map = mapping; 692 if (buffer_dirty(bh)) { 693 get_bh(bh); 694 spin_unlock(lock); 695 /* 696 * Ensure any pending I/O completes so that 697 * write_dirty_buffer() actually writes the 698 * current contents - it is a noop if I/O is 699 * still in flight on potentially older 700 * contents. 701 */ 702 write_dirty_buffer(bh, REQ_SYNC); 703 704 /* 705 * Kick off IO for the previous mapping. Note 706 * that we will not run the very last mapping, 707 * wait_on_buffer() will do that for us 708 * through sync_buffer(). 709 */ 710 brelse(bh); 711 spin_lock(lock); 712 } 713 } 714 } 715 716 spin_unlock(lock); 717 blk_finish_plug(&plug); 718 spin_lock(lock); 719 720 while (!list_empty(&tmp)) { 721 bh = BH_ENTRY(tmp.prev); 722 get_bh(bh); 723 mapping = bh->b_assoc_map; 724 __remove_assoc_queue(bh); 725 /* Avoid race with mark_buffer_dirty_inode() which does 726 * a lockless check and we rely on seeing the dirty bit */ 727 smp_mb(); 728 if (buffer_dirty(bh)) { 729 list_add(&bh->b_assoc_buffers, 730 &mapping->private_list); 731 bh->b_assoc_map = mapping; 732 } 733 spin_unlock(lock); 734 wait_on_buffer(bh); 735 if (!buffer_uptodate(bh)) 736 err = -EIO; 737 brelse(bh); 738 spin_lock(lock); 739 } 740 741 spin_unlock(lock); 742 err2 = osync_buffers_list(lock, list); 743 if (err) 744 return err; 745 else 746 return err2; 747 } 748 749 /* 750 * Invalidate any and all dirty buffers on a given inode. We are 751 * probably unmounting the fs, but that doesn't mean we have already 752 * done a sync(). Just drop the buffers from the inode list. 753 * 754 * NOTE: we take the inode's blockdev's mapping's private_lock. Which 755 * assumes that all the buffers are against the blockdev. Not true 756 * for reiserfs. 757 */ 758 void invalidate_inode_buffers(struct inode *inode) 759 { 760 if (inode_has_buffers(inode)) { 761 struct address_space *mapping = &inode->i_data; 762 struct list_head *list = &mapping->private_list; 763 struct address_space *buffer_mapping = mapping->private_data; 764 765 spin_lock(&buffer_mapping->private_lock); 766 while (!list_empty(list)) 767 __remove_assoc_queue(BH_ENTRY(list->next)); 768 spin_unlock(&buffer_mapping->private_lock); 769 } 770 } 771 EXPORT_SYMBOL(invalidate_inode_buffers); 772 773 /* 774 * Remove any clean buffers from the inode's buffer list. This is called 775 * when we're trying to free the inode itself. Those buffers can pin it. 776 * 777 * Returns true if all buffers were removed. 778 */ 779 int remove_inode_buffers(struct inode *inode) 780 { 781 int ret = 1; 782 783 if (inode_has_buffers(inode)) { 784 struct address_space *mapping = &inode->i_data; 785 struct list_head *list = &mapping->private_list; 786 struct address_space *buffer_mapping = mapping->private_data; 787 788 spin_lock(&buffer_mapping->private_lock); 789 while (!list_empty(list)) { 790 struct buffer_head *bh = BH_ENTRY(list->next); 791 if (buffer_dirty(bh)) { 792 ret = 0; 793 break; 794 } 795 __remove_assoc_queue(bh); 796 } 797 spin_unlock(&buffer_mapping->private_lock); 798 } 799 return ret; 800 } 801 802 /* 803 * Create the appropriate buffers when given a page for data area and 804 * the size of each buffer.. Use the bh->b_this_page linked list to 805 * follow the buffers created. Return NULL if unable to create more 806 * buffers. 807 * 808 * The retry flag is used to differentiate async IO (paging, swapping) 809 * which may not fail from ordinary buffer allocations. 810 */ 811 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, 812 bool retry) 813 { 814 struct buffer_head *bh, *head; 815 gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT; 816 long offset; 817 struct mem_cgroup *memcg, *old_memcg; 818 819 if (retry) 820 gfp |= __GFP_NOFAIL; 821 822 /* The page lock pins the memcg */ 823 memcg = page_memcg(page); 824 old_memcg = set_active_memcg(memcg); 825 826 head = NULL; 827 offset = PAGE_SIZE; 828 while ((offset -= size) >= 0) { 829 bh = alloc_buffer_head(gfp); 830 if (!bh) 831 goto no_grow; 832 833 bh->b_this_page = head; 834 bh->b_blocknr = -1; 835 head = bh; 836 837 bh->b_size = size; 838 839 /* Link the buffer to its page */ 840 set_bh_page(bh, page, offset); 841 } 842 out: 843 set_active_memcg(old_memcg); 844 return head; 845 /* 846 * In case anything failed, we just free everything we got. 847 */ 848 no_grow: 849 if (head) { 850 do { 851 bh = head; 852 head = head->b_this_page; 853 free_buffer_head(bh); 854 } while (head); 855 } 856 857 goto out; 858 } 859 EXPORT_SYMBOL_GPL(alloc_page_buffers); 860 861 static inline void 862 link_dev_buffers(struct page *page, struct buffer_head *head) 863 { 864 struct buffer_head *bh, *tail; 865 866 bh = head; 867 do { 868 tail = bh; 869 bh = bh->b_this_page; 870 } while (bh); 871 tail->b_this_page = head; 872 attach_page_private(page, head); 873 } 874 875 static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size) 876 { 877 sector_t retval = ~((sector_t)0); 878 loff_t sz = bdev_nr_bytes(bdev); 879 880 if (sz) { 881 unsigned int sizebits = blksize_bits(size); 882 retval = (sz >> sizebits); 883 } 884 return retval; 885 } 886 887 /* 888 * Initialise the state of a blockdev page's buffers. 889 */ 890 static sector_t 891 init_page_buffers(struct page *page, struct block_device *bdev, 892 sector_t block, int size) 893 { 894 struct buffer_head *head = page_buffers(page); 895 struct buffer_head *bh = head; 896 int uptodate = PageUptodate(page); 897 sector_t end_block = blkdev_max_block(bdev, size); 898 899 do { 900 if (!buffer_mapped(bh)) { 901 bh->b_end_io = NULL; 902 bh->b_private = NULL; 903 bh->b_bdev = bdev; 904 bh->b_blocknr = block; 905 if (uptodate) 906 set_buffer_uptodate(bh); 907 if (block < end_block) 908 set_buffer_mapped(bh); 909 } 910 block++; 911 bh = bh->b_this_page; 912 } while (bh != head); 913 914 /* 915 * Caller needs to validate requested block against end of device. 916 */ 917 return end_block; 918 } 919 920 /* 921 * Create the page-cache page that contains the requested block. 922 * 923 * This is used purely for blockdev mappings. 924 */ 925 static int 926 grow_dev_page(struct block_device *bdev, sector_t block, 927 pgoff_t index, int size, int sizebits, gfp_t gfp) 928 { 929 struct inode *inode = bdev->bd_inode; 930 struct page *page; 931 struct buffer_head *bh; 932 sector_t end_block; 933 int ret = 0; 934 gfp_t gfp_mask; 935 936 gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp; 937 938 /* 939 * XXX: __getblk_slow() can not really deal with failure and 940 * will endlessly loop on improvised global reclaim. Prefer 941 * looping in the allocator rather than here, at least that 942 * code knows what it's doing. 943 */ 944 gfp_mask |= __GFP_NOFAIL; 945 946 page = find_or_create_page(inode->i_mapping, index, gfp_mask); 947 948 BUG_ON(!PageLocked(page)); 949 950 if (page_has_buffers(page)) { 951 bh = page_buffers(page); 952 if (bh->b_size == size) { 953 end_block = init_page_buffers(page, bdev, 954 (sector_t)index << sizebits, 955 size); 956 goto done; 957 } 958 if (!try_to_free_buffers(page_folio(page))) 959 goto failed; 960 } 961 962 /* 963 * Allocate some buffers for this page 964 */ 965 bh = alloc_page_buffers(page, size, true); 966 967 /* 968 * Link the page to the buffers and initialise them. Take the 969 * lock to be atomic wrt __find_get_block(), which does not 970 * run under the page lock. 971 */ 972 spin_lock(&inode->i_mapping->private_lock); 973 link_dev_buffers(page, bh); 974 end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits, 975 size); 976 spin_unlock(&inode->i_mapping->private_lock); 977 done: 978 ret = (block < end_block) ? 1 : -ENXIO; 979 failed: 980 unlock_page(page); 981 put_page(page); 982 return ret; 983 } 984 985 /* 986 * Create buffers for the specified block device block's page. If 987 * that page was dirty, the buffers are set dirty also. 988 */ 989 static int 990 grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp) 991 { 992 pgoff_t index; 993 int sizebits; 994 995 sizebits = PAGE_SHIFT - __ffs(size); 996 index = block >> sizebits; 997 998 /* 999 * Check for a block which wants to lie outside our maximum possible 1000 * pagecache index. (this comparison is done using sector_t types). 1001 */ 1002 if (unlikely(index != block >> sizebits)) { 1003 printk(KERN_ERR "%s: requested out-of-range block %llu for " 1004 "device %pg\n", 1005 __func__, (unsigned long long)block, 1006 bdev); 1007 return -EIO; 1008 } 1009 1010 /* Create a page with the proper size buffers.. */ 1011 return grow_dev_page(bdev, block, index, size, sizebits, gfp); 1012 } 1013 1014 static struct buffer_head * 1015 __getblk_slow(struct block_device *bdev, sector_t block, 1016 unsigned size, gfp_t gfp) 1017 { 1018 /* Size must be multiple of hard sectorsize */ 1019 if (unlikely(size & (bdev_logical_block_size(bdev)-1) || 1020 (size < 512 || size > PAGE_SIZE))) { 1021 printk(KERN_ERR "getblk(): invalid block size %d requested\n", 1022 size); 1023 printk(KERN_ERR "logical block size: %d\n", 1024 bdev_logical_block_size(bdev)); 1025 1026 dump_stack(); 1027 return NULL; 1028 } 1029 1030 for (;;) { 1031 struct buffer_head *bh; 1032 int ret; 1033 1034 bh = __find_get_block(bdev, block, size); 1035 if (bh) 1036 return bh; 1037 1038 ret = grow_buffers(bdev, block, size, gfp); 1039 if (ret < 0) 1040 return NULL; 1041 } 1042 } 1043 1044 /* 1045 * The relationship between dirty buffers and dirty pages: 1046 * 1047 * Whenever a page has any dirty buffers, the page's dirty bit is set, and 1048 * the page is tagged dirty in the page cache. 1049 * 1050 * At all times, the dirtiness of the buffers represents the dirtiness of 1051 * subsections of the page. If the page has buffers, the page dirty bit is 1052 * merely a hint about the true dirty state. 1053 * 1054 * When a page is set dirty in its entirety, all its buffers are marked dirty 1055 * (if the page has buffers). 1056 * 1057 * When a buffer is marked dirty, its page is dirtied, but the page's other 1058 * buffers are not. 1059 * 1060 * Also. When blockdev buffers are explicitly read with bread(), they 1061 * individually become uptodate. But their backing page remains not 1062 * uptodate - even if all of its buffers are uptodate. A subsequent 1063 * block_read_full_folio() against that folio will discover all the uptodate 1064 * buffers, will set the folio uptodate and will perform no I/O. 1065 */ 1066 1067 /** 1068 * mark_buffer_dirty - mark a buffer_head as needing writeout 1069 * @bh: the buffer_head to mark dirty 1070 * 1071 * mark_buffer_dirty() will set the dirty bit against the buffer, then set 1072 * its backing page dirty, then tag the page as dirty in the page cache 1073 * and then attach the address_space's inode to its superblock's dirty 1074 * inode list. 1075 * 1076 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock, 1077 * i_pages lock and mapping->host->i_lock. 1078 */ 1079 void mark_buffer_dirty(struct buffer_head *bh) 1080 { 1081 WARN_ON_ONCE(!buffer_uptodate(bh)); 1082 1083 trace_block_dirty_buffer(bh); 1084 1085 /* 1086 * Very *carefully* optimize the it-is-already-dirty case. 1087 * 1088 * Don't let the final "is it dirty" escape to before we 1089 * perhaps modified the buffer. 1090 */ 1091 if (buffer_dirty(bh)) { 1092 smp_mb(); 1093 if (buffer_dirty(bh)) 1094 return; 1095 } 1096 1097 if (!test_set_buffer_dirty(bh)) { 1098 struct page *page = bh->b_page; 1099 struct address_space *mapping = NULL; 1100 1101 lock_page_memcg(page); 1102 if (!TestSetPageDirty(page)) { 1103 mapping = page_mapping(page); 1104 if (mapping) 1105 __set_page_dirty(page, mapping, 0); 1106 } 1107 unlock_page_memcg(page); 1108 if (mapping) 1109 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 1110 } 1111 } 1112 EXPORT_SYMBOL(mark_buffer_dirty); 1113 1114 void mark_buffer_write_io_error(struct buffer_head *bh) 1115 { 1116 struct super_block *sb; 1117 1118 set_buffer_write_io_error(bh); 1119 /* FIXME: do we need to set this in both places? */ 1120 if (bh->b_page && bh->b_page->mapping) 1121 mapping_set_error(bh->b_page->mapping, -EIO); 1122 if (bh->b_assoc_map) 1123 mapping_set_error(bh->b_assoc_map, -EIO); 1124 rcu_read_lock(); 1125 sb = READ_ONCE(bh->b_bdev->bd_super); 1126 if (sb) 1127 errseq_set(&sb->s_wb_err, -EIO); 1128 rcu_read_unlock(); 1129 } 1130 EXPORT_SYMBOL(mark_buffer_write_io_error); 1131 1132 /* 1133 * Decrement a buffer_head's reference count. If all buffers against a page 1134 * have zero reference count, are clean and unlocked, and if the page is clean 1135 * and unlocked then try_to_free_buffers() may strip the buffers from the page 1136 * in preparation for freeing it (sometimes, rarely, buffers are removed from 1137 * a page but it ends up not being freed, and buffers may later be reattached). 1138 */ 1139 void __brelse(struct buffer_head * buf) 1140 { 1141 if (atomic_read(&buf->b_count)) { 1142 put_bh(buf); 1143 return; 1144 } 1145 WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n"); 1146 } 1147 EXPORT_SYMBOL(__brelse); 1148 1149 /* 1150 * bforget() is like brelse(), except it discards any 1151 * potentially dirty data. 1152 */ 1153 void __bforget(struct buffer_head *bh) 1154 { 1155 clear_buffer_dirty(bh); 1156 if (bh->b_assoc_map) { 1157 struct address_space *buffer_mapping = bh->b_page->mapping; 1158 1159 spin_lock(&buffer_mapping->private_lock); 1160 list_del_init(&bh->b_assoc_buffers); 1161 bh->b_assoc_map = NULL; 1162 spin_unlock(&buffer_mapping->private_lock); 1163 } 1164 __brelse(bh); 1165 } 1166 EXPORT_SYMBOL(__bforget); 1167 1168 static struct buffer_head *__bread_slow(struct buffer_head *bh) 1169 { 1170 lock_buffer(bh); 1171 if (buffer_uptodate(bh)) { 1172 unlock_buffer(bh); 1173 return bh; 1174 } else { 1175 get_bh(bh); 1176 bh->b_end_io = end_buffer_read_sync; 1177 submit_bh(REQ_OP_READ, bh); 1178 wait_on_buffer(bh); 1179 if (buffer_uptodate(bh)) 1180 return bh; 1181 } 1182 brelse(bh); 1183 return NULL; 1184 } 1185 1186 /* 1187 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block(). 1188 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their 1189 * refcount elevated by one when they're in an LRU. A buffer can only appear 1190 * once in a particular CPU's LRU. A single buffer can be present in multiple 1191 * CPU's LRUs at the same time. 1192 * 1193 * This is a transparent caching front-end to sb_bread(), sb_getblk() and 1194 * sb_find_get_block(). 1195 * 1196 * The LRUs themselves only need locking against invalidate_bh_lrus. We use 1197 * a local interrupt disable for that. 1198 */ 1199 1200 #define BH_LRU_SIZE 16 1201 1202 struct bh_lru { 1203 struct buffer_head *bhs[BH_LRU_SIZE]; 1204 }; 1205 1206 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }}; 1207 1208 #ifdef CONFIG_SMP 1209 #define bh_lru_lock() local_irq_disable() 1210 #define bh_lru_unlock() local_irq_enable() 1211 #else 1212 #define bh_lru_lock() preempt_disable() 1213 #define bh_lru_unlock() preempt_enable() 1214 #endif 1215 1216 static inline void check_irqs_on(void) 1217 { 1218 #ifdef irqs_disabled 1219 BUG_ON(irqs_disabled()); 1220 #endif 1221 } 1222 1223 /* 1224 * Install a buffer_head into this cpu's LRU. If not already in the LRU, it is 1225 * inserted at the front, and the buffer_head at the back if any is evicted. 1226 * Or, if already in the LRU it is moved to the front. 1227 */ 1228 static void bh_lru_install(struct buffer_head *bh) 1229 { 1230 struct buffer_head *evictee = bh; 1231 struct bh_lru *b; 1232 int i; 1233 1234 check_irqs_on(); 1235 bh_lru_lock(); 1236 1237 /* 1238 * the refcount of buffer_head in bh_lru prevents dropping the 1239 * attached page(i.e., try_to_free_buffers) so it could cause 1240 * failing page migration. 1241 * Skip putting upcoming bh into bh_lru until migration is done. 1242 */ 1243 if (lru_cache_disabled()) { 1244 bh_lru_unlock(); 1245 return; 1246 } 1247 1248 b = this_cpu_ptr(&bh_lrus); 1249 for (i = 0; i < BH_LRU_SIZE; i++) { 1250 swap(evictee, b->bhs[i]); 1251 if (evictee == bh) { 1252 bh_lru_unlock(); 1253 return; 1254 } 1255 } 1256 1257 get_bh(bh); 1258 bh_lru_unlock(); 1259 brelse(evictee); 1260 } 1261 1262 /* 1263 * Look up the bh in this cpu's LRU. If it's there, move it to the head. 1264 */ 1265 static struct buffer_head * 1266 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size) 1267 { 1268 struct buffer_head *ret = NULL; 1269 unsigned int i; 1270 1271 check_irqs_on(); 1272 bh_lru_lock(); 1273 for (i = 0; i < BH_LRU_SIZE; i++) { 1274 struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]); 1275 1276 if (bh && bh->b_blocknr == block && bh->b_bdev == bdev && 1277 bh->b_size == size) { 1278 if (i) { 1279 while (i) { 1280 __this_cpu_write(bh_lrus.bhs[i], 1281 __this_cpu_read(bh_lrus.bhs[i - 1])); 1282 i--; 1283 } 1284 __this_cpu_write(bh_lrus.bhs[0], bh); 1285 } 1286 get_bh(bh); 1287 ret = bh; 1288 break; 1289 } 1290 } 1291 bh_lru_unlock(); 1292 return ret; 1293 } 1294 1295 /* 1296 * Perform a pagecache lookup for the matching buffer. If it's there, refresh 1297 * it in the LRU and mark it as accessed. If it is not present then return 1298 * NULL 1299 */ 1300 struct buffer_head * 1301 __find_get_block(struct block_device *bdev, sector_t block, unsigned size) 1302 { 1303 struct buffer_head *bh = lookup_bh_lru(bdev, block, size); 1304 1305 if (bh == NULL) { 1306 /* __find_get_block_slow will mark the page accessed */ 1307 bh = __find_get_block_slow(bdev, block); 1308 if (bh) 1309 bh_lru_install(bh); 1310 } else 1311 touch_buffer(bh); 1312 1313 return bh; 1314 } 1315 EXPORT_SYMBOL(__find_get_block); 1316 1317 /* 1318 * __getblk_gfp() will locate (and, if necessary, create) the buffer_head 1319 * which corresponds to the passed block_device, block and size. The 1320 * returned buffer has its reference count incremented. 1321 * 1322 * __getblk_gfp() will lock up the machine if grow_dev_page's 1323 * try_to_free_buffers() attempt is failing. FIXME, perhaps? 1324 */ 1325 struct buffer_head * 1326 __getblk_gfp(struct block_device *bdev, sector_t block, 1327 unsigned size, gfp_t gfp) 1328 { 1329 struct buffer_head *bh = __find_get_block(bdev, block, size); 1330 1331 might_sleep(); 1332 if (bh == NULL) 1333 bh = __getblk_slow(bdev, block, size, gfp); 1334 return bh; 1335 } 1336 EXPORT_SYMBOL(__getblk_gfp); 1337 1338 /* 1339 * Do async read-ahead on a buffer.. 1340 */ 1341 void __breadahead(struct block_device *bdev, sector_t block, unsigned size) 1342 { 1343 struct buffer_head *bh = __getblk(bdev, block, size); 1344 if (likely(bh)) { 1345 ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, &bh); 1346 brelse(bh); 1347 } 1348 } 1349 EXPORT_SYMBOL(__breadahead); 1350 1351 void __breadahead_gfp(struct block_device *bdev, sector_t block, unsigned size, 1352 gfp_t gfp) 1353 { 1354 struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp); 1355 if (likely(bh)) { 1356 ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, &bh); 1357 brelse(bh); 1358 } 1359 } 1360 EXPORT_SYMBOL(__breadahead_gfp); 1361 1362 /** 1363 * __bread_gfp() - reads a specified block and returns the bh 1364 * @bdev: the block_device to read from 1365 * @block: number of block 1366 * @size: size (in bytes) to read 1367 * @gfp: page allocation flag 1368 * 1369 * Reads a specified block, and returns buffer head that contains it. 1370 * The page cache can be allocated from non-movable area 1371 * not to prevent page migration if you set gfp to zero. 1372 * It returns NULL if the block was unreadable. 1373 */ 1374 struct buffer_head * 1375 __bread_gfp(struct block_device *bdev, sector_t block, 1376 unsigned size, gfp_t gfp) 1377 { 1378 struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp); 1379 1380 if (likely(bh) && !buffer_uptodate(bh)) 1381 bh = __bread_slow(bh); 1382 return bh; 1383 } 1384 EXPORT_SYMBOL(__bread_gfp); 1385 1386 static void __invalidate_bh_lrus(struct bh_lru *b) 1387 { 1388 int i; 1389 1390 for (i = 0; i < BH_LRU_SIZE; i++) { 1391 brelse(b->bhs[i]); 1392 b->bhs[i] = NULL; 1393 } 1394 } 1395 /* 1396 * invalidate_bh_lrus() is called rarely - but not only at unmount. 1397 * This doesn't race because it runs in each cpu either in irq 1398 * or with preempt disabled. 1399 */ 1400 static void invalidate_bh_lru(void *arg) 1401 { 1402 struct bh_lru *b = &get_cpu_var(bh_lrus); 1403 1404 __invalidate_bh_lrus(b); 1405 put_cpu_var(bh_lrus); 1406 } 1407 1408 bool has_bh_in_lru(int cpu, void *dummy) 1409 { 1410 struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu); 1411 int i; 1412 1413 for (i = 0; i < BH_LRU_SIZE; i++) { 1414 if (b->bhs[i]) 1415 return true; 1416 } 1417 1418 return false; 1419 } 1420 1421 void invalidate_bh_lrus(void) 1422 { 1423 on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1); 1424 } 1425 EXPORT_SYMBOL_GPL(invalidate_bh_lrus); 1426 1427 /* 1428 * It's called from workqueue context so we need a bh_lru_lock to close 1429 * the race with preemption/irq. 1430 */ 1431 void invalidate_bh_lrus_cpu(void) 1432 { 1433 struct bh_lru *b; 1434 1435 bh_lru_lock(); 1436 b = this_cpu_ptr(&bh_lrus); 1437 __invalidate_bh_lrus(b); 1438 bh_lru_unlock(); 1439 } 1440 1441 void set_bh_page(struct buffer_head *bh, 1442 struct page *page, unsigned long offset) 1443 { 1444 bh->b_page = page; 1445 BUG_ON(offset >= PAGE_SIZE); 1446 if (PageHighMem(page)) 1447 /* 1448 * This catches illegal uses and preserves the offset: 1449 */ 1450 bh->b_data = (char *)(0 + offset); 1451 else 1452 bh->b_data = page_address(page) + offset; 1453 } 1454 EXPORT_SYMBOL(set_bh_page); 1455 1456 /* 1457 * Called when truncating a buffer on a page completely. 1458 */ 1459 1460 /* Bits that are cleared during an invalidate */ 1461 #define BUFFER_FLAGS_DISCARD \ 1462 (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \ 1463 1 << BH_Delay | 1 << BH_Unwritten) 1464 1465 static void discard_buffer(struct buffer_head * bh) 1466 { 1467 unsigned long b_state; 1468 1469 lock_buffer(bh); 1470 clear_buffer_dirty(bh); 1471 bh->b_bdev = NULL; 1472 b_state = READ_ONCE(bh->b_state); 1473 do { 1474 } while (!try_cmpxchg(&bh->b_state, &b_state, 1475 b_state & ~BUFFER_FLAGS_DISCARD)); 1476 unlock_buffer(bh); 1477 } 1478 1479 /** 1480 * block_invalidate_folio - Invalidate part or all of a buffer-backed folio. 1481 * @folio: The folio which is affected. 1482 * @offset: start of the range to invalidate 1483 * @length: length of the range to invalidate 1484 * 1485 * block_invalidate_folio() is called when all or part of the folio has been 1486 * invalidated by a truncate operation. 1487 * 1488 * block_invalidate_folio() does not have to release all buffers, but it must 1489 * ensure that no dirty buffer is left outside @offset and that no I/O 1490 * is underway against any of the blocks which are outside the truncation 1491 * point. Because the caller is about to free (and possibly reuse) those 1492 * blocks on-disk. 1493 */ 1494 void block_invalidate_folio(struct folio *folio, size_t offset, size_t length) 1495 { 1496 struct buffer_head *head, *bh, *next; 1497 size_t curr_off = 0; 1498 size_t stop = length + offset; 1499 1500 BUG_ON(!folio_test_locked(folio)); 1501 1502 /* 1503 * Check for overflow 1504 */ 1505 BUG_ON(stop > folio_size(folio) || stop < length); 1506 1507 head = folio_buffers(folio); 1508 if (!head) 1509 return; 1510 1511 bh = head; 1512 do { 1513 size_t next_off = curr_off + bh->b_size; 1514 next = bh->b_this_page; 1515 1516 /* 1517 * Are we still fully in range ? 1518 */ 1519 if (next_off > stop) 1520 goto out; 1521 1522 /* 1523 * is this block fully invalidated? 1524 */ 1525 if (offset <= curr_off) 1526 discard_buffer(bh); 1527 curr_off = next_off; 1528 bh = next; 1529 } while (bh != head); 1530 1531 /* 1532 * We release buffers only if the entire folio is being invalidated. 1533 * The get_block cached value has been unconditionally invalidated, 1534 * so real IO is not possible anymore. 1535 */ 1536 if (length == folio_size(folio)) 1537 filemap_release_folio(folio, 0); 1538 out: 1539 return; 1540 } 1541 EXPORT_SYMBOL(block_invalidate_folio); 1542 1543 1544 /* 1545 * We attach and possibly dirty the buffers atomically wrt 1546 * block_dirty_folio() via private_lock. try_to_free_buffers 1547 * is already excluded via the page lock. 1548 */ 1549 void create_empty_buffers(struct page *page, 1550 unsigned long blocksize, unsigned long b_state) 1551 { 1552 struct buffer_head *bh, *head, *tail; 1553 1554 head = alloc_page_buffers(page, blocksize, true); 1555 bh = head; 1556 do { 1557 bh->b_state |= b_state; 1558 tail = bh; 1559 bh = bh->b_this_page; 1560 } while (bh); 1561 tail->b_this_page = head; 1562 1563 spin_lock(&page->mapping->private_lock); 1564 if (PageUptodate(page) || PageDirty(page)) { 1565 bh = head; 1566 do { 1567 if (PageDirty(page)) 1568 set_buffer_dirty(bh); 1569 if (PageUptodate(page)) 1570 set_buffer_uptodate(bh); 1571 bh = bh->b_this_page; 1572 } while (bh != head); 1573 } 1574 attach_page_private(page, head); 1575 spin_unlock(&page->mapping->private_lock); 1576 } 1577 EXPORT_SYMBOL(create_empty_buffers); 1578 1579 /** 1580 * clean_bdev_aliases: clean a range of buffers in block device 1581 * @bdev: Block device to clean buffers in 1582 * @block: Start of a range of blocks to clean 1583 * @len: Number of blocks to clean 1584 * 1585 * We are taking a range of blocks for data and we don't want writeback of any 1586 * buffer-cache aliases starting from return from this function and until the 1587 * moment when something will explicitly mark the buffer dirty (hopefully that 1588 * will not happen until we will free that block ;-) We don't even need to mark 1589 * it not-uptodate - nobody can expect anything from a newly allocated buffer 1590 * anyway. We used to use unmap_buffer() for such invalidation, but that was 1591 * wrong. We definitely don't want to mark the alias unmapped, for example - it 1592 * would confuse anyone who might pick it with bread() afterwards... 1593 * 1594 * Also.. Note that bforget() doesn't lock the buffer. So there can be 1595 * writeout I/O going on against recently-freed buffers. We don't wait on that 1596 * I/O in bforget() - it's more efficient to wait on the I/O only if we really 1597 * need to. That happens here. 1598 */ 1599 void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len) 1600 { 1601 struct inode *bd_inode = bdev->bd_inode; 1602 struct address_space *bd_mapping = bd_inode->i_mapping; 1603 struct folio_batch fbatch; 1604 pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits); 1605 pgoff_t end; 1606 int i, count; 1607 struct buffer_head *bh; 1608 struct buffer_head *head; 1609 1610 end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits); 1611 folio_batch_init(&fbatch); 1612 while (filemap_get_folios(bd_mapping, &index, end, &fbatch)) { 1613 count = folio_batch_count(&fbatch); 1614 for (i = 0; i < count; i++) { 1615 struct folio *folio = fbatch.folios[i]; 1616 1617 if (!folio_buffers(folio)) 1618 continue; 1619 /* 1620 * We use folio lock instead of bd_mapping->private_lock 1621 * to pin buffers here since we can afford to sleep and 1622 * it scales better than a global spinlock lock. 1623 */ 1624 folio_lock(folio); 1625 /* Recheck when the folio is locked which pins bhs */ 1626 head = folio_buffers(folio); 1627 if (!head) 1628 goto unlock_page; 1629 bh = head; 1630 do { 1631 if (!buffer_mapped(bh) || (bh->b_blocknr < block)) 1632 goto next; 1633 if (bh->b_blocknr >= block + len) 1634 break; 1635 clear_buffer_dirty(bh); 1636 wait_on_buffer(bh); 1637 clear_buffer_req(bh); 1638 next: 1639 bh = bh->b_this_page; 1640 } while (bh != head); 1641 unlock_page: 1642 folio_unlock(folio); 1643 } 1644 folio_batch_release(&fbatch); 1645 cond_resched(); 1646 /* End of range already reached? */ 1647 if (index > end || !index) 1648 break; 1649 } 1650 } 1651 EXPORT_SYMBOL(clean_bdev_aliases); 1652 1653 /* 1654 * Size is a power-of-two in the range 512..PAGE_SIZE, 1655 * and the case we care about most is PAGE_SIZE. 1656 * 1657 * So this *could* possibly be written with those 1658 * constraints in mind (relevant mostly if some 1659 * architecture has a slow bit-scan instruction) 1660 */ 1661 static inline int block_size_bits(unsigned int blocksize) 1662 { 1663 return ilog2(blocksize); 1664 } 1665 1666 static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state) 1667 { 1668 BUG_ON(!PageLocked(page)); 1669 1670 if (!page_has_buffers(page)) 1671 create_empty_buffers(page, 1 << READ_ONCE(inode->i_blkbits), 1672 b_state); 1673 return page_buffers(page); 1674 } 1675 1676 /* 1677 * NOTE! All mapped/uptodate combinations are valid: 1678 * 1679 * Mapped Uptodate Meaning 1680 * 1681 * No No "unknown" - must do get_block() 1682 * No Yes "hole" - zero-filled 1683 * Yes No "allocated" - allocated on disk, not read in 1684 * Yes Yes "valid" - allocated and up-to-date in memory. 1685 * 1686 * "Dirty" is valid only with the last case (mapped+uptodate). 1687 */ 1688 1689 /* 1690 * While block_write_full_page is writing back the dirty buffers under 1691 * the page lock, whoever dirtied the buffers may decide to clean them 1692 * again at any time. We handle that by only looking at the buffer 1693 * state inside lock_buffer(). 1694 * 1695 * If block_write_full_page() is called for regular writeback 1696 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a 1697 * locked buffer. This only can happen if someone has written the buffer 1698 * directly, with submit_bh(). At the address_space level PageWriteback 1699 * prevents this contention from occurring. 1700 * 1701 * If block_write_full_page() is called with wbc->sync_mode == 1702 * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this 1703 * causes the writes to be flagged as synchronous writes. 1704 */ 1705 int __block_write_full_page(struct inode *inode, struct page *page, 1706 get_block_t *get_block, struct writeback_control *wbc, 1707 bh_end_io_t *handler) 1708 { 1709 int err; 1710 sector_t block; 1711 sector_t last_block; 1712 struct buffer_head *bh, *head; 1713 unsigned int blocksize, bbits; 1714 int nr_underway = 0; 1715 blk_opf_t write_flags = wbc_to_write_flags(wbc); 1716 1717 head = create_page_buffers(page, inode, 1718 (1 << BH_Dirty)|(1 << BH_Uptodate)); 1719 1720 /* 1721 * Be very careful. We have no exclusion from block_dirty_folio 1722 * here, and the (potentially unmapped) buffers may become dirty at 1723 * any time. If a buffer becomes dirty here after we've inspected it 1724 * then we just miss that fact, and the page stays dirty. 1725 * 1726 * Buffers outside i_size may be dirtied by block_dirty_folio; 1727 * handle that here by just cleaning them. 1728 */ 1729 1730 bh = head; 1731 blocksize = bh->b_size; 1732 bbits = block_size_bits(blocksize); 1733 1734 block = (sector_t)page->index << (PAGE_SHIFT - bbits); 1735 last_block = (i_size_read(inode) - 1) >> bbits; 1736 1737 /* 1738 * Get all the dirty buffers mapped to disk addresses and 1739 * handle any aliases from the underlying blockdev's mapping. 1740 */ 1741 do { 1742 if (block > last_block) { 1743 /* 1744 * mapped buffers outside i_size will occur, because 1745 * this page can be outside i_size when there is a 1746 * truncate in progress. 1747 */ 1748 /* 1749 * The buffer was zeroed by block_write_full_page() 1750 */ 1751 clear_buffer_dirty(bh); 1752 set_buffer_uptodate(bh); 1753 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) && 1754 buffer_dirty(bh)) { 1755 WARN_ON(bh->b_size != blocksize); 1756 err = get_block(inode, block, bh, 1); 1757 if (err) 1758 goto recover; 1759 clear_buffer_delay(bh); 1760 if (buffer_new(bh)) { 1761 /* blockdev mappings never come here */ 1762 clear_buffer_new(bh); 1763 clean_bdev_bh_alias(bh); 1764 } 1765 } 1766 bh = bh->b_this_page; 1767 block++; 1768 } while (bh != head); 1769 1770 do { 1771 if (!buffer_mapped(bh)) 1772 continue; 1773 /* 1774 * If it's a fully non-blocking write attempt and we cannot 1775 * lock the buffer then redirty the page. Note that this can 1776 * potentially cause a busy-wait loop from writeback threads 1777 * and kswapd activity, but those code paths have their own 1778 * higher-level throttling. 1779 */ 1780 if (wbc->sync_mode != WB_SYNC_NONE) { 1781 lock_buffer(bh); 1782 } else if (!trylock_buffer(bh)) { 1783 redirty_page_for_writepage(wbc, page); 1784 continue; 1785 } 1786 if (test_clear_buffer_dirty(bh)) { 1787 mark_buffer_async_write_endio(bh, handler); 1788 } else { 1789 unlock_buffer(bh); 1790 } 1791 } while ((bh = bh->b_this_page) != head); 1792 1793 /* 1794 * The page and its buffers are protected by PageWriteback(), so we can 1795 * drop the bh refcounts early. 1796 */ 1797 BUG_ON(PageWriteback(page)); 1798 set_page_writeback(page); 1799 1800 do { 1801 struct buffer_head *next = bh->b_this_page; 1802 if (buffer_async_write(bh)) { 1803 submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, wbc); 1804 nr_underway++; 1805 } 1806 bh = next; 1807 } while (bh != head); 1808 unlock_page(page); 1809 1810 err = 0; 1811 done: 1812 if (nr_underway == 0) { 1813 /* 1814 * The page was marked dirty, but the buffers were 1815 * clean. Someone wrote them back by hand with 1816 * ll_rw_block/submit_bh. A rare case. 1817 */ 1818 end_page_writeback(page); 1819 1820 /* 1821 * The page and buffer_heads can be released at any time from 1822 * here on. 1823 */ 1824 } 1825 return err; 1826 1827 recover: 1828 /* 1829 * ENOSPC, or some other error. We may already have added some 1830 * blocks to the file, so we need to write these out to avoid 1831 * exposing stale data. 1832 * The page is currently locked and not marked for writeback 1833 */ 1834 bh = head; 1835 /* Recovery: lock and submit the mapped buffers */ 1836 do { 1837 if (buffer_mapped(bh) && buffer_dirty(bh) && 1838 !buffer_delay(bh)) { 1839 lock_buffer(bh); 1840 mark_buffer_async_write_endio(bh, handler); 1841 } else { 1842 /* 1843 * The buffer may have been set dirty during 1844 * attachment to a dirty page. 1845 */ 1846 clear_buffer_dirty(bh); 1847 } 1848 } while ((bh = bh->b_this_page) != head); 1849 SetPageError(page); 1850 BUG_ON(PageWriteback(page)); 1851 mapping_set_error(page->mapping, err); 1852 set_page_writeback(page); 1853 do { 1854 struct buffer_head *next = bh->b_this_page; 1855 if (buffer_async_write(bh)) { 1856 clear_buffer_dirty(bh); 1857 submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, wbc); 1858 nr_underway++; 1859 } 1860 bh = next; 1861 } while (bh != head); 1862 unlock_page(page); 1863 goto done; 1864 } 1865 EXPORT_SYMBOL(__block_write_full_page); 1866 1867 /* 1868 * If a page has any new buffers, zero them out here, and mark them uptodate 1869 * and dirty so they'll be written out (in order to prevent uninitialised 1870 * block data from leaking). And clear the new bit. 1871 */ 1872 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to) 1873 { 1874 unsigned int block_start, block_end; 1875 struct buffer_head *head, *bh; 1876 1877 BUG_ON(!PageLocked(page)); 1878 if (!page_has_buffers(page)) 1879 return; 1880 1881 bh = head = page_buffers(page); 1882 block_start = 0; 1883 do { 1884 block_end = block_start + bh->b_size; 1885 1886 if (buffer_new(bh)) { 1887 if (block_end > from && block_start < to) { 1888 if (!PageUptodate(page)) { 1889 unsigned start, size; 1890 1891 start = max(from, block_start); 1892 size = min(to, block_end) - start; 1893 1894 zero_user(page, start, size); 1895 set_buffer_uptodate(bh); 1896 } 1897 1898 clear_buffer_new(bh); 1899 mark_buffer_dirty(bh); 1900 } 1901 } 1902 1903 block_start = block_end; 1904 bh = bh->b_this_page; 1905 } while (bh != head); 1906 } 1907 EXPORT_SYMBOL(page_zero_new_buffers); 1908 1909 static void 1910 iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh, 1911 const struct iomap *iomap) 1912 { 1913 loff_t offset = block << inode->i_blkbits; 1914 1915 bh->b_bdev = iomap->bdev; 1916 1917 /* 1918 * Block points to offset in file we need to map, iomap contains 1919 * the offset at which the map starts. If the map ends before the 1920 * current block, then do not map the buffer and let the caller 1921 * handle it. 1922 */ 1923 BUG_ON(offset >= iomap->offset + iomap->length); 1924 1925 switch (iomap->type) { 1926 case IOMAP_HOLE: 1927 /* 1928 * If the buffer is not up to date or beyond the current EOF, 1929 * we need to mark it as new to ensure sub-block zeroing is 1930 * executed if necessary. 1931 */ 1932 if (!buffer_uptodate(bh) || 1933 (offset >= i_size_read(inode))) 1934 set_buffer_new(bh); 1935 break; 1936 case IOMAP_DELALLOC: 1937 if (!buffer_uptodate(bh) || 1938 (offset >= i_size_read(inode))) 1939 set_buffer_new(bh); 1940 set_buffer_uptodate(bh); 1941 set_buffer_mapped(bh); 1942 set_buffer_delay(bh); 1943 break; 1944 case IOMAP_UNWRITTEN: 1945 /* 1946 * For unwritten regions, we always need to ensure that regions 1947 * in the block we are not writing to are zeroed. Mark the 1948 * buffer as new to ensure this. 1949 */ 1950 set_buffer_new(bh); 1951 set_buffer_unwritten(bh); 1952 fallthrough; 1953 case IOMAP_MAPPED: 1954 if ((iomap->flags & IOMAP_F_NEW) || 1955 offset >= i_size_read(inode)) 1956 set_buffer_new(bh); 1957 bh->b_blocknr = (iomap->addr + offset - iomap->offset) >> 1958 inode->i_blkbits; 1959 set_buffer_mapped(bh); 1960 break; 1961 } 1962 } 1963 1964 int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len, 1965 get_block_t *get_block, const struct iomap *iomap) 1966 { 1967 unsigned from = pos & (PAGE_SIZE - 1); 1968 unsigned to = from + len; 1969 struct inode *inode = folio->mapping->host; 1970 unsigned block_start, block_end; 1971 sector_t block; 1972 int err = 0; 1973 unsigned blocksize, bbits; 1974 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait; 1975 1976 BUG_ON(!folio_test_locked(folio)); 1977 BUG_ON(from > PAGE_SIZE); 1978 BUG_ON(to > PAGE_SIZE); 1979 BUG_ON(from > to); 1980 1981 head = create_page_buffers(&folio->page, inode, 0); 1982 blocksize = head->b_size; 1983 bbits = block_size_bits(blocksize); 1984 1985 block = (sector_t)folio->index << (PAGE_SHIFT - bbits); 1986 1987 for(bh = head, block_start = 0; bh != head || !block_start; 1988 block++, block_start=block_end, bh = bh->b_this_page) { 1989 block_end = block_start + blocksize; 1990 if (block_end <= from || block_start >= to) { 1991 if (folio_test_uptodate(folio)) { 1992 if (!buffer_uptodate(bh)) 1993 set_buffer_uptodate(bh); 1994 } 1995 continue; 1996 } 1997 if (buffer_new(bh)) 1998 clear_buffer_new(bh); 1999 if (!buffer_mapped(bh)) { 2000 WARN_ON(bh->b_size != blocksize); 2001 if (get_block) { 2002 err = get_block(inode, block, bh, 1); 2003 if (err) 2004 break; 2005 } else { 2006 iomap_to_bh(inode, block, bh, iomap); 2007 } 2008 2009 if (buffer_new(bh)) { 2010 clean_bdev_bh_alias(bh); 2011 if (folio_test_uptodate(folio)) { 2012 clear_buffer_new(bh); 2013 set_buffer_uptodate(bh); 2014 mark_buffer_dirty(bh); 2015 continue; 2016 } 2017 if (block_end > to || block_start < from) 2018 folio_zero_segments(folio, 2019 to, block_end, 2020 block_start, from); 2021 continue; 2022 } 2023 } 2024 if (folio_test_uptodate(folio)) { 2025 if (!buffer_uptodate(bh)) 2026 set_buffer_uptodate(bh); 2027 continue; 2028 } 2029 if (!buffer_uptodate(bh) && !buffer_delay(bh) && 2030 !buffer_unwritten(bh) && 2031 (block_start < from || block_end > to)) { 2032 ll_rw_block(REQ_OP_READ, 1, &bh); 2033 *wait_bh++=bh; 2034 } 2035 } 2036 /* 2037 * If we issued read requests - let them complete. 2038 */ 2039 while(wait_bh > wait) { 2040 wait_on_buffer(*--wait_bh); 2041 if (!buffer_uptodate(*wait_bh)) 2042 err = -EIO; 2043 } 2044 if (unlikely(err)) 2045 page_zero_new_buffers(&folio->page, from, to); 2046 return err; 2047 } 2048 2049 int __block_write_begin(struct page *page, loff_t pos, unsigned len, 2050 get_block_t *get_block) 2051 { 2052 return __block_write_begin_int(page_folio(page), pos, len, get_block, 2053 NULL); 2054 } 2055 EXPORT_SYMBOL(__block_write_begin); 2056 2057 static int __block_commit_write(struct inode *inode, struct page *page, 2058 unsigned from, unsigned to) 2059 { 2060 unsigned block_start, block_end; 2061 int partial = 0; 2062 unsigned blocksize; 2063 struct buffer_head *bh, *head; 2064 2065 bh = head = page_buffers(page); 2066 blocksize = bh->b_size; 2067 2068 block_start = 0; 2069 do { 2070 block_end = block_start + blocksize; 2071 if (block_end <= from || block_start >= to) { 2072 if (!buffer_uptodate(bh)) 2073 partial = 1; 2074 } else { 2075 set_buffer_uptodate(bh); 2076 mark_buffer_dirty(bh); 2077 } 2078 if (buffer_new(bh)) 2079 clear_buffer_new(bh); 2080 2081 block_start = block_end; 2082 bh = bh->b_this_page; 2083 } while (bh != head); 2084 2085 /* 2086 * If this is a partial write which happened to make all buffers 2087 * uptodate then we can optimize away a bogus read_folio() for 2088 * the next read(). Here we 'discover' whether the page went 2089 * uptodate as a result of this (potentially partial) write. 2090 */ 2091 if (!partial) 2092 SetPageUptodate(page); 2093 return 0; 2094 } 2095 2096 /* 2097 * block_write_begin takes care of the basic task of block allocation and 2098 * bringing partial write blocks uptodate first. 2099 * 2100 * The filesystem needs to handle block truncation upon failure. 2101 */ 2102 int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len, 2103 struct page **pagep, get_block_t *get_block) 2104 { 2105 pgoff_t index = pos >> PAGE_SHIFT; 2106 struct page *page; 2107 int status; 2108 2109 page = grab_cache_page_write_begin(mapping, index); 2110 if (!page) 2111 return -ENOMEM; 2112 2113 status = __block_write_begin(page, pos, len, get_block); 2114 if (unlikely(status)) { 2115 unlock_page(page); 2116 put_page(page); 2117 page = NULL; 2118 } 2119 2120 *pagep = page; 2121 return status; 2122 } 2123 EXPORT_SYMBOL(block_write_begin); 2124 2125 int block_write_end(struct file *file, struct address_space *mapping, 2126 loff_t pos, unsigned len, unsigned copied, 2127 struct page *page, void *fsdata) 2128 { 2129 struct inode *inode = mapping->host; 2130 unsigned start; 2131 2132 start = pos & (PAGE_SIZE - 1); 2133 2134 if (unlikely(copied < len)) { 2135 /* 2136 * The buffers that were written will now be uptodate, so 2137 * we don't have to worry about a read_folio reading them 2138 * and overwriting a partial write. However if we have 2139 * encountered a short write and only partially written 2140 * into a buffer, it will not be marked uptodate, so a 2141 * read_folio might come in and destroy our partial write. 2142 * 2143 * Do the simplest thing, and just treat any short write to a 2144 * non uptodate page as a zero-length write, and force the 2145 * caller to redo the whole thing. 2146 */ 2147 if (!PageUptodate(page)) 2148 copied = 0; 2149 2150 page_zero_new_buffers(page, start+copied, start+len); 2151 } 2152 flush_dcache_page(page); 2153 2154 /* This could be a short (even 0-length) commit */ 2155 __block_commit_write(inode, page, start, start+copied); 2156 2157 return copied; 2158 } 2159 EXPORT_SYMBOL(block_write_end); 2160 2161 int generic_write_end(struct file *file, struct address_space *mapping, 2162 loff_t pos, unsigned len, unsigned copied, 2163 struct page *page, void *fsdata) 2164 { 2165 struct inode *inode = mapping->host; 2166 loff_t old_size = inode->i_size; 2167 bool i_size_changed = false; 2168 2169 copied = block_write_end(file, mapping, pos, len, copied, page, fsdata); 2170 2171 /* 2172 * No need to use i_size_read() here, the i_size cannot change under us 2173 * because we hold i_rwsem. 2174 * 2175 * But it's important to update i_size while still holding page lock: 2176 * page writeout could otherwise come in and zero beyond i_size. 2177 */ 2178 if (pos + copied > inode->i_size) { 2179 i_size_write(inode, pos + copied); 2180 i_size_changed = true; 2181 } 2182 2183 unlock_page(page); 2184 put_page(page); 2185 2186 if (old_size < pos) 2187 pagecache_isize_extended(inode, old_size, pos); 2188 /* 2189 * Don't mark the inode dirty under page lock. First, it unnecessarily 2190 * makes the holding time of page lock longer. Second, it forces lock 2191 * ordering of page lock and transaction start for journaling 2192 * filesystems. 2193 */ 2194 if (i_size_changed) 2195 mark_inode_dirty(inode); 2196 return copied; 2197 } 2198 EXPORT_SYMBOL(generic_write_end); 2199 2200 /* 2201 * block_is_partially_uptodate checks whether buffers within a folio are 2202 * uptodate or not. 2203 * 2204 * Returns true if all buffers which correspond to the specified part 2205 * of the folio are uptodate. 2206 */ 2207 bool block_is_partially_uptodate(struct folio *folio, size_t from, size_t count) 2208 { 2209 unsigned block_start, block_end, blocksize; 2210 unsigned to; 2211 struct buffer_head *bh, *head; 2212 bool ret = true; 2213 2214 head = folio_buffers(folio); 2215 if (!head) 2216 return false; 2217 blocksize = head->b_size; 2218 to = min_t(unsigned, folio_size(folio) - from, count); 2219 to = from + to; 2220 if (from < blocksize && to > folio_size(folio) - blocksize) 2221 return false; 2222 2223 bh = head; 2224 block_start = 0; 2225 do { 2226 block_end = block_start + blocksize; 2227 if (block_end > from && block_start < to) { 2228 if (!buffer_uptodate(bh)) { 2229 ret = false; 2230 break; 2231 } 2232 if (block_end >= to) 2233 break; 2234 } 2235 block_start = block_end; 2236 bh = bh->b_this_page; 2237 } while (bh != head); 2238 2239 return ret; 2240 } 2241 EXPORT_SYMBOL(block_is_partially_uptodate); 2242 2243 /* 2244 * Generic "read_folio" function for block devices that have the normal 2245 * get_block functionality. This is most of the block device filesystems. 2246 * Reads the folio asynchronously --- the unlock_buffer() and 2247 * set/clear_buffer_uptodate() functions propagate buffer state into the 2248 * folio once IO has completed. 2249 */ 2250 int block_read_full_folio(struct folio *folio, get_block_t *get_block) 2251 { 2252 struct inode *inode = folio->mapping->host; 2253 sector_t iblock, lblock; 2254 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE]; 2255 unsigned int blocksize, bbits; 2256 int nr, i; 2257 int fully_mapped = 1; 2258 bool page_error = false; 2259 2260 VM_BUG_ON_FOLIO(folio_test_large(folio), folio); 2261 2262 head = create_page_buffers(&folio->page, inode, 0); 2263 blocksize = head->b_size; 2264 bbits = block_size_bits(blocksize); 2265 2266 iblock = (sector_t)folio->index << (PAGE_SHIFT - bbits); 2267 lblock = (i_size_read(inode)+blocksize-1) >> bbits; 2268 bh = head; 2269 nr = 0; 2270 i = 0; 2271 2272 do { 2273 if (buffer_uptodate(bh)) 2274 continue; 2275 2276 if (!buffer_mapped(bh)) { 2277 int err = 0; 2278 2279 fully_mapped = 0; 2280 if (iblock < lblock) { 2281 WARN_ON(bh->b_size != blocksize); 2282 err = get_block(inode, iblock, bh, 0); 2283 if (err) { 2284 folio_set_error(folio); 2285 page_error = true; 2286 } 2287 } 2288 if (!buffer_mapped(bh)) { 2289 folio_zero_range(folio, i * blocksize, 2290 blocksize); 2291 if (!err) 2292 set_buffer_uptodate(bh); 2293 continue; 2294 } 2295 /* 2296 * get_block() might have updated the buffer 2297 * synchronously 2298 */ 2299 if (buffer_uptodate(bh)) 2300 continue; 2301 } 2302 arr[nr++] = bh; 2303 } while (i++, iblock++, (bh = bh->b_this_page) != head); 2304 2305 if (fully_mapped) 2306 folio_set_mappedtodisk(folio); 2307 2308 if (!nr) { 2309 /* 2310 * All buffers are uptodate - we can set the folio uptodate 2311 * as well. But not if get_block() returned an error. 2312 */ 2313 if (!page_error) 2314 folio_mark_uptodate(folio); 2315 folio_unlock(folio); 2316 return 0; 2317 } 2318 2319 /* Stage two: lock the buffers */ 2320 for (i = 0; i < nr; i++) { 2321 bh = arr[i]; 2322 lock_buffer(bh); 2323 mark_buffer_async_read(bh); 2324 } 2325 2326 /* 2327 * Stage 3: start the IO. Check for uptodateness 2328 * inside the buffer lock in case another process reading 2329 * the underlying blockdev brought it uptodate (the sct fix). 2330 */ 2331 for (i = 0; i < nr; i++) { 2332 bh = arr[i]; 2333 if (buffer_uptodate(bh)) 2334 end_buffer_async_read(bh, 1); 2335 else 2336 submit_bh(REQ_OP_READ, bh); 2337 } 2338 return 0; 2339 } 2340 EXPORT_SYMBOL(block_read_full_folio); 2341 2342 /* utility function for filesystems that need to do work on expanding 2343 * truncates. Uses filesystem pagecache writes to allow the filesystem to 2344 * deal with the hole. 2345 */ 2346 int generic_cont_expand_simple(struct inode *inode, loff_t size) 2347 { 2348 struct address_space *mapping = inode->i_mapping; 2349 const struct address_space_operations *aops = mapping->a_ops; 2350 struct page *page; 2351 void *fsdata; 2352 int err; 2353 2354 err = inode_newsize_ok(inode, size); 2355 if (err) 2356 goto out; 2357 2358 err = aops->write_begin(NULL, mapping, size, 0, &page, &fsdata); 2359 if (err) 2360 goto out; 2361 2362 err = aops->write_end(NULL, mapping, size, 0, 0, page, fsdata); 2363 BUG_ON(err > 0); 2364 2365 out: 2366 return err; 2367 } 2368 EXPORT_SYMBOL(generic_cont_expand_simple); 2369 2370 static int cont_expand_zero(struct file *file, struct address_space *mapping, 2371 loff_t pos, loff_t *bytes) 2372 { 2373 struct inode *inode = mapping->host; 2374 const struct address_space_operations *aops = mapping->a_ops; 2375 unsigned int blocksize = i_blocksize(inode); 2376 struct page *page; 2377 void *fsdata; 2378 pgoff_t index, curidx; 2379 loff_t curpos; 2380 unsigned zerofrom, offset, len; 2381 int err = 0; 2382 2383 index = pos >> PAGE_SHIFT; 2384 offset = pos & ~PAGE_MASK; 2385 2386 while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) { 2387 zerofrom = curpos & ~PAGE_MASK; 2388 if (zerofrom & (blocksize-1)) { 2389 *bytes |= (blocksize-1); 2390 (*bytes)++; 2391 } 2392 len = PAGE_SIZE - zerofrom; 2393 2394 err = aops->write_begin(file, mapping, curpos, len, 2395 &page, &fsdata); 2396 if (err) 2397 goto out; 2398 zero_user(page, zerofrom, len); 2399 err = aops->write_end(file, mapping, curpos, len, len, 2400 page, fsdata); 2401 if (err < 0) 2402 goto out; 2403 BUG_ON(err != len); 2404 err = 0; 2405 2406 balance_dirty_pages_ratelimited(mapping); 2407 2408 if (fatal_signal_pending(current)) { 2409 err = -EINTR; 2410 goto out; 2411 } 2412 } 2413 2414 /* page covers the boundary, find the boundary offset */ 2415 if (index == curidx) { 2416 zerofrom = curpos & ~PAGE_MASK; 2417 /* if we will expand the thing last block will be filled */ 2418 if (offset <= zerofrom) { 2419 goto out; 2420 } 2421 if (zerofrom & (blocksize-1)) { 2422 *bytes |= (blocksize-1); 2423 (*bytes)++; 2424 } 2425 len = offset - zerofrom; 2426 2427 err = aops->write_begin(file, mapping, curpos, len, 2428 &page, &fsdata); 2429 if (err) 2430 goto out; 2431 zero_user(page, zerofrom, len); 2432 err = aops->write_end(file, mapping, curpos, len, len, 2433 page, fsdata); 2434 if (err < 0) 2435 goto out; 2436 BUG_ON(err != len); 2437 err = 0; 2438 } 2439 out: 2440 return err; 2441 } 2442 2443 /* 2444 * For moronic filesystems that do not allow holes in file. 2445 * We may have to extend the file. 2446 */ 2447 int cont_write_begin(struct file *file, struct address_space *mapping, 2448 loff_t pos, unsigned len, 2449 struct page **pagep, void **fsdata, 2450 get_block_t *get_block, loff_t *bytes) 2451 { 2452 struct inode *inode = mapping->host; 2453 unsigned int blocksize = i_blocksize(inode); 2454 unsigned int zerofrom; 2455 int err; 2456 2457 err = cont_expand_zero(file, mapping, pos, bytes); 2458 if (err) 2459 return err; 2460 2461 zerofrom = *bytes & ~PAGE_MASK; 2462 if (pos+len > *bytes && zerofrom & (blocksize-1)) { 2463 *bytes |= (blocksize-1); 2464 (*bytes)++; 2465 } 2466 2467 return block_write_begin(mapping, pos, len, pagep, get_block); 2468 } 2469 EXPORT_SYMBOL(cont_write_begin); 2470 2471 int block_commit_write(struct page *page, unsigned from, unsigned to) 2472 { 2473 struct inode *inode = page->mapping->host; 2474 __block_commit_write(inode,page,from,to); 2475 return 0; 2476 } 2477 EXPORT_SYMBOL(block_commit_write); 2478 2479 /* 2480 * block_page_mkwrite() is not allowed to change the file size as it gets 2481 * called from a page fault handler when a page is first dirtied. Hence we must 2482 * be careful to check for EOF conditions here. We set the page up correctly 2483 * for a written page which means we get ENOSPC checking when writing into 2484 * holes and correct delalloc and unwritten extent mapping on filesystems that 2485 * support these features. 2486 * 2487 * We are not allowed to take the i_mutex here so we have to play games to 2488 * protect against truncate races as the page could now be beyond EOF. Because 2489 * truncate writes the inode size before removing pages, once we have the 2490 * page lock we can determine safely if the page is beyond EOF. If it is not 2491 * beyond EOF, then the page is guaranteed safe against truncation until we 2492 * unlock the page. 2493 * 2494 * Direct callers of this function should protect against filesystem freezing 2495 * using sb_start_pagefault() - sb_end_pagefault() functions. 2496 */ 2497 int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, 2498 get_block_t get_block) 2499 { 2500 struct page *page = vmf->page; 2501 struct inode *inode = file_inode(vma->vm_file); 2502 unsigned long end; 2503 loff_t size; 2504 int ret; 2505 2506 lock_page(page); 2507 size = i_size_read(inode); 2508 if ((page->mapping != inode->i_mapping) || 2509 (page_offset(page) > size)) { 2510 /* We overload EFAULT to mean page got truncated */ 2511 ret = -EFAULT; 2512 goto out_unlock; 2513 } 2514 2515 /* page is wholly or partially inside EOF */ 2516 if (((page->index + 1) << PAGE_SHIFT) > size) 2517 end = size & ~PAGE_MASK; 2518 else 2519 end = PAGE_SIZE; 2520 2521 ret = __block_write_begin(page, 0, end, get_block); 2522 if (!ret) 2523 ret = block_commit_write(page, 0, end); 2524 2525 if (unlikely(ret < 0)) 2526 goto out_unlock; 2527 set_page_dirty(page); 2528 wait_for_stable_page(page); 2529 return 0; 2530 out_unlock: 2531 unlock_page(page); 2532 return ret; 2533 } 2534 EXPORT_SYMBOL(block_page_mkwrite); 2535 2536 int block_truncate_page(struct address_space *mapping, 2537 loff_t from, get_block_t *get_block) 2538 { 2539 pgoff_t index = from >> PAGE_SHIFT; 2540 unsigned offset = from & (PAGE_SIZE-1); 2541 unsigned blocksize; 2542 sector_t iblock; 2543 unsigned length, pos; 2544 struct inode *inode = mapping->host; 2545 struct page *page; 2546 struct buffer_head *bh; 2547 int err; 2548 2549 blocksize = i_blocksize(inode); 2550 length = offset & (blocksize - 1); 2551 2552 /* Block boundary? Nothing to do */ 2553 if (!length) 2554 return 0; 2555 2556 length = blocksize - length; 2557 iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits); 2558 2559 page = grab_cache_page(mapping, index); 2560 err = -ENOMEM; 2561 if (!page) 2562 goto out; 2563 2564 if (!page_has_buffers(page)) 2565 create_empty_buffers(page, blocksize, 0); 2566 2567 /* Find the buffer that contains "offset" */ 2568 bh = page_buffers(page); 2569 pos = blocksize; 2570 while (offset >= pos) { 2571 bh = bh->b_this_page; 2572 iblock++; 2573 pos += blocksize; 2574 } 2575 2576 err = 0; 2577 if (!buffer_mapped(bh)) { 2578 WARN_ON(bh->b_size != blocksize); 2579 err = get_block(inode, iblock, bh, 0); 2580 if (err) 2581 goto unlock; 2582 /* unmapped? It's a hole - nothing to do */ 2583 if (!buffer_mapped(bh)) 2584 goto unlock; 2585 } 2586 2587 /* Ok, it's mapped. Make sure it's up-to-date */ 2588 if (PageUptodate(page)) 2589 set_buffer_uptodate(bh); 2590 2591 if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) { 2592 err = -EIO; 2593 ll_rw_block(REQ_OP_READ, 1, &bh); 2594 wait_on_buffer(bh); 2595 /* Uhhuh. Read error. Complain and punt. */ 2596 if (!buffer_uptodate(bh)) 2597 goto unlock; 2598 } 2599 2600 zero_user(page, offset, length); 2601 mark_buffer_dirty(bh); 2602 err = 0; 2603 2604 unlock: 2605 unlock_page(page); 2606 put_page(page); 2607 out: 2608 return err; 2609 } 2610 EXPORT_SYMBOL(block_truncate_page); 2611 2612 /* 2613 * The generic ->writepage function for buffer-backed address_spaces 2614 */ 2615 int block_write_full_page(struct page *page, get_block_t *get_block, 2616 struct writeback_control *wbc) 2617 { 2618 struct inode * const inode = page->mapping->host; 2619 loff_t i_size = i_size_read(inode); 2620 const pgoff_t end_index = i_size >> PAGE_SHIFT; 2621 unsigned offset; 2622 2623 /* Is the page fully inside i_size? */ 2624 if (page->index < end_index) 2625 return __block_write_full_page(inode, page, get_block, wbc, 2626 end_buffer_async_write); 2627 2628 /* Is the page fully outside i_size? (truncate in progress) */ 2629 offset = i_size & (PAGE_SIZE-1); 2630 if (page->index >= end_index+1 || !offset) { 2631 unlock_page(page); 2632 return 0; /* don't care */ 2633 } 2634 2635 /* 2636 * The page straddles i_size. It must be zeroed out on each and every 2637 * writepage invocation because it may be mmapped. "A file is mapped 2638 * in multiples of the page size. For a file that is not a multiple of 2639 * the page size, the remaining memory is zeroed when mapped, and 2640 * writes to that region are not written out to the file." 2641 */ 2642 zero_user_segment(page, offset, PAGE_SIZE); 2643 return __block_write_full_page(inode, page, get_block, wbc, 2644 end_buffer_async_write); 2645 } 2646 EXPORT_SYMBOL(block_write_full_page); 2647 2648 sector_t generic_block_bmap(struct address_space *mapping, sector_t block, 2649 get_block_t *get_block) 2650 { 2651 struct inode *inode = mapping->host; 2652 struct buffer_head tmp = { 2653 .b_size = i_blocksize(inode), 2654 }; 2655 2656 get_block(inode, block, &tmp, 0); 2657 return tmp.b_blocknr; 2658 } 2659 EXPORT_SYMBOL(generic_block_bmap); 2660 2661 static void end_bio_bh_io_sync(struct bio *bio) 2662 { 2663 struct buffer_head *bh = bio->bi_private; 2664 2665 if (unlikely(bio_flagged(bio, BIO_QUIET))) 2666 set_bit(BH_Quiet, &bh->b_state); 2667 2668 bh->b_end_io(bh, !bio->bi_status); 2669 bio_put(bio); 2670 } 2671 2672 static int submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, 2673 struct writeback_control *wbc) 2674 { 2675 const enum req_op op = opf & REQ_OP_MASK; 2676 struct bio *bio; 2677 2678 BUG_ON(!buffer_locked(bh)); 2679 BUG_ON(!buffer_mapped(bh)); 2680 BUG_ON(!bh->b_end_io); 2681 BUG_ON(buffer_delay(bh)); 2682 BUG_ON(buffer_unwritten(bh)); 2683 2684 /* 2685 * Only clear out a write error when rewriting 2686 */ 2687 if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE)) 2688 clear_buffer_write_io_error(bh); 2689 2690 if (buffer_meta(bh)) 2691 opf |= REQ_META; 2692 if (buffer_prio(bh)) 2693 opf |= REQ_PRIO; 2694 2695 bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO); 2696 2697 fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); 2698 2699 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); 2700 2701 bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh)); 2702 BUG_ON(bio->bi_iter.bi_size != bh->b_size); 2703 2704 bio->bi_end_io = end_bio_bh_io_sync; 2705 bio->bi_private = bh; 2706 2707 /* Take care of bh's that straddle the end of the device */ 2708 guard_bio_eod(bio); 2709 2710 if (wbc) { 2711 wbc_init_bio(wbc, bio); 2712 wbc_account_cgroup_owner(wbc, bh->b_page, bh->b_size); 2713 } 2714 2715 submit_bio(bio); 2716 return 0; 2717 } 2718 2719 int submit_bh(blk_opf_t opf, struct buffer_head *bh) 2720 { 2721 return submit_bh_wbc(opf, bh, NULL); 2722 } 2723 EXPORT_SYMBOL(submit_bh); 2724 2725 /** 2726 * ll_rw_block: low-level access to block devices (DEPRECATED) 2727 * @opf: block layer request operation and flags. 2728 * @nr: number of &struct buffer_heads in the array 2729 * @bhs: array of pointers to &struct buffer_head 2730 * 2731 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and 2732 * requests an I/O operation on them, either a %REQ_OP_READ or a %REQ_OP_WRITE. 2733 * @opf contains flags modifying the detailed I/O behavior, most notably 2734 * %REQ_RAHEAD. 2735 * 2736 * This function drops any buffer that it cannot get a lock on (with the 2737 * BH_Lock state bit), any buffer that appears to be clean when doing a write 2738 * request, and any buffer that appears to be up-to-date when doing read 2739 * request. Further it marks as clean buffers that are processed for 2740 * writing (the buffer cache won't assume that they are actually clean 2741 * until the buffer gets unlocked). 2742 * 2743 * ll_rw_block sets b_end_io to simple completion handler that marks 2744 * the buffer up-to-date (if appropriate), unlocks the buffer and wakes 2745 * any waiters. 2746 * 2747 * All of the buffers must be for the same device, and must also be a 2748 * multiple of the current approved size for the device. 2749 */ 2750 void ll_rw_block(const blk_opf_t opf, int nr, struct buffer_head *bhs[]) 2751 { 2752 const enum req_op op = opf & REQ_OP_MASK; 2753 int i; 2754 2755 for (i = 0; i < nr; i++) { 2756 struct buffer_head *bh = bhs[i]; 2757 2758 if (!trylock_buffer(bh)) 2759 continue; 2760 if (op == REQ_OP_WRITE) { 2761 if (test_clear_buffer_dirty(bh)) { 2762 bh->b_end_io = end_buffer_write_sync; 2763 get_bh(bh); 2764 submit_bh(opf, bh); 2765 continue; 2766 } 2767 } else { 2768 if (!buffer_uptodate(bh)) { 2769 bh->b_end_io = end_buffer_read_sync; 2770 get_bh(bh); 2771 submit_bh(opf, bh); 2772 continue; 2773 } 2774 } 2775 unlock_buffer(bh); 2776 } 2777 } 2778 EXPORT_SYMBOL(ll_rw_block); 2779 2780 void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) 2781 { 2782 lock_buffer(bh); 2783 if (!test_clear_buffer_dirty(bh)) { 2784 unlock_buffer(bh); 2785 return; 2786 } 2787 bh->b_end_io = end_buffer_write_sync; 2788 get_bh(bh); 2789 submit_bh(REQ_OP_WRITE | op_flags, bh); 2790 } 2791 EXPORT_SYMBOL(write_dirty_buffer); 2792 2793 /* 2794 * For a data-integrity writeout, we need to wait upon any in-progress I/O 2795 * and then start new I/O and then wait upon it. The caller must have a ref on 2796 * the buffer_head. 2797 */ 2798 int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) 2799 { 2800 int ret = 0; 2801 2802 WARN_ON(atomic_read(&bh->b_count) < 1); 2803 lock_buffer(bh); 2804 if (test_clear_buffer_dirty(bh)) { 2805 /* 2806 * The bh should be mapped, but it might not be if the 2807 * device was hot-removed. Not much we can do but fail the I/O. 2808 */ 2809 if (!buffer_mapped(bh)) { 2810 unlock_buffer(bh); 2811 return -EIO; 2812 } 2813 2814 get_bh(bh); 2815 bh->b_end_io = end_buffer_write_sync; 2816 ret = submit_bh(REQ_OP_WRITE | op_flags, bh); 2817 wait_on_buffer(bh); 2818 if (!ret && !buffer_uptodate(bh)) 2819 ret = -EIO; 2820 } else { 2821 unlock_buffer(bh); 2822 } 2823 return ret; 2824 } 2825 EXPORT_SYMBOL(__sync_dirty_buffer); 2826 2827 int sync_dirty_buffer(struct buffer_head *bh) 2828 { 2829 return __sync_dirty_buffer(bh, REQ_SYNC); 2830 } 2831 EXPORT_SYMBOL(sync_dirty_buffer); 2832 2833 /* 2834 * try_to_free_buffers() checks if all the buffers on this particular folio 2835 * are unused, and releases them if so. 2836 * 2837 * Exclusion against try_to_free_buffers may be obtained by either 2838 * locking the folio or by holding its mapping's private_lock. 2839 * 2840 * If the folio is dirty but all the buffers are clean then we need to 2841 * be sure to mark the folio clean as well. This is because the folio 2842 * may be against a block device, and a later reattachment of buffers 2843 * to a dirty folio will set *all* buffers dirty. Which would corrupt 2844 * filesystem data on the same device. 2845 * 2846 * The same applies to regular filesystem folios: if all the buffers are 2847 * clean then we set the folio clean and proceed. To do that, we require 2848 * total exclusion from block_dirty_folio(). That is obtained with 2849 * private_lock. 2850 * 2851 * try_to_free_buffers() is non-blocking. 2852 */ 2853 static inline int buffer_busy(struct buffer_head *bh) 2854 { 2855 return atomic_read(&bh->b_count) | 2856 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock))); 2857 } 2858 2859 static bool 2860 drop_buffers(struct folio *folio, struct buffer_head **buffers_to_free) 2861 { 2862 struct buffer_head *head = folio_buffers(folio); 2863 struct buffer_head *bh; 2864 2865 bh = head; 2866 do { 2867 if (buffer_busy(bh)) 2868 goto failed; 2869 bh = bh->b_this_page; 2870 } while (bh != head); 2871 2872 do { 2873 struct buffer_head *next = bh->b_this_page; 2874 2875 if (bh->b_assoc_map) 2876 __remove_assoc_queue(bh); 2877 bh = next; 2878 } while (bh != head); 2879 *buffers_to_free = head; 2880 folio_detach_private(folio); 2881 return true; 2882 failed: 2883 return false; 2884 } 2885 2886 bool try_to_free_buffers(struct folio *folio) 2887 { 2888 struct address_space * const mapping = folio->mapping; 2889 struct buffer_head *buffers_to_free = NULL; 2890 bool ret = 0; 2891 2892 BUG_ON(!folio_test_locked(folio)); 2893 if (folio_test_writeback(folio)) 2894 return false; 2895 2896 if (mapping == NULL) { /* can this still happen? */ 2897 ret = drop_buffers(folio, &buffers_to_free); 2898 goto out; 2899 } 2900 2901 spin_lock(&mapping->private_lock); 2902 ret = drop_buffers(folio, &buffers_to_free); 2903 2904 /* 2905 * If the filesystem writes its buffers by hand (eg ext3) 2906 * then we can have clean buffers against a dirty folio. We 2907 * clean the folio here; otherwise the VM will never notice 2908 * that the filesystem did any IO at all. 2909 * 2910 * Also, during truncate, discard_buffer will have marked all 2911 * the folio's buffers clean. We discover that here and clean 2912 * the folio also. 2913 * 2914 * private_lock must be held over this entire operation in order 2915 * to synchronise against block_dirty_folio and prevent the 2916 * dirty bit from being lost. 2917 */ 2918 if (ret) 2919 folio_cancel_dirty(folio); 2920 spin_unlock(&mapping->private_lock); 2921 out: 2922 if (buffers_to_free) { 2923 struct buffer_head *bh = buffers_to_free; 2924 2925 do { 2926 struct buffer_head *next = bh->b_this_page; 2927 free_buffer_head(bh); 2928 bh = next; 2929 } while (bh != buffers_to_free); 2930 } 2931 return ret; 2932 } 2933 EXPORT_SYMBOL(try_to_free_buffers); 2934 2935 /* 2936 * Buffer-head allocation 2937 */ 2938 static struct kmem_cache *bh_cachep __read_mostly; 2939 2940 /* 2941 * Once the number of bh's in the machine exceeds this level, we start 2942 * stripping them in writeback. 2943 */ 2944 static unsigned long max_buffer_heads; 2945 2946 int buffer_heads_over_limit; 2947 2948 struct bh_accounting { 2949 int nr; /* Number of live bh's */ 2950 int ratelimit; /* Limit cacheline bouncing */ 2951 }; 2952 2953 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0}; 2954 2955 static void recalc_bh_state(void) 2956 { 2957 int i; 2958 int tot = 0; 2959 2960 if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096) 2961 return; 2962 __this_cpu_write(bh_accounting.ratelimit, 0); 2963 for_each_online_cpu(i) 2964 tot += per_cpu(bh_accounting, i).nr; 2965 buffer_heads_over_limit = (tot > max_buffer_heads); 2966 } 2967 2968 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags) 2969 { 2970 struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags); 2971 if (ret) { 2972 INIT_LIST_HEAD(&ret->b_assoc_buffers); 2973 spin_lock_init(&ret->b_uptodate_lock); 2974 preempt_disable(); 2975 __this_cpu_inc(bh_accounting.nr); 2976 recalc_bh_state(); 2977 preempt_enable(); 2978 } 2979 return ret; 2980 } 2981 EXPORT_SYMBOL(alloc_buffer_head); 2982 2983 void free_buffer_head(struct buffer_head *bh) 2984 { 2985 BUG_ON(!list_empty(&bh->b_assoc_buffers)); 2986 kmem_cache_free(bh_cachep, bh); 2987 preempt_disable(); 2988 __this_cpu_dec(bh_accounting.nr); 2989 recalc_bh_state(); 2990 preempt_enable(); 2991 } 2992 EXPORT_SYMBOL(free_buffer_head); 2993 2994 static int buffer_exit_cpu_dead(unsigned int cpu) 2995 { 2996 int i; 2997 struct bh_lru *b = &per_cpu(bh_lrus, cpu); 2998 2999 for (i = 0; i < BH_LRU_SIZE; i++) { 3000 brelse(b->bhs[i]); 3001 b->bhs[i] = NULL; 3002 } 3003 this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr); 3004 per_cpu(bh_accounting, cpu).nr = 0; 3005 return 0; 3006 } 3007 3008 /** 3009 * bh_uptodate_or_lock - Test whether the buffer is uptodate 3010 * @bh: struct buffer_head 3011 * 3012 * Return true if the buffer is up-to-date and false, 3013 * with the buffer locked, if not. 3014 */ 3015 int bh_uptodate_or_lock(struct buffer_head *bh) 3016 { 3017 if (!buffer_uptodate(bh)) { 3018 lock_buffer(bh); 3019 if (!buffer_uptodate(bh)) 3020 return 0; 3021 unlock_buffer(bh); 3022 } 3023 return 1; 3024 } 3025 EXPORT_SYMBOL(bh_uptodate_or_lock); 3026 3027 /** 3028 * bh_submit_read - Submit a locked buffer for reading 3029 * @bh: struct buffer_head 3030 * 3031 * Returns zero on success and -EIO on error. 3032 */ 3033 int bh_submit_read(struct buffer_head *bh) 3034 { 3035 BUG_ON(!buffer_locked(bh)); 3036 3037 if (buffer_uptodate(bh)) { 3038 unlock_buffer(bh); 3039 return 0; 3040 } 3041 3042 get_bh(bh); 3043 bh->b_end_io = end_buffer_read_sync; 3044 submit_bh(REQ_OP_READ, bh); 3045 wait_on_buffer(bh); 3046 if (buffer_uptodate(bh)) 3047 return 0; 3048 return -EIO; 3049 } 3050 EXPORT_SYMBOL(bh_submit_read); 3051 3052 void __init buffer_init(void) 3053 { 3054 unsigned long nrpages; 3055 int ret; 3056 3057 bh_cachep = kmem_cache_create("buffer_head", 3058 sizeof(struct buffer_head), 0, 3059 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC| 3060 SLAB_MEM_SPREAD), 3061 NULL); 3062 3063 /* 3064 * Limit the bh occupancy to 10% of ZONE_NORMAL 3065 */ 3066 nrpages = (nr_free_buffer_pages() * 10) / 100; 3067 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head)); 3068 ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead", 3069 NULL, buffer_exit_cpu_dead); 3070 WARN_ON(ret < 0); 3071 } 3072