1 /* 2 * linux/fs/buffer.c 3 * 4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds 5 */ 6 7 /* 8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95 9 * 10 * Removed a lot of unnecessary code and simplified things now that 11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96 12 * 13 * Speed up hash, lru, and free list operations. Use gfp() for allocating 14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM 15 * 16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK 17 * 18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de> 19 */ 20 21 #include <linux/kernel.h> 22 #include <linux/syscalls.h> 23 #include <linux/fs.h> 24 #include <linux/mm.h> 25 #include <linux/percpu.h> 26 #include <linux/slab.h> 27 #include <linux/smp_lock.h> 28 #include <linux/capability.h> 29 #include <linux/blkdev.h> 30 #include <linux/file.h> 31 #include <linux/quotaops.h> 32 #include <linux/highmem.h> 33 #include <linux/module.h> 34 #include <linux/writeback.h> 35 #include <linux/hash.h> 36 #include <linux/suspend.h> 37 #include <linux/buffer_head.h> 38 #include <linux/bio.h> 39 #include <linux/notifier.h> 40 #include <linux/cpu.h> 41 #include <linux/bitops.h> 42 #include <linux/mpage.h> 43 #include <linux/bit_spinlock.h> 44 45 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list); 46 static void invalidate_bh_lrus(void); 47 48 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) 49 50 inline void 51 init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private) 52 { 53 bh->b_end_io = handler; 54 bh->b_private = private; 55 } 56 57 static int sync_buffer(void *word) 58 { 59 struct block_device *bd; 60 struct buffer_head *bh 61 = container_of(word, struct buffer_head, b_state); 62 63 smp_mb(); 64 bd = bh->b_bdev; 65 if (bd) 66 blk_run_address_space(bd->bd_inode->i_mapping); 67 io_schedule(); 68 return 0; 69 } 70 71 void fastcall __lock_buffer(struct buffer_head *bh) 72 { 73 wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer, 74 TASK_UNINTERRUPTIBLE); 75 } 76 EXPORT_SYMBOL(__lock_buffer); 77 78 void fastcall unlock_buffer(struct buffer_head *bh) 79 { 80 clear_buffer_locked(bh); 81 smp_mb__after_clear_bit(); 82 wake_up_bit(&bh->b_state, BH_Lock); 83 } 84 85 /* 86 * Block until a buffer comes unlocked. This doesn't stop it 87 * from becoming locked again - you have to lock it yourself 88 * if you want to preserve its state. 89 */ 90 void __wait_on_buffer(struct buffer_head * bh) 91 { 92 wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE); 93 } 94 95 static void 96 __clear_page_buffers(struct page *page) 97 { 98 ClearPagePrivate(page); 99 set_page_private(page, 0); 100 page_cache_release(page); 101 } 102 103 static void buffer_io_error(struct buffer_head *bh) 104 { 105 char b[BDEVNAME_SIZE]; 106 107 printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n", 108 bdevname(bh->b_bdev, b), 109 (unsigned long long)bh->b_blocknr); 110 } 111 112 /* 113 * Default synchronous end-of-IO handler.. Just mark it up-to-date and 114 * unlock the buffer. This is what ll_rw_block uses too. 115 */ 116 void end_buffer_read_sync(struct buffer_head *bh, int uptodate) 117 { 118 if (uptodate) { 119 set_buffer_uptodate(bh); 120 } else { 121 /* This happens, due to failed READA attempts. */ 122 clear_buffer_uptodate(bh); 123 } 124 unlock_buffer(bh); 125 put_bh(bh); 126 } 127 128 void end_buffer_write_sync(struct buffer_head *bh, int uptodate) 129 { 130 char b[BDEVNAME_SIZE]; 131 132 if (uptodate) { 133 set_buffer_uptodate(bh); 134 } else { 135 if (!buffer_eopnotsupp(bh) && printk_ratelimit()) { 136 buffer_io_error(bh); 137 printk(KERN_WARNING "lost page write due to " 138 "I/O error on %s\n", 139 bdevname(bh->b_bdev, b)); 140 } 141 set_buffer_write_io_error(bh); 142 clear_buffer_uptodate(bh); 143 } 144 unlock_buffer(bh); 145 put_bh(bh); 146 } 147 148 /* 149 * Write out and wait upon all the dirty data associated with a block 150 * device via its mapping. Does not take the superblock lock. 151 */ 152 int sync_blockdev(struct block_device *bdev) 153 { 154 int ret = 0; 155 156 if (bdev) 157 ret = filemap_write_and_wait(bdev->bd_inode->i_mapping); 158 return ret; 159 } 160 EXPORT_SYMBOL(sync_blockdev); 161 162 /* 163 * Write out and wait upon all dirty data associated with this 164 * device. Filesystem data as well as the underlying block 165 * device. Takes the superblock lock. 166 */ 167 int fsync_bdev(struct block_device *bdev) 168 { 169 struct super_block *sb = get_super(bdev); 170 if (sb) { 171 int res = fsync_super(sb); 172 drop_super(sb); 173 return res; 174 } 175 return sync_blockdev(bdev); 176 } 177 178 /** 179 * freeze_bdev -- lock a filesystem and force it into a consistent state 180 * @bdev: blockdevice to lock 181 * 182 * This takes the block device bd_mount_mutex to make sure no new mounts 183 * happen on bdev until thaw_bdev() is called. 184 * If a superblock is found on this device, we take the s_umount semaphore 185 * on it to make sure nobody unmounts until the snapshot creation is done. 186 */ 187 struct super_block *freeze_bdev(struct block_device *bdev) 188 { 189 struct super_block *sb; 190 191 mutex_lock(&bdev->bd_mount_mutex); 192 sb = get_super(bdev); 193 if (sb && !(sb->s_flags & MS_RDONLY)) { 194 sb->s_frozen = SB_FREEZE_WRITE; 195 smp_wmb(); 196 197 __fsync_super(sb); 198 199 sb->s_frozen = SB_FREEZE_TRANS; 200 smp_wmb(); 201 202 sync_blockdev(sb->s_bdev); 203 204 if (sb->s_op->write_super_lockfs) 205 sb->s_op->write_super_lockfs(sb); 206 } 207 208 sync_blockdev(bdev); 209 return sb; /* thaw_bdev releases s->s_umount and bd_mount_sem */ 210 } 211 EXPORT_SYMBOL(freeze_bdev); 212 213 /** 214 * thaw_bdev -- unlock filesystem 215 * @bdev: blockdevice to unlock 216 * @sb: associated superblock 217 * 218 * Unlocks the filesystem and marks it writeable again after freeze_bdev(). 219 */ 220 void thaw_bdev(struct block_device *bdev, struct super_block *sb) 221 { 222 if (sb) { 223 BUG_ON(sb->s_bdev != bdev); 224 225 if (sb->s_op->unlockfs) 226 sb->s_op->unlockfs(sb); 227 sb->s_frozen = SB_UNFROZEN; 228 smp_wmb(); 229 wake_up(&sb->s_wait_unfrozen); 230 drop_super(sb); 231 } 232 233 mutex_unlock(&bdev->bd_mount_mutex); 234 } 235 EXPORT_SYMBOL(thaw_bdev); 236 237 /* 238 * Various filesystems appear to want __find_get_block to be non-blocking. 239 * But it's the page lock which protects the buffers. To get around this, 240 * we get exclusion from try_to_free_buffers with the blockdev mapping's 241 * private_lock. 242 * 243 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention 244 * may be quite high. This code could TryLock the page, and if that 245 * succeeds, there is no need to take private_lock. (But if 246 * private_lock is contended then so is mapping->tree_lock). 247 */ 248 static struct buffer_head * 249 __find_get_block_slow(struct block_device *bdev, sector_t block) 250 { 251 struct inode *bd_inode = bdev->bd_inode; 252 struct address_space *bd_mapping = bd_inode->i_mapping; 253 struct buffer_head *ret = NULL; 254 pgoff_t index; 255 struct buffer_head *bh; 256 struct buffer_head *head; 257 struct page *page; 258 int all_mapped = 1; 259 260 index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits); 261 page = find_get_page(bd_mapping, index); 262 if (!page) 263 goto out; 264 265 spin_lock(&bd_mapping->private_lock); 266 if (!page_has_buffers(page)) 267 goto out_unlock; 268 head = page_buffers(page); 269 bh = head; 270 do { 271 if (bh->b_blocknr == block) { 272 ret = bh; 273 get_bh(bh); 274 goto out_unlock; 275 } 276 if (!buffer_mapped(bh)) 277 all_mapped = 0; 278 bh = bh->b_this_page; 279 } while (bh != head); 280 281 /* we might be here because some of the buffers on this page are 282 * not mapped. This is due to various races between 283 * file io on the block device and getblk. It gets dealt with 284 * elsewhere, don't buffer_error if we had some unmapped buffers 285 */ 286 if (all_mapped) { 287 printk("__find_get_block_slow() failed. " 288 "block=%llu, b_blocknr=%llu\n", 289 (unsigned long long)block, 290 (unsigned long long)bh->b_blocknr); 291 printk("b_state=0x%08lx, b_size=%zu\n", 292 bh->b_state, bh->b_size); 293 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits); 294 } 295 out_unlock: 296 spin_unlock(&bd_mapping->private_lock); 297 page_cache_release(page); 298 out: 299 return ret; 300 } 301 302 /* If invalidate_buffers() will trash dirty buffers, it means some kind 303 of fs corruption is going on. Trashing dirty data always imply losing 304 information that was supposed to be just stored on the physical layer 305 by the user. 306 307 Thus invalidate_buffers in general usage is not allwowed to trash 308 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to 309 be preserved. These buffers are simply skipped. 310 311 We also skip buffers which are still in use. For example this can 312 happen if a userspace program is reading the block device. 313 314 NOTE: In the case where the user removed a removable-media-disk even if 315 there's still dirty data not synced on disk (due a bug in the device driver 316 or due an error of the user), by not destroying the dirty buffers we could 317 generate corruption also on the next media inserted, thus a parameter is 318 necessary to handle this case in the most safe way possible (trying 319 to not corrupt also the new disk inserted with the data belonging to 320 the old now corrupted disk). Also for the ramdisk the natural thing 321 to do in order to release the ramdisk memory is to destroy dirty buffers. 322 323 These are two special cases. Normal usage imply the device driver 324 to issue a sync on the device (without waiting I/O completion) and 325 then an invalidate_buffers call that doesn't trash dirty buffers. 326 327 For handling cache coherency with the blkdev pagecache the 'update' case 328 is been introduced. It is needed to re-read from disk any pinned 329 buffer. NOTE: re-reading from disk is destructive so we can do it only 330 when we assume nobody is changing the buffercache under our I/O and when 331 we think the disk contains more recent information than the buffercache. 332 The update == 1 pass marks the buffers we need to update, the update == 2 333 pass does the actual I/O. */ 334 void invalidate_bdev(struct block_device *bdev, int destroy_dirty_buffers) 335 { 336 struct address_space *mapping = bdev->bd_inode->i_mapping; 337 338 if (mapping->nrpages == 0) 339 return; 340 341 invalidate_bh_lrus(); 342 /* 343 * FIXME: what about destroy_dirty_buffers? 344 * We really want to use invalidate_inode_pages2() for 345 * that, but not until that's cleaned up. 346 */ 347 invalidate_inode_pages(mapping); 348 } 349 350 /* 351 * Kick pdflush then try to free up some ZONE_NORMAL memory. 352 */ 353 static void free_more_memory(void) 354 { 355 struct zone **zones; 356 pg_data_t *pgdat; 357 358 wakeup_pdflush(1024); 359 yield(); 360 361 for_each_online_pgdat(pgdat) { 362 zones = pgdat->node_zonelists[gfp_zone(GFP_NOFS)].zones; 363 if (*zones) 364 try_to_free_pages(zones, GFP_NOFS); 365 } 366 } 367 368 /* 369 * I/O completion handler for block_read_full_page() - pages 370 * which come unlocked at the end of I/O. 371 */ 372 static void end_buffer_async_read(struct buffer_head *bh, int uptodate) 373 { 374 unsigned long flags; 375 struct buffer_head *first; 376 struct buffer_head *tmp; 377 struct page *page; 378 int page_uptodate = 1; 379 380 BUG_ON(!buffer_async_read(bh)); 381 382 page = bh->b_page; 383 if (uptodate) { 384 set_buffer_uptodate(bh); 385 } else { 386 clear_buffer_uptodate(bh); 387 if (printk_ratelimit()) 388 buffer_io_error(bh); 389 SetPageError(page); 390 } 391 392 /* 393 * Be _very_ careful from here on. Bad things can happen if 394 * two buffer heads end IO at almost the same time and both 395 * decide that the page is now completely done. 396 */ 397 first = page_buffers(page); 398 local_irq_save(flags); 399 bit_spin_lock(BH_Uptodate_Lock, &first->b_state); 400 clear_buffer_async_read(bh); 401 unlock_buffer(bh); 402 tmp = bh; 403 do { 404 if (!buffer_uptodate(tmp)) 405 page_uptodate = 0; 406 if (buffer_async_read(tmp)) { 407 BUG_ON(!buffer_locked(tmp)); 408 goto still_busy; 409 } 410 tmp = tmp->b_this_page; 411 } while (tmp != bh); 412 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); 413 local_irq_restore(flags); 414 415 /* 416 * If none of the buffers had errors and they are all 417 * uptodate then we can set the page uptodate. 418 */ 419 if (page_uptodate && !PageError(page)) 420 SetPageUptodate(page); 421 unlock_page(page); 422 return; 423 424 still_busy: 425 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); 426 local_irq_restore(flags); 427 return; 428 } 429 430 /* 431 * Completion handler for block_write_full_page() - pages which are unlocked 432 * during I/O, and which have PageWriteback cleared upon I/O completion. 433 */ 434 static void end_buffer_async_write(struct buffer_head *bh, int uptodate) 435 { 436 char b[BDEVNAME_SIZE]; 437 unsigned long flags; 438 struct buffer_head *first; 439 struct buffer_head *tmp; 440 struct page *page; 441 442 BUG_ON(!buffer_async_write(bh)); 443 444 page = bh->b_page; 445 if (uptodate) { 446 set_buffer_uptodate(bh); 447 } else { 448 if (printk_ratelimit()) { 449 buffer_io_error(bh); 450 printk(KERN_WARNING "lost page write due to " 451 "I/O error on %s\n", 452 bdevname(bh->b_bdev, b)); 453 } 454 set_bit(AS_EIO, &page->mapping->flags); 455 set_buffer_write_io_error(bh); 456 clear_buffer_uptodate(bh); 457 SetPageError(page); 458 } 459 460 first = page_buffers(page); 461 local_irq_save(flags); 462 bit_spin_lock(BH_Uptodate_Lock, &first->b_state); 463 464 clear_buffer_async_write(bh); 465 unlock_buffer(bh); 466 tmp = bh->b_this_page; 467 while (tmp != bh) { 468 if (buffer_async_write(tmp)) { 469 BUG_ON(!buffer_locked(tmp)); 470 goto still_busy; 471 } 472 tmp = tmp->b_this_page; 473 } 474 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); 475 local_irq_restore(flags); 476 end_page_writeback(page); 477 return; 478 479 still_busy: 480 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); 481 local_irq_restore(flags); 482 return; 483 } 484 485 /* 486 * If a page's buffers are under async readin (end_buffer_async_read 487 * completion) then there is a possibility that another thread of 488 * control could lock one of the buffers after it has completed 489 * but while some of the other buffers have not completed. This 490 * locked buffer would confuse end_buffer_async_read() into not unlocking 491 * the page. So the absence of BH_Async_Read tells end_buffer_async_read() 492 * that this buffer is not under async I/O. 493 * 494 * The page comes unlocked when it has no locked buffer_async buffers 495 * left. 496 * 497 * PageLocked prevents anyone starting new async I/O reads any of 498 * the buffers. 499 * 500 * PageWriteback is used to prevent simultaneous writeout of the same 501 * page. 502 * 503 * PageLocked prevents anyone from starting writeback of a page which is 504 * under read I/O (PageWriteback is only ever set against a locked page). 505 */ 506 static void mark_buffer_async_read(struct buffer_head *bh) 507 { 508 bh->b_end_io = end_buffer_async_read; 509 set_buffer_async_read(bh); 510 } 511 512 void mark_buffer_async_write(struct buffer_head *bh) 513 { 514 bh->b_end_io = end_buffer_async_write; 515 set_buffer_async_write(bh); 516 } 517 EXPORT_SYMBOL(mark_buffer_async_write); 518 519 520 /* 521 * fs/buffer.c contains helper functions for buffer-backed address space's 522 * fsync functions. A common requirement for buffer-based filesystems is 523 * that certain data from the backing blockdev needs to be written out for 524 * a successful fsync(). For example, ext2 indirect blocks need to be 525 * written back and waited upon before fsync() returns. 526 * 527 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(), 528 * inode_has_buffers() and invalidate_inode_buffers() are provided for the 529 * management of a list of dependent buffers at ->i_mapping->private_list. 530 * 531 * Locking is a little subtle: try_to_free_buffers() will remove buffers 532 * from their controlling inode's queue when they are being freed. But 533 * try_to_free_buffers() will be operating against the *blockdev* mapping 534 * at the time, not against the S_ISREG file which depends on those buffers. 535 * So the locking for private_list is via the private_lock in the address_space 536 * which backs the buffers. Which is different from the address_space 537 * against which the buffers are listed. So for a particular address_space, 538 * mapping->private_lock does *not* protect mapping->private_list! In fact, 539 * mapping->private_list will always be protected by the backing blockdev's 540 * ->private_lock. 541 * 542 * Which introduces a requirement: all buffers on an address_space's 543 * ->private_list must be from the same address_space: the blockdev's. 544 * 545 * address_spaces which do not place buffers at ->private_list via these 546 * utility functions are free to use private_lock and private_list for 547 * whatever they want. The only requirement is that list_empty(private_list) 548 * be true at clear_inode() time. 549 * 550 * FIXME: clear_inode should not call invalidate_inode_buffers(). The 551 * filesystems should do that. invalidate_inode_buffers() should just go 552 * BUG_ON(!list_empty). 553 * 554 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should 555 * take an address_space, not an inode. And it should be called 556 * mark_buffer_dirty_fsync() to clearly define why those buffers are being 557 * queued up. 558 * 559 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the 560 * list if it is already on a list. Because if the buffer is on a list, 561 * it *must* already be on the right one. If not, the filesystem is being 562 * silly. This will save a ton of locking. But first we have to ensure 563 * that buffers are taken *off* the old inode's list when they are freed 564 * (presumably in truncate). That requires careful auditing of all 565 * filesystems (do it inside bforget()). It could also be done by bringing 566 * b_inode back. 567 */ 568 569 /* 570 * The buffer's backing address_space's private_lock must be held 571 */ 572 static inline void __remove_assoc_queue(struct buffer_head *bh) 573 { 574 list_del_init(&bh->b_assoc_buffers); 575 WARN_ON(!bh->b_assoc_map); 576 if (buffer_write_io_error(bh)) 577 set_bit(AS_EIO, &bh->b_assoc_map->flags); 578 bh->b_assoc_map = NULL; 579 } 580 581 int inode_has_buffers(struct inode *inode) 582 { 583 return !list_empty(&inode->i_data.private_list); 584 } 585 586 /* 587 * osync is designed to support O_SYNC io. It waits synchronously for 588 * all already-submitted IO to complete, but does not queue any new 589 * writes to the disk. 590 * 591 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as 592 * you dirty the buffers, and then use osync_inode_buffers to wait for 593 * completion. Any other dirty buffers which are not yet queued for 594 * write will not be flushed to disk by the osync. 595 */ 596 static int osync_buffers_list(spinlock_t *lock, struct list_head *list) 597 { 598 struct buffer_head *bh; 599 struct list_head *p; 600 int err = 0; 601 602 spin_lock(lock); 603 repeat: 604 list_for_each_prev(p, list) { 605 bh = BH_ENTRY(p); 606 if (buffer_locked(bh)) { 607 get_bh(bh); 608 spin_unlock(lock); 609 wait_on_buffer(bh); 610 if (!buffer_uptodate(bh)) 611 err = -EIO; 612 brelse(bh); 613 spin_lock(lock); 614 goto repeat; 615 } 616 } 617 spin_unlock(lock); 618 return err; 619 } 620 621 /** 622 * sync_mapping_buffers - write out and wait upon a mapping's "associated" 623 * buffers 624 * @mapping: the mapping which wants those buffers written 625 * 626 * Starts I/O against the buffers at mapping->private_list, and waits upon 627 * that I/O. 628 * 629 * Basically, this is a convenience function for fsync(). 630 * @mapping is a file or directory which needs those buffers to be written for 631 * a successful fsync(). 632 */ 633 int sync_mapping_buffers(struct address_space *mapping) 634 { 635 struct address_space *buffer_mapping = mapping->assoc_mapping; 636 637 if (buffer_mapping == NULL || list_empty(&mapping->private_list)) 638 return 0; 639 640 return fsync_buffers_list(&buffer_mapping->private_lock, 641 &mapping->private_list); 642 } 643 EXPORT_SYMBOL(sync_mapping_buffers); 644 645 /* 646 * Called when we've recently written block `bblock', and it is known that 647 * `bblock' was for a buffer_boundary() buffer. This means that the block at 648 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's 649 * dirty, schedule it for IO. So that indirects merge nicely with their data. 650 */ 651 void write_boundary_block(struct block_device *bdev, 652 sector_t bblock, unsigned blocksize) 653 { 654 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize); 655 if (bh) { 656 if (buffer_dirty(bh)) 657 ll_rw_block(WRITE, 1, &bh); 658 put_bh(bh); 659 } 660 } 661 662 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode) 663 { 664 struct address_space *mapping = inode->i_mapping; 665 struct address_space *buffer_mapping = bh->b_page->mapping; 666 667 mark_buffer_dirty(bh); 668 if (!mapping->assoc_mapping) { 669 mapping->assoc_mapping = buffer_mapping; 670 } else { 671 BUG_ON(mapping->assoc_mapping != buffer_mapping); 672 } 673 if (list_empty(&bh->b_assoc_buffers)) { 674 spin_lock(&buffer_mapping->private_lock); 675 list_move_tail(&bh->b_assoc_buffers, 676 &mapping->private_list); 677 bh->b_assoc_map = mapping; 678 spin_unlock(&buffer_mapping->private_lock); 679 } 680 } 681 EXPORT_SYMBOL(mark_buffer_dirty_inode); 682 683 /* 684 * Add a page to the dirty page list. 685 * 686 * It is a sad fact of life that this function is called from several places 687 * deeply under spinlocking. It may not sleep. 688 * 689 * If the page has buffers, the uptodate buffers are set dirty, to preserve 690 * dirty-state coherency between the page and the buffers. It the page does 691 * not have buffers then when they are later attached they will all be set 692 * dirty. 693 * 694 * The buffers are dirtied before the page is dirtied. There's a small race 695 * window in which a writepage caller may see the page cleanness but not the 696 * buffer dirtiness. That's fine. If this code were to set the page dirty 697 * before the buffers, a concurrent writepage caller could clear the page dirty 698 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean 699 * page on the dirty page list. 700 * 701 * We use private_lock to lock against try_to_free_buffers while using the 702 * page's buffer list. Also use this to protect against clean buffers being 703 * added to the page after it was set dirty. 704 * 705 * FIXME: may need to call ->reservepage here as well. That's rather up to the 706 * address_space though. 707 */ 708 int __set_page_dirty_buffers(struct page *page) 709 { 710 struct address_space * const mapping = page_mapping(page); 711 712 if (unlikely(!mapping)) 713 return !TestSetPageDirty(page); 714 715 spin_lock(&mapping->private_lock); 716 if (page_has_buffers(page)) { 717 struct buffer_head *head = page_buffers(page); 718 struct buffer_head *bh = head; 719 720 do { 721 set_buffer_dirty(bh); 722 bh = bh->b_this_page; 723 } while (bh != head); 724 } 725 spin_unlock(&mapping->private_lock); 726 727 if (!TestSetPageDirty(page)) { 728 write_lock_irq(&mapping->tree_lock); 729 if (page->mapping) { /* Race with truncate? */ 730 if (mapping_cap_account_dirty(mapping)) 731 __inc_zone_page_state(page, NR_FILE_DIRTY); 732 radix_tree_tag_set(&mapping->page_tree, 733 page_index(page), 734 PAGECACHE_TAG_DIRTY); 735 } 736 write_unlock_irq(&mapping->tree_lock); 737 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 738 return 1; 739 } 740 return 0; 741 } 742 EXPORT_SYMBOL(__set_page_dirty_buffers); 743 744 /* 745 * Write out and wait upon a list of buffers. 746 * 747 * We have conflicting pressures: we want to make sure that all 748 * initially dirty buffers get waited on, but that any subsequently 749 * dirtied buffers don't. After all, we don't want fsync to last 750 * forever if somebody is actively writing to the file. 751 * 752 * Do this in two main stages: first we copy dirty buffers to a 753 * temporary inode list, queueing the writes as we go. Then we clean 754 * up, waiting for those writes to complete. 755 * 756 * During this second stage, any subsequent updates to the file may end 757 * up refiling the buffer on the original inode's dirty list again, so 758 * there is a chance we will end up with a buffer queued for write but 759 * not yet completed on that list. So, as a final cleanup we go through 760 * the osync code to catch these locked, dirty buffers without requeuing 761 * any newly dirty buffers for write. 762 */ 763 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list) 764 { 765 struct buffer_head *bh; 766 struct list_head tmp; 767 int err = 0, err2; 768 769 INIT_LIST_HEAD(&tmp); 770 771 spin_lock(lock); 772 while (!list_empty(list)) { 773 bh = BH_ENTRY(list->next); 774 __remove_assoc_queue(bh); 775 if (buffer_dirty(bh) || buffer_locked(bh)) { 776 list_add(&bh->b_assoc_buffers, &tmp); 777 if (buffer_dirty(bh)) { 778 get_bh(bh); 779 spin_unlock(lock); 780 /* 781 * Ensure any pending I/O completes so that 782 * ll_rw_block() actually writes the current 783 * contents - it is a noop if I/O is still in 784 * flight on potentially older contents. 785 */ 786 ll_rw_block(SWRITE, 1, &bh); 787 brelse(bh); 788 spin_lock(lock); 789 } 790 } 791 } 792 793 while (!list_empty(&tmp)) { 794 bh = BH_ENTRY(tmp.prev); 795 list_del_init(&bh->b_assoc_buffers); 796 get_bh(bh); 797 spin_unlock(lock); 798 wait_on_buffer(bh); 799 if (!buffer_uptodate(bh)) 800 err = -EIO; 801 brelse(bh); 802 spin_lock(lock); 803 } 804 805 spin_unlock(lock); 806 err2 = osync_buffers_list(lock, list); 807 if (err) 808 return err; 809 else 810 return err2; 811 } 812 813 /* 814 * Invalidate any and all dirty buffers on a given inode. We are 815 * probably unmounting the fs, but that doesn't mean we have already 816 * done a sync(). Just drop the buffers from the inode list. 817 * 818 * NOTE: we take the inode's blockdev's mapping's private_lock. Which 819 * assumes that all the buffers are against the blockdev. Not true 820 * for reiserfs. 821 */ 822 void invalidate_inode_buffers(struct inode *inode) 823 { 824 if (inode_has_buffers(inode)) { 825 struct address_space *mapping = &inode->i_data; 826 struct list_head *list = &mapping->private_list; 827 struct address_space *buffer_mapping = mapping->assoc_mapping; 828 829 spin_lock(&buffer_mapping->private_lock); 830 while (!list_empty(list)) 831 __remove_assoc_queue(BH_ENTRY(list->next)); 832 spin_unlock(&buffer_mapping->private_lock); 833 } 834 } 835 836 /* 837 * Remove any clean buffers from the inode's buffer list. This is called 838 * when we're trying to free the inode itself. Those buffers can pin it. 839 * 840 * Returns true if all buffers were removed. 841 */ 842 int remove_inode_buffers(struct inode *inode) 843 { 844 int ret = 1; 845 846 if (inode_has_buffers(inode)) { 847 struct address_space *mapping = &inode->i_data; 848 struct list_head *list = &mapping->private_list; 849 struct address_space *buffer_mapping = mapping->assoc_mapping; 850 851 spin_lock(&buffer_mapping->private_lock); 852 while (!list_empty(list)) { 853 struct buffer_head *bh = BH_ENTRY(list->next); 854 if (buffer_dirty(bh)) { 855 ret = 0; 856 break; 857 } 858 __remove_assoc_queue(bh); 859 } 860 spin_unlock(&buffer_mapping->private_lock); 861 } 862 return ret; 863 } 864 865 /* 866 * Create the appropriate buffers when given a page for data area and 867 * the size of each buffer.. Use the bh->b_this_page linked list to 868 * follow the buffers created. Return NULL if unable to create more 869 * buffers. 870 * 871 * The retry flag is used to differentiate async IO (paging, swapping) 872 * which may not fail from ordinary buffer allocations. 873 */ 874 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size, 875 int retry) 876 { 877 struct buffer_head *bh, *head; 878 long offset; 879 880 try_again: 881 head = NULL; 882 offset = PAGE_SIZE; 883 while ((offset -= size) >= 0) { 884 bh = alloc_buffer_head(GFP_NOFS); 885 if (!bh) 886 goto no_grow; 887 888 bh->b_bdev = NULL; 889 bh->b_this_page = head; 890 bh->b_blocknr = -1; 891 head = bh; 892 893 bh->b_state = 0; 894 atomic_set(&bh->b_count, 0); 895 bh->b_private = NULL; 896 bh->b_size = size; 897 898 /* Link the buffer to its page */ 899 set_bh_page(bh, page, offset); 900 901 init_buffer(bh, NULL, NULL); 902 } 903 return head; 904 /* 905 * In case anything failed, we just free everything we got. 906 */ 907 no_grow: 908 if (head) { 909 do { 910 bh = head; 911 head = head->b_this_page; 912 free_buffer_head(bh); 913 } while (head); 914 } 915 916 /* 917 * Return failure for non-async IO requests. Async IO requests 918 * are not allowed to fail, so we have to wait until buffer heads 919 * become available. But we don't want tasks sleeping with 920 * partially complete buffers, so all were released above. 921 */ 922 if (!retry) 923 return NULL; 924 925 /* We're _really_ low on memory. Now we just 926 * wait for old buffer heads to become free due to 927 * finishing IO. Since this is an async request and 928 * the reserve list is empty, we're sure there are 929 * async buffer heads in use. 930 */ 931 free_more_memory(); 932 goto try_again; 933 } 934 EXPORT_SYMBOL_GPL(alloc_page_buffers); 935 936 static inline void 937 link_dev_buffers(struct page *page, struct buffer_head *head) 938 { 939 struct buffer_head *bh, *tail; 940 941 bh = head; 942 do { 943 tail = bh; 944 bh = bh->b_this_page; 945 } while (bh); 946 tail->b_this_page = head; 947 attach_page_buffers(page, head); 948 } 949 950 /* 951 * Initialise the state of a blockdev page's buffers. 952 */ 953 static void 954 init_page_buffers(struct page *page, struct block_device *bdev, 955 sector_t block, int size) 956 { 957 struct buffer_head *head = page_buffers(page); 958 struct buffer_head *bh = head; 959 int uptodate = PageUptodate(page); 960 961 do { 962 if (!buffer_mapped(bh)) { 963 init_buffer(bh, NULL, NULL); 964 bh->b_bdev = bdev; 965 bh->b_blocknr = block; 966 if (uptodate) 967 set_buffer_uptodate(bh); 968 set_buffer_mapped(bh); 969 } 970 block++; 971 bh = bh->b_this_page; 972 } while (bh != head); 973 } 974 975 /* 976 * Create the page-cache page that contains the requested block. 977 * 978 * This is user purely for blockdev mappings. 979 */ 980 static struct page * 981 grow_dev_page(struct block_device *bdev, sector_t block, 982 pgoff_t index, int size) 983 { 984 struct inode *inode = bdev->bd_inode; 985 struct page *page; 986 struct buffer_head *bh; 987 988 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS); 989 if (!page) 990 return NULL; 991 992 BUG_ON(!PageLocked(page)); 993 994 if (page_has_buffers(page)) { 995 bh = page_buffers(page); 996 if (bh->b_size == size) { 997 init_page_buffers(page, bdev, block, size); 998 return page; 999 } 1000 if (!try_to_free_buffers(page)) 1001 goto failed; 1002 } 1003 1004 /* 1005 * Allocate some buffers for this page 1006 */ 1007 bh = alloc_page_buffers(page, size, 0); 1008 if (!bh) 1009 goto failed; 1010 1011 /* 1012 * Link the page to the buffers and initialise them. Take the 1013 * lock to be atomic wrt __find_get_block(), which does not 1014 * run under the page lock. 1015 */ 1016 spin_lock(&inode->i_mapping->private_lock); 1017 link_dev_buffers(page, bh); 1018 init_page_buffers(page, bdev, block, size); 1019 spin_unlock(&inode->i_mapping->private_lock); 1020 return page; 1021 1022 failed: 1023 BUG(); 1024 unlock_page(page); 1025 page_cache_release(page); 1026 return NULL; 1027 } 1028 1029 /* 1030 * Create buffers for the specified block device block's page. If 1031 * that page was dirty, the buffers are set dirty also. 1032 * 1033 * Except that's a bug. Attaching dirty buffers to a dirty 1034 * blockdev's page can result in filesystem corruption, because 1035 * some of those buffers may be aliases of filesystem data. 1036 * grow_dev_page() will go BUG() if this happens. 1037 */ 1038 static int 1039 grow_buffers(struct block_device *bdev, sector_t block, int size) 1040 { 1041 struct page *page; 1042 pgoff_t index; 1043 int sizebits; 1044 1045 sizebits = -1; 1046 do { 1047 sizebits++; 1048 } while ((size << sizebits) < PAGE_SIZE); 1049 1050 index = block >> sizebits; 1051 1052 /* 1053 * Check for a block which wants to lie outside our maximum possible 1054 * pagecache index. (this comparison is done using sector_t types). 1055 */ 1056 if (unlikely(index != block >> sizebits)) { 1057 char b[BDEVNAME_SIZE]; 1058 1059 printk(KERN_ERR "%s: requested out-of-range block %llu for " 1060 "device %s\n", 1061 __FUNCTION__, (unsigned long long)block, 1062 bdevname(bdev, b)); 1063 return -EIO; 1064 } 1065 block = index << sizebits; 1066 /* Create a page with the proper size buffers.. */ 1067 page = grow_dev_page(bdev, block, index, size); 1068 if (!page) 1069 return 0; 1070 unlock_page(page); 1071 page_cache_release(page); 1072 return 1; 1073 } 1074 1075 static struct buffer_head * 1076 __getblk_slow(struct block_device *bdev, sector_t block, int size) 1077 { 1078 /* Size must be multiple of hard sectorsize */ 1079 if (unlikely(size & (bdev_hardsect_size(bdev)-1) || 1080 (size < 512 || size > PAGE_SIZE))) { 1081 printk(KERN_ERR "getblk(): invalid block size %d requested\n", 1082 size); 1083 printk(KERN_ERR "hardsect size: %d\n", 1084 bdev_hardsect_size(bdev)); 1085 1086 dump_stack(); 1087 return NULL; 1088 } 1089 1090 for (;;) { 1091 struct buffer_head * bh; 1092 int ret; 1093 1094 bh = __find_get_block(bdev, block, size); 1095 if (bh) 1096 return bh; 1097 1098 ret = grow_buffers(bdev, block, size); 1099 if (ret < 0) 1100 return NULL; 1101 if (ret == 0) 1102 free_more_memory(); 1103 } 1104 } 1105 1106 /* 1107 * The relationship between dirty buffers and dirty pages: 1108 * 1109 * Whenever a page has any dirty buffers, the page's dirty bit is set, and 1110 * the page is tagged dirty in its radix tree. 1111 * 1112 * At all times, the dirtiness of the buffers represents the dirtiness of 1113 * subsections of the page. If the page has buffers, the page dirty bit is 1114 * merely a hint about the true dirty state. 1115 * 1116 * When a page is set dirty in its entirety, all its buffers are marked dirty 1117 * (if the page has buffers). 1118 * 1119 * When a buffer is marked dirty, its page is dirtied, but the page's other 1120 * buffers are not. 1121 * 1122 * Also. When blockdev buffers are explicitly read with bread(), they 1123 * individually become uptodate. But their backing page remains not 1124 * uptodate - even if all of its buffers are uptodate. A subsequent 1125 * block_read_full_page() against that page will discover all the uptodate 1126 * buffers, will set the page uptodate and will perform no I/O. 1127 */ 1128 1129 /** 1130 * mark_buffer_dirty - mark a buffer_head as needing writeout 1131 * @bh: the buffer_head to mark dirty 1132 * 1133 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its 1134 * backing page dirty, then tag the page as dirty in its address_space's radix 1135 * tree and then attach the address_space's inode to its superblock's dirty 1136 * inode list. 1137 * 1138 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock, 1139 * mapping->tree_lock and the global inode_lock. 1140 */ 1141 void fastcall mark_buffer_dirty(struct buffer_head *bh) 1142 { 1143 if (!buffer_dirty(bh) && !test_set_buffer_dirty(bh)) 1144 __set_page_dirty_nobuffers(bh->b_page); 1145 } 1146 1147 /* 1148 * Decrement a buffer_head's reference count. If all buffers against a page 1149 * have zero reference count, are clean and unlocked, and if the page is clean 1150 * and unlocked then try_to_free_buffers() may strip the buffers from the page 1151 * in preparation for freeing it (sometimes, rarely, buffers are removed from 1152 * a page but it ends up not being freed, and buffers may later be reattached). 1153 */ 1154 void __brelse(struct buffer_head * buf) 1155 { 1156 if (atomic_read(&buf->b_count)) { 1157 put_bh(buf); 1158 return; 1159 } 1160 printk(KERN_ERR "VFS: brelse: Trying to free free buffer\n"); 1161 WARN_ON(1); 1162 } 1163 1164 /* 1165 * bforget() is like brelse(), except it discards any 1166 * potentially dirty data. 1167 */ 1168 void __bforget(struct buffer_head *bh) 1169 { 1170 clear_buffer_dirty(bh); 1171 if (!list_empty(&bh->b_assoc_buffers)) { 1172 struct address_space *buffer_mapping = bh->b_page->mapping; 1173 1174 spin_lock(&buffer_mapping->private_lock); 1175 list_del_init(&bh->b_assoc_buffers); 1176 bh->b_assoc_map = NULL; 1177 spin_unlock(&buffer_mapping->private_lock); 1178 } 1179 __brelse(bh); 1180 } 1181 1182 static struct buffer_head *__bread_slow(struct buffer_head *bh) 1183 { 1184 lock_buffer(bh); 1185 if (buffer_uptodate(bh)) { 1186 unlock_buffer(bh); 1187 return bh; 1188 } else { 1189 get_bh(bh); 1190 bh->b_end_io = end_buffer_read_sync; 1191 submit_bh(READ, bh); 1192 wait_on_buffer(bh); 1193 if (buffer_uptodate(bh)) 1194 return bh; 1195 } 1196 brelse(bh); 1197 return NULL; 1198 } 1199 1200 /* 1201 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block(). 1202 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their 1203 * refcount elevated by one when they're in an LRU. A buffer can only appear 1204 * once in a particular CPU's LRU. A single buffer can be present in multiple 1205 * CPU's LRUs at the same time. 1206 * 1207 * This is a transparent caching front-end to sb_bread(), sb_getblk() and 1208 * sb_find_get_block(). 1209 * 1210 * The LRUs themselves only need locking against invalidate_bh_lrus. We use 1211 * a local interrupt disable for that. 1212 */ 1213 1214 #define BH_LRU_SIZE 8 1215 1216 struct bh_lru { 1217 struct buffer_head *bhs[BH_LRU_SIZE]; 1218 }; 1219 1220 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }}; 1221 1222 #ifdef CONFIG_SMP 1223 #define bh_lru_lock() local_irq_disable() 1224 #define bh_lru_unlock() local_irq_enable() 1225 #else 1226 #define bh_lru_lock() preempt_disable() 1227 #define bh_lru_unlock() preempt_enable() 1228 #endif 1229 1230 static inline void check_irqs_on(void) 1231 { 1232 #ifdef irqs_disabled 1233 BUG_ON(irqs_disabled()); 1234 #endif 1235 } 1236 1237 /* 1238 * The LRU management algorithm is dopey-but-simple. Sorry. 1239 */ 1240 static void bh_lru_install(struct buffer_head *bh) 1241 { 1242 struct buffer_head *evictee = NULL; 1243 struct bh_lru *lru; 1244 1245 check_irqs_on(); 1246 bh_lru_lock(); 1247 lru = &__get_cpu_var(bh_lrus); 1248 if (lru->bhs[0] != bh) { 1249 struct buffer_head *bhs[BH_LRU_SIZE]; 1250 int in; 1251 int out = 0; 1252 1253 get_bh(bh); 1254 bhs[out++] = bh; 1255 for (in = 0; in < BH_LRU_SIZE; in++) { 1256 struct buffer_head *bh2 = lru->bhs[in]; 1257 1258 if (bh2 == bh) { 1259 __brelse(bh2); 1260 } else { 1261 if (out >= BH_LRU_SIZE) { 1262 BUG_ON(evictee != NULL); 1263 evictee = bh2; 1264 } else { 1265 bhs[out++] = bh2; 1266 } 1267 } 1268 } 1269 while (out < BH_LRU_SIZE) 1270 bhs[out++] = NULL; 1271 memcpy(lru->bhs, bhs, sizeof(bhs)); 1272 } 1273 bh_lru_unlock(); 1274 1275 if (evictee) 1276 __brelse(evictee); 1277 } 1278 1279 /* 1280 * Look up the bh in this cpu's LRU. If it's there, move it to the head. 1281 */ 1282 static struct buffer_head * 1283 lookup_bh_lru(struct block_device *bdev, sector_t block, int size) 1284 { 1285 struct buffer_head *ret = NULL; 1286 struct bh_lru *lru; 1287 int i; 1288 1289 check_irqs_on(); 1290 bh_lru_lock(); 1291 lru = &__get_cpu_var(bh_lrus); 1292 for (i = 0; i < BH_LRU_SIZE; i++) { 1293 struct buffer_head *bh = lru->bhs[i]; 1294 1295 if (bh && bh->b_bdev == bdev && 1296 bh->b_blocknr == block && bh->b_size == size) { 1297 if (i) { 1298 while (i) { 1299 lru->bhs[i] = lru->bhs[i - 1]; 1300 i--; 1301 } 1302 lru->bhs[0] = bh; 1303 } 1304 get_bh(bh); 1305 ret = bh; 1306 break; 1307 } 1308 } 1309 bh_lru_unlock(); 1310 return ret; 1311 } 1312 1313 /* 1314 * Perform a pagecache lookup for the matching buffer. If it's there, refresh 1315 * it in the LRU and mark it as accessed. If it is not present then return 1316 * NULL 1317 */ 1318 struct buffer_head * 1319 __find_get_block(struct block_device *bdev, sector_t block, int size) 1320 { 1321 struct buffer_head *bh = lookup_bh_lru(bdev, block, size); 1322 1323 if (bh == NULL) { 1324 bh = __find_get_block_slow(bdev, block); 1325 if (bh) 1326 bh_lru_install(bh); 1327 } 1328 if (bh) 1329 touch_buffer(bh); 1330 return bh; 1331 } 1332 EXPORT_SYMBOL(__find_get_block); 1333 1334 /* 1335 * __getblk will locate (and, if necessary, create) the buffer_head 1336 * which corresponds to the passed block_device, block and size. The 1337 * returned buffer has its reference count incremented. 1338 * 1339 * __getblk() cannot fail - it just keeps trying. If you pass it an 1340 * illegal block number, __getblk() will happily return a buffer_head 1341 * which represents the non-existent block. Very weird. 1342 * 1343 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers() 1344 * attempt is failing. FIXME, perhaps? 1345 */ 1346 struct buffer_head * 1347 __getblk(struct block_device *bdev, sector_t block, int size) 1348 { 1349 struct buffer_head *bh = __find_get_block(bdev, block, size); 1350 1351 might_sleep(); 1352 if (bh == NULL) 1353 bh = __getblk_slow(bdev, block, size); 1354 return bh; 1355 } 1356 EXPORT_SYMBOL(__getblk); 1357 1358 /* 1359 * Do async read-ahead on a buffer.. 1360 */ 1361 void __breadahead(struct block_device *bdev, sector_t block, int size) 1362 { 1363 struct buffer_head *bh = __getblk(bdev, block, size); 1364 if (likely(bh)) { 1365 ll_rw_block(READA, 1, &bh); 1366 brelse(bh); 1367 } 1368 } 1369 EXPORT_SYMBOL(__breadahead); 1370 1371 /** 1372 * __bread() - reads a specified block and returns the bh 1373 * @bdev: the block_device to read from 1374 * @block: number of block 1375 * @size: size (in bytes) to read 1376 * 1377 * Reads a specified block, and returns buffer head that contains it. 1378 * It returns NULL if the block was unreadable. 1379 */ 1380 struct buffer_head * 1381 __bread(struct block_device *bdev, sector_t block, int size) 1382 { 1383 struct buffer_head *bh = __getblk(bdev, block, size); 1384 1385 if (likely(bh) && !buffer_uptodate(bh)) 1386 bh = __bread_slow(bh); 1387 return bh; 1388 } 1389 EXPORT_SYMBOL(__bread); 1390 1391 /* 1392 * invalidate_bh_lrus() is called rarely - but not only at unmount. 1393 * This doesn't race because it runs in each cpu either in irq 1394 * or with preempt disabled. 1395 */ 1396 static void invalidate_bh_lru(void *arg) 1397 { 1398 struct bh_lru *b = &get_cpu_var(bh_lrus); 1399 int i; 1400 1401 for (i = 0; i < BH_LRU_SIZE; i++) { 1402 brelse(b->bhs[i]); 1403 b->bhs[i] = NULL; 1404 } 1405 put_cpu_var(bh_lrus); 1406 } 1407 1408 static void invalidate_bh_lrus(void) 1409 { 1410 on_each_cpu(invalidate_bh_lru, NULL, 1, 1); 1411 } 1412 1413 void set_bh_page(struct buffer_head *bh, 1414 struct page *page, unsigned long offset) 1415 { 1416 bh->b_page = page; 1417 BUG_ON(offset >= PAGE_SIZE); 1418 if (PageHighMem(page)) 1419 /* 1420 * This catches illegal uses and preserves the offset: 1421 */ 1422 bh->b_data = (char *)(0 + offset); 1423 else 1424 bh->b_data = page_address(page) + offset; 1425 } 1426 EXPORT_SYMBOL(set_bh_page); 1427 1428 /* 1429 * Called when truncating a buffer on a page completely. 1430 */ 1431 static void discard_buffer(struct buffer_head * bh) 1432 { 1433 lock_buffer(bh); 1434 clear_buffer_dirty(bh); 1435 bh->b_bdev = NULL; 1436 clear_buffer_mapped(bh); 1437 clear_buffer_req(bh); 1438 clear_buffer_new(bh); 1439 clear_buffer_delay(bh); 1440 unlock_buffer(bh); 1441 } 1442 1443 /** 1444 * block_invalidatepage - invalidate part of all of a buffer-backed page 1445 * 1446 * @page: the page which is affected 1447 * @offset: the index of the truncation point 1448 * 1449 * block_invalidatepage() is called when all or part of the page has become 1450 * invalidatedby a truncate operation. 1451 * 1452 * block_invalidatepage() does not have to release all buffers, but it must 1453 * ensure that no dirty buffer is left outside @offset and that no I/O 1454 * is underway against any of the blocks which are outside the truncation 1455 * point. Because the caller is about to free (and possibly reuse) those 1456 * blocks on-disk. 1457 */ 1458 void block_invalidatepage(struct page *page, unsigned long offset) 1459 { 1460 struct buffer_head *head, *bh, *next; 1461 unsigned int curr_off = 0; 1462 1463 BUG_ON(!PageLocked(page)); 1464 if (!page_has_buffers(page)) 1465 goto out; 1466 1467 head = page_buffers(page); 1468 bh = head; 1469 do { 1470 unsigned int next_off = curr_off + bh->b_size; 1471 next = bh->b_this_page; 1472 1473 /* 1474 * is this block fully invalidated? 1475 */ 1476 if (offset <= curr_off) 1477 discard_buffer(bh); 1478 curr_off = next_off; 1479 bh = next; 1480 } while (bh != head); 1481 1482 /* 1483 * We release buffers only if the entire page is being invalidated. 1484 * The get_block cached value has been unconditionally invalidated, 1485 * so real IO is not possible anymore. 1486 */ 1487 if (offset == 0) 1488 try_to_release_page(page, 0); 1489 out: 1490 return; 1491 } 1492 EXPORT_SYMBOL(block_invalidatepage); 1493 1494 /* 1495 * We attach and possibly dirty the buffers atomically wrt 1496 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers 1497 * is already excluded via the page lock. 1498 */ 1499 void create_empty_buffers(struct page *page, 1500 unsigned long blocksize, unsigned long b_state) 1501 { 1502 struct buffer_head *bh, *head, *tail; 1503 1504 head = alloc_page_buffers(page, blocksize, 1); 1505 bh = head; 1506 do { 1507 bh->b_state |= b_state; 1508 tail = bh; 1509 bh = bh->b_this_page; 1510 } while (bh); 1511 tail->b_this_page = head; 1512 1513 spin_lock(&page->mapping->private_lock); 1514 if (PageUptodate(page) || PageDirty(page)) { 1515 bh = head; 1516 do { 1517 if (PageDirty(page)) 1518 set_buffer_dirty(bh); 1519 if (PageUptodate(page)) 1520 set_buffer_uptodate(bh); 1521 bh = bh->b_this_page; 1522 } while (bh != head); 1523 } 1524 attach_page_buffers(page, head); 1525 spin_unlock(&page->mapping->private_lock); 1526 } 1527 EXPORT_SYMBOL(create_empty_buffers); 1528 1529 /* 1530 * We are taking a block for data and we don't want any output from any 1531 * buffer-cache aliases starting from return from that function and 1532 * until the moment when something will explicitly mark the buffer 1533 * dirty (hopefully that will not happen until we will free that block ;-) 1534 * We don't even need to mark it not-uptodate - nobody can expect 1535 * anything from a newly allocated buffer anyway. We used to used 1536 * unmap_buffer() for such invalidation, but that was wrong. We definitely 1537 * don't want to mark the alias unmapped, for example - it would confuse 1538 * anyone who might pick it with bread() afterwards... 1539 * 1540 * Also.. Note that bforget() doesn't lock the buffer. So there can 1541 * be writeout I/O going on against recently-freed buffers. We don't 1542 * wait on that I/O in bforget() - it's more efficient to wait on the I/O 1543 * only if we really need to. That happens here. 1544 */ 1545 void unmap_underlying_metadata(struct block_device *bdev, sector_t block) 1546 { 1547 struct buffer_head *old_bh; 1548 1549 might_sleep(); 1550 1551 old_bh = __find_get_block_slow(bdev, block); 1552 if (old_bh) { 1553 clear_buffer_dirty(old_bh); 1554 wait_on_buffer(old_bh); 1555 clear_buffer_req(old_bh); 1556 __brelse(old_bh); 1557 } 1558 } 1559 EXPORT_SYMBOL(unmap_underlying_metadata); 1560 1561 /* 1562 * NOTE! All mapped/uptodate combinations are valid: 1563 * 1564 * Mapped Uptodate Meaning 1565 * 1566 * No No "unknown" - must do get_block() 1567 * No Yes "hole" - zero-filled 1568 * Yes No "allocated" - allocated on disk, not read in 1569 * Yes Yes "valid" - allocated and up-to-date in memory. 1570 * 1571 * "Dirty" is valid only with the last case (mapped+uptodate). 1572 */ 1573 1574 /* 1575 * While block_write_full_page is writing back the dirty buffers under 1576 * the page lock, whoever dirtied the buffers may decide to clean them 1577 * again at any time. We handle that by only looking at the buffer 1578 * state inside lock_buffer(). 1579 * 1580 * If block_write_full_page() is called for regular writeback 1581 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a 1582 * locked buffer. This only can happen if someone has written the buffer 1583 * directly, with submit_bh(). At the address_space level PageWriteback 1584 * prevents this contention from occurring. 1585 */ 1586 static int __block_write_full_page(struct inode *inode, struct page *page, 1587 get_block_t *get_block, struct writeback_control *wbc) 1588 { 1589 int err; 1590 sector_t block; 1591 sector_t last_block; 1592 struct buffer_head *bh, *head; 1593 const unsigned blocksize = 1 << inode->i_blkbits; 1594 int nr_underway = 0; 1595 1596 BUG_ON(!PageLocked(page)); 1597 1598 last_block = (i_size_read(inode) - 1) >> inode->i_blkbits; 1599 1600 if (!page_has_buffers(page)) { 1601 create_empty_buffers(page, blocksize, 1602 (1 << BH_Dirty)|(1 << BH_Uptodate)); 1603 } 1604 1605 /* 1606 * Be very careful. We have no exclusion from __set_page_dirty_buffers 1607 * here, and the (potentially unmapped) buffers may become dirty at 1608 * any time. If a buffer becomes dirty here after we've inspected it 1609 * then we just miss that fact, and the page stays dirty. 1610 * 1611 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers; 1612 * handle that here by just cleaning them. 1613 */ 1614 1615 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits); 1616 head = page_buffers(page); 1617 bh = head; 1618 1619 /* 1620 * Get all the dirty buffers mapped to disk addresses and 1621 * handle any aliases from the underlying blockdev's mapping. 1622 */ 1623 do { 1624 if (block > last_block) { 1625 /* 1626 * mapped buffers outside i_size will occur, because 1627 * this page can be outside i_size when there is a 1628 * truncate in progress. 1629 */ 1630 /* 1631 * The buffer was zeroed by block_write_full_page() 1632 */ 1633 clear_buffer_dirty(bh); 1634 set_buffer_uptodate(bh); 1635 } else if (!buffer_mapped(bh) && buffer_dirty(bh)) { 1636 WARN_ON(bh->b_size != blocksize); 1637 err = get_block(inode, block, bh, 1); 1638 if (err) 1639 goto recover; 1640 if (buffer_new(bh)) { 1641 /* blockdev mappings never come here */ 1642 clear_buffer_new(bh); 1643 unmap_underlying_metadata(bh->b_bdev, 1644 bh->b_blocknr); 1645 } 1646 } 1647 bh = bh->b_this_page; 1648 block++; 1649 } while (bh != head); 1650 1651 do { 1652 if (!buffer_mapped(bh)) 1653 continue; 1654 /* 1655 * If it's a fully non-blocking write attempt and we cannot 1656 * lock the buffer then redirty the page. Note that this can 1657 * potentially cause a busy-wait loop from pdflush and kswapd 1658 * activity, but those code paths have their own higher-level 1659 * throttling. 1660 */ 1661 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) { 1662 lock_buffer(bh); 1663 } else if (test_set_buffer_locked(bh)) { 1664 redirty_page_for_writepage(wbc, page); 1665 continue; 1666 } 1667 if (test_clear_buffer_dirty(bh)) { 1668 mark_buffer_async_write(bh); 1669 } else { 1670 unlock_buffer(bh); 1671 } 1672 } while ((bh = bh->b_this_page) != head); 1673 1674 /* 1675 * The page and its buffers are protected by PageWriteback(), so we can 1676 * drop the bh refcounts early. 1677 */ 1678 BUG_ON(PageWriteback(page)); 1679 set_page_writeback(page); 1680 1681 do { 1682 struct buffer_head *next = bh->b_this_page; 1683 if (buffer_async_write(bh)) { 1684 submit_bh(WRITE, bh); 1685 nr_underway++; 1686 } 1687 bh = next; 1688 } while (bh != head); 1689 unlock_page(page); 1690 1691 err = 0; 1692 done: 1693 if (nr_underway == 0) { 1694 /* 1695 * The page was marked dirty, but the buffers were 1696 * clean. Someone wrote them back by hand with 1697 * ll_rw_block/submit_bh. A rare case. 1698 */ 1699 int uptodate = 1; 1700 do { 1701 if (!buffer_uptodate(bh)) { 1702 uptodate = 0; 1703 break; 1704 } 1705 bh = bh->b_this_page; 1706 } while (bh != head); 1707 if (uptodate) 1708 SetPageUptodate(page); 1709 end_page_writeback(page); 1710 /* 1711 * The page and buffer_heads can be released at any time from 1712 * here on. 1713 */ 1714 wbc->pages_skipped++; /* We didn't write this page */ 1715 } 1716 return err; 1717 1718 recover: 1719 /* 1720 * ENOSPC, or some other error. We may already have added some 1721 * blocks to the file, so we need to write these out to avoid 1722 * exposing stale data. 1723 * The page is currently locked and not marked for writeback 1724 */ 1725 bh = head; 1726 /* Recovery: lock and submit the mapped buffers */ 1727 do { 1728 if (buffer_mapped(bh) && buffer_dirty(bh)) { 1729 lock_buffer(bh); 1730 mark_buffer_async_write(bh); 1731 } else { 1732 /* 1733 * The buffer may have been set dirty during 1734 * attachment to a dirty page. 1735 */ 1736 clear_buffer_dirty(bh); 1737 } 1738 } while ((bh = bh->b_this_page) != head); 1739 SetPageError(page); 1740 BUG_ON(PageWriteback(page)); 1741 set_page_writeback(page); 1742 unlock_page(page); 1743 do { 1744 struct buffer_head *next = bh->b_this_page; 1745 if (buffer_async_write(bh)) { 1746 clear_buffer_dirty(bh); 1747 submit_bh(WRITE, bh); 1748 nr_underway++; 1749 } 1750 bh = next; 1751 } while (bh != head); 1752 goto done; 1753 } 1754 1755 static int __block_prepare_write(struct inode *inode, struct page *page, 1756 unsigned from, unsigned to, get_block_t *get_block) 1757 { 1758 unsigned block_start, block_end; 1759 sector_t block; 1760 int err = 0; 1761 unsigned blocksize, bbits; 1762 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait; 1763 1764 BUG_ON(!PageLocked(page)); 1765 BUG_ON(from > PAGE_CACHE_SIZE); 1766 BUG_ON(to > PAGE_CACHE_SIZE); 1767 BUG_ON(from > to); 1768 1769 blocksize = 1 << inode->i_blkbits; 1770 if (!page_has_buffers(page)) 1771 create_empty_buffers(page, blocksize, 0); 1772 head = page_buffers(page); 1773 1774 bbits = inode->i_blkbits; 1775 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits); 1776 1777 for(bh = head, block_start = 0; bh != head || !block_start; 1778 block++, block_start=block_end, bh = bh->b_this_page) { 1779 block_end = block_start + blocksize; 1780 if (block_end <= from || block_start >= to) { 1781 if (PageUptodate(page)) { 1782 if (!buffer_uptodate(bh)) 1783 set_buffer_uptodate(bh); 1784 } 1785 continue; 1786 } 1787 if (buffer_new(bh)) 1788 clear_buffer_new(bh); 1789 if (!buffer_mapped(bh)) { 1790 WARN_ON(bh->b_size != blocksize); 1791 err = get_block(inode, block, bh, 1); 1792 if (err) 1793 break; 1794 if (buffer_new(bh)) { 1795 unmap_underlying_metadata(bh->b_bdev, 1796 bh->b_blocknr); 1797 if (PageUptodate(page)) { 1798 set_buffer_uptodate(bh); 1799 continue; 1800 } 1801 if (block_end > to || block_start < from) { 1802 void *kaddr; 1803 1804 kaddr = kmap_atomic(page, KM_USER0); 1805 if (block_end > to) 1806 memset(kaddr+to, 0, 1807 block_end-to); 1808 if (block_start < from) 1809 memset(kaddr+block_start, 1810 0, from-block_start); 1811 flush_dcache_page(page); 1812 kunmap_atomic(kaddr, KM_USER0); 1813 } 1814 continue; 1815 } 1816 } 1817 if (PageUptodate(page)) { 1818 if (!buffer_uptodate(bh)) 1819 set_buffer_uptodate(bh); 1820 continue; 1821 } 1822 if (!buffer_uptodate(bh) && !buffer_delay(bh) && 1823 (block_start < from || block_end > to)) { 1824 ll_rw_block(READ, 1, &bh); 1825 *wait_bh++=bh; 1826 } 1827 } 1828 /* 1829 * If we issued read requests - let them complete. 1830 */ 1831 while(wait_bh > wait) { 1832 wait_on_buffer(*--wait_bh); 1833 if (!buffer_uptodate(*wait_bh)) 1834 err = -EIO; 1835 } 1836 if (!err) { 1837 bh = head; 1838 do { 1839 if (buffer_new(bh)) 1840 clear_buffer_new(bh); 1841 } while ((bh = bh->b_this_page) != head); 1842 return 0; 1843 } 1844 /* Error case: */ 1845 /* 1846 * Zero out any newly allocated blocks to avoid exposing stale 1847 * data. If BH_New is set, we know that the block was newly 1848 * allocated in the above loop. 1849 */ 1850 bh = head; 1851 block_start = 0; 1852 do { 1853 block_end = block_start+blocksize; 1854 if (block_end <= from) 1855 goto next_bh; 1856 if (block_start >= to) 1857 break; 1858 if (buffer_new(bh)) { 1859 void *kaddr; 1860 1861 clear_buffer_new(bh); 1862 kaddr = kmap_atomic(page, KM_USER0); 1863 memset(kaddr+block_start, 0, bh->b_size); 1864 flush_dcache_page(page); 1865 kunmap_atomic(kaddr, KM_USER0); 1866 set_buffer_uptodate(bh); 1867 mark_buffer_dirty(bh); 1868 } 1869 next_bh: 1870 block_start = block_end; 1871 bh = bh->b_this_page; 1872 } while (bh != head); 1873 return err; 1874 } 1875 1876 static int __block_commit_write(struct inode *inode, struct page *page, 1877 unsigned from, unsigned to) 1878 { 1879 unsigned block_start, block_end; 1880 int partial = 0; 1881 unsigned blocksize; 1882 struct buffer_head *bh, *head; 1883 1884 blocksize = 1 << inode->i_blkbits; 1885 1886 for(bh = head = page_buffers(page), block_start = 0; 1887 bh != head || !block_start; 1888 block_start=block_end, bh = bh->b_this_page) { 1889 block_end = block_start + blocksize; 1890 if (block_end <= from || block_start >= to) { 1891 if (!buffer_uptodate(bh)) 1892 partial = 1; 1893 } else { 1894 set_buffer_uptodate(bh); 1895 mark_buffer_dirty(bh); 1896 } 1897 } 1898 1899 /* 1900 * If this is a partial write which happened to make all buffers 1901 * uptodate then we can optimize away a bogus readpage() for 1902 * the next read(). Here we 'discover' whether the page went 1903 * uptodate as a result of this (potentially partial) write. 1904 */ 1905 if (!partial) 1906 SetPageUptodate(page); 1907 return 0; 1908 } 1909 1910 /* 1911 * Generic "read page" function for block devices that have the normal 1912 * get_block functionality. This is most of the block device filesystems. 1913 * Reads the page asynchronously --- the unlock_buffer() and 1914 * set/clear_buffer_uptodate() functions propagate buffer state into the 1915 * page struct once IO has completed. 1916 */ 1917 int block_read_full_page(struct page *page, get_block_t *get_block) 1918 { 1919 struct inode *inode = page->mapping->host; 1920 sector_t iblock, lblock; 1921 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE]; 1922 unsigned int blocksize; 1923 int nr, i; 1924 int fully_mapped = 1; 1925 1926 BUG_ON(!PageLocked(page)); 1927 blocksize = 1 << inode->i_blkbits; 1928 if (!page_has_buffers(page)) 1929 create_empty_buffers(page, blocksize, 0); 1930 head = page_buffers(page); 1931 1932 iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits); 1933 lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits; 1934 bh = head; 1935 nr = 0; 1936 i = 0; 1937 1938 do { 1939 if (buffer_uptodate(bh)) 1940 continue; 1941 1942 if (!buffer_mapped(bh)) { 1943 int err = 0; 1944 1945 fully_mapped = 0; 1946 if (iblock < lblock) { 1947 WARN_ON(bh->b_size != blocksize); 1948 err = get_block(inode, iblock, bh, 0); 1949 if (err) 1950 SetPageError(page); 1951 } 1952 if (!buffer_mapped(bh)) { 1953 void *kaddr = kmap_atomic(page, KM_USER0); 1954 memset(kaddr + i * blocksize, 0, blocksize); 1955 flush_dcache_page(page); 1956 kunmap_atomic(kaddr, KM_USER0); 1957 if (!err) 1958 set_buffer_uptodate(bh); 1959 continue; 1960 } 1961 /* 1962 * get_block() might have updated the buffer 1963 * synchronously 1964 */ 1965 if (buffer_uptodate(bh)) 1966 continue; 1967 } 1968 arr[nr++] = bh; 1969 } while (i++, iblock++, (bh = bh->b_this_page) != head); 1970 1971 if (fully_mapped) 1972 SetPageMappedToDisk(page); 1973 1974 if (!nr) { 1975 /* 1976 * All buffers are uptodate - we can set the page uptodate 1977 * as well. But not if get_block() returned an error. 1978 */ 1979 if (!PageError(page)) 1980 SetPageUptodate(page); 1981 unlock_page(page); 1982 return 0; 1983 } 1984 1985 /* Stage two: lock the buffers */ 1986 for (i = 0; i < nr; i++) { 1987 bh = arr[i]; 1988 lock_buffer(bh); 1989 mark_buffer_async_read(bh); 1990 } 1991 1992 /* 1993 * Stage 3: start the IO. Check for uptodateness 1994 * inside the buffer lock in case another process reading 1995 * the underlying blockdev brought it uptodate (the sct fix). 1996 */ 1997 for (i = 0; i < nr; i++) { 1998 bh = arr[i]; 1999 if (buffer_uptodate(bh)) 2000 end_buffer_async_read(bh, 1); 2001 else 2002 submit_bh(READ, bh); 2003 } 2004 return 0; 2005 } 2006 2007 /* utility function for filesystems that need to do work on expanding 2008 * truncates. Uses prepare/commit_write to allow the filesystem to 2009 * deal with the hole. 2010 */ 2011 static int __generic_cont_expand(struct inode *inode, loff_t size, 2012 pgoff_t index, unsigned int offset) 2013 { 2014 struct address_space *mapping = inode->i_mapping; 2015 struct page *page; 2016 unsigned long limit; 2017 int err; 2018 2019 err = -EFBIG; 2020 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur; 2021 if (limit != RLIM_INFINITY && size > (loff_t)limit) { 2022 send_sig(SIGXFSZ, current, 0); 2023 goto out; 2024 } 2025 if (size > inode->i_sb->s_maxbytes) 2026 goto out; 2027 2028 err = -ENOMEM; 2029 page = grab_cache_page(mapping, index); 2030 if (!page) 2031 goto out; 2032 err = mapping->a_ops->prepare_write(NULL, page, offset, offset); 2033 if (err) { 2034 /* 2035 * ->prepare_write() may have instantiated a few blocks 2036 * outside i_size. Trim these off again. 2037 */ 2038 unlock_page(page); 2039 page_cache_release(page); 2040 vmtruncate(inode, inode->i_size); 2041 goto out; 2042 } 2043 2044 err = mapping->a_ops->commit_write(NULL, page, offset, offset); 2045 2046 unlock_page(page); 2047 page_cache_release(page); 2048 if (err > 0) 2049 err = 0; 2050 out: 2051 return err; 2052 } 2053 2054 int generic_cont_expand(struct inode *inode, loff_t size) 2055 { 2056 pgoff_t index; 2057 unsigned int offset; 2058 2059 offset = (size & (PAGE_CACHE_SIZE - 1)); /* Within page */ 2060 2061 /* ugh. in prepare/commit_write, if from==to==start of block, we 2062 ** skip the prepare. make sure we never send an offset for the start 2063 ** of a block 2064 */ 2065 if ((offset & (inode->i_sb->s_blocksize - 1)) == 0) { 2066 /* caller must handle this extra byte. */ 2067 offset++; 2068 } 2069 index = size >> PAGE_CACHE_SHIFT; 2070 2071 return __generic_cont_expand(inode, size, index, offset); 2072 } 2073 2074 int generic_cont_expand_simple(struct inode *inode, loff_t size) 2075 { 2076 loff_t pos = size - 1; 2077 pgoff_t index = pos >> PAGE_CACHE_SHIFT; 2078 unsigned int offset = (pos & (PAGE_CACHE_SIZE - 1)) + 1; 2079 2080 /* prepare/commit_write can handle even if from==to==start of block. */ 2081 return __generic_cont_expand(inode, size, index, offset); 2082 } 2083 2084 /* 2085 * For moronic filesystems that do not allow holes in file. 2086 * We may have to extend the file. 2087 */ 2088 2089 int cont_prepare_write(struct page *page, unsigned offset, 2090 unsigned to, get_block_t *get_block, loff_t *bytes) 2091 { 2092 struct address_space *mapping = page->mapping; 2093 struct inode *inode = mapping->host; 2094 struct page *new_page; 2095 pgoff_t pgpos; 2096 long status; 2097 unsigned zerofrom; 2098 unsigned blocksize = 1 << inode->i_blkbits; 2099 void *kaddr; 2100 2101 while(page->index > (pgpos = *bytes>>PAGE_CACHE_SHIFT)) { 2102 status = -ENOMEM; 2103 new_page = grab_cache_page(mapping, pgpos); 2104 if (!new_page) 2105 goto out; 2106 /* we might sleep */ 2107 if (*bytes>>PAGE_CACHE_SHIFT != pgpos) { 2108 unlock_page(new_page); 2109 page_cache_release(new_page); 2110 continue; 2111 } 2112 zerofrom = *bytes & ~PAGE_CACHE_MASK; 2113 if (zerofrom & (blocksize-1)) { 2114 *bytes |= (blocksize-1); 2115 (*bytes)++; 2116 } 2117 status = __block_prepare_write(inode, new_page, zerofrom, 2118 PAGE_CACHE_SIZE, get_block); 2119 if (status) 2120 goto out_unmap; 2121 kaddr = kmap_atomic(new_page, KM_USER0); 2122 memset(kaddr+zerofrom, 0, PAGE_CACHE_SIZE-zerofrom); 2123 flush_dcache_page(new_page); 2124 kunmap_atomic(kaddr, KM_USER0); 2125 generic_commit_write(NULL, new_page, zerofrom, PAGE_CACHE_SIZE); 2126 unlock_page(new_page); 2127 page_cache_release(new_page); 2128 } 2129 2130 if (page->index < pgpos) { 2131 /* completely inside the area */ 2132 zerofrom = offset; 2133 } else { 2134 /* page covers the boundary, find the boundary offset */ 2135 zerofrom = *bytes & ~PAGE_CACHE_MASK; 2136 2137 /* if we will expand the thing last block will be filled */ 2138 if (to > zerofrom && (zerofrom & (blocksize-1))) { 2139 *bytes |= (blocksize-1); 2140 (*bytes)++; 2141 } 2142 2143 /* starting below the boundary? Nothing to zero out */ 2144 if (offset <= zerofrom) 2145 zerofrom = offset; 2146 } 2147 status = __block_prepare_write(inode, page, zerofrom, to, get_block); 2148 if (status) 2149 goto out1; 2150 if (zerofrom < offset) { 2151 kaddr = kmap_atomic(page, KM_USER0); 2152 memset(kaddr+zerofrom, 0, offset-zerofrom); 2153 flush_dcache_page(page); 2154 kunmap_atomic(kaddr, KM_USER0); 2155 __block_commit_write(inode, page, zerofrom, offset); 2156 } 2157 return 0; 2158 out1: 2159 ClearPageUptodate(page); 2160 return status; 2161 2162 out_unmap: 2163 ClearPageUptodate(new_page); 2164 unlock_page(new_page); 2165 page_cache_release(new_page); 2166 out: 2167 return status; 2168 } 2169 2170 int block_prepare_write(struct page *page, unsigned from, unsigned to, 2171 get_block_t *get_block) 2172 { 2173 struct inode *inode = page->mapping->host; 2174 int err = __block_prepare_write(inode, page, from, to, get_block); 2175 if (err) 2176 ClearPageUptodate(page); 2177 return err; 2178 } 2179 2180 int block_commit_write(struct page *page, unsigned from, unsigned to) 2181 { 2182 struct inode *inode = page->mapping->host; 2183 __block_commit_write(inode,page,from,to); 2184 return 0; 2185 } 2186 2187 int generic_commit_write(struct file *file, struct page *page, 2188 unsigned from, unsigned to) 2189 { 2190 struct inode *inode = page->mapping->host; 2191 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to; 2192 __block_commit_write(inode,page,from,to); 2193 /* 2194 * No need to use i_size_read() here, the i_size 2195 * cannot change under us because we hold i_mutex. 2196 */ 2197 if (pos > inode->i_size) { 2198 i_size_write(inode, pos); 2199 mark_inode_dirty(inode); 2200 } 2201 return 0; 2202 } 2203 2204 2205 /* 2206 * nobh_prepare_write()'s prereads are special: the buffer_heads are freed 2207 * immediately, while under the page lock. So it needs a special end_io 2208 * handler which does not touch the bh after unlocking it. 2209 * 2210 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but 2211 * a race there is benign: unlock_buffer() only use the bh's address for 2212 * hashing after unlocking the buffer, so it doesn't actually touch the bh 2213 * itself. 2214 */ 2215 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate) 2216 { 2217 if (uptodate) { 2218 set_buffer_uptodate(bh); 2219 } else { 2220 /* This happens, due to failed READA attempts. */ 2221 clear_buffer_uptodate(bh); 2222 } 2223 unlock_buffer(bh); 2224 } 2225 2226 /* 2227 * On entry, the page is fully not uptodate. 2228 * On exit the page is fully uptodate in the areas outside (from,to) 2229 */ 2230 int nobh_prepare_write(struct page *page, unsigned from, unsigned to, 2231 get_block_t *get_block) 2232 { 2233 struct inode *inode = page->mapping->host; 2234 const unsigned blkbits = inode->i_blkbits; 2235 const unsigned blocksize = 1 << blkbits; 2236 struct buffer_head map_bh; 2237 struct buffer_head *read_bh[MAX_BUF_PER_PAGE]; 2238 unsigned block_in_page; 2239 unsigned block_start; 2240 sector_t block_in_file; 2241 char *kaddr; 2242 int nr_reads = 0; 2243 int i; 2244 int ret = 0; 2245 int is_mapped_to_disk = 1; 2246 int dirtied_it = 0; 2247 2248 if (PageMappedToDisk(page)) 2249 return 0; 2250 2251 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits); 2252 map_bh.b_page = page; 2253 2254 /* 2255 * We loop across all blocks in the page, whether or not they are 2256 * part of the affected region. This is so we can discover if the 2257 * page is fully mapped-to-disk. 2258 */ 2259 for (block_start = 0, block_in_page = 0; 2260 block_start < PAGE_CACHE_SIZE; 2261 block_in_page++, block_start += blocksize) { 2262 unsigned block_end = block_start + blocksize; 2263 int create; 2264 2265 map_bh.b_state = 0; 2266 create = 1; 2267 if (block_start >= to) 2268 create = 0; 2269 map_bh.b_size = blocksize; 2270 ret = get_block(inode, block_in_file + block_in_page, 2271 &map_bh, create); 2272 if (ret) 2273 goto failed; 2274 if (!buffer_mapped(&map_bh)) 2275 is_mapped_to_disk = 0; 2276 if (buffer_new(&map_bh)) 2277 unmap_underlying_metadata(map_bh.b_bdev, 2278 map_bh.b_blocknr); 2279 if (PageUptodate(page)) 2280 continue; 2281 if (buffer_new(&map_bh) || !buffer_mapped(&map_bh)) { 2282 kaddr = kmap_atomic(page, KM_USER0); 2283 if (block_start < from) { 2284 memset(kaddr+block_start, 0, from-block_start); 2285 dirtied_it = 1; 2286 } 2287 if (block_end > to) { 2288 memset(kaddr + to, 0, block_end - to); 2289 dirtied_it = 1; 2290 } 2291 flush_dcache_page(page); 2292 kunmap_atomic(kaddr, KM_USER0); 2293 continue; 2294 } 2295 if (buffer_uptodate(&map_bh)) 2296 continue; /* reiserfs does this */ 2297 if (block_start < from || block_end > to) { 2298 struct buffer_head *bh = alloc_buffer_head(GFP_NOFS); 2299 2300 if (!bh) { 2301 ret = -ENOMEM; 2302 goto failed; 2303 } 2304 bh->b_state = map_bh.b_state; 2305 atomic_set(&bh->b_count, 0); 2306 bh->b_this_page = NULL; 2307 bh->b_page = page; 2308 bh->b_blocknr = map_bh.b_blocknr; 2309 bh->b_size = blocksize; 2310 bh->b_data = (char *)(long)block_start; 2311 bh->b_bdev = map_bh.b_bdev; 2312 bh->b_private = NULL; 2313 read_bh[nr_reads++] = bh; 2314 } 2315 } 2316 2317 if (nr_reads) { 2318 struct buffer_head *bh; 2319 2320 /* 2321 * The page is locked, so these buffers are protected from 2322 * any VM or truncate activity. Hence we don't need to care 2323 * for the buffer_head refcounts. 2324 */ 2325 for (i = 0; i < nr_reads; i++) { 2326 bh = read_bh[i]; 2327 lock_buffer(bh); 2328 bh->b_end_io = end_buffer_read_nobh; 2329 submit_bh(READ, bh); 2330 } 2331 for (i = 0; i < nr_reads; i++) { 2332 bh = read_bh[i]; 2333 wait_on_buffer(bh); 2334 if (!buffer_uptodate(bh)) 2335 ret = -EIO; 2336 free_buffer_head(bh); 2337 read_bh[i] = NULL; 2338 } 2339 if (ret) 2340 goto failed; 2341 } 2342 2343 if (is_mapped_to_disk) 2344 SetPageMappedToDisk(page); 2345 SetPageUptodate(page); 2346 2347 /* 2348 * Setting the page dirty here isn't necessary for the prepare_write 2349 * function - commit_write will do that. But if/when this function is 2350 * used within the pagefault handler to ensure that all mmapped pages 2351 * have backing space in the filesystem, we will need to dirty the page 2352 * if its contents were altered. 2353 */ 2354 if (dirtied_it) 2355 set_page_dirty(page); 2356 2357 return 0; 2358 2359 failed: 2360 for (i = 0; i < nr_reads; i++) { 2361 if (read_bh[i]) 2362 free_buffer_head(read_bh[i]); 2363 } 2364 2365 /* 2366 * Error recovery is pretty slack. Clear the page and mark it dirty 2367 * so we'll later zero out any blocks which _were_ allocated. 2368 */ 2369 kaddr = kmap_atomic(page, KM_USER0); 2370 memset(kaddr, 0, PAGE_CACHE_SIZE); 2371 flush_dcache_page(page); 2372 kunmap_atomic(kaddr, KM_USER0); 2373 SetPageUptodate(page); 2374 set_page_dirty(page); 2375 return ret; 2376 } 2377 EXPORT_SYMBOL(nobh_prepare_write); 2378 2379 int nobh_commit_write(struct file *file, struct page *page, 2380 unsigned from, unsigned to) 2381 { 2382 struct inode *inode = page->mapping->host; 2383 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to; 2384 2385 set_page_dirty(page); 2386 if (pos > inode->i_size) { 2387 i_size_write(inode, pos); 2388 mark_inode_dirty(inode); 2389 } 2390 return 0; 2391 } 2392 EXPORT_SYMBOL(nobh_commit_write); 2393 2394 /* 2395 * nobh_writepage() - based on block_full_write_page() except 2396 * that it tries to operate without attaching bufferheads to 2397 * the page. 2398 */ 2399 int nobh_writepage(struct page *page, get_block_t *get_block, 2400 struct writeback_control *wbc) 2401 { 2402 struct inode * const inode = page->mapping->host; 2403 loff_t i_size = i_size_read(inode); 2404 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT; 2405 unsigned offset; 2406 void *kaddr; 2407 int ret; 2408 2409 /* Is the page fully inside i_size? */ 2410 if (page->index < end_index) 2411 goto out; 2412 2413 /* Is the page fully outside i_size? (truncate in progress) */ 2414 offset = i_size & (PAGE_CACHE_SIZE-1); 2415 if (page->index >= end_index+1 || !offset) { 2416 /* 2417 * The page may have dirty, unmapped buffers. For example, 2418 * they may have been added in ext3_writepage(). Make them 2419 * freeable here, so the page does not leak. 2420 */ 2421 #if 0 2422 /* Not really sure about this - do we need this ? */ 2423 if (page->mapping->a_ops->invalidatepage) 2424 page->mapping->a_ops->invalidatepage(page, offset); 2425 #endif 2426 unlock_page(page); 2427 return 0; /* don't care */ 2428 } 2429 2430 /* 2431 * The page straddles i_size. It must be zeroed out on each and every 2432 * writepage invocation because it may be mmapped. "A file is mapped 2433 * in multiples of the page size. For a file that is not a multiple of 2434 * the page size, the remaining memory is zeroed when mapped, and 2435 * writes to that region are not written out to the file." 2436 */ 2437 kaddr = kmap_atomic(page, KM_USER0); 2438 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset); 2439 flush_dcache_page(page); 2440 kunmap_atomic(kaddr, KM_USER0); 2441 out: 2442 ret = mpage_writepage(page, get_block, wbc); 2443 if (ret == -EAGAIN) 2444 ret = __block_write_full_page(inode, page, get_block, wbc); 2445 return ret; 2446 } 2447 EXPORT_SYMBOL(nobh_writepage); 2448 2449 /* 2450 * This function assumes that ->prepare_write() uses nobh_prepare_write(). 2451 */ 2452 int nobh_truncate_page(struct address_space *mapping, loff_t from) 2453 { 2454 struct inode *inode = mapping->host; 2455 unsigned blocksize = 1 << inode->i_blkbits; 2456 pgoff_t index = from >> PAGE_CACHE_SHIFT; 2457 unsigned offset = from & (PAGE_CACHE_SIZE-1); 2458 unsigned to; 2459 struct page *page; 2460 const struct address_space_operations *a_ops = mapping->a_ops; 2461 char *kaddr; 2462 int ret = 0; 2463 2464 if ((offset & (blocksize - 1)) == 0) 2465 goto out; 2466 2467 ret = -ENOMEM; 2468 page = grab_cache_page(mapping, index); 2469 if (!page) 2470 goto out; 2471 2472 to = (offset + blocksize) & ~(blocksize - 1); 2473 ret = a_ops->prepare_write(NULL, page, offset, to); 2474 if (ret == 0) { 2475 kaddr = kmap_atomic(page, KM_USER0); 2476 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset); 2477 flush_dcache_page(page); 2478 kunmap_atomic(kaddr, KM_USER0); 2479 set_page_dirty(page); 2480 } 2481 unlock_page(page); 2482 page_cache_release(page); 2483 out: 2484 return ret; 2485 } 2486 EXPORT_SYMBOL(nobh_truncate_page); 2487 2488 int block_truncate_page(struct address_space *mapping, 2489 loff_t from, get_block_t *get_block) 2490 { 2491 pgoff_t index = from >> PAGE_CACHE_SHIFT; 2492 unsigned offset = from & (PAGE_CACHE_SIZE-1); 2493 unsigned blocksize; 2494 sector_t iblock; 2495 unsigned length, pos; 2496 struct inode *inode = mapping->host; 2497 struct page *page; 2498 struct buffer_head *bh; 2499 void *kaddr; 2500 int err; 2501 2502 blocksize = 1 << inode->i_blkbits; 2503 length = offset & (blocksize - 1); 2504 2505 /* Block boundary? Nothing to do */ 2506 if (!length) 2507 return 0; 2508 2509 length = blocksize - length; 2510 iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits); 2511 2512 page = grab_cache_page(mapping, index); 2513 err = -ENOMEM; 2514 if (!page) 2515 goto out; 2516 2517 if (!page_has_buffers(page)) 2518 create_empty_buffers(page, blocksize, 0); 2519 2520 /* Find the buffer that contains "offset" */ 2521 bh = page_buffers(page); 2522 pos = blocksize; 2523 while (offset >= pos) { 2524 bh = bh->b_this_page; 2525 iblock++; 2526 pos += blocksize; 2527 } 2528 2529 err = 0; 2530 if (!buffer_mapped(bh)) { 2531 WARN_ON(bh->b_size != blocksize); 2532 err = get_block(inode, iblock, bh, 0); 2533 if (err) 2534 goto unlock; 2535 /* unmapped? It's a hole - nothing to do */ 2536 if (!buffer_mapped(bh)) 2537 goto unlock; 2538 } 2539 2540 /* Ok, it's mapped. Make sure it's up-to-date */ 2541 if (PageUptodate(page)) 2542 set_buffer_uptodate(bh); 2543 2544 if (!buffer_uptodate(bh) && !buffer_delay(bh)) { 2545 err = -EIO; 2546 ll_rw_block(READ, 1, &bh); 2547 wait_on_buffer(bh); 2548 /* Uhhuh. Read error. Complain and punt. */ 2549 if (!buffer_uptodate(bh)) 2550 goto unlock; 2551 } 2552 2553 kaddr = kmap_atomic(page, KM_USER0); 2554 memset(kaddr + offset, 0, length); 2555 flush_dcache_page(page); 2556 kunmap_atomic(kaddr, KM_USER0); 2557 2558 mark_buffer_dirty(bh); 2559 err = 0; 2560 2561 unlock: 2562 unlock_page(page); 2563 page_cache_release(page); 2564 out: 2565 return err; 2566 } 2567 2568 /* 2569 * The generic ->writepage function for buffer-backed address_spaces 2570 */ 2571 int block_write_full_page(struct page *page, get_block_t *get_block, 2572 struct writeback_control *wbc) 2573 { 2574 struct inode * const inode = page->mapping->host; 2575 loff_t i_size = i_size_read(inode); 2576 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT; 2577 unsigned offset; 2578 void *kaddr; 2579 2580 /* Is the page fully inside i_size? */ 2581 if (page->index < end_index) 2582 return __block_write_full_page(inode, page, get_block, wbc); 2583 2584 /* Is the page fully outside i_size? (truncate in progress) */ 2585 offset = i_size & (PAGE_CACHE_SIZE-1); 2586 if (page->index >= end_index+1 || !offset) { 2587 /* 2588 * The page may have dirty, unmapped buffers. For example, 2589 * they may have been added in ext3_writepage(). Make them 2590 * freeable here, so the page does not leak. 2591 */ 2592 do_invalidatepage(page, 0); 2593 unlock_page(page); 2594 return 0; /* don't care */ 2595 } 2596 2597 /* 2598 * The page straddles i_size. It must be zeroed out on each and every 2599 * writepage invokation because it may be mmapped. "A file is mapped 2600 * in multiples of the page size. For a file that is not a multiple of 2601 * the page size, the remaining memory is zeroed when mapped, and 2602 * writes to that region are not written out to the file." 2603 */ 2604 kaddr = kmap_atomic(page, KM_USER0); 2605 memset(kaddr + offset, 0, PAGE_CACHE_SIZE - offset); 2606 flush_dcache_page(page); 2607 kunmap_atomic(kaddr, KM_USER0); 2608 return __block_write_full_page(inode, page, get_block, wbc); 2609 } 2610 2611 sector_t generic_block_bmap(struct address_space *mapping, sector_t block, 2612 get_block_t *get_block) 2613 { 2614 struct buffer_head tmp; 2615 struct inode *inode = mapping->host; 2616 tmp.b_state = 0; 2617 tmp.b_blocknr = 0; 2618 tmp.b_size = 1 << inode->i_blkbits; 2619 get_block(inode, block, &tmp, 0); 2620 return tmp.b_blocknr; 2621 } 2622 2623 static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err) 2624 { 2625 struct buffer_head *bh = bio->bi_private; 2626 2627 if (bio->bi_size) 2628 return 1; 2629 2630 if (err == -EOPNOTSUPP) { 2631 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags); 2632 set_bit(BH_Eopnotsupp, &bh->b_state); 2633 } 2634 2635 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags)); 2636 bio_put(bio); 2637 return 0; 2638 } 2639 2640 int submit_bh(int rw, struct buffer_head * bh) 2641 { 2642 struct bio *bio; 2643 int ret = 0; 2644 2645 BUG_ON(!buffer_locked(bh)); 2646 BUG_ON(!buffer_mapped(bh)); 2647 BUG_ON(!bh->b_end_io); 2648 2649 if (buffer_ordered(bh) && (rw == WRITE)) 2650 rw = WRITE_BARRIER; 2651 2652 /* 2653 * Only clear out a write error when rewriting, should this 2654 * include WRITE_SYNC as well? 2655 */ 2656 if (test_set_buffer_req(bh) && (rw == WRITE || rw == WRITE_BARRIER)) 2657 clear_buffer_write_io_error(bh); 2658 2659 /* 2660 * from here on down, it's all bio -- do the initial mapping, 2661 * submit_bio -> generic_make_request may further map this bio around 2662 */ 2663 bio = bio_alloc(GFP_NOIO, 1); 2664 2665 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9); 2666 bio->bi_bdev = bh->b_bdev; 2667 bio->bi_io_vec[0].bv_page = bh->b_page; 2668 bio->bi_io_vec[0].bv_len = bh->b_size; 2669 bio->bi_io_vec[0].bv_offset = bh_offset(bh); 2670 2671 bio->bi_vcnt = 1; 2672 bio->bi_idx = 0; 2673 bio->bi_size = bh->b_size; 2674 2675 bio->bi_end_io = end_bio_bh_io_sync; 2676 bio->bi_private = bh; 2677 2678 bio_get(bio); 2679 submit_bio(rw, bio); 2680 2681 if (bio_flagged(bio, BIO_EOPNOTSUPP)) 2682 ret = -EOPNOTSUPP; 2683 2684 bio_put(bio); 2685 return ret; 2686 } 2687 2688 /** 2689 * ll_rw_block: low-level access to block devices (DEPRECATED) 2690 * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead) 2691 * @nr: number of &struct buffer_heads in the array 2692 * @bhs: array of pointers to &struct buffer_head 2693 * 2694 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and 2695 * requests an I/O operation on them, either a %READ or a %WRITE. The third 2696 * %SWRITE is like %WRITE only we make sure that the *current* data in buffers 2697 * are sent to disk. The fourth %READA option is described in the documentation 2698 * for generic_make_request() which ll_rw_block() calls. 2699 * 2700 * This function drops any buffer that it cannot get a lock on (with the 2701 * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be 2702 * clean when doing a write request, and any buffer that appears to be 2703 * up-to-date when doing read request. Further it marks as clean buffers that 2704 * are processed for writing (the buffer cache won't assume that they are 2705 * actually clean until the buffer gets unlocked). 2706 * 2707 * ll_rw_block sets b_end_io to simple completion handler that marks 2708 * the buffer up-to-date (if approriate), unlocks the buffer and wakes 2709 * any waiters. 2710 * 2711 * All of the buffers must be for the same device, and must also be a 2712 * multiple of the current approved size for the device. 2713 */ 2714 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[]) 2715 { 2716 int i; 2717 2718 for (i = 0; i < nr; i++) { 2719 struct buffer_head *bh = bhs[i]; 2720 2721 if (rw == SWRITE) 2722 lock_buffer(bh); 2723 else if (test_set_buffer_locked(bh)) 2724 continue; 2725 2726 if (rw == WRITE || rw == SWRITE) { 2727 if (test_clear_buffer_dirty(bh)) { 2728 bh->b_end_io = end_buffer_write_sync; 2729 get_bh(bh); 2730 submit_bh(WRITE, bh); 2731 continue; 2732 } 2733 } else { 2734 if (!buffer_uptodate(bh)) { 2735 bh->b_end_io = end_buffer_read_sync; 2736 get_bh(bh); 2737 submit_bh(rw, bh); 2738 continue; 2739 } 2740 } 2741 unlock_buffer(bh); 2742 } 2743 } 2744 2745 /* 2746 * For a data-integrity writeout, we need to wait upon any in-progress I/O 2747 * and then start new I/O and then wait upon it. The caller must have a ref on 2748 * the buffer_head. 2749 */ 2750 int sync_dirty_buffer(struct buffer_head *bh) 2751 { 2752 int ret = 0; 2753 2754 WARN_ON(atomic_read(&bh->b_count) < 1); 2755 lock_buffer(bh); 2756 if (test_clear_buffer_dirty(bh)) { 2757 get_bh(bh); 2758 bh->b_end_io = end_buffer_write_sync; 2759 ret = submit_bh(WRITE, bh); 2760 wait_on_buffer(bh); 2761 if (buffer_eopnotsupp(bh)) { 2762 clear_buffer_eopnotsupp(bh); 2763 ret = -EOPNOTSUPP; 2764 } 2765 if (!ret && !buffer_uptodate(bh)) 2766 ret = -EIO; 2767 } else { 2768 unlock_buffer(bh); 2769 } 2770 return ret; 2771 } 2772 2773 /* 2774 * try_to_free_buffers() checks if all the buffers on this particular page 2775 * are unused, and releases them if so. 2776 * 2777 * Exclusion against try_to_free_buffers may be obtained by either 2778 * locking the page or by holding its mapping's private_lock. 2779 * 2780 * If the page is dirty but all the buffers are clean then we need to 2781 * be sure to mark the page clean as well. This is because the page 2782 * may be against a block device, and a later reattachment of buffers 2783 * to a dirty page will set *all* buffers dirty. Which would corrupt 2784 * filesystem data on the same device. 2785 * 2786 * The same applies to regular filesystem pages: if all the buffers are 2787 * clean then we set the page clean and proceed. To do that, we require 2788 * total exclusion from __set_page_dirty_buffers(). That is obtained with 2789 * private_lock. 2790 * 2791 * try_to_free_buffers() is non-blocking. 2792 */ 2793 static inline int buffer_busy(struct buffer_head *bh) 2794 { 2795 return atomic_read(&bh->b_count) | 2796 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock))); 2797 } 2798 2799 static int 2800 drop_buffers(struct page *page, struct buffer_head **buffers_to_free) 2801 { 2802 struct buffer_head *head = page_buffers(page); 2803 struct buffer_head *bh; 2804 2805 bh = head; 2806 do { 2807 if (buffer_write_io_error(bh) && page->mapping) 2808 set_bit(AS_EIO, &page->mapping->flags); 2809 if (buffer_busy(bh)) 2810 goto failed; 2811 bh = bh->b_this_page; 2812 } while (bh != head); 2813 2814 do { 2815 struct buffer_head *next = bh->b_this_page; 2816 2817 if (!list_empty(&bh->b_assoc_buffers)) 2818 __remove_assoc_queue(bh); 2819 bh = next; 2820 } while (bh != head); 2821 *buffers_to_free = head; 2822 __clear_page_buffers(page); 2823 return 1; 2824 failed: 2825 return 0; 2826 } 2827 2828 int try_to_free_buffers(struct page *page) 2829 { 2830 struct address_space * const mapping = page->mapping; 2831 struct buffer_head *buffers_to_free = NULL; 2832 int ret = 0; 2833 2834 BUG_ON(!PageLocked(page)); 2835 if (PageWriteback(page)) 2836 return 0; 2837 2838 if (mapping == NULL) { /* can this still happen? */ 2839 ret = drop_buffers(page, &buffers_to_free); 2840 goto out; 2841 } 2842 2843 spin_lock(&mapping->private_lock); 2844 ret = drop_buffers(page, &buffers_to_free); 2845 spin_unlock(&mapping->private_lock); 2846 if (ret) { 2847 /* 2848 * If the filesystem writes its buffers by hand (eg ext3) 2849 * then we can have clean buffers against a dirty page. We 2850 * clean the page here; otherwise later reattachment of buffers 2851 * could encounter a non-uptodate page, which is unresolvable. 2852 * This only applies in the rare case where try_to_free_buffers 2853 * succeeds but the page is not freed. 2854 */ 2855 clear_page_dirty(page); 2856 } 2857 out: 2858 if (buffers_to_free) { 2859 struct buffer_head *bh = buffers_to_free; 2860 2861 do { 2862 struct buffer_head *next = bh->b_this_page; 2863 free_buffer_head(bh); 2864 bh = next; 2865 } while (bh != buffers_to_free); 2866 } 2867 return ret; 2868 } 2869 EXPORT_SYMBOL(try_to_free_buffers); 2870 2871 void block_sync_page(struct page *page) 2872 { 2873 struct address_space *mapping; 2874 2875 smp_mb(); 2876 mapping = page_mapping(page); 2877 if (mapping) 2878 blk_run_backing_dev(mapping->backing_dev_info, page); 2879 } 2880 2881 /* 2882 * There are no bdflush tunables left. But distributions are 2883 * still running obsolete flush daemons, so we terminate them here. 2884 * 2885 * Use of bdflush() is deprecated and will be removed in a future kernel. 2886 * The `pdflush' kernel threads fully replace bdflush daemons and this call. 2887 */ 2888 asmlinkage long sys_bdflush(int func, long data) 2889 { 2890 static int msg_count; 2891 2892 if (!capable(CAP_SYS_ADMIN)) 2893 return -EPERM; 2894 2895 if (msg_count < 5) { 2896 msg_count++; 2897 printk(KERN_INFO 2898 "warning: process `%s' used the obsolete bdflush" 2899 " system call\n", current->comm); 2900 printk(KERN_INFO "Fix your initscripts?\n"); 2901 } 2902 2903 if (func == 1) 2904 do_exit(0); 2905 return 0; 2906 } 2907 2908 /* 2909 * Buffer-head allocation 2910 */ 2911 static struct kmem_cache *bh_cachep; 2912 2913 /* 2914 * Once the number of bh's in the machine exceeds this level, we start 2915 * stripping them in writeback. 2916 */ 2917 static int max_buffer_heads; 2918 2919 int buffer_heads_over_limit; 2920 2921 struct bh_accounting { 2922 int nr; /* Number of live bh's */ 2923 int ratelimit; /* Limit cacheline bouncing */ 2924 }; 2925 2926 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0}; 2927 2928 static void recalc_bh_state(void) 2929 { 2930 int i; 2931 int tot = 0; 2932 2933 if (__get_cpu_var(bh_accounting).ratelimit++ < 4096) 2934 return; 2935 __get_cpu_var(bh_accounting).ratelimit = 0; 2936 for_each_online_cpu(i) 2937 tot += per_cpu(bh_accounting, i).nr; 2938 buffer_heads_over_limit = (tot > max_buffer_heads); 2939 } 2940 2941 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags) 2942 { 2943 struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags); 2944 if (ret) { 2945 get_cpu_var(bh_accounting).nr++; 2946 recalc_bh_state(); 2947 put_cpu_var(bh_accounting); 2948 } 2949 return ret; 2950 } 2951 EXPORT_SYMBOL(alloc_buffer_head); 2952 2953 void free_buffer_head(struct buffer_head *bh) 2954 { 2955 BUG_ON(!list_empty(&bh->b_assoc_buffers)); 2956 kmem_cache_free(bh_cachep, bh); 2957 get_cpu_var(bh_accounting).nr--; 2958 recalc_bh_state(); 2959 put_cpu_var(bh_accounting); 2960 } 2961 EXPORT_SYMBOL(free_buffer_head); 2962 2963 static void 2964 init_buffer_head(void *data, struct kmem_cache *cachep, unsigned long flags) 2965 { 2966 if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) == 2967 SLAB_CTOR_CONSTRUCTOR) { 2968 struct buffer_head * bh = (struct buffer_head *)data; 2969 2970 memset(bh, 0, sizeof(*bh)); 2971 INIT_LIST_HEAD(&bh->b_assoc_buffers); 2972 } 2973 } 2974 2975 static void buffer_exit_cpu(int cpu) 2976 { 2977 int i; 2978 struct bh_lru *b = &per_cpu(bh_lrus, cpu); 2979 2980 for (i = 0; i < BH_LRU_SIZE; i++) { 2981 brelse(b->bhs[i]); 2982 b->bhs[i] = NULL; 2983 } 2984 get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr; 2985 per_cpu(bh_accounting, cpu).nr = 0; 2986 put_cpu_var(bh_accounting); 2987 } 2988 2989 static int buffer_cpu_notify(struct notifier_block *self, 2990 unsigned long action, void *hcpu) 2991 { 2992 if (action == CPU_DEAD) 2993 buffer_exit_cpu((unsigned long)hcpu); 2994 return NOTIFY_OK; 2995 } 2996 2997 void __init buffer_init(void) 2998 { 2999 int nrpages; 3000 3001 bh_cachep = kmem_cache_create("buffer_head", 3002 sizeof(struct buffer_head), 0, 3003 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC| 3004 SLAB_MEM_SPREAD), 3005 init_buffer_head, 3006 NULL); 3007 3008 /* 3009 * Limit the bh occupancy to 10% of ZONE_NORMAL 3010 */ 3011 nrpages = (nr_free_buffer_pages() * 10) / 100; 3012 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head)); 3013 hotcpu_notifier(buffer_cpu_notify, 0); 3014 } 3015 3016 EXPORT_SYMBOL(__bforget); 3017 EXPORT_SYMBOL(__brelse); 3018 EXPORT_SYMBOL(__wait_on_buffer); 3019 EXPORT_SYMBOL(block_commit_write); 3020 EXPORT_SYMBOL(block_prepare_write); 3021 EXPORT_SYMBOL(block_read_full_page); 3022 EXPORT_SYMBOL(block_sync_page); 3023 EXPORT_SYMBOL(block_truncate_page); 3024 EXPORT_SYMBOL(block_write_full_page); 3025 EXPORT_SYMBOL(cont_prepare_write); 3026 EXPORT_SYMBOL(end_buffer_read_sync); 3027 EXPORT_SYMBOL(end_buffer_write_sync); 3028 EXPORT_SYMBOL(file_fsync); 3029 EXPORT_SYMBOL(fsync_bdev); 3030 EXPORT_SYMBOL(generic_block_bmap); 3031 EXPORT_SYMBOL(generic_commit_write); 3032 EXPORT_SYMBOL(generic_cont_expand); 3033 EXPORT_SYMBOL(generic_cont_expand_simple); 3034 EXPORT_SYMBOL(init_buffer); 3035 EXPORT_SYMBOL(invalidate_bdev); 3036 EXPORT_SYMBOL(ll_rw_block); 3037 EXPORT_SYMBOL(mark_buffer_dirty); 3038 EXPORT_SYMBOL(submit_bh); 3039 EXPORT_SYMBOL(sync_dirty_buffer); 3040 EXPORT_SYMBOL(unlock_buffer); 3041