xref: /openbmc/linux/fs/buffer.c (revision b6dcefde)
1 /*
2  *  linux/fs/buffer.c
3  *
4  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
5  */
6 
7 /*
8  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9  *
10  * Removed a lot of unnecessary code and simplified things now that
11  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12  *
13  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
14  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
15  *
16  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17  *
18  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19  */
20 
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
23 #include <linux/fs.h>
24 #include <linux/mm.h>
25 #include <linux/percpu.h>
26 #include <linux/slab.h>
27 #include <linux/capability.h>
28 #include <linux/blkdev.h>
29 #include <linux/file.h>
30 #include <linux/quotaops.h>
31 #include <linux/highmem.h>
32 #include <linux/module.h>
33 #include <linux/writeback.h>
34 #include <linux/hash.h>
35 #include <linux/suspend.h>
36 #include <linux/buffer_head.h>
37 #include <linux/task_io_accounting_ops.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
44 
45 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
46 
47 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
48 
49 inline void
50 init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
51 {
52 	bh->b_end_io = handler;
53 	bh->b_private = private;
54 }
55 EXPORT_SYMBOL(init_buffer);
56 
57 static int sync_buffer(void *word)
58 {
59 	struct block_device *bd;
60 	struct buffer_head *bh
61 		= container_of(word, struct buffer_head, b_state);
62 
63 	smp_mb();
64 	bd = bh->b_bdev;
65 	if (bd)
66 		blk_run_address_space(bd->bd_inode->i_mapping);
67 	io_schedule();
68 	return 0;
69 }
70 
71 void __lock_buffer(struct buffer_head *bh)
72 {
73 	wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
74 							TASK_UNINTERRUPTIBLE);
75 }
76 EXPORT_SYMBOL(__lock_buffer);
77 
78 void unlock_buffer(struct buffer_head *bh)
79 {
80 	clear_bit_unlock(BH_Lock, &bh->b_state);
81 	smp_mb__after_clear_bit();
82 	wake_up_bit(&bh->b_state, BH_Lock);
83 }
84 EXPORT_SYMBOL(unlock_buffer);
85 
86 /*
87  * Block until a buffer comes unlocked.  This doesn't stop it
88  * from becoming locked again - you have to lock it yourself
89  * if you want to preserve its state.
90  */
91 void __wait_on_buffer(struct buffer_head * bh)
92 {
93 	wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
94 }
95 EXPORT_SYMBOL(__wait_on_buffer);
96 
97 static void
98 __clear_page_buffers(struct page *page)
99 {
100 	ClearPagePrivate(page);
101 	set_page_private(page, 0);
102 	page_cache_release(page);
103 }
104 
105 
106 static int quiet_error(struct buffer_head *bh)
107 {
108 	if (!test_bit(BH_Quiet, &bh->b_state) && printk_ratelimit())
109 		return 0;
110 	return 1;
111 }
112 
113 
114 static void buffer_io_error(struct buffer_head *bh)
115 {
116 	char b[BDEVNAME_SIZE];
117 	printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
118 			bdevname(bh->b_bdev, b),
119 			(unsigned long long)bh->b_blocknr);
120 }
121 
122 /*
123  * End-of-IO handler helper function which does not touch the bh after
124  * unlocking it.
125  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
126  * a race there is benign: unlock_buffer() only use the bh's address for
127  * hashing after unlocking the buffer, so it doesn't actually touch the bh
128  * itself.
129  */
130 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
131 {
132 	if (uptodate) {
133 		set_buffer_uptodate(bh);
134 	} else {
135 		/* This happens, due to failed READA attempts. */
136 		clear_buffer_uptodate(bh);
137 	}
138 	unlock_buffer(bh);
139 }
140 
141 /*
142  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
143  * unlock the buffer. This is what ll_rw_block uses too.
144  */
145 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
146 {
147 	__end_buffer_read_notouch(bh, uptodate);
148 	put_bh(bh);
149 }
150 EXPORT_SYMBOL(end_buffer_read_sync);
151 
152 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
153 {
154 	char b[BDEVNAME_SIZE];
155 
156 	if (uptodate) {
157 		set_buffer_uptodate(bh);
158 	} else {
159 		if (!buffer_eopnotsupp(bh) && !quiet_error(bh)) {
160 			buffer_io_error(bh);
161 			printk(KERN_WARNING "lost page write due to "
162 					"I/O error on %s\n",
163 				       bdevname(bh->b_bdev, b));
164 		}
165 		set_buffer_write_io_error(bh);
166 		clear_buffer_uptodate(bh);
167 	}
168 	unlock_buffer(bh);
169 	put_bh(bh);
170 }
171 EXPORT_SYMBOL(end_buffer_write_sync);
172 
173 /*
174  * Various filesystems appear to want __find_get_block to be non-blocking.
175  * But it's the page lock which protects the buffers.  To get around this,
176  * we get exclusion from try_to_free_buffers with the blockdev mapping's
177  * private_lock.
178  *
179  * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
180  * may be quite high.  This code could TryLock the page, and if that
181  * succeeds, there is no need to take private_lock. (But if
182  * private_lock is contended then so is mapping->tree_lock).
183  */
184 static struct buffer_head *
185 __find_get_block_slow(struct block_device *bdev, sector_t block)
186 {
187 	struct inode *bd_inode = bdev->bd_inode;
188 	struct address_space *bd_mapping = bd_inode->i_mapping;
189 	struct buffer_head *ret = NULL;
190 	pgoff_t index;
191 	struct buffer_head *bh;
192 	struct buffer_head *head;
193 	struct page *page;
194 	int all_mapped = 1;
195 
196 	index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
197 	page = find_get_page(bd_mapping, index);
198 	if (!page)
199 		goto out;
200 
201 	spin_lock(&bd_mapping->private_lock);
202 	if (!page_has_buffers(page))
203 		goto out_unlock;
204 	head = page_buffers(page);
205 	bh = head;
206 	do {
207 		if (!buffer_mapped(bh))
208 			all_mapped = 0;
209 		else if (bh->b_blocknr == block) {
210 			ret = bh;
211 			get_bh(bh);
212 			goto out_unlock;
213 		}
214 		bh = bh->b_this_page;
215 	} while (bh != head);
216 
217 	/* we might be here because some of the buffers on this page are
218 	 * not mapped.  This is due to various races between
219 	 * file io on the block device and getblk.  It gets dealt with
220 	 * elsewhere, don't buffer_error if we had some unmapped buffers
221 	 */
222 	if (all_mapped) {
223 		printk("__find_get_block_slow() failed. "
224 			"block=%llu, b_blocknr=%llu\n",
225 			(unsigned long long)block,
226 			(unsigned long long)bh->b_blocknr);
227 		printk("b_state=0x%08lx, b_size=%zu\n",
228 			bh->b_state, bh->b_size);
229 		printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
230 	}
231 out_unlock:
232 	spin_unlock(&bd_mapping->private_lock);
233 	page_cache_release(page);
234 out:
235 	return ret;
236 }
237 
238 /* If invalidate_buffers() will trash dirty buffers, it means some kind
239    of fs corruption is going on. Trashing dirty data always imply losing
240    information that was supposed to be just stored on the physical layer
241    by the user.
242 
243    Thus invalidate_buffers in general usage is not allwowed to trash
244    dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
245    be preserved.  These buffers are simply skipped.
246 
247    We also skip buffers which are still in use.  For example this can
248    happen if a userspace program is reading the block device.
249 
250    NOTE: In the case where the user removed a removable-media-disk even if
251    there's still dirty data not synced on disk (due a bug in the device driver
252    or due an error of the user), by not destroying the dirty buffers we could
253    generate corruption also on the next media inserted, thus a parameter is
254    necessary to handle this case in the most safe way possible (trying
255    to not corrupt also the new disk inserted with the data belonging to
256    the old now corrupted disk). Also for the ramdisk the natural thing
257    to do in order to release the ramdisk memory is to destroy dirty buffers.
258 
259    These are two special cases. Normal usage imply the device driver
260    to issue a sync on the device (without waiting I/O completion) and
261    then an invalidate_buffers call that doesn't trash dirty buffers.
262 
263    For handling cache coherency with the blkdev pagecache the 'update' case
264    is been introduced. It is needed to re-read from disk any pinned
265    buffer. NOTE: re-reading from disk is destructive so we can do it only
266    when we assume nobody is changing the buffercache under our I/O and when
267    we think the disk contains more recent information than the buffercache.
268    The update == 1 pass marks the buffers we need to update, the update == 2
269    pass does the actual I/O. */
270 void invalidate_bdev(struct block_device *bdev)
271 {
272 	struct address_space *mapping = bdev->bd_inode->i_mapping;
273 
274 	if (mapping->nrpages == 0)
275 		return;
276 
277 	invalidate_bh_lrus();
278 	invalidate_mapping_pages(mapping, 0, -1);
279 }
280 EXPORT_SYMBOL(invalidate_bdev);
281 
282 /*
283  * Kick the writeback threads then try to free up some ZONE_NORMAL memory.
284  */
285 static void free_more_memory(void)
286 {
287 	struct zone *zone;
288 	int nid;
289 
290 	wakeup_flusher_threads(1024);
291 	yield();
292 
293 	for_each_online_node(nid) {
294 		(void)first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
295 						gfp_zone(GFP_NOFS), NULL,
296 						&zone);
297 		if (zone)
298 			try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
299 						GFP_NOFS, NULL);
300 	}
301 }
302 
303 /*
304  * I/O completion handler for block_read_full_page() - pages
305  * which come unlocked at the end of I/O.
306  */
307 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
308 {
309 	unsigned long flags;
310 	struct buffer_head *first;
311 	struct buffer_head *tmp;
312 	struct page *page;
313 	int page_uptodate = 1;
314 
315 	BUG_ON(!buffer_async_read(bh));
316 
317 	page = bh->b_page;
318 	if (uptodate) {
319 		set_buffer_uptodate(bh);
320 	} else {
321 		clear_buffer_uptodate(bh);
322 		if (!quiet_error(bh))
323 			buffer_io_error(bh);
324 		SetPageError(page);
325 	}
326 
327 	/*
328 	 * Be _very_ careful from here on. Bad things can happen if
329 	 * two buffer heads end IO at almost the same time and both
330 	 * decide that the page is now completely done.
331 	 */
332 	first = page_buffers(page);
333 	local_irq_save(flags);
334 	bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
335 	clear_buffer_async_read(bh);
336 	unlock_buffer(bh);
337 	tmp = bh;
338 	do {
339 		if (!buffer_uptodate(tmp))
340 			page_uptodate = 0;
341 		if (buffer_async_read(tmp)) {
342 			BUG_ON(!buffer_locked(tmp));
343 			goto still_busy;
344 		}
345 		tmp = tmp->b_this_page;
346 	} while (tmp != bh);
347 	bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
348 	local_irq_restore(flags);
349 
350 	/*
351 	 * If none of the buffers had errors and they are all
352 	 * uptodate then we can set the page uptodate.
353 	 */
354 	if (page_uptodate && !PageError(page))
355 		SetPageUptodate(page);
356 	unlock_page(page);
357 	return;
358 
359 still_busy:
360 	bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
361 	local_irq_restore(flags);
362 	return;
363 }
364 
365 /*
366  * Completion handler for block_write_full_page() - pages which are unlocked
367  * during I/O, and which have PageWriteback cleared upon I/O completion.
368  */
369 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
370 {
371 	char b[BDEVNAME_SIZE];
372 	unsigned long flags;
373 	struct buffer_head *first;
374 	struct buffer_head *tmp;
375 	struct page *page;
376 
377 	BUG_ON(!buffer_async_write(bh));
378 
379 	page = bh->b_page;
380 	if (uptodate) {
381 		set_buffer_uptodate(bh);
382 	} else {
383 		if (!quiet_error(bh)) {
384 			buffer_io_error(bh);
385 			printk(KERN_WARNING "lost page write due to "
386 					"I/O error on %s\n",
387 			       bdevname(bh->b_bdev, b));
388 		}
389 		set_bit(AS_EIO, &page->mapping->flags);
390 		set_buffer_write_io_error(bh);
391 		clear_buffer_uptodate(bh);
392 		SetPageError(page);
393 	}
394 
395 	first = page_buffers(page);
396 	local_irq_save(flags);
397 	bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
398 
399 	clear_buffer_async_write(bh);
400 	unlock_buffer(bh);
401 	tmp = bh->b_this_page;
402 	while (tmp != bh) {
403 		if (buffer_async_write(tmp)) {
404 			BUG_ON(!buffer_locked(tmp));
405 			goto still_busy;
406 		}
407 		tmp = tmp->b_this_page;
408 	}
409 	bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
410 	local_irq_restore(flags);
411 	end_page_writeback(page);
412 	return;
413 
414 still_busy:
415 	bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
416 	local_irq_restore(flags);
417 	return;
418 }
419 EXPORT_SYMBOL(end_buffer_async_write);
420 
421 /*
422  * If a page's buffers are under async readin (end_buffer_async_read
423  * completion) then there is a possibility that another thread of
424  * control could lock one of the buffers after it has completed
425  * but while some of the other buffers have not completed.  This
426  * locked buffer would confuse end_buffer_async_read() into not unlocking
427  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
428  * that this buffer is not under async I/O.
429  *
430  * The page comes unlocked when it has no locked buffer_async buffers
431  * left.
432  *
433  * PageLocked prevents anyone starting new async I/O reads any of
434  * the buffers.
435  *
436  * PageWriteback is used to prevent simultaneous writeout of the same
437  * page.
438  *
439  * PageLocked prevents anyone from starting writeback of a page which is
440  * under read I/O (PageWriteback is only ever set against a locked page).
441  */
442 static void mark_buffer_async_read(struct buffer_head *bh)
443 {
444 	bh->b_end_io = end_buffer_async_read;
445 	set_buffer_async_read(bh);
446 }
447 
448 static void mark_buffer_async_write_endio(struct buffer_head *bh,
449 					  bh_end_io_t *handler)
450 {
451 	bh->b_end_io = handler;
452 	set_buffer_async_write(bh);
453 }
454 
455 void mark_buffer_async_write(struct buffer_head *bh)
456 {
457 	mark_buffer_async_write_endio(bh, end_buffer_async_write);
458 }
459 EXPORT_SYMBOL(mark_buffer_async_write);
460 
461 
462 /*
463  * fs/buffer.c contains helper functions for buffer-backed address space's
464  * fsync functions.  A common requirement for buffer-based filesystems is
465  * that certain data from the backing blockdev needs to be written out for
466  * a successful fsync().  For example, ext2 indirect blocks need to be
467  * written back and waited upon before fsync() returns.
468  *
469  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
470  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
471  * management of a list of dependent buffers at ->i_mapping->private_list.
472  *
473  * Locking is a little subtle: try_to_free_buffers() will remove buffers
474  * from their controlling inode's queue when they are being freed.  But
475  * try_to_free_buffers() will be operating against the *blockdev* mapping
476  * at the time, not against the S_ISREG file which depends on those buffers.
477  * So the locking for private_list is via the private_lock in the address_space
478  * which backs the buffers.  Which is different from the address_space
479  * against which the buffers are listed.  So for a particular address_space,
480  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
481  * mapping->private_list will always be protected by the backing blockdev's
482  * ->private_lock.
483  *
484  * Which introduces a requirement: all buffers on an address_space's
485  * ->private_list must be from the same address_space: the blockdev's.
486  *
487  * address_spaces which do not place buffers at ->private_list via these
488  * utility functions are free to use private_lock and private_list for
489  * whatever they want.  The only requirement is that list_empty(private_list)
490  * be true at clear_inode() time.
491  *
492  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
493  * filesystems should do that.  invalidate_inode_buffers() should just go
494  * BUG_ON(!list_empty).
495  *
496  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
497  * take an address_space, not an inode.  And it should be called
498  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
499  * queued up.
500  *
501  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
502  * list if it is already on a list.  Because if the buffer is on a list,
503  * it *must* already be on the right one.  If not, the filesystem is being
504  * silly.  This will save a ton of locking.  But first we have to ensure
505  * that buffers are taken *off* the old inode's list when they are freed
506  * (presumably in truncate).  That requires careful auditing of all
507  * filesystems (do it inside bforget()).  It could also be done by bringing
508  * b_inode back.
509  */
510 
511 /*
512  * The buffer's backing address_space's private_lock must be held
513  */
514 static void __remove_assoc_queue(struct buffer_head *bh)
515 {
516 	list_del_init(&bh->b_assoc_buffers);
517 	WARN_ON(!bh->b_assoc_map);
518 	if (buffer_write_io_error(bh))
519 		set_bit(AS_EIO, &bh->b_assoc_map->flags);
520 	bh->b_assoc_map = NULL;
521 }
522 
523 int inode_has_buffers(struct inode *inode)
524 {
525 	return !list_empty(&inode->i_data.private_list);
526 }
527 
528 /*
529  * osync is designed to support O_SYNC io.  It waits synchronously for
530  * all already-submitted IO to complete, but does not queue any new
531  * writes to the disk.
532  *
533  * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
534  * you dirty the buffers, and then use osync_inode_buffers to wait for
535  * completion.  Any other dirty buffers which are not yet queued for
536  * write will not be flushed to disk by the osync.
537  */
538 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
539 {
540 	struct buffer_head *bh;
541 	struct list_head *p;
542 	int err = 0;
543 
544 	spin_lock(lock);
545 repeat:
546 	list_for_each_prev(p, list) {
547 		bh = BH_ENTRY(p);
548 		if (buffer_locked(bh)) {
549 			get_bh(bh);
550 			spin_unlock(lock);
551 			wait_on_buffer(bh);
552 			if (!buffer_uptodate(bh))
553 				err = -EIO;
554 			brelse(bh);
555 			spin_lock(lock);
556 			goto repeat;
557 		}
558 	}
559 	spin_unlock(lock);
560 	return err;
561 }
562 
563 static void do_thaw_all(struct work_struct *work)
564 {
565 	struct super_block *sb;
566 	char b[BDEVNAME_SIZE];
567 
568 	spin_lock(&sb_lock);
569 restart:
570 	list_for_each_entry(sb, &super_blocks, s_list) {
571 		sb->s_count++;
572 		spin_unlock(&sb_lock);
573 		down_read(&sb->s_umount);
574 		while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
575 			printk(KERN_WARNING "Emergency Thaw on %s\n",
576 			       bdevname(sb->s_bdev, b));
577 		up_read(&sb->s_umount);
578 		spin_lock(&sb_lock);
579 		if (__put_super_and_need_restart(sb))
580 			goto restart;
581 	}
582 	spin_unlock(&sb_lock);
583 	kfree(work);
584 	printk(KERN_WARNING "Emergency Thaw complete\n");
585 }
586 
587 /**
588  * emergency_thaw_all -- forcibly thaw every frozen filesystem
589  *
590  * Used for emergency unfreeze of all filesystems via SysRq
591  */
592 void emergency_thaw_all(void)
593 {
594 	struct work_struct *work;
595 
596 	work = kmalloc(sizeof(*work), GFP_ATOMIC);
597 	if (work) {
598 		INIT_WORK(work, do_thaw_all);
599 		schedule_work(work);
600 	}
601 }
602 
603 /**
604  * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
605  * @mapping: the mapping which wants those buffers written
606  *
607  * Starts I/O against the buffers at mapping->private_list, and waits upon
608  * that I/O.
609  *
610  * Basically, this is a convenience function for fsync().
611  * @mapping is a file or directory which needs those buffers to be written for
612  * a successful fsync().
613  */
614 int sync_mapping_buffers(struct address_space *mapping)
615 {
616 	struct address_space *buffer_mapping = mapping->assoc_mapping;
617 
618 	if (buffer_mapping == NULL || list_empty(&mapping->private_list))
619 		return 0;
620 
621 	return fsync_buffers_list(&buffer_mapping->private_lock,
622 					&mapping->private_list);
623 }
624 EXPORT_SYMBOL(sync_mapping_buffers);
625 
626 /*
627  * Called when we've recently written block `bblock', and it is known that
628  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
629  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
630  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
631  */
632 void write_boundary_block(struct block_device *bdev,
633 			sector_t bblock, unsigned blocksize)
634 {
635 	struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
636 	if (bh) {
637 		if (buffer_dirty(bh))
638 			ll_rw_block(WRITE, 1, &bh);
639 		put_bh(bh);
640 	}
641 }
642 
643 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
644 {
645 	struct address_space *mapping = inode->i_mapping;
646 	struct address_space *buffer_mapping = bh->b_page->mapping;
647 
648 	mark_buffer_dirty(bh);
649 	if (!mapping->assoc_mapping) {
650 		mapping->assoc_mapping = buffer_mapping;
651 	} else {
652 		BUG_ON(mapping->assoc_mapping != buffer_mapping);
653 	}
654 	if (!bh->b_assoc_map) {
655 		spin_lock(&buffer_mapping->private_lock);
656 		list_move_tail(&bh->b_assoc_buffers,
657 				&mapping->private_list);
658 		bh->b_assoc_map = mapping;
659 		spin_unlock(&buffer_mapping->private_lock);
660 	}
661 }
662 EXPORT_SYMBOL(mark_buffer_dirty_inode);
663 
664 /*
665  * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
666  * dirty.
667  *
668  * If warn is true, then emit a warning if the page is not uptodate and has
669  * not been truncated.
670  */
671 static void __set_page_dirty(struct page *page,
672 		struct address_space *mapping, int warn)
673 {
674 	spin_lock_irq(&mapping->tree_lock);
675 	if (page->mapping) {	/* Race with truncate? */
676 		WARN_ON_ONCE(warn && !PageUptodate(page));
677 		account_page_dirtied(page, mapping);
678 		radix_tree_tag_set(&mapping->page_tree,
679 				page_index(page), PAGECACHE_TAG_DIRTY);
680 	}
681 	spin_unlock_irq(&mapping->tree_lock);
682 	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
683 }
684 
685 /*
686  * Add a page to the dirty page list.
687  *
688  * It is a sad fact of life that this function is called from several places
689  * deeply under spinlocking.  It may not sleep.
690  *
691  * If the page has buffers, the uptodate buffers are set dirty, to preserve
692  * dirty-state coherency between the page and the buffers.  It the page does
693  * not have buffers then when they are later attached they will all be set
694  * dirty.
695  *
696  * The buffers are dirtied before the page is dirtied.  There's a small race
697  * window in which a writepage caller may see the page cleanness but not the
698  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
699  * before the buffers, a concurrent writepage caller could clear the page dirty
700  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
701  * page on the dirty page list.
702  *
703  * We use private_lock to lock against try_to_free_buffers while using the
704  * page's buffer list.  Also use this to protect against clean buffers being
705  * added to the page after it was set dirty.
706  *
707  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
708  * address_space though.
709  */
710 int __set_page_dirty_buffers(struct page *page)
711 {
712 	int newly_dirty;
713 	struct address_space *mapping = page_mapping(page);
714 
715 	if (unlikely(!mapping))
716 		return !TestSetPageDirty(page);
717 
718 	spin_lock(&mapping->private_lock);
719 	if (page_has_buffers(page)) {
720 		struct buffer_head *head = page_buffers(page);
721 		struct buffer_head *bh = head;
722 
723 		do {
724 			set_buffer_dirty(bh);
725 			bh = bh->b_this_page;
726 		} while (bh != head);
727 	}
728 	newly_dirty = !TestSetPageDirty(page);
729 	spin_unlock(&mapping->private_lock);
730 
731 	if (newly_dirty)
732 		__set_page_dirty(page, mapping, 1);
733 	return newly_dirty;
734 }
735 EXPORT_SYMBOL(__set_page_dirty_buffers);
736 
737 /*
738  * Write out and wait upon a list of buffers.
739  *
740  * We have conflicting pressures: we want to make sure that all
741  * initially dirty buffers get waited on, but that any subsequently
742  * dirtied buffers don't.  After all, we don't want fsync to last
743  * forever if somebody is actively writing to the file.
744  *
745  * Do this in two main stages: first we copy dirty buffers to a
746  * temporary inode list, queueing the writes as we go.  Then we clean
747  * up, waiting for those writes to complete.
748  *
749  * During this second stage, any subsequent updates to the file may end
750  * up refiling the buffer on the original inode's dirty list again, so
751  * there is a chance we will end up with a buffer queued for write but
752  * not yet completed on that list.  So, as a final cleanup we go through
753  * the osync code to catch these locked, dirty buffers without requeuing
754  * any newly dirty buffers for write.
755  */
756 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
757 {
758 	struct buffer_head *bh;
759 	struct list_head tmp;
760 	struct address_space *mapping, *prev_mapping = NULL;
761 	int err = 0, err2;
762 
763 	INIT_LIST_HEAD(&tmp);
764 
765 	spin_lock(lock);
766 	while (!list_empty(list)) {
767 		bh = BH_ENTRY(list->next);
768 		mapping = bh->b_assoc_map;
769 		__remove_assoc_queue(bh);
770 		/* Avoid race with mark_buffer_dirty_inode() which does
771 		 * a lockless check and we rely on seeing the dirty bit */
772 		smp_mb();
773 		if (buffer_dirty(bh) || buffer_locked(bh)) {
774 			list_add(&bh->b_assoc_buffers, &tmp);
775 			bh->b_assoc_map = mapping;
776 			if (buffer_dirty(bh)) {
777 				get_bh(bh);
778 				spin_unlock(lock);
779 				/*
780 				 * Ensure any pending I/O completes so that
781 				 * ll_rw_block() actually writes the current
782 				 * contents - it is a noop if I/O is still in
783 				 * flight on potentially older contents.
784 				 */
785 				ll_rw_block(SWRITE_SYNC_PLUG, 1, &bh);
786 
787 				/*
788 				 * Kick off IO for the previous mapping. Note
789 				 * that we will not run the very last mapping,
790 				 * wait_on_buffer() will do that for us
791 				 * through sync_buffer().
792 				 */
793 				if (prev_mapping && prev_mapping != mapping)
794 					blk_run_address_space(prev_mapping);
795 				prev_mapping = mapping;
796 
797 				brelse(bh);
798 				spin_lock(lock);
799 			}
800 		}
801 	}
802 
803 	while (!list_empty(&tmp)) {
804 		bh = BH_ENTRY(tmp.prev);
805 		get_bh(bh);
806 		mapping = bh->b_assoc_map;
807 		__remove_assoc_queue(bh);
808 		/* Avoid race with mark_buffer_dirty_inode() which does
809 		 * a lockless check and we rely on seeing the dirty bit */
810 		smp_mb();
811 		if (buffer_dirty(bh)) {
812 			list_add(&bh->b_assoc_buffers,
813 				 &mapping->private_list);
814 			bh->b_assoc_map = mapping;
815 		}
816 		spin_unlock(lock);
817 		wait_on_buffer(bh);
818 		if (!buffer_uptodate(bh))
819 			err = -EIO;
820 		brelse(bh);
821 		spin_lock(lock);
822 	}
823 
824 	spin_unlock(lock);
825 	err2 = osync_buffers_list(lock, list);
826 	if (err)
827 		return err;
828 	else
829 		return err2;
830 }
831 
832 /*
833  * Invalidate any and all dirty buffers on a given inode.  We are
834  * probably unmounting the fs, but that doesn't mean we have already
835  * done a sync().  Just drop the buffers from the inode list.
836  *
837  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
838  * assumes that all the buffers are against the blockdev.  Not true
839  * for reiserfs.
840  */
841 void invalidate_inode_buffers(struct inode *inode)
842 {
843 	if (inode_has_buffers(inode)) {
844 		struct address_space *mapping = &inode->i_data;
845 		struct list_head *list = &mapping->private_list;
846 		struct address_space *buffer_mapping = mapping->assoc_mapping;
847 
848 		spin_lock(&buffer_mapping->private_lock);
849 		while (!list_empty(list))
850 			__remove_assoc_queue(BH_ENTRY(list->next));
851 		spin_unlock(&buffer_mapping->private_lock);
852 	}
853 }
854 EXPORT_SYMBOL(invalidate_inode_buffers);
855 
856 /*
857  * Remove any clean buffers from the inode's buffer list.  This is called
858  * when we're trying to free the inode itself.  Those buffers can pin it.
859  *
860  * Returns true if all buffers were removed.
861  */
862 int remove_inode_buffers(struct inode *inode)
863 {
864 	int ret = 1;
865 
866 	if (inode_has_buffers(inode)) {
867 		struct address_space *mapping = &inode->i_data;
868 		struct list_head *list = &mapping->private_list;
869 		struct address_space *buffer_mapping = mapping->assoc_mapping;
870 
871 		spin_lock(&buffer_mapping->private_lock);
872 		while (!list_empty(list)) {
873 			struct buffer_head *bh = BH_ENTRY(list->next);
874 			if (buffer_dirty(bh)) {
875 				ret = 0;
876 				break;
877 			}
878 			__remove_assoc_queue(bh);
879 		}
880 		spin_unlock(&buffer_mapping->private_lock);
881 	}
882 	return ret;
883 }
884 
885 /*
886  * Create the appropriate buffers when given a page for data area and
887  * the size of each buffer.. Use the bh->b_this_page linked list to
888  * follow the buffers created.  Return NULL if unable to create more
889  * buffers.
890  *
891  * The retry flag is used to differentiate async IO (paging, swapping)
892  * which may not fail from ordinary buffer allocations.
893  */
894 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
895 		int retry)
896 {
897 	struct buffer_head *bh, *head;
898 	long offset;
899 
900 try_again:
901 	head = NULL;
902 	offset = PAGE_SIZE;
903 	while ((offset -= size) >= 0) {
904 		bh = alloc_buffer_head(GFP_NOFS);
905 		if (!bh)
906 			goto no_grow;
907 
908 		bh->b_bdev = NULL;
909 		bh->b_this_page = head;
910 		bh->b_blocknr = -1;
911 		head = bh;
912 
913 		bh->b_state = 0;
914 		atomic_set(&bh->b_count, 0);
915 		bh->b_private = NULL;
916 		bh->b_size = size;
917 
918 		/* Link the buffer to its page */
919 		set_bh_page(bh, page, offset);
920 
921 		init_buffer(bh, NULL, NULL);
922 	}
923 	return head;
924 /*
925  * In case anything failed, we just free everything we got.
926  */
927 no_grow:
928 	if (head) {
929 		do {
930 			bh = head;
931 			head = head->b_this_page;
932 			free_buffer_head(bh);
933 		} while (head);
934 	}
935 
936 	/*
937 	 * Return failure for non-async IO requests.  Async IO requests
938 	 * are not allowed to fail, so we have to wait until buffer heads
939 	 * become available.  But we don't want tasks sleeping with
940 	 * partially complete buffers, so all were released above.
941 	 */
942 	if (!retry)
943 		return NULL;
944 
945 	/* We're _really_ low on memory. Now we just
946 	 * wait for old buffer heads to become free due to
947 	 * finishing IO.  Since this is an async request and
948 	 * the reserve list is empty, we're sure there are
949 	 * async buffer heads in use.
950 	 */
951 	free_more_memory();
952 	goto try_again;
953 }
954 EXPORT_SYMBOL_GPL(alloc_page_buffers);
955 
956 static inline void
957 link_dev_buffers(struct page *page, struct buffer_head *head)
958 {
959 	struct buffer_head *bh, *tail;
960 
961 	bh = head;
962 	do {
963 		tail = bh;
964 		bh = bh->b_this_page;
965 	} while (bh);
966 	tail->b_this_page = head;
967 	attach_page_buffers(page, head);
968 }
969 
970 /*
971  * Initialise the state of a blockdev page's buffers.
972  */
973 static void
974 init_page_buffers(struct page *page, struct block_device *bdev,
975 			sector_t block, int size)
976 {
977 	struct buffer_head *head = page_buffers(page);
978 	struct buffer_head *bh = head;
979 	int uptodate = PageUptodate(page);
980 
981 	do {
982 		if (!buffer_mapped(bh)) {
983 			init_buffer(bh, NULL, NULL);
984 			bh->b_bdev = bdev;
985 			bh->b_blocknr = block;
986 			if (uptodate)
987 				set_buffer_uptodate(bh);
988 			set_buffer_mapped(bh);
989 		}
990 		block++;
991 		bh = bh->b_this_page;
992 	} while (bh != head);
993 }
994 
995 /*
996  * Create the page-cache page that contains the requested block.
997  *
998  * This is user purely for blockdev mappings.
999  */
1000 static struct page *
1001 grow_dev_page(struct block_device *bdev, sector_t block,
1002 		pgoff_t index, int size)
1003 {
1004 	struct inode *inode = bdev->bd_inode;
1005 	struct page *page;
1006 	struct buffer_head *bh;
1007 
1008 	page = find_or_create_page(inode->i_mapping, index,
1009 		(mapping_gfp_mask(inode->i_mapping) & ~__GFP_FS)|__GFP_MOVABLE);
1010 	if (!page)
1011 		return NULL;
1012 
1013 	BUG_ON(!PageLocked(page));
1014 
1015 	if (page_has_buffers(page)) {
1016 		bh = page_buffers(page);
1017 		if (bh->b_size == size) {
1018 			init_page_buffers(page, bdev, block, size);
1019 			return page;
1020 		}
1021 		if (!try_to_free_buffers(page))
1022 			goto failed;
1023 	}
1024 
1025 	/*
1026 	 * Allocate some buffers for this page
1027 	 */
1028 	bh = alloc_page_buffers(page, size, 0);
1029 	if (!bh)
1030 		goto failed;
1031 
1032 	/*
1033 	 * Link the page to the buffers and initialise them.  Take the
1034 	 * lock to be atomic wrt __find_get_block(), which does not
1035 	 * run under the page lock.
1036 	 */
1037 	spin_lock(&inode->i_mapping->private_lock);
1038 	link_dev_buffers(page, bh);
1039 	init_page_buffers(page, bdev, block, size);
1040 	spin_unlock(&inode->i_mapping->private_lock);
1041 	return page;
1042 
1043 failed:
1044 	BUG();
1045 	unlock_page(page);
1046 	page_cache_release(page);
1047 	return NULL;
1048 }
1049 
1050 /*
1051  * Create buffers for the specified block device block's page.  If
1052  * that page was dirty, the buffers are set dirty also.
1053  */
1054 static int
1055 grow_buffers(struct block_device *bdev, sector_t block, int size)
1056 {
1057 	struct page *page;
1058 	pgoff_t index;
1059 	int sizebits;
1060 
1061 	sizebits = -1;
1062 	do {
1063 		sizebits++;
1064 	} while ((size << sizebits) < PAGE_SIZE);
1065 
1066 	index = block >> sizebits;
1067 
1068 	/*
1069 	 * Check for a block which wants to lie outside our maximum possible
1070 	 * pagecache index.  (this comparison is done using sector_t types).
1071 	 */
1072 	if (unlikely(index != block >> sizebits)) {
1073 		char b[BDEVNAME_SIZE];
1074 
1075 		printk(KERN_ERR "%s: requested out-of-range block %llu for "
1076 			"device %s\n",
1077 			__func__, (unsigned long long)block,
1078 			bdevname(bdev, b));
1079 		return -EIO;
1080 	}
1081 	block = index << sizebits;
1082 	/* Create a page with the proper size buffers.. */
1083 	page = grow_dev_page(bdev, block, index, size);
1084 	if (!page)
1085 		return 0;
1086 	unlock_page(page);
1087 	page_cache_release(page);
1088 	return 1;
1089 }
1090 
1091 static struct buffer_head *
1092 __getblk_slow(struct block_device *bdev, sector_t block, int size)
1093 {
1094 	/* Size must be multiple of hard sectorsize */
1095 	if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1096 			(size < 512 || size > PAGE_SIZE))) {
1097 		printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1098 					size);
1099 		printk(KERN_ERR "logical block size: %d\n",
1100 					bdev_logical_block_size(bdev));
1101 
1102 		dump_stack();
1103 		return NULL;
1104 	}
1105 
1106 	for (;;) {
1107 		struct buffer_head * bh;
1108 		int ret;
1109 
1110 		bh = __find_get_block(bdev, block, size);
1111 		if (bh)
1112 			return bh;
1113 
1114 		ret = grow_buffers(bdev, block, size);
1115 		if (ret < 0)
1116 			return NULL;
1117 		if (ret == 0)
1118 			free_more_memory();
1119 	}
1120 }
1121 
1122 /*
1123  * The relationship between dirty buffers and dirty pages:
1124  *
1125  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1126  * the page is tagged dirty in its radix tree.
1127  *
1128  * At all times, the dirtiness of the buffers represents the dirtiness of
1129  * subsections of the page.  If the page has buffers, the page dirty bit is
1130  * merely a hint about the true dirty state.
1131  *
1132  * When a page is set dirty in its entirety, all its buffers are marked dirty
1133  * (if the page has buffers).
1134  *
1135  * When a buffer is marked dirty, its page is dirtied, but the page's other
1136  * buffers are not.
1137  *
1138  * Also.  When blockdev buffers are explicitly read with bread(), they
1139  * individually become uptodate.  But their backing page remains not
1140  * uptodate - even if all of its buffers are uptodate.  A subsequent
1141  * block_read_full_page() against that page will discover all the uptodate
1142  * buffers, will set the page uptodate and will perform no I/O.
1143  */
1144 
1145 /**
1146  * mark_buffer_dirty - mark a buffer_head as needing writeout
1147  * @bh: the buffer_head to mark dirty
1148  *
1149  * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1150  * backing page dirty, then tag the page as dirty in its address_space's radix
1151  * tree and then attach the address_space's inode to its superblock's dirty
1152  * inode list.
1153  *
1154  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1155  * mapping->tree_lock and the global inode_lock.
1156  */
1157 void mark_buffer_dirty(struct buffer_head *bh)
1158 {
1159 	WARN_ON_ONCE(!buffer_uptodate(bh));
1160 
1161 	/*
1162 	 * Very *carefully* optimize the it-is-already-dirty case.
1163 	 *
1164 	 * Don't let the final "is it dirty" escape to before we
1165 	 * perhaps modified the buffer.
1166 	 */
1167 	if (buffer_dirty(bh)) {
1168 		smp_mb();
1169 		if (buffer_dirty(bh))
1170 			return;
1171 	}
1172 
1173 	if (!test_set_buffer_dirty(bh)) {
1174 		struct page *page = bh->b_page;
1175 		if (!TestSetPageDirty(page)) {
1176 			struct address_space *mapping = page_mapping(page);
1177 			if (mapping)
1178 				__set_page_dirty(page, mapping, 0);
1179 		}
1180 	}
1181 }
1182 EXPORT_SYMBOL(mark_buffer_dirty);
1183 
1184 /*
1185  * Decrement a buffer_head's reference count.  If all buffers against a page
1186  * have zero reference count, are clean and unlocked, and if the page is clean
1187  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1188  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1189  * a page but it ends up not being freed, and buffers may later be reattached).
1190  */
1191 void __brelse(struct buffer_head * buf)
1192 {
1193 	if (atomic_read(&buf->b_count)) {
1194 		put_bh(buf);
1195 		return;
1196 	}
1197 	WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1198 }
1199 EXPORT_SYMBOL(__brelse);
1200 
1201 /*
1202  * bforget() is like brelse(), except it discards any
1203  * potentially dirty data.
1204  */
1205 void __bforget(struct buffer_head *bh)
1206 {
1207 	clear_buffer_dirty(bh);
1208 	if (bh->b_assoc_map) {
1209 		struct address_space *buffer_mapping = bh->b_page->mapping;
1210 
1211 		spin_lock(&buffer_mapping->private_lock);
1212 		list_del_init(&bh->b_assoc_buffers);
1213 		bh->b_assoc_map = NULL;
1214 		spin_unlock(&buffer_mapping->private_lock);
1215 	}
1216 	__brelse(bh);
1217 }
1218 EXPORT_SYMBOL(__bforget);
1219 
1220 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1221 {
1222 	lock_buffer(bh);
1223 	if (buffer_uptodate(bh)) {
1224 		unlock_buffer(bh);
1225 		return bh;
1226 	} else {
1227 		get_bh(bh);
1228 		bh->b_end_io = end_buffer_read_sync;
1229 		submit_bh(READ, bh);
1230 		wait_on_buffer(bh);
1231 		if (buffer_uptodate(bh))
1232 			return bh;
1233 	}
1234 	brelse(bh);
1235 	return NULL;
1236 }
1237 
1238 /*
1239  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1240  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1241  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1242  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1243  * CPU's LRUs at the same time.
1244  *
1245  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1246  * sb_find_get_block().
1247  *
1248  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1249  * a local interrupt disable for that.
1250  */
1251 
1252 #define BH_LRU_SIZE	8
1253 
1254 struct bh_lru {
1255 	struct buffer_head *bhs[BH_LRU_SIZE];
1256 };
1257 
1258 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1259 
1260 #ifdef CONFIG_SMP
1261 #define bh_lru_lock()	local_irq_disable()
1262 #define bh_lru_unlock()	local_irq_enable()
1263 #else
1264 #define bh_lru_lock()	preempt_disable()
1265 #define bh_lru_unlock()	preempt_enable()
1266 #endif
1267 
1268 static inline void check_irqs_on(void)
1269 {
1270 #ifdef irqs_disabled
1271 	BUG_ON(irqs_disabled());
1272 #endif
1273 }
1274 
1275 /*
1276  * The LRU management algorithm is dopey-but-simple.  Sorry.
1277  */
1278 static void bh_lru_install(struct buffer_head *bh)
1279 {
1280 	struct buffer_head *evictee = NULL;
1281 	struct bh_lru *lru;
1282 
1283 	check_irqs_on();
1284 	bh_lru_lock();
1285 	lru = &__get_cpu_var(bh_lrus);
1286 	if (lru->bhs[0] != bh) {
1287 		struct buffer_head *bhs[BH_LRU_SIZE];
1288 		int in;
1289 		int out = 0;
1290 
1291 		get_bh(bh);
1292 		bhs[out++] = bh;
1293 		for (in = 0; in < BH_LRU_SIZE; in++) {
1294 			struct buffer_head *bh2 = lru->bhs[in];
1295 
1296 			if (bh2 == bh) {
1297 				__brelse(bh2);
1298 			} else {
1299 				if (out >= BH_LRU_SIZE) {
1300 					BUG_ON(evictee != NULL);
1301 					evictee = bh2;
1302 				} else {
1303 					bhs[out++] = bh2;
1304 				}
1305 			}
1306 		}
1307 		while (out < BH_LRU_SIZE)
1308 			bhs[out++] = NULL;
1309 		memcpy(lru->bhs, bhs, sizeof(bhs));
1310 	}
1311 	bh_lru_unlock();
1312 
1313 	if (evictee)
1314 		__brelse(evictee);
1315 }
1316 
1317 /*
1318  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1319  */
1320 static struct buffer_head *
1321 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1322 {
1323 	struct buffer_head *ret = NULL;
1324 	struct bh_lru *lru;
1325 	unsigned int i;
1326 
1327 	check_irqs_on();
1328 	bh_lru_lock();
1329 	lru = &__get_cpu_var(bh_lrus);
1330 	for (i = 0; i < BH_LRU_SIZE; i++) {
1331 		struct buffer_head *bh = lru->bhs[i];
1332 
1333 		if (bh && bh->b_bdev == bdev &&
1334 				bh->b_blocknr == block && bh->b_size == size) {
1335 			if (i) {
1336 				while (i) {
1337 					lru->bhs[i] = lru->bhs[i - 1];
1338 					i--;
1339 				}
1340 				lru->bhs[0] = bh;
1341 			}
1342 			get_bh(bh);
1343 			ret = bh;
1344 			break;
1345 		}
1346 	}
1347 	bh_lru_unlock();
1348 	return ret;
1349 }
1350 
1351 /*
1352  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1353  * it in the LRU and mark it as accessed.  If it is not present then return
1354  * NULL
1355  */
1356 struct buffer_head *
1357 __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1358 {
1359 	struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1360 
1361 	if (bh == NULL) {
1362 		bh = __find_get_block_slow(bdev, block);
1363 		if (bh)
1364 			bh_lru_install(bh);
1365 	}
1366 	if (bh)
1367 		touch_buffer(bh);
1368 	return bh;
1369 }
1370 EXPORT_SYMBOL(__find_get_block);
1371 
1372 /*
1373  * __getblk will locate (and, if necessary, create) the buffer_head
1374  * which corresponds to the passed block_device, block and size. The
1375  * returned buffer has its reference count incremented.
1376  *
1377  * __getblk() cannot fail - it just keeps trying.  If you pass it an
1378  * illegal block number, __getblk() will happily return a buffer_head
1379  * which represents the non-existent block.  Very weird.
1380  *
1381  * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1382  * attempt is failing.  FIXME, perhaps?
1383  */
1384 struct buffer_head *
1385 __getblk(struct block_device *bdev, sector_t block, unsigned size)
1386 {
1387 	struct buffer_head *bh = __find_get_block(bdev, block, size);
1388 
1389 	might_sleep();
1390 	if (bh == NULL)
1391 		bh = __getblk_slow(bdev, block, size);
1392 	return bh;
1393 }
1394 EXPORT_SYMBOL(__getblk);
1395 
1396 /*
1397  * Do async read-ahead on a buffer..
1398  */
1399 void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1400 {
1401 	struct buffer_head *bh = __getblk(bdev, block, size);
1402 	if (likely(bh)) {
1403 		ll_rw_block(READA, 1, &bh);
1404 		brelse(bh);
1405 	}
1406 }
1407 EXPORT_SYMBOL(__breadahead);
1408 
1409 /**
1410  *  __bread() - reads a specified block and returns the bh
1411  *  @bdev: the block_device to read from
1412  *  @block: number of block
1413  *  @size: size (in bytes) to read
1414  *
1415  *  Reads a specified block, and returns buffer head that contains it.
1416  *  It returns NULL if the block was unreadable.
1417  */
1418 struct buffer_head *
1419 __bread(struct block_device *bdev, sector_t block, unsigned size)
1420 {
1421 	struct buffer_head *bh = __getblk(bdev, block, size);
1422 
1423 	if (likely(bh) && !buffer_uptodate(bh))
1424 		bh = __bread_slow(bh);
1425 	return bh;
1426 }
1427 EXPORT_SYMBOL(__bread);
1428 
1429 /*
1430  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1431  * This doesn't race because it runs in each cpu either in irq
1432  * or with preempt disabled.
1433  */
1434 static void invalidate_bh_lru(void *arg)
1435 {
1436 	struct bh_lru *b = &get_cpu_var(bh_lrus);
1437 	int i;
1438 
1439 	for (i = 0; i < BH_LRU_SIZE; i++) {
1440 		brelse(b->bhs[i]);
1441 		b->bhs[i] = NULL;
1442 	}
1443 	put_cpu_var(bh_lrus);
1444 }
1445 
1446 void invalidate_bh_lrus(void)
1447 {
1448 	on_each_cpu(invalidate_bh_lru, NULL, 1);
1449 }
1450 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1451 
1452 void set_bh_page(struct buffer_head *bh,
1453 		struct page *page, unsigned long offset)
1454 {
1455 	bh->b_page = page;
1456 	BUG_ON(offset >= PAGE_SIZE);
1457 	if (PageHighMem(page))
1458 		/*
1459 		 * This catches illegal uses and preserves the offset:
1460 		 */
1461 		bh->b_data = (char *)(0 + offset);
1462 	else
1463 		bh->b_data = page_address(page) + offset;
1464 }
1465 EXPORT_SYMBOL(set_bh_page);
1466 
1467 /*
1468  * Called when truncating a buffer on a page completely.
1469  */
1470 static void discard_buffer(struct buffer_head * bh)
1471 {
1472 	lock_buffer(bh);
1473 	clear_buffer_dirty(bh);
1474 	bh->b_bdev = NULL;
1475 	clear_buffer_mapped(bh);
1476 	clear_buffer_req(bh);
1477 	clear_buffer_new(bh);
1478 	clear_buffer_delay(bh);
1479 	clear_buffer_unwritten(bh);
1480 	unlock_buffer(bh);
1481 }
1482 
1483 /**
1484  * block_invalidatepage - invalidate part of all of a buffer-backed page
1485  *
1486  * @page: the page which is affected
1487  * @offset: the index of the truncation point
1488  *
1489  * block_invalidatepage() is called when all or part of the page has become
1490  * invalidatedby a truncate operation.
1491  *
1492  * block_invalidatepage() does not have to release all buffers, but it must
1493  * ensure that no dirty buffer is left outside @offset and that no I/O
1494  * is underway against any of the blocks which are outside the truncation
1495  * point.  Because the caller is about to free (and possibly reuse) those
1496  * blocks on-disk.
1497  */
1498 void block_invalidatepage(struct page *page, unsigned long offset)
1499 {
1500 	struct buffer_head *head, *bh, *next;
1501 	unsigned int curr_off = 0;
1502 
1503 	BUG_ON(!PageLocked(page));
1504 	if (!page_has_buffers(page))
1505 		goto out;
1506 
1507 	head = page_buffers(page);
1508 	bh = head;
1509 	do {
1510 		unsigned int next_off = curr_off + bh->b_size;
1511 		next = bh->b_this_page;
1512 
1513 		/*
1514 		 * is this block fully invalidated?
1515 		 */
1516 		if (offset <= curr_off)
1517 			discard_buffer(bh);
1518 		curr_off = next_off;
1519 		bh = next;
1520 	} while (bh != head);
1521 
1522 	/*
1523 	 * We release buffers only if the entire page is being invalidated.
1524 	 * The get_block cached value has been unconditionally invalidated,
1525 	 * so real IO is not possible anymore.
1526 	 */
1527 	if (offset == 0)
1528 		try_to_release_page(page, 0);
1529 out:
1530 	return;
1531 }
1532 EXPORT_SYMBOL(block_invalidatepage);
1533 
1534 /*
1535  * We attach and possibly dirty the buffers atomically wrt
1536  * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1537  * is already excluded via the page lock.
1538  */
1539 void create_empty_buffers(struct page *page,
1540 			unsigned long blocksize, unsigned long b_state)
1541 {
1542 	struct buffer_head *bh, *head, *tail;
1543 
1544 	head = alloc_page_buffers(page, blocksize, 1);
1545 	bh = head;
1546 	do {
1547 		bh->b_state |= b_state;
1548 		tail = bh;
1549 		bh = bh->b_this_page;
1550 	} while (bh);
1551 	tail->b_this_page = head;
1552 
1553 	spin_lock(&page->mapping->private_lock);
1554 	if (PageUptodate(page) || PageDirty(page)) {
1555 		bh = head;
1556 		do {
1557 			if (PageDirty(page))
1558 				set_buffer_dirty(bh);
1559 			if (PageUptodate(page))
1560 				set_buffer_uptodate(bh);
1561 			bh = bh->b_this_page;
1562 		} while (bh != head);
1563 	}
1564 	attach_page_buffers(page, head);
1565 	spin_unlock(&page->mapping->private_lock);
1566 }
1567 EXPORT_SYMBOL(create_empty_buffers);
1568 
1569 /*
1570  * We are taking a block for data and we don't want any output from any
1571  * buffer-cache aliases starting from return from that function and
1572  * until the moment when something will explicitly mark the buffer
1573  * dirty (hopefully that will not happen until we will free that block ;-)
1574  * We don't even need to mark it not-uptodate - nobody can expect
1575  * anything from a newly allocated buffer anyway. We used to used
1576  * unmap_buffer() for such invalidation, but that was wrong. We definitely
1577  * don't want to mark the alias unmapped, for example - it would confuse
1578  * anyone who might pick it with bread() afterwards...
1579  *
1580  * Also..  Note that bforget() doesn't lock the buffer.  So there can
1581  * be writeout I/O going on against recently-freed buffers.  We don't
1582  * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1583  * only if we really need to.  That happens here.
1584  */
1585 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1586 {
1587 	struct buffer_head *old_bh;
1588 
1589 	might_sleep();
1590 
1591 	old_bh = __find_get_block_slow(bdev, block);
1592 	if (old_bh) {
1593 		clear_buffer_dirty(old_bh);
1594 		wait_on_buffer(old_bh);
1595 		clear_buffer_req(old_bh);
1596 		__brelse(old_bh);
1597 	}
1598 }
1599 EXPORT_SYMBOL(unmap_underlying_metadata);
1600 
1601 /*
1602  * NOTE! All mapped/uptodate combinations are valid:
1603  *
1604  *	Mapped	Uptodate	Meaning
1605  *
1606  *	No	No		"unknown" - must do get_block()
1607  *	No	Yes		"hole" - zero-filled
1608  *	Yes	No		"allocated" - allocated on disk, not read in
1609  *	Yes	Yes		"valid" - allocated and up-to-date in memory.
1610  *
1611  * "Dirty" is valid only with the last case (mapped+uptodate).
1612  */
1613 
1614 /*
1615  * While block_write_full_page is writing back the dirty buffers under
1616  * the page lock, whoever dirtied the buffers may decide to clean them
1617  * again at any time.  We handle that by only looking at the buffer
1618  * state inside lock_buffer().
1619  *
1620  * If block_write_full_page() is called for regular writeback
1621  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1622  * locked buffer.   This only can happen if someone has written the buffer
1623  * directly, with submit_bh().  At the address_space level PageWriteback
1624  * prevents this contention from occurring.
1625  *
1626  * If block_write_full_page() is called with wbc->sync_mode ==
1627  * WB_SYNC_ALL, the writes are posted using WRITE_SYNC_PLUG; this
1628  * causes the writes to be flagged as synchronous writes, but the
1629  * block device queue will NOT be unplugged, since usually many pages
1630  * will be pushed to the out before the higher-level caller actually
1631  * waits for the writes to be completed.  The various wait functions,
1632  * such as wait_on_writeback_range() will ultimately call sync_page()
1633  * which will ultimately call blk_run_backing_dev(), which will end up
1634  * unplugging the device queue.
1635  */
1636 static int __block_write_full_page(struct inode *inode, struct page *page,
1637 			get_block_t *get_block, struct writeback_control *wbc,
1638 			bh_end_io_t *handler)
1639 {
1640 	int err;
1641 	sector_t block;
1642 	sector_t last_block;
1643 	struct buffer_head *bh, *head;
1644 	const unsigned blocksize = 1 << inode->i_blkbits;
1645 	int nr_underway = 0;
1646 	int write_op = (wbc->sync_mode == WB_SYNC_ALL ?
1647 			WRITE_SYNC_PLUG : WRITE);
1648 
1649 	BUG_ON(!PageLocked(page));
1650 
1651 	last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1652 
1653 	if (!page_has_buffers(page)) {
1654 		create_empty_buffers(page, blocksize,
1655 					(1 << BH_Dirty)|(1 << BH_Uptodate));
1656 	}
1657 
1658 	/*
1659 	 * Be very careful.  We have no exclusion from __set_page_dirty_buffers
1660 	 * here, and the (potentially unmapped) buffers may become dirty at
1661 	 * any time.  If a buffer becomes dirty here after we've inspected it
1662 	 * then we just miss that fact, and the page stays dirty.
1663 	 *
1664 	 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1665 	 * handle that here by just cleaning them.
1666 	 */
1667 
1668 	block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1669 	head = page_buffers(page);
1670 	bh = head;
1671 
1672 	/*
1673 	 * Get all the dirty buffers mapped to disk addresses and
1674 	 * handle any aliases from the underlying blockdev's mapping.
1675 	 */
1676 	do {
1677 		if (block > last_block) {
1678 			/*
1679 			 * mapped buffers outside i_size will occur, because
1680 			 * this page can be outside i_size when there is a
1681 			 * truncate in progress.
1682 			 */
1683 			/*
1684 			 * The buffer was zeroed by block_write_full_page()
1685 			 */
1686 			clear_buffer_dirty(bh);
1687 			set_buffer_uptodate(bh);
1688 		} else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1689 			   buffer_dirty(bh)) {
1690 			WARN_ON(bh->b_size != blocksize);
1691 			err = get_block(inode, block, bh, 1);
1692 			if (err)
1693 				goto recover;
1694 			clear_buffer_delay(bh);
1695 			if (buffer_new(bh)) {
1696 				/* blockdev mappings never come here */
1697 				clear_buffer_new(bh);
1698 				unmap_underlying_metadata(bh->b_bdev,
1699 							bh->b_blocknr);
1700 			}
1701 		}
1702 		bh = bh->b_this_page;
1703 		block++;
1704 	} while (bh != head);
1705 
1706 	do {
1707 		if (!buffer_mapped(bh))
1708 			continue;
1709 		/*
1710 		 * If it's a fully non-blocking write attempt and we cannot
1711 		 * lock the buffer then redirty the page.  Note that this can
1712 		 * potentially cause a busy-wait loop from writeback threads
1713 		 * and kswapd activity, but those code paths have their own
1714 		 * higher-level throttling.
1715 		 */
1716 		if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1717 			lock_buffer(bh);
1718 		} else if (!trylock_buffer(bh)) {
1719 			redirty_page_for_writepage(wbc, page);
1720 			continue;
1721 		}
1722 		if (test_clear_buffer_dirty(bh)) {
1723 			mark_buffer_async_write_endio(bh, handler);
1724 		} else {
1725 			unlock_buffer(bh);
1726 		}
1727 	} while ((bh = bh->b_this_page) != head);
1728 
1729 	/*
1730 	 * The page and its buffers are protected by PageWriteback(), so we can
1731 	 * drop the bh refcounts early.
1732 	 */
1733 	BUG_ON(PageWriteback(page));
1734 	set_page_writeback(page);
1735 
1736 	do {
1737 		struct buffer_head *next = bh->b_this_page;
1738 		if (buffer_async_write(bh)) {
1739 			submit_bh(write_op, bh);
1740 			nr_underway++;
1741 		}
1742 		bh = next;
1743 	} while (bh != head);
1744 	unlock_page(page);
1745 
1746 	err = 0;
1747 done:
1748 	if (nr_underway == 0) {
1749 		/*
1750 		 * The page was marked dirty, but the buffers were
1751 		 * clean.  Someone wrote them back by hand with
1752 		 * ll_rw_block/submit_bh.  A rare case.
1753 		 */
1754 		end_page_writeback(page);
1755 
1756 		/*
1757 		 * The page and buffer_heads can be released at any time from
1758 		 * here on.
1759 		 */
1760 	}
1761 	return err;
1762 
1763 recover:
1764 	/*
1765 	 * ENOSPC, or some other error.  We may already have added some
1766 	 * blocks to the file, so we need to write these out to avoid
1767 	 * exposing stale data.
1768 	 * The page is currently locked and not marked for writeback
1769 	 */
1770 	bh = head;
1771 	/* Recovery: lock and submit the mapped buffers */
1772 	do {
1773 		if (buffer_mapped(bh) && buffer_dirty(bh) &&
1774 		    !buffer_delay(bh)) {
1775 			lock_buffer(bh);
1776 			mark_buffer_async_write_endio(bh, handler);
1777 		} else {
1778 			/*
1779 			 * The buffer may have been set dirty during
1780 			 * attachment to a dirty page.
1781 			 */
1782 			clear_buffer_dirty(bh);
1783 		}
1784 	} while ((bh = bh->b_this_page) != head);
1785 	SetPageError(page);
1786 	BUG_ON(PageWriteback(page));
1787 	mapping_set_error(page->mapping, err);
1788 	set_page_writeback(page);
1789 	do {
1790 		struct buffer_head *next = bh->b_this_page;
1791 		if (buffer_async_write(bh)) {
1792 			clear_buffer_dirty(bh);
1793 			submit_bh(write_op, bh);
1794 			nr_underway++;
1795 		}
1796 		bh = next;
1797 	} while (bh != head);
1798 	unlock_page(page);
1799 	goto done;
1800 }
1801 
1802 /*
1803  * If a page has any new buffers, zero them out here, and mark them uptodate
1804  * and dirty so they'll be written out (in order to prevent uninitialised
1805  * block data from leaking). And clear the new bit.
1806  */
1807 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1808 {
1809 	unsigned int block_start, block_end;
1810 	struct buffer_head *head, *bh;
1811 
1812 	BUG_ON(!PageLocked(page));
1813 	if (!page_has_buffers(page))
1814 		return;
1815 
1816 	bh = head = page_buffers(page);
1817 	block_start = 0;
1818 	do {
1819 		block_end = block_start + bh->b_size;
1820 
1821 		if (buffer_new(bh)) {
1822 			if (block_end > from && block_start < to) {
1823 				if (!PageUptodate(page)) {
1824 					unsigned start, size;
1825 
1826 					start = max(from, block_start);
1827 					size = min(to, block_end) - start;
1828 
1829 					zero_user(page, start, size);
1830 					set_buffer_uptodate(bh);
1831 				}
1832 
1833 				clear_buffer_new(bh);
1834 				mark_buffer_dirty(bh);
1835 			}
1836 		}
1837 
1838 		block_start = block_end;
1839 		bh = bh->b_this_page;
1840 	} while (bh != head);
1841 }
1842 EXPORT_SYMBOL(page_zero_new_buffers);
1843 
1844 static int __block_prepare_write(struct inode *inode, struct page *page,
1845 		unsigned from, unsigned to, get_block_t *get_block)
1846 {
1847 	unsigned block_start, block_end;
1848 	sector_t block;
1849 	int err = 0;
1850 	unsigned blocksize, bbits;
1851 	struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1852 
1853 	BUG_ON(!PageLocked(page));
1854 	BUG_ON(from > PAGE_CACHE_SIZE);
1855 	BUG_ON(to > PAGE_CACHE_SIZE);
1856 	BUG_ON(from > to);
1857 
1858 	blocksize = 1 << inode->i_blkbits;
1859 	if (!page_has_buffers(page))
1860 		create_empty_buffers(page, blocksize, 0);
1861 	head = page_buffers(page);
1862 
1863 	bbits = inode->i_blkbits;
1864 	block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1865 
1866 	for(bh = head, block_start = 0; bh != head || !block_start;
1867 	    block++, block_start=block_end, bh = bh->b_this_page) {
1868 		block_end = block_start + blocksize;
1869 		if (block_end <= from || block_start >= to) {
1870 			if (PageUptodate(page)) {
1871 				if (!buffer_uptodate(bh))
1872 					set_buffer_uptodate(bh);
1873 			}
1874 			continue;
1875 		}
1876 		if (buffer_new(bh))
1877 			clear_buffer_new(bh);
1878 		if (!buffer_mapped(bh)) {
1879 			WARN_ON(bh->b_size != blocksize);
1880 			err = get_block(inode, block, bh, 1);
1881 			if (err)
1882 				break;
1883 			if (buffer_new(bh)) {
1884 				unmap_underlying_metadata(bh->b_bdev,
1885 							bh->b_blocknr);
1886 				if (PageUptodate(page)) {
1887 					clear_buffer_new(bh);
1888 					set_buffer_uptodate(bh);
1889 					mark_buffer_dirty(bh);
1890 					continue;
1891 				}
1892 				if (block_end > to || block_start < from)
1893 					zero_user_segments(page,
1894 						to, block_end,
1895 						block_start, from);
1896 				continue;
1897 			}
1898 		}
1899 		if (PageUptodate(page)) {
1900 			if (!buffer_uptodate(bh))
1901 				set_buffer_uptodate(bh);
1902 			continue;
1903 		}
1904 		if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1905 		    !buffer_unwritten(bh) &&
1906 		     (block_start < from || block_end > to)) {
1907 			ll_rw_block(READ, 1, &bh);
1908 			*wait_bh++=bh;
1909 		}
1910 	}
1911 	/*
1912 	 * If we issued read requests - let them complete.
1913 	 */
1914 	while(wait_bh > wait) {
1915 		wait_on_buffer(*--wait_bh);
1916 		if (!buffer_uptodate(*wait_bh))
1917 			err = -EIO;
1918 	}
1919 	if (unlikely(err))
1920 		page_zero_new_buffers(page, from, to);
1921 	return err;
1922 }
1923 
1924 static int __block_commit_write(struct inode *inode, struct page *page,
1925 		unsigned from, unsigned to)
1926 {
1927 	unsigned block_start, block_end;
1928 	int partial = 0;
1929 	unsigned blocksize;
1930 	struct buffer_head *bh, *head;
1931 
1932 	blocksize = 1 << inode->i_blkbits;
1933 
1934 	for(bh = head = page_buffers(page), block_start = 0;
1935 	    bh != head || !block_start;
1936 	    block_start=block_end, bh = bh->b_this_page) {
1937 		block_end = block_start + blocksize;
1938 		if (block_end <= from || block_start >= to) {
1939 			if (!buffer_uptodate(bh))
1940 				partial = 1;
1941 		} else {
1942 			set_buffer_uptodate(bh);
1943 			mark_buffer_dirty(bh);
1944 		}
1945 		clear_buffer_new(bh);
1946 	}
1947 
1948 	/*
1949 	 * If this is a partial write which happened to make all buffers
1950 	 * uptodate then we can optimize away a bogus readpage() for
1951 	 * the next read(). Here we 'discover' whether the page went
1952 	 * uptodate as a result of this (potentially partial) write.
1953 	 */
1954 	if (!partial)
1955 		SetPageUptodate(page);
1956 	return 0;
1957 }
1958 
1959 /*
1960  * block_write_begin takes care of the basic task of block allocation and
1961  * bringing partial write blocks uptodate first.
1962  *
1963  * If *pagep is not NULL, then block_write_begin uses the locked page
1964  * at *pagep rather than allocating its own. In this case, the page will
1965  * not be unlocked or deallocated on failure.
1966  */
1967 int block_write_begin(struct file *file, struct address_space *mapping,
1968 			loff_t pos, unsigned len, unsigned flags,
1969 			struct page **pagep, void **fsdata,
1970 			get_block_t *get_block)
1971 {
1972 	struct inode *inode = mapping->host;
1973 	int status = 0;
1974 	struct page *page;
1975 	pgoff_t index;
1976 	unsigned start, end;
1977 	int ownpage = 0;
1978 
1979 	index = pos >> PAGE_CACHE_SHIFT;
1980 	start = pos & (PAGE_CACHE_SIZE - 1);
1981 	end = start + len;
1982 
1983 	page = *pagep;
1984 	if (page == NULL) {
1985 		ownpage = 1;
1986 		page = grab_cache_page_write_begin(mapping, index, flags);
1987 		if (!page) {
1988 			status = -ENOMEM;
1989 			goto out;
1990 		}
1991 		*pagep = page;
1992 	} else
1993 		BUG_ON(!PageLocked(page));
1994 
1995 	status = __block_prepare_write(inode, page, start, end, get_block);
1996 	if (unlikely(status)) {
1997 		ClearPageUptodate(page);
1998 
1999 		if (ownpage) {
2000 			unlock_page(page);
2001 			page_cache_release(page);
2002 			*pagep = NULL;
2003 
2004 			/*
2005 			 * prepare_write() may have instantiated a few blocks
2006 			 * outside i_size.  Trim these off again. Don't need
2007 			 * i_size_read because we hold i_mutex.
2008 			 */
2009 			if (pos + len > inode->i_size)
2010 				vmtruncate(inode, inode->i_size);
2011 		}
2012 	}
2013 
2014 out:
2015 	return status;
2016 }
2017 EXPORT_SYMBOL(block_write_begin);
2018 
2019 int block_write_end(struct file *file, struct address_space *mapping,
2020 			loff_t pos, unsigned len, unsigned copied,
2021 			struct page *page, void *fsdata)
2022 {
2023 	struct inode *inode = mapping->host;
2024 	unsigned start;
2025 
2026 	start = pos & (PAGE_CACHE_SIZE - 1);
2027 
2028 	if (unlikely(copied < len)) {
2029 		/*
2030 		 * The buffers that were written will now be uptodate, so we
2031 		 * don't have to worry about a readpage reading them and
2032 		 * overwriting a partial write. However if we have encountered
2033 		 * a short write and only partially written into a buffer, it
2034 		 * will not be marked uptodate, so a readpage might come in and
2035 		 * destroy our partial write.
2036 		 *
2037 		 * Do the simplest thing, and just treat any short write to a
2038 		 * non uptodate page as a zero-length write, and force the
2039 		 * caller to redo the whole thing.
2040 		 */
2041 		if (!PageUptodate(page))
2042 			copied = 0;
2043 
2044 		page_zero_new_buffers(page, start+copied, start+len);
2045 	}
2046 	flush_dcache_page(page);
2047 
2048 	/* This could be a short (even 0-length) commit */
2049 	__block_commit_write(inode, page, start, start+copied);
2050 
2051 	return copied;
2052 }
2053 EXPORT_SYMBOL(block_write_end);
2054 
2055 int generic_write_end(struct file *file, struct address_space *mapping,
2056 			loff_t pos, unsigned len, unsigned copied,
2057 			struct page *page, void *fsdata)
2058 {
2059 	struct inode *inode = mapping->host;
2060 	int i_size_changed = 0;
2061 
2062 	copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2063 
2064 	/*
2065 	 * No need to use i_size_read() here, the i_size
2066 	 * cannot change under us because we hold i_mutex.
2067 	 *
2068 	 * But it's important to update i_size while still holding page lock:
2069 	 * page writeout could otherwise come in and zero beyond i_size.
2070 	 */
2071 	if (pos+copied > inode->i_size) {
2072 		i_size_write(inode, pos+copied);
2073 		i_size_changed = 1;
2074 	}
2075 
2076 	unlock_page(page);
2077 	page_cache_release(page);
2078 
2079 	/*
2080 	 * Don't mark the inode dirty under page lock. First, it unnecessarily
2081 	 * makes the holding time of page lock longer. Second, it forces lock
2082 	 * ordering of page lock and transaction start for journaling
2083 	 * filesystems.
2084 	 */
2085 	if (i_size_changed)
2086 		mark_inode_dirty(inode);
2087 
2088 	return copied;
2089 }
2090 EXPORT_SYMBOL(generic_write_end);
2091 
2092 /*
2093  * block_is_partially_uptodate checks whether buffers within a page are
2094  * uptodate or not.
2095  *
2096  * Returns true if all buffers which correspond to a file portion
2097  * we want to read are uptodate.
2098  */
2099 int block_is_partially_uptodate(struct page *page, read_descriptor_t *desc,
2100 					unsigned long from)
2101 {
2102 	struct inode *inode = page->mapping->host;
2103 	unsigned block_start, block_end, blocksize;
2104 	unsigned to;
2105 	struct buffer_head *bh, *head;
2106 	int ret = 1;
2107 
2108 	if (!page_has_buffers(page))
2109 		return 0;
2110 
2111 	blocksize = 1 << inode->i_blkbits;
2112 	to = min_t(unsigned, PAGE_CACHE_SIZE - from, desc->count);
2113 	to = from + to;
2114 	if (from < blocksize && to > PAGE_CACHE_SIZE - blocksize)
2115 		return 0;
2116 
2117 	head = page_buffers(page);
2118 	bh = head;
2119 	block_start = 0;
2120 	do {
2121 		block_end = block_start + blocksize;
2122 		if (block_end > from && block_start < to) {
2123 			if (!buffer_uptodate(bh)) {
2124 				ret = 0;
2125 				break;
2126 			}
2127 			if (block_end >= to)
2128 				break;
2129 		}
2130 		block_start = block_end;
2131 		bh = bh->b_this_page;
2132 	} while (bh != head);
2133 
2134 	return ret;
2135 }
2136 EXPORT_SYMBOL(block_is_partially_uptodate);
2137 
2138 /*
2139  * Generic "read page" function for block devices that have the normal
2140  * get_block functionality. This is most of the block device filesystems.
2141  * Reads the page asynchronously --- the unlock_buffer() and
2142  * set/clear_buffer_uptodate() functions propagate buffer state into the
2143  * page struct once IO has completed.
2144  */
2145 int block_read_full_page(struct page *page, get_block_t *get_block)
2146 {
2147 	struct inode *inode = page->mapping->host;
2148 	sector_t iblock, lblock;
2149 	struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2150 	unsigned int blocksize;
2151 	int nr, i;
2152 	int fully_mapped = 1;
2153 
2154 	BUG_ON(!PageLocked(page));
2155 	blocksize = 1 << inode->i_blkbits;
2156 	if (!page_has_buffers(page))
2157 		create_empty_buffers(page, blocksize, 0);
2158 	head = page_buffers(page);
2159 
2160 	iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2161 	lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2162 	bh = head;
2163 	nr = 0;
2164 	i = 0;
2165 
2166 	do {
2167 		if (buffer_uptodate(bh))
2168 			continue;
2169 
2170 		if (!buffer_mapped(bh)) {
2171 			int err = 0;
2172 
2173 			fully_mapped = 0;
2174 			if (iblock < lblock) {
2175 				WARN_ON(bh->b_size != blocksize);
2176 				err = get_block(inode, iblock, bh, 0);
2177 				if (err)
2178 					SetPageError(page);
2179 			}
2180 			if (!buffer_mapped(bh)) {
2181 				zero_user(page, i * blocksize, blocksize);
2182 				if (!err)
2183 					set_buffer_uptodate(bh);
2184 				continue;
2185 			}
2186 			/*
2187 			 * get_block() might have updated the buffer
2188 			 * synchronously
2189 			 */
2190 			if (buffer_uptodate(bh))
2191 				continue;
2192 		}
2193 		arr[nr++] = bh;
2194 	} while (i++, iblock++, (bh = bh->b_this_page) != head);
2195 
2196 	if (fully_mapped)
2197 		SetPageMappedToDisk(page);
2198 
2199 	if (!nr) {
2200 		/*
2201 		 * All buffers are uptodate - we can set the page uptodate
2202 		 * as well. But not if get_block() returned an error.
2203 		 */
2204 		if (!PageError(page))
2205 			SetPageUptodate(page);
2206 		unlock_page(page);
2207 		return 0;
2208 	}
2209 
2210 	/* Stage two: lock the buffers */
2211 	for (i = 0; i < nr; i++) {
2212 		bh = arr[i];
2213 		lock_buffer(bh);
2214 		mark_buffer_async_read(bh);
2215 	}
2216 
2217 	/*
2218 	 * Stage 3: start the IO.  Check for uptodateness
2219 	 * inside the buffer lock in case another process reading
2220 	 * the underlying blockdev brought it uptodate (the sct fix).
2221 	 */
2222 	for (i = 0; i < nr; i++) {
2223 		bh = arr[i];
2224 		if (buffer_uptodate(bh))
2225 			end_buffer_async_read(bh, 1);
2226 		else
2227 			submit_bh(READ, bh);
2228 	}
2229 	return 0;
2230 }
2231 EXPORT_SYMBOL(block_read_full_page);
2232 
2233 /* utility function for filesystems that need to do work on expanding
2234  * truncates.  Uses filesystem pagecache writes to allow the filesystem to
2235  * deal with the hole.
2236  */
2237 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2238 {
2239 	struct address_space *mapping = inode->i_mapping;
2240 	struct page *page;
2241 	void *fsdata;
2242 	int err;
2243 
2244 	err = inode_newsize_ok(inode, size);
2245 	if (err)
2246 		goto out;
2247 
2248 	err = pagecache_write_begin(NULL, mapping, size, 0,
2249 				AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
2250 				&page, &fsdata);
2251 	if (err)
2252 		goto out;
2253 
2254 	err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2255 	BUG_ON(err > 0);
2256 
2257 out:
2258 	return err;
2259 }
2260 EXPORT_SYMBOL(generic_cont_expand_simple);
2261 
2262 static int cont_expand_zero(struct file *file, struct address_space *mapping,
2263 			    loff_t pos, loff_t *bytes)
2264 {
2265 	struct inode *inode = mapping->host;
2266 	unsigned blocksize = 1 << inode->i_blkbits;
2267 	struct page *page;
2268 	void *fsdata;
2269 	pgoff_t index, curidx;
2270 	loff_t curpos;
2271 	unsigned zerofrom, offset, len;
2272 	int err = 0;
2273 
2274 	index = pos >> PAGE_CACHE_SHIFT;
2275 	offset = pos & ~PAGE_CACHE_MASK;
2276 
2277 	while (index > (curidx = (curpos = *bytes)>>PAGE_CACHE_SHIFT)) {
2278 		zerofrom = curpos & ~PAGE_CACHE_MASK;
2279 		if (zerofrom & (blocksize-1)) {
2280 			*bytes |= (blocksize-1);
2281 			(*bytes)++;
2282 		}
2283 		len = PAGE_CACHE_SIZE - zerofrom;
2284 
2285 		err = pagecache_write_begin(file, mapping, curpos, len,
2286 						AOP_FLAG_UNINTERRUPTIBLE,
2287 						&page, &fsdata);
2288 		if (err)
2289 			goto out;
2290 		zero_user(page, zerofrom, len);
2291 		err = pagecache_write_end(file, mapping, curpos, len, len,
2292 						page, fsdata);
2293 		if (err < 0)
2294 			goto out;
2295 		BUG_ON(err != len);
2296 		err = 0;
2297 
2298 		balance_dirty_pages_ratelimited(mapping);
2299 	}
2300 
2301 	/* page covers the boundary, find the boundary offset */
2302 	if (index == curidx) {
2303 		zerofrom = curpos & ~PAGE_CACHE_MASK;
2304 		/* if we will expand the thing last block will be filled */
2305 		if (offset <= zerofrom) {
2306 			goto out;
2307 		}
2308 		if (zerofrom & (blocksize-1)) {
2309 			*bytes |= (blocksize-1);
2310 			(*bytes)++;
2311 		}
2312 		len = offset - zerofrom;
2313 
2314 		err = pagecache_write_begin(file, mapping, curpos, len,
2315 						AOP_FLAG_UNINTERRUPTIBLE,
2316 						&page, &fsdata);
2317 		if (err)
2318 			goto out;
2319 		zero_user(page, zerofrom, len);
2320 		err = pagecache_write_end(file, mapping, curpos, len, len,
2321 						page, fsdata);
2322 		if (err < 0)
2323 			goto out;
2324 		BUG_ON(err != len);
2325 		err = 0;
2326 	}
2327 out:
2328 	return err;
2329 }
2330 
2331 /*
2332  * For moronic filesystems that do not allow holes in file.
2333  * We may have to extend the file.
2334  */
2335 int cont_write_begin(struct file *file, struct address_space *mapping,
2336 			loff_t pos, unsigned len, unsigned flags,
2337 			struct page **pagep, void **fsdata,
2338 			get_block_t *get_block, loff_t *bytes)
2339 {
2340 	struct inode *inode = mapping->host;
2341 	unsigned blocksize = 1 << inode->i_blkbits;
2342 	unsigned zerofrom;
2343 	int err;
2344 
2345 	err = cont_expand_zero(file, mapping, pos, bytes);
2346 	if (err)
2347 		goto out;
2348 
2349 	zerofrom = *bytes & ~PAGE_CACHE_MASK;
2350 	if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2351 		*bytes |= (blocksize-1);
2352 		(*bytes)++;
2353 	}
2354 
2355 	*pagep = NULL;
2356 	err = block_write_begin(file, mapping, pos, len,
2357 				flags, pagep, fsdata, get_block);
2358 out:
2359 	return err;
2360 }
2361 EXPORT_SYMBOL(cont_write_begin);
2362 
2363 int block_prepare_write(struct page *page, unsigned from, unsigned to,
2364 			get_block_t *get_block)
2365 {
2366 	struct inode *inode = page->mapping->host;
2367 	int err = __block_prepare_write(inode, page, from, to, get_block);
2368 	if (err)
2369 		ClearPageUptodate(page);
2370 	return err;
2371 }
2372 EXPORT_SYMBOL(block_prepare_write);
2373 
2374 int block_commit_write(struct page *page, unsigned from, unsigned to)
2375 {
2376 	struct inode *inode = page->mapping->host;
2377 	__block_commit_write(inode,page,from,to);
2378 	return 0;
2379 }
2380 EXPORT_SYMBOL(block_commit_write);
2381 
2382 /*
2383  * block_page_mkwrite() is not allowed to change the file size as it gets
2384  * called from a page fault handler when a page is first dirtied. Hence we must
2385  * be careful to check for EOF conditions here. We set the page up correctly
2386  * for a written page which means we get ENOSPC checking when writing into
2387  * holes and correct delalloc and unwritten extent mapping on filesystems that
2388  * support these features.
2389  *
2390  * We are not allowed to take the i_mutex here so we have to play games to
2391  * protect against truncate races as the page could now be beyond EOF.  Because
2392  * vmtruncate() writes the inode size before removing pages, once we have the
2393  * page lock we can determine safely if the page is beyond EOF. If it is not
2394  * beyond EOF, then the page is guaranteed safe against truncation until we
2395  * unlock the page.
2396  */
2397 int
2398 block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2399 		   get_block_t get_block)
2400 {
2401 	struct page *page = vmf->page;
2402 	struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
2403 	unsigned long end;
2404 	loff_t size;
2405 	int ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
2406 
2407 	lock_page(page);
2408 	size = i_size_read(inode);
2409 	if ((page->mapping != inode->i_mapping) ||
2410 	    (page_offset(page) > size)) {
2411 		/* page got truncated out from underneath us */
2412 		unlock_page(page);
2413 		goto out;
2414 	}
2415 
2416 	/* page is wholly or partially inside EOF */
2417 	if (((page->index + 1) << PAGE_CACHE_SHIFT) > size)
2418 		end = size & ~PAGE_CACHE_MASK;
2419 	else
2420 		end = PAGE_CACHE_SIZE;
2421 
2422 	ret = block_prepare_write(page, 0, end, get_block);
2423 	if (!ret)
2424 		ret = block_commit_write(page, 0, end);
2425 
2426 	if (unlikely(ret)) {
2427 		unlock_page(page);
2428 		if (ret == -ENOMEM)
2429 			ret = VM_FAULT_OOM;
2430 		else /* -ENOSPC, -EIO, etc */
2431 			ret = VM_FAULT_SIGBUS;
2432 	} else
2433 		ret = VM_FAULT_LOCKED;
2434 
2435 out:
2436 	return ret;
2437 }
2438 EXPORT_SYMBOL(block_page_mkwrite);
2439 
2440 /*
2441  * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2442  * immediately, while under the page lock.  So it needs a special end_io
2443  * handler which does not touch the bh after unlocking it.
2444  */
2445 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2446 {
2447 	__end_buffer_read_notouch(bh, uptodate);
2448 }
2449 
2450 /*
2451  * Attach the singly-linked list of buffers created by nobh_write_begin, to
2452  * the page (converting it to circular linked list and taking care of page
2453  * dirty races).
2454  */
2455 static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
2456 {
2457 	struct buffer_head *bh;
2458 
2459 	BUG_ON(!PageLocked(page));
2460 
2461 	spin_lock(&page->mapping->private_lock);
2462 	bh = head;
2463 	do {
2464 		if (PageDirty(page))
2465 			set_buffer_dirty(bh);
2466 		if (!bh->b_this_page)
2467 			bh->b_this_page = head;
2468 		bh = bh->b_this_page;
2469 	} while (bh != head);
2470 	attach_page_buffers(page, head);
2471 	spin_unlock(&page->mapping->private_lock);
2472 }
2473 
2474 /*
2475  * On entry, the page is fully not uptodate.
2476  * On exit the page is fully uptodate in the areas outside (from,to)
2477  */
2478 int nobh_write_begin(struct file *file, struct address_space *mapping,
2479 			loff_t pos, unsigned len, unsigned flags,
2480 			struct page **pagep, void **fsdata,
2481 			get_block_t *get_block)
2482 {
2483 	struct inode *inode = mapping->host;
2484 	const unsigned blkbits = inode->i_blkbits;
2485 	const unsigned blocksize = 1 << blkbits;
2486 	struct buffer_head *head, *bh;
2487 	struct page *page;
2488 	pgoff_t index;
2489 	unsigned from, to;
2490 	unsigned block_in_page;
2491 	unsigned block_start, block_end;
2492 	sector_t block_in_file;
2493 	int nr_reads = 0;
2494 	int ret = 0;
2495 	int is_mapped_to_disk = 1;
2496 
2497 	index = pos >> PAGE_CACHE_SHIFT;
2498 	from = pos & (PAGE_CACHE_SIZE - 1);
2499 	to = from + len;
2500 
2501 	page = grab_cache_page_write_begin(mapping, index, flags);
2502 	if (!page)
2503 		return -ENOMEM;
2504 	*pagep = page;
2505 	*fsdata = NULL;
2506 
2507 	if (page_has_buffers(page)) {
2508 		unlock_page(page);
2509 		page_cache_release(page);
2510 		*pagep = NULL;
2511 		return block_write_begin(file, mapping, pos, len, flags, pagep,
2512 					fsdata, get_block);
2513 	}
2514 
2515 	if (PageMappedToDisk(page))
2516 		return 0;
2517 
2518 	/*
2519 	 * Allocate buffers so that we can keep track of state, and potentially
2520 	 * attach them to the page if an error occurs. In the common case of
2521 	 * no error, they will just be freed again without ever being attached
2522 	 * to the page (which is all OK, because we're under the page lock).
2523 	 *
2524 	 * Be careful: the buffer linked list is a NULL terminated one, rather
2525 	 * than the circular one we're used to.
2526 	 */
2527 	head = alloc_page_buffers(page, blocksize, 0);
2528 	if (!head) {
2529 		ret = -ENOMEM;
2530 		goto out_release;
2531 	}
2532 
2533 	block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2534 
2535 	/*
2536 	 * We loop across all blocks in the page, whether or not they are
2537 	 * part of the affected region.  This is so we can discover if the
2538 	 * page is fully mapped-to-disk.
2539 	 */
2540 	for (block_start = 0, block_in_page = 0, bh = head;
2541 		  block_start < PAGE_CACHE_SIZE;
2542 		  block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
2543 		int create;
2544 
2545 		block_end = block_start + blocksize;
2546 		bh->b_state = 0;
2547 		create = 1;
2548 		if (block_start >= to)
2549 			create = 0;
2550 		ret = get_block(inode, block_in_file + block_in_page,
2551 					bh, create);
2552 		if (ret)
2553 			goto failed;
2554 		if (!buffer_mapped(bh))
2555 			is_mapped_to_disk = 0;
2556 		if (buffer_new(bh))
2557 			unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
2558 		if (PageUptodate(page)) {
2559 			set_buffer_uptodate(bh);
2560 			continue;
2561 		}
2562 		if (buffer_new(bh) || !buffer_mapped(bh)) {
2563 			zero_user_segments(page, block_start, from,
2564 							to, block_end);
2565 			continue;
2566 		}
2567 		if (buffer_uptodate(bh))
2568 			continue;	/* reiserfs does this */
2569 		if (block_start < from || block_end > to) {
2570 			lock_buffer(bh);
2571 			bh->b_end_io = end_buffer_read_nobh;
2572 			submit_bh(READ, bh);
2573 			nr_reads++;
2574 		}
2575 	}
2576 
2577 	if (nr_reads) {
2578 		/*
2579 		 * The page is locked, so these buffers are protected from
2580 		 * any VM or truncate activity.  Hence we don't need to care
2581 		 * for the buffer_head refcounts.
2582 		 */
2583 		for (bh = head; bh; bh = bh->b_this_page) {
2584 			wait_on_buffer(bh);
2585 			if (!buffer_uptodate(bh))
2586 				ret = -EIO;
2587 		}
2588 		if (ret)
2589 			goto failed;
2590 	}
2591 
2592 	if (is_mapped_to_disk)
2593 		SetPageMappedToDisk(page);
2594 
2595 	*fsdata = head; /* to be released by nobh_write_end */
2596 
2597 	return 0;
2598 
2599 failed:
2600 	BUG_ON(!ret);
2601 	/*
2602 	 * Error recovery is a bit difficult. We need to zero out blocks that
2603 	 * were newly allocated, and dirty them to ensure they get written out.
2604 	 * Buffers need to be attached to the page at this point, otherwise
2605 	 * the handling of potential IO errors during writeout would be hard
2606 	 * (could try doing synchronous writeout, but what if that fails too?)
2607 	 */
2608 	attach_nobh_buffers(page, head);
2609 	page_zero_new_buffers(page, from, to);
2610 
2611 out_release:
2612 	unlock_page(page);
2613 	page_cache_release(page);
2614 	*pagep = NULL;
2615 
2616 	if (pos + len > inode->i_size)
2617 		vmtruncate(inode, inode->i_size);
2618 
2619 	return ret;
2620 }
2621 EXPORT_SYMBOL(nobh_write_begin);
2622 
2623 int nobh_write_end(struct file *file, struct address_space *mapping,
2624 			loff_t pos, unsigned len, unsigned copied,
2625 			struct page *page, void *fsdata)
2626 {
2627 	struct inode *inode = page->mapping->host;
2628 	struct buffer_head *head = fsdata;
2629 	struct buffer_head *bh;
2630 	BUG_ON(fsdata != NULL && page_has_buffers(page));
2631 
2632 	if (unlikely(copied < len) && head)
2633 		attach_nobh_buffers(page, head);
2634 	if (page_has_buffers(page))
2635 		return generic_write_end(file, mapping, pos, len,
2636 					copied, page, fsdata);
2637 
2638 	SetPageUptodate(page);
2639 	set_page_dirty(page);
2640 	if (pos+copied > inode->i_size) {
2641 		i_size_write(inode, pos+copied);
2642 		mark_inode_dirty(inode);
2643 	}
2644 
2645 	unlock_page(page);
2646 	page_cache_release(page);
2647 
2648 	while (head) {
2649 		bh = head;
2650 		head = head->b_this_page;
2651 		free_buffer_head(bh);
2652 	}
2653 
2654 	return copied;
2655 }
2656 EXPORT_SYMBOL(nobh_write_end);
2657 
2658 /*
2659  * nobh_writepage() - based on block_full_write_page() except
2660  * that it tries to operate without attaching bufferheads to
2661  * the page.
2662  */
2663 int nobh_writepage(struct page *page, get_block_t *get_block,
2664 			struct writeback_control *wbc)
2665 {
2666 	struct inode * const inode = page->mapping->host;
2667 	loff_t i_size = i_size_read(inode);
2668 	const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2669 	unsigned offset;
2670 	int ret;
2671 
2672 	/* Is the page fully inside i_size? */
2673 	if (page->index < end_index)
2674 		goto out;
2675 
2676 	/* Is the page fully outside i_size? (truncate in progress) */
2677 	offset = i_size & (PAGE_CACHE_SIZE-1);
2678 	if (page->index >= end_index+1 || !offset) {
2679 		/*
2680 		 * The page may have dirty, unmapped buffers.  For example,
2681 		 * they may have been added in ext3_writepage().  Make them
2682 		 * freeable here, so the page does not leak.
2683 		 */
2684 #if 0
2685 		/* Not really sure about this  - do we need this ? */
2686 		if (page->mapping->a_ops->invalidatepage)
2687 			page->mapping->a_ops->invalidatepage(page, offset);
2688 #endif
2689 		unlock_page(page);
2690 		return 0; /* don't care */
2691 	}
2692 
2693 	/*
2694 	 * The page straddles i_size.  It must be zeroed out on each and every
2695 	 * writepage invocation because it may be mmapped.  "A file is mapped
2696 	 * in multiples of the page size.  For a file that is not a multiple of
2697 	 * the  page size, the remaining memory is zeroed when mapped, and
2698 	 * writes to that region are not written out to the file."
2699 	 */
2700 	zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2701 out:
2702 	ret = mpage_writepage(page, get_block, wbc);
2703 	if (ret == -EAGAIN)
2704 		ret = __block_write_full_page(inode, page, get_block, wbc,
2705 					      end_buffer_async_write);
2706 	return ret;
2707 }
2708 EXPORT_SYMBOL(nobh_writepage);
2709 
2710 int nobh_truncate_page(struct address_space *mapping,
2711 			loff_t from, get_block_t *get_block)
2712 {
2713 	pgoff_t index = from >> PAGE_CACHE_SHIFT;
2714 	unsigned offset = from & (PAGE_CACHE_SIZE-1);
2715 	unsigned blocksize;
2716 	sector_t iblock;
2717 	unsigned length, pos;
2718 	struct inode *inode = mapping->host;
2719 	struct page *page;
2720 	struct buffer_head map_bh;
2721 	int err;
2722 
2723 	blocksize = 1 << inode->i_blkbits;
2724 	length = offset & (blocksize - 1);
2725 
2726 	/* Block boundary? Nothing to do */
2727 	if (!length)
2728 		return 0;
2729 
2730 	length = blocksize - length;
2731 	iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2732 
2733 	page = grab_cache_page(mapping, index);
2734 	err = -ENOMEM;
2735 	if (!page)
2736 		goto out;
2737 
2738 	if (page_has_buffers(page)) {
2739 has_buffers:
2740 		unlock_page(page);
2741 		page_cache_release(page);
2742 		return block_truncate_page(mapping, from, get_block);
2743 	}
2744 
2745 	/* Find the buffer that contains "offset" */
2746 	pos = blocksize;
2747 	while (offset >= pos) {
2748 		iblock++;
2749 		pos += blocksize;
2750 	}
2751 
2752 	map_bh.b_size = blocksize;
2753 	map_bh.b_state = 0;
2754 	err = get_block(inode, iblock, &map_bh, 0);
2755 	if (err)
2756 		goto unlock;
2757 	/* unmapped? It's a hole - nothing to do */
2758 	if (!buffer_mapped(&map_bh))
2759 		goto unlock;
2760 
2761 	/* Ok, it's mapped. Make sure it's up-to-date */
2762 	if (!PageUptodate(page)) {
2763 		err = mapping->a_ops->readpage(NULL, page);
2764 		if (err) {
2765 			page_cache_release(page);
2766 			goto out;
2767 		}
2768 		lock_page(page);
2769 		if (!PageUptodate(page)) {
2770 			err = -EIO;
2771 			goto unlock;
2772 		}
2773 		if (page_has_buffers(page))
2774 			goto has_buffers;
2775 	}
2776 	zero_user(page, offset, length);
2777 	set_page_dirty(page);
2778 	err = 0;
2779 
2780 unlock:
2781 	unlock_page(page);
2782 	page_cache_release(page);
2783 out:
2784 	return err;
2785 }
2786 EXPORT_SYMBOL(nobh_truncate_page);
2787 
2788 int block_truncate_page(struct address_space *mapping,
2789 			loff_t from, get_block_t *get_block)
2790 {
2791 	pgoff_t index = from >> PAGE_CACHE_SHIFT;
2792 	unsigned offset = from & (PAGE_CACHE_SIZE-1);
2793 	unsigned blocksize;
2794 	sector_t iblock;
2795 	unsigned length, pos;
2796 	struct inode *inode = mapping->host;
2797 	struct page *page;
2798 	struct buffer_head *bh;
2799 	int err;
2800 
2801 	blocksize = 1 << inode->i_blkbits;
2802 	length = offset & (blocksize - 1);
2803 
2804 	/* Block boundary? Nothing to do */
2805 	if (!length)
2806 		return 0;
2807 
2808 	length = blocksize - length;
2809 	iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2810 
2811 	page = grab_cache_page(mapping, index);
2812 	err = -ENOMEM;
2813 	if (!page)
2814 		goto out;
2815 
2816 	if (!page_has_buffers(page))
2817 		create_empty_buffers(page, blocksize, 0);
2818 
2819 	/* Find the buffer that contains "offset" */
2820 	bh = page_buffers(page);
2821 	pos = blocksize;
2822 	while (offset >= pos) {
2823 		bh = bh->b_this_page;
2824 		iblock++;
2825 		pos += blocksize;
2826 	}
2827 
2828 	err = 0;
2829 	if (!buffer_mapped(bh)) {
2830 		WARN_ON(bh->b_size != blocksize);
2831 		err = get_block(inode, iblock, bh, 0);
2832 		if (err)
2833 			goto unlock;
2834 		/* unmapped? It's a hole - nothing to do */
2835 		if (!buffer_mapped(bh))
2836 			goto unlock;
2837 	}
2838 
2839 	/* Ok, it's mapped. Make sure it's up-to-date */
2840 	if (PageUptodate(page))
2841 		set_buffer_uptodate(bh);
2842 
2843 	if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2844 		err = -EIO;
2845 		ll_rw_block(READ, 1, &bh);
2846 		wait_on_buffer(bh);
2847 		/* Uhhuh. Read error. Complain and punt. */
2848 		if (!buffer_uptodate(bh))
2849 			goto unlock;
2850 	}
2851 
2852 	zero_user(page, offset, length);
2853 	mark_buffer_dirty(bh);
2854 	err = 0;
2855 
2856 unlock:
2857 	unlock_page(page);
2858 	page_cache_release(page);
2859 out:
2860 	return err;
2861 }
2862 EXPORT_SYMBOL(block_truncate_page);
2863 
2864 /*
2865  * The generic ->writepage function for buffer-backed address_spaces
2866  * this form passes in the end_io handler used to finish the IO.
2867  */
2868 int block_write_full_page_endio(struct page *page, get_block_t *get_block,
2869 			struct writeback_control *wbc, bh_end_io_t *handler)
2870 {
2871 	struct inode * const inode = page->mapping->host;
2872 	loff_t i_size = i_size_read(inode);
2873 	const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2874 	unsigned offset;
2875 
2876 	/* Is the page fully inside i_size? */
2877 	if (page->index < end_index)
2878 		return __block_write_full_page(inode, page, get_block, wbc,
2879 					       handler);
2880 
2881 	/* Is the page fully outside i_size? (truncate in progress) */
2882 	offset = i_size & (PAGE_CACHE_SIZE-1);
2883 	if (page->index >= end_index+1 || !offset) {
2884 		/*
2885 		 * The page may have dirty, unmapped buffers.  For example,
2886 		 * they may have been added in ext3_writepage().  Make them
2887 		 * freeable here, so the page does not leak.
2888 		 */
2889 		do_invalidatepage(page, 0);
2890 		unlock_page(page);
2891 		return 0; /* don't care */
2892 	}
2893 
2894 	/*
2895 	 * The page straddles i_size.  It must be zeroed out on each and every
2896 	 * writepage invokation because it may be mmapped.  "A file is mapped
2897 	 * in multiples of the page size.  For a file that is not a multiple of
2898 	 * the  page size, the remaining memory is zeroed when mapped, and
2899 	 * writes to that region are not written out to the file."
2900 	 */
2901 	zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2902 	return __block_write_full_page(inode, page, get_block, wbc, handler);
2903 }
2904 EXPORT_SYMBOL(block_write_full_page_endio);
2905 
2906 /*
2907  * The generic ->writepage function for buffer-backed address_spaces
2908  */
2909 int block_write_full_page(struct page *page, get_block_t *get_block,
2910 			struct writeback_control *wbc)
2911 {
2912 	return block_write_full_page_endio(page, get_block, wbc,
2913 					   end_buffer_async_write);
2914 }
2915 EXPORT_SYMBOL(block_write_full_page);
2916 
2917 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2918 			    get_block_t *get_block)
2919 {
2920 	struct buffer_head tmp;
2921 	struct inode *inode = mapping->host;
2922 	tmp.b_state = 0;
2923 	tmp.b_blocknr = 0;
2924 	tmp.b_size = 1 << inode->i_blkbits;
2925 	get_block(inode, block, &tmp, 0);
2926 	return tmp.b_blocknr;
2927 }
2928 EXPORT_SYMBOL(generic_block_bmap);
2929 
2930 static void end_bio_bh_io_sync(struct bio *bio, int err)
2931 {
2932 	struct buffer_head *bh = bio->bi_private;
2933 
2934 	if (err == -EOPNOTSUPP) {
2935 		set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2936 		set_bit(BH_Eopnotsupp, &bh->b_state);
2937 	}
2938 
2939 	if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags)))
2940 		set_bit(BH_Quiet, &bh->b_state);
2941 
2942 	bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2943 	bio_put(bio);
2944 }
2945 
2946 int submit_bh(int rw, struct buffer_head * bh)
2947 {
2948 	struct bio *bio;
2949 	int ret = 0;
2950 
2951 	BUG_ON(!buffer_locked(bh));
2952 	BUG_ON(!buffer_mapped(bh));
2953 	BUG_ON(!bh->b_end_io);
2954 	BUG_ON(buffer_delay(bh));
2955 	BUG_ON(buffer_unwritten(bh));
2956 
2957 	/*
2958 	 * Mask in barrier bit for a write (could be either a WRITE or a
2959 	 * WRITE_SYNC
2960 	 */
2961 	if (buffer_ordered(bh) && (rw & WRITE))
2962 		rw |= WRITE_BARRIER;
2963 
2964 	/*
2965 	 * Only clear out a write error when rewriting
2966 	 */
2967 	if (test_set_buffer_req(bh) && (rw & WRITE))
2968 		clear_buffer_write_io_error(bh);
2969 
2970 	/*
2971 	 * from here on down, it's all bio -- do the initial mapping,
2972 	 * submit_bio -> generic_make_request may further map this bio around
2973 	 */
2974 	bio = bio_alloc(GFP_NOIO, 1);
2975 
2976 	bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2977 	bio->bi_bdev = bh->b_bdev;
2978 	bio->bi_io_vec[0].bv_page = bh->b_page;
2979 	bio->bi_io_vec[0].bv_len = bh->b_size;
2980 	bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2981 
2982 	bio->bi_vcnt = 1;
2983 	bio->bi_idx = 0;
2984 	bio->bi_size = bh->b_size;
2985 
2986 	bio->bi_end_io = end_bio_bh_io_sync;
2987 	bio->bi_private = bh;
2988 
2989 	bio_get(bio);
2990 	submit_bio(rw, bio);
2991 
2992 	if (bio_flagged(bio, BIO_EOPNOTSUPP))
2993 		ret = -EOPNOTSUPP;
2994 
2995 	bio_put(bio);
2996 	return ret;
2997 }
2998 EXPORT_SYMBOL(submit_bh);
2999 
3000 /**
3001  * ll_rw_block: low-level access to block devices (DEPRECATED)
3002  * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
3003  * @nr: number of &struct buffer_heads in the array
3004  * @bhs: array of pointers to &struct buffer_head
3005  *
3006  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3007  * requests an I/O operation on them, either a %READ or a %WRITE.  The third
3008  * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
3009  * are sent to disk. The fourth %READA option is described in the documentation
3010  * for generic_make_request() which ll_rw_block() calls.
3011  *
3012  * This function drops any buffer that it cannot get a lock on (with the
3013  * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
3014  * clean when doing a write request, and any buffer that appears to be
3015  * up-to-date when doing read request.  Further it marks as clean buffers that
3016  * are processed for writing (the buffer cache won't assume that they are
3017  * actually clean until the buffer gets unlocked).
3018  *
3019  * ll_rw_block sets b_end_io to simple completion handler that marks
3020  * the buffer up-to-date (if approriate), unlocks the buffer and wakes
3021  * any waiters.
3022  *
3023  * All of the buffers must be for the same device, and must also be a
3024  * multiple of the current approved size for the device.
3025  */
3026 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
3027 {
3028 	int i;
3029 
3030 	for (i = 0; i < nr; i++) {
3031 		struct buffer_head *bh = bhs[i];
3032 
3033 		if (rw == SWRITE || rw == SWRITE_SYNC || rw == SWRITE_SYNC_PLUG)
3034 			lock_buffer(bh);
3035 		else if (!trylock_buffer(bh))
3036 			continue;
3037 
3038 		if (rw == WRITE || rw == SWRITE || rw == SWRITE_SYNC ||
3039 		    rw == SWRITE_SYNC_PLUG) {
3040 			if (test_clear_buffer_dirty(bh)) {
3041 				bh->b_end_io = end_buffer_write_sync;
3042 				get_bh(bh);
3043 				if (rw == SWRITE_SYNC)
3044 					submit_bh(WRITE_SYNC, bh);
3045 				else
3046 					submit_bh(WRITE, bh);
3047 				continue;
3048 			}
3049 		} else {
3050 			if (!buffer_uptodate(bh)) {
3051 				bh->b_end_io = end_buffer_read_sync;
3052 				get_bh(bh);
3053 				submit_bh(rw, bh);
3054 				continue;
3055 			}
3056 		}
3057 		unlock_buffer(bh);
3058 	}
3059 }
3060 EXPORT_SYMBOL(ll_rw_block);
3061 
3062 /*
3063  * For a data-integrity writeout, we need to wait upon any in-progress I/O
3064  * and then start new I/O and then wait upon it.  The caller must have a ref on
3065  * the buffer_head.
3066  */
3067 int sync_dirty_buffer(struct buffer_head *bh)
3068 {
3069 	int ret = 0;
3070 
3071 	WARN_ON(atomic_read(&bh->b_count) < 1);
3072 	lock_buffer(bh);
3073 	if (test_clear_buffer_dirty(bh)) {
3074 		get_bh(bh);
3075 		bh->b_end_io = end_buffer_write_sync;
3076 		ret = submit_bh(WRITE_SYNC, bh);
3077 		wait_on_buffer(bh);
3078 		if (buffer_eopnotsupp(bh)) {
3079 			clear_buffer_eopnotsupp(bh);
3080 			ret = -EOPNOTSUPP;
3081 		}
3082 		if (!ret && !buffer_uptodate(bh))
3083 			ret = -EIO;
3084 	} else {
3085 		unlock_buffer(bh);
3086 	}
3087 	return ret;
3088 }
3089 EXPORT_SYMBOL(sync_dirty_buffer);
3090 
3091 /*
3092  * try_to_free_buffers() checks if all the buffers on this particular page
3093  * are unused, and releases them if so.
3094  *
3095  * Exclusion against try_to_free_buffers may be obtained by either
3096  * locking the page or by holding its mapping's private_lock.
3097  *
3098  * If the page is dirty but all the buffers are clean then we need to
3099  * be sure to mark the page clean as well.  This is because the page
3100  * may be against a block device, and a later reattachment of buffers
3101  * to a dirty page will set *all* buffers dirty.  Which would corrupt
3102  * filesystem data on the same device.
3103  *
3104  * The same applies to regular filesystem pages: if all the buffers are
3105  * clean then we set the page clean and proceed.  To do that, we require
3106  * total exclusion from __set_page_dirty_buffers().  That is obtained with
3107  * private_lock.
3108  *
3109  * try_to_free_buffers() is non-blocking.
3110  */
3111 static inline int buffer_busy(struct buffer_head *bh)
3112 {
3113 	return atomic_read(&bh->b_count) |
3114 		(bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
3115 }
3116 
3117 static int
3118 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
3119 {
3120 	struct buffer_head *head = page_buffers(page);
3121 	struct buffer_head *bh;
3122 
3123 	bh = head;
3124 	do {
3125 		if (buffer_write_io_error(bh) && page->mapping)
3126 			set_bit(AS_EIO, &page->mapping->flags);
3127 		if (buffer_busy(bh))
3128 			goto failed;
3129 		bh = bh->b_this_page;
3130 	} while (bh != head);
3131 
3132 	do {
3133 		struct buffer_head *next = bh->b_this_page;
3134 
3135 		if (bh->b_assoc_map)
3136 			__remove_assoc_queue(bh);
3137 		bh = next;
3138 	} while (bh != head);
3139 	*buffers_to_free = head;
3140 	__clear_page_buffers(page);
3141 	return 1;
3142 failed:
3143 	return 0;
3144 }
3145 
3146 int try_to_free_buffers(struct page *page)
3147 {
3148 	struct address_space * const mapping = page->mapping;
3149 	struct buffer_head *buffers_to_free = NULL;
3150 	int ret = 0;
3151 
3152 	BUG_ON(!PageLocked(page));
3153 	if (PageWriteback(page))
3154 		return 0;
3155 
3156 	if (mapping == NULL) {		/* can this still happen? */
3157 		ret = drop_buffers(page, &buffers_to_free);
3158 		goto out;
3159 	}
3160 
3161 	spin_lock(&mapping->private_lock);
3162 	ret = drop_buffers(page, &buffers_to_free);
3163 
3164 	/*
3165 	 * If the filesystem writes its buffers by hand (eg ext3)
3166 	 * then we can have clean buffers against a dirty page.  We
3167 	 * clean the page here; otherwise the VM will never notice
3168 	 * that the filesystem did any IO at all.
3169 	 *
3170 	 * Also, during truncate, discard_buffer will have marked all
3171 	 * the page's buffers clean.  We discover that here and clean
3172 	 * the page also.
3173 	 *
3174 	 * private_lock must be held over this entire operation in order
3175 	 * to synchronise against __set_page_dirty_buffers and prevent the
3176 	 * dirty bit from being lost.
3177 	 */
3178 	if (ret)
3179 		cancel_dirty_page(page, PAGE_CACHE_SIZE);
3180 	spin_unlock(&mapping->private_lock);
3181 out:
3182 	if (buffers_to_free) {
3183 		struct buffer_head *bh = buffers_to_free;
3184 
3185 		do {
3186 			struct buffer_head *next = bh->b_this_page;
3187 			free_buffer_head(bh);
3188 			bh = next;
3189 		} while (bh != buffers_to_free);
3190 	}
3191 	return ret;
3192 }
3193 EXPORT_SYMBOL(try_to_free_buffers);
3194 
3195 void block_sync_page(struct page *page)
3196 {
3197 	struct address_space *mapping;
3198 
3199 	smp_mb();
3200 	mapping = page_mapping(page);
3201 	if (mapping)
3202 		blk_run_backing_dev(mapping->backing_dev_info, page);
3203 }
3204 EXPORT_SYMBOL(block_sync_page);
3205 
3206 /*
3207  * There are no bdflush tunables left.  But distributions are
3208  * still running obsolete flush daemons, so we terminate them here.
3209  *
3210  * Use of bdflush() is deprecated and will be removed in a future kernel.
3211  * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3212  */
3213 SYSCALL_DEFINE2(bdflush, int, func, long, data)
3214 {
3215 	static int msg_count;
3216 
3217 	if (!capable(CAP_SYS_ADMIN))
3218 		return -EPERM;
3219 
3220 	if (msg_count < 5) {
3221 		msg_count++;
3222 		printk(KERN_INFO
3223 			"warning: process `%s' used the obsolete bdflush"
3224 			" system call\n", current->comm);
3225 		printk(KERN_INFO "Fix your initscripts?\n");
3226 	}
3227 
3228 	if (func == 1)
3229 		do_exit(0);
3230 	return 0;
3231 }
3232 
3233 /*
3234  * Buffer-head allocation
3235  */
3236 static struct kmem_cache *bh_cachep;
3237 
3238 /*
3239  * Once the number of bh's in the machine exceeds this level, we start
3240  * stripping them in writeback.
3241  */
3242 static int max_buffer_heads;
3243 
3244 int buffer_heads_over_limit;
3245 
3246 struct bh_accounting {
3247 	int nr;			/* Number of live bh's */
3248 	int ratelimit;		/* Limit cacheline bouncing */
3249 };
3250 
3251 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3252 
3253 static void recalc_bh_state(void)
3254 {
3255 	int i;
3256 	int tot = 0;
3257 
3258 	if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3259 		return;
3260 	__get_cpu_var(bh_accounting).ratelimit = 0;
3261 	for_each_online_cpu(i)
3262 		tot += per_cpu(bh_accounting, i).nr;
3263 	buffer_heads_over_limit = (tot > max_buffer_heads);
3264 }
3265 
3266 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3267 {
3268 	struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3269 	if (ret) {
3270 		INIT_LIST_HEAD(&ret->b_assoc_buffers);
3271 		get_cpu_var(bh_accounting).nr++;
3272 		recalc_bh_state();
3273 		put_cpu_var(bh_accounting);
3274 	}
3275 	return ret;
3276 }
3277 EXPORT_SYMBOL(alloc_buffer_head);
3278 
3279 void free_buffer_head(struct buffer_head *bh)
3280 {
3281 	BUG_ON(!list_empty(&bh->b_assoc_buffers));
3282 	kmem_cache_free(bh_cachep, bh);
3283 	get_cpu_var(bh_accounting).nr--;
3284 	recalc_bh_state();
3285 	put_cpu_var(bh_accounting);
3286 }
3287 EXPORT_SYMBOL(free_buffer_head);
3288 
3289 static void buffer_exit_cpu(int cpu)
3290 {
3291 	int i;
3292 	struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3293 
3294 	for (i = 0; i < BH_LRU_SIZE; i++) {
3295 		brelse(b->bhs[i]);
3296 		b->bhs[i] = NULL;
3297 	}
3298 	get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr;
3299 	per_cpu(bh_accounting, cpu).nr = 0;
3300 	put_cpu_var(bh_accounting);
3301 }
3302 
3303 static int buffer_cpu_notify(struct notifier_block *self,
3304 			      unsigned long action, void *hcpu)
3305 {
3306 	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
3307 		buffer_exit_cpu((unsigned long)hcpu);
3308 	return NOTIFY_OK;
3309 }
3310 
3311 /**
3312  * bh_uptodate_or_lock - Test whether the buffer is uptodate
3313  * @bh: struct buffer_head
3314  *
3315  * Return true if the buffer is up-to-date and false,
3316  * with the buffer locked, if not.
3317  */
3318 int bh_uptodate_or_lock(struct buffer_head *bh)
3319 {
3320 	if (!buffer_uptodate(bh)) {
3321 		lock_buffer(bh);
3322 		if (!buffer_uptodate(bh))
3323 			return 0;
3324 		unlock_buffer(bh);
3325 	}
3326 	return 1;
3327 }
3328 EXPORT_SYMBOL(bh_uptodate_or_lock);
3329 
3330 /**
3331  * bh_submit_read - Submit a locked buffer for reading
3332  * @bh: struct buffer_head
3333  *
3334  * Returns zero on success and -EIO on error.
3335  */
3336 int bh_submit_read(struct buffer_head *bh)
3337 {
3338 	BUG_ON(!buffer_locked(bh));
3339 
3340 	if (buffer_uptodate(bh)) {
3341 		unlock_buffer(bh);
3342 		return 0;
3343 	}
3344 
3345 	get_bh(bh);
3346 	bh->b_end_io = end_buffer_read_sync;
3347 	submit_bh(READ, bh);
3348 	wait_on_buffer(bh);
3349 	if (buffer_uptodate(bh))
3350 		return 0;
3351 	return -EIO;
3352 }
3353 EXPORT_SYMBOL(bh_submit_read);
3354 
3355 static void
3356 init_buffer_head(void *data)
3357 {
3358 	struct buffer_head *bh = data;
3359 
3360 	memset(bh, 0, sizeof(*bh));
3361 	INIT_LIST_HEAD(&bh->b_assoc_buffers);
3362 }
3363 
3364 void __init buffer_init(void)
3365 {
3366 	int nrpages;
3367 
3368 	bh_cachep = kmem_cache_create("buffer_head",
3369 			sizeof(struct buffer_head), 0,
3370 				(SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3371 				SLAB_MEM_SPREAD),
3372 				init_buffer_head);
3373 
3374 	/*
3375 	 * Limit the bh occupancy to 10% of ZONE_NORMAL
3376 	 */
3377 	nrpages = (nr_free_buffer_pages() * 10) / 100;
3378 	max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3379 	hotcpu_notifier(buffer_cpu_notify, 0);
3380 }
3381