1 /* 2 * linux/mm/filemap.c 3 * 4 * Copyright (C) 1994-1999 Linus Torvalds 5 */ 6 7 /* 8 * This file handles the generic file mmap semantics used by 9 * most "normal" filesystems (but you don't /have/ to use this: 10 * the NFS filesystem used to do this differently, for example) 11 */ 12 #include <linux/module.h> 13 #include <linux/compiler.h> 14 #include <linux/fs.h> 15 #include <linux/uaccess.h> 16 #include <linux/aio.h> 17 #include <linux/capability.h> 18 #include <linux/kernel_stat.h> 19 #include <linux/gfp.h> 20 #include <linux/mm.h> 21 #include <linux/swap.h> 22 #include <linux/mman.h> 23 #include <linux/pagemap.h> 24 #include <linux/file.h> 25 #include <linux/uio.h> 26 #include <linux/hash.h> 27 #include <linux/writeback.h> 28 #include <linux/backing-dev.h> 29 #include <linux/pagevec.h> 30 #include <linux/blkdev.h> 31 #include <linux/security.h> 32 #include <linux/syscalls.h> 33 #include <linux/cpuset.h> 34 #include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */ 35 #include <linux/memcontrol.h> 36 #include <linux/cleancache.h> 37 #include "internal.h" 38 39 /* 40 * FIXME: remove all knowledge of the buffer layer from the core VM 41 */ 42 #include <linux/buffer_head.h> /* for try_to_free_buffers */ 43 44 #include <asm/mman.h> 45 46 /* 47 * Shared mappings implemented 30.11.1994. It's not fully working yet, 48 * though. 49 * 50 * Shared mappings now work. 15.8.1995 Bruno. 51 * 52 * finished 'unifying' the page and buffer cache and SMP-threaded the 53 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com> 54 * 55 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de> 56 */ 57 58 /* 59 * Lock ordering: 60 * 61 * ->i_mmap_mutex (truncate_pagecache) 62 * ->private_lock (__free_pte->__set_page_dirty_buffers) 63 * ->swap_lock (exclusive_swap_page, others) 64 * ->mapping->tree_lock 65 * 66 * ->i_mutex 67 * ->i_mmap_mutex (truncate->unmap_mapping_range) 68 * 69 * ->mmap_sem 70 * ->i_mmap_mutex 71 * ->page_table_lock or pte_lock (various, mainly in memory.c) 72 * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock) 73 * 74 * ->mmap_sem 75 * ->lock_page (access_process_vm) 76 * 77 * ->i_mutex (generic_file_buffered_write) 78 * ->mmap_sem (fault_in_pages_readable->do_page_fault) 79 * 80 * bdi->wb.list_lock 81 * sb_lock (fs/fs-writeback.c) 82 * ->mapping->tree_lock (__sync_single_inode) 83 * 84 * ->i_mmap_mutex 85 * ->anon_vma.lock (vma_adjust) 86 * 87 * ->anon_vma.lock 88 * ->page_table_lock or pte_lock (anon_vma_prepare and various) 89 * 90 * ->page_table_lock or pte_lock 91 * ->swap_lock (try_to_unmap_one) 92 * ->private_lock (try_to_unmap_one) 93 * ->tree_lock (try_to_unmap_one) 94 * ->zone.lru_lock (follow_page->mark_page_accessed) 95 * ->zone.lru_lock (check_pte_range->isolate_lru_page) 96 * ->private_lock (page_remove_rmap->set_page_dirty) 97 * ->tree_lock (page_remove_rmap->set_page_dirty) 98 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) 99 * ->inode->i_lock (page_remove_rmap->set_page_dirty) 100 * bdi.wb->list_lock (zap_pte_range->set_page_dirty) 101 * ->inode->i_lock (zap_pte_range->set_page_dirty) 102 * ->private_lock (zap_pte_range->__set_page_dirty_buffers) 103 * 104 * (code doesn't rely on that order, so you could switch it around) 105 * ->tasklist_lock (memory_failure, collect_procs_ao) 106 * ->i_mmap_mutex 107 */ 108 109 /* 110 * Delete a page from the page cache and free it. Caller has to make 111 * sure the page is locked and that nobody else uses it - or that usage 112 * is safe. The caller must hold the mapping's tree_lock. 113 */ 114 void __delete_from_page_cache(struct page *page) 115 { 116 struct address_space *mapping = page->mapping; 117 118 /* 119 * if we're uptodate, flush out into the cleancache, otherwise 120 * invalidate any existing cleancache entries. We can't leave 121 * stale data around in the cleancache once our page is gone 122 */ 123 if (PageUptodate(page) && PageMappedToDisk(page)) 124 cleancache_put_page(page); 125 else 126 cleancache_flush_page(mapping, page); 127 128 radix_tree_delete(&mapping->page_tree, page->index); 129 page->mapping = NULL; 130 /* Leave page->index set: truncation lookup relies upon it */ 131 mapping->nrpages--; 132 __dec_zone_page_state(page, NR_FILE_PAGES); 133 if (PageSwapBacked(page)) 134 __dec_zone_page_state(page, NR_SHMEM); 135 BUG_ON(page_mapped(page)); 136 137 /* 138 * Some filesystems seem to re-dirty the page even after 139 * the VM has canceled the dirty bit (eg ext3 journaling). 140 * 141 * Fix it up by doing a final dirty accounting check after 142 * having removed the page entirely. 143 */ 144 if (PageDirty(page) && mapping_cap_account_dirty(mapping)) { 145 dec_zone_page_state(page, NR_FILE_DIRTY); 146 dec_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE); 147 } 148 } 149 150 /** 151 * delete_from_page_cache - delete page from page cache 152 * @page: the page which the kernel is trying to remove from page cache 153 * 154 * This must be called only on pages that have been verified to be in the page 155 * cache and locked. It will never put the page into the free list, the caller 156 * has a reference on the page. 157 */ 158 void delete_from_page_cache(struct page *page) 159 { 160 struct address_space *mapping = page->mapping; 161 void (*freepage)(struct page *); 162 163 BUG_ON(!PageLocked(page)); 164 165 freepage = mapping->a_ops->freepage; 166 spin_lock_irq(&mapping->tree_lock); 167 __delete_from_page_cache(page); 168 spin_unlock_irq(&mapping->tree_lock); 169 mem_cgroup_uncharge_cache_page(page); 170 171 if (freepage) 172 freepage(page); 173 page_cache_release(page); 174 } 175 EXPORT_SYMBOL(delete_from_page_cache); 176 177 static int sleep_on_page(void *word) 178 { 179 io_schedule(); 180 return 0; 181 } 182 183 static int sleep_on_page_killable(void *word) 184 { 185 sleep_on_page(word); 186 return fatal_signal_pending(current) ? -EINTR : 0; 187 } 188 189 /** 190 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range 191 * @mapping: address space structure to write 192 * @start: offset in bytes where the range starts 193 * @end: offset in bytes where the range ends (inclusive) 194 * @sync_mode: enable synchronous operation 195 * 196 * Start writeback against all of a mapping's dirty pages that lie 197 * within the byte offsets <start, end> inclusive. 198 * 199 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as 200 * opposed to a regular memory cleansing writeback. The difference between 201 * these two operations is that if a dirty page/buffer is encountered, it must 202 * be waited upon, and not just skipped over. 203 */ 204 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 205 loff_t end, int sync_mode) 206 { 207 int ret; 208 struct writeback_control wbc = { 209 .sync_mode = sync_mode, 210 .nr_to_write = LONG_MAX, 211 .range_start = start, 212 .range_end = end, 213 }; 214 215 if (!mapping_cap_writeback_dirty(mapping)) 216 return 0; 217 218 ret = do_writepages(mapping, &wbc); 219 return ret; 220 } 221 222 static inline int __filemap_fdatawrite(struct address_space *mapping, 223 int sync_mode) 224 { 225 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode); 226 } 227 228 int filemap_fdatawrite(struct address_space *mapping) 229 { 230 return __filemap_fdatawrite(mapping, WB_SYNC_ALL); 231 } 232 EXPORT_SYMBOL(filemap_fdatawrite); 233 234 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 235 loff_t end) 236 { 237 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); 238 } 239 EXPORT_SYMBOL(filemap_fdatawrite_range); 240 241 /** 242 * filemap_flush - mostly a non-blocking flush 243 * @mapping: target address_space 244 * 245 * This is a mostly non-blocking flush. Not suitable for data-integrity 246 * purposes - I/O may not be started against all dirty pages. 247 */ 248 int filemap_flush(struct address_space *mapping) 249 { 250 return __filemap_fdatawrite(mapping, WB_SYNC_NONE); 251 } 252 EXPORT_SYMBOL(filemap_flush); 253 254 /** 255 * filemap_fdatawait_range - wait for writeback to complete 256 * @mapping: address space structure to wait for 257 * @start_byte: offset in bytes where the range starts 258 * @end_byte: offset in bytes where the range ends (inclusive) 259 * 260 * Walk the list of under-writeback pages of the given address space 261 * in the given range and wait for all of them. 262 */ 263 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte, 264 loff_t end_byte) 265 { 266 pgoff_t index = start_byte >> PAGE_CACHE_SHIFT; 267 pgoff_t end = end_byte >> PAGE_CACHE_SHIFT; 268 struct pagevec pvec; 269 int nr_pages; 270 int ret = 0; 271 272 if (end_byte < start_byte) 273 return 0; 274 275 pagevec_init(&pvec, 0); 276 while ((index <= end) && 277 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, 278 PAGECACHE_TAG_WRITEBACK, 279 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) { 280 unsigned i; 281 282 for (i = 0; i < nr_pages; i++) { 283 struct page *page = pvec.pages[i]; 284 285 /* until radix tree lookup accepts end_index */ 286 if (page->index > end) 287 continue; 288 289 wait_on_page_writeback(page); 290 if (TestClearPageError(page)) 291 ret = -EIO; 292 } 293 pagevec_release(&pvec); 294 cond_resched(); 295 } 296 297 /* Check for outstanding write errors */ 298 if (test_and_clear_bit(AS_ENOSPC, &mapping->flags)) 299 ret = -ENOSPC; 300 if (test_and_clear_bit(AS_EIO, &mapping->flags)) 301 ret = -EIO; 302 303 return ret; 304 } 305 EXPORT_SYMBOL(filemap_fdatawait_range); 306 307 /** 308 * filemap_fdatawait - wait for all under-writeback pages to complete 309 * @mapping: address space structure to wait for 310 * 311 * Walk the list of under-writeback pages of the given address space 312 * and wait for all of them. 313 */ 314 int filemap_fdatawait(struct address_space *mapping) 315 { 316 loff_t i_size = i_size_read(mapping->host); 317 318 if (i_size == 0) 319 return 0; 320 321 return filemap_fdatawait_range(mapping, 0, i_size - 1); 322 } 323 EXPORT_SYMBOL(filemap_fdatawait); 324 325 int filemap_write_and_wait(struct address_space *mapping) 326 { 327 int err = 0; 328 329 if (mapping->nrpages) { 330 err = filemap_fdatawrite(mapping); 331 /* 332 * Even if the above returned error, the pages may be 333 * written partially (e.g. -ENOSPC), so we wait for it. 334 * But the -EIO is special case, it may indicate the worst 335 * thing (e.g. bug) happened, so we avoid waiting for it. 336 */ 337 if (err != -EIO) { 338 int err2 = filemap_fdatawait(mapping); 339 if (!err) 340 err = err2; 341 } 342 } 343 return err; 344 } 345 EXPORT_SYMBOL(filemap_write_and_wait); 346 347 /** 348 * filemap_write_and_wait_range - write out & wait on a file range 349 * @mapping: the address_space for the pages 350 * @lstart: offset in bytes where the range starts 351 * @lend: offset in bytes where the range ends (inclusive) 352 * 353 * Write out and wait upon file offsets lstart->lend, inclusive. 354 * 355 * Note that `lend' is inclusive (describes the last byte to be written) so 356 * that this function can be used to write to the very end-of-file (end = -1). 357 */ 358 int filemap_write_and_wait_range(struct address_space *mapping, 359 loff_t lstart, loff_t lend) 360 { 361 int err = 0; 362 363 if (mapping->nrpages) { 364 err = __filemap_fdatawrite_range(mapping, lstart, lend, 365 WB_SYNC_ALL); 366 /* See comment of filemap_write_and_wait() */ 367 if (err != -EIO) { 368 int err2 = filemap_fdatawait_range(mapping, 369 lstart, lend); 370 if (!err) 371 err = err2; 372 } 373 } 374 return err; 375 } 376 EXPORT_SYMBOL(filemap_write_and_wait_range); 377 378 /** 379 * replace_page_cache_page - replace a pagecache page with a new one 380 * @old: page to be replaced 381 * @new: page to replace with 382 * @gfp_mask: allocation mode 383 * 384 * This function replaces a page in the pagecache with a new one. On 385 * success it acquires the pagecache reference for the new page and 386 * drops it for the old page. Both the old and new pages must be 387 * locked. This function does not add the new page to the LRU, the 388 * caller must do that. 389 * 390 * The remove + add is atomic. The only way this function can fail is 391 * memory allocation failure. 392 */ 393 int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) 394 { 395 int error; 396 struct mem_cgroup *memcg = NULL; 397 398 VM_BUG_ON(!PageLocked(old)); 399 VM_BUG_ON(!PageLocked(new)); 400 VM_BUG_ON(new->mapping); 401 402 /* 403 * This is not page migration, but prepare_migration and 404 * end_migration does enough work for charge replacement. 405 * 406 * In the longer term we probably want a specialized function 407 * for moving the charge from old to new in a more efficient 408 * manner. 409 */ 410 error = mem_cgroup_prepare_migration(old, new, &memcg, gfp_mask); 411 if (error) 412 return error; 413 414 error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM); 415 if (!error) { 416 struct address_space *mapping = old->mapping; 417 void (*freepage)(struct page *); 418 419 pgoff_t offset = old->index; 420 freepage = mapping->a_ops->freepage; 421 422 page_cache_get(new); 423 new->mapping = mapping; 424 new->index = offset; 425 426 spin_lock_irq(&mapping->tree_lock); 427 __delete_from_page_cache(old); 428 error = radix_tree_insert(&mapping->page_tree, offset, new); 429 BUG_ON(error); 430 mapping->nrpages++; 431 __inc_zone_page_state(new, NR_FILE_PAGES); 432 if (PageSwapBacked(new)) 433 __inc_zone_page_state(new, NR_SHMEM); 434 spin_unlock_irq(&mapping->tree_lock); 435 radix_tree_preload_end(); 436 if (freepage) 437 freepage(old); 438 page_cache_release(old); 439 mem_cgroup_end_migration(memcg, old, new, true); 440 } else { 441 mem_cgroup_end_migration(memcg, old, new, false); 442 } 443 444 return error; 445 } 446 EXPORT_SYMBOL_GPL(replace_page_cache_page); 447 448 /** 449 * add_to_page_cache_locked - add a locked page to the pagecache 450 * @page: page to add 451 * @mapping: the page's address_space 452 * @offset: page index 453 * @gfp_mask: page allocation mode 454 * 455 * This function is used to add a page to the pagecache. It must be locked. 456 * This function does not add the page to the LRU. The caller must do that. 457 */ 458 int add_to_page_cache_locked(struct page *page, struct address_space *mapping, 459 pgoff_t offset, gfp_t gfp_mask) 460 { 461 int error; 462 463 VM_BUG_ON(!PageLocked(page)); 464 VM_BUG_ON(PageSwapBacked(page)); 465 466 error = mem_cgroup_cache_charge(page, current->mm, 467 gfp_mask & GFP_RECLAIM_MASK); 468 if (error) 469 goto out; 470 471 error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM); 472 if (error == 0) { 473 page_cache_get(page); 474 page->mapping = mapping; 475 page->index = offset; 476 477 spin_lock_irq(&mapping->tree_lock); 478 error = radix_tree_insert(&mapping->page_tree, offset, page); 479 if (likely(!error)) { 480 mapping->nrpages++; 481 __inc_zone_page_state(page, NR_FILE_PAGES); 482 spin_unlock_irq(&mapping->tree_lock); 483 } else { 484 page->mapping = NULL; 485 /* Leave page->index set: truncation relies upon it */ 486 spin_unlock_irq(&mapping->tree_lock); 487 mem_cgroup_uncharge_cache_page(page); 488 page_cache_release(page); 489 } 490 radix_tree_preload_end(); 491 } else 492 mem_cgroup_uncharge_cache_page(page); 493 out: 494 return error; 495 } 496 EXPORT_SYMBOL(add_to_page_cache_locked); 497 498 int add_to_page_cache_lru(struct page *page, struct address_space *mapping, 499 pgoff_t offset, gfp_t gfp_mask) 500 { 501 int ret; 502 503 ret = add_to_page_cache(page, mapping, offset, gfp_mask); 504 if (ret == 0) 505 lru_cache_add_file(page); 506 return ret; 507 } 508 EXPORT_SYMBOL_GPL(add_to_page_cache_lru); 509 510 #ifdef CONFIG_NUMA 511 struct page *__page_cache_alloc(gfp_t gfp) 512 { 513 int n; 514 struct page *page; 515 516 if (cpuset_do_page_mem_spread()) { 517 get_mems_allowed(); 518 n = cpuset_mem_spread_node(); 519 page = alloc_pages_exact_node(n, gfp, 0); 520 put_mems_allowed(); 521 return page; 522 } 523 return alloc_pages(gfp, 0); 524 } 525 EXPORT_SYMBOL(__page_cache_alloc); 526 #endif 527 528 /* 529 * In order to wait for pages to become available there must be 530 * waitqueues associated with pages. By using a hash table of 531 * waitqueues where the bucket discipline is to maintain all 532 * waiters on the same queue and wake all when any of the pages 533 * become available, and for the woken contexts to check to be 534 * sure the appropriate page became available, this saves space 535 * at a cost of "thundering herd" phenomena during rare hash 536 * collisions. 537 */ 538 static wait_queue_head_t *page_waitqueue(struct page *page) 539 { 540 const struct zone *zone = page_zone(page); 541 542 return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)]; 543 } 544 545 static inline void wake_up_page(struct page *page, int bit) 546 { 547 __wake_up_bit(page_waitqueue(page), &page->flags, bit); 548 } 549 550 void wait_on_page_bit(struct page *page, int bit_nr) 551 { 552 DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); 553 554 if (test_bit(bit_nr, &page->flags)) 555 __wait_on_bit(page_waitqueue(page), &wait, sleep_on_page, 556 TASK_UNINTERRUPTIBLE); 557 } 558 EXPORT_SYMBOL(wait_on_page_bit); 559 560 int wait_on_page_bit_killable(struct page *page, int bit_nr) 561 { 562 DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); 563 564 if (!test_bit(bit_nr, &page->flags)) 565 return 0; 566 567 return __wait_on_bit(page_waitqueue(page), &wait, 568 sleep_on_page_killable, TASK_KILLABLE); 569 } 570 571 /** 572 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue 573 * @page: Page defining the wait queue of interest 574 * @waiter: Waiter to add to the queue 575 * 576 * Add an arbitrary @waiter to the wait queue for the nominated @page. 577 */ 578 void add_page_wait_queue(struct page *page, wait_queue_t *waiter) 579 { 580 wait_queue_head_t *q = page_waitqueue(page); 581 unsigned long flags; 582 583 spin_lock_irqsave(&q->lock, flags); 584 __add_wait_queue(q, waiter); 585 spin_unlock_irqrestore(&q->lock, flags); 586 } 587 EXPORT_SYMBOL_GPL(add_page_wait_queue); 588 589 /** 590 * unlock_page - unlock a locked page 591 * @page: the page 592 * 593 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked(). 594 * Also wakes sleepers in wait_on_page_writeback() because the wakeup 595 * mechananism between PageLocked pages and PageWriteback pages is shared. 596 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep. 597 * 598 * The mb is necessary to enforce ordering between the clear_bit and the read 599 * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()). 600 */ 601 void unlock_page(struct page *page) 602 { 603 VM_BUG_ON(!PageLocked(page)); 604 clear_bit_unlock(PG_locked, &page->flags); 605 smp_mb__after_clear_bit(); 606 wake_up_page(page, PG_locked); 607 } 608 EXPORT_SYMBOL(unlock_page); 609 610 /** 611 * end_page_writeback - end writeback against a page 612 * @page: the page 613 */ 614 void end_page_writeback(struct page *page) 615 { 616 if (TestClearPageReclaim(page)) 617 rotate_reclaimable_page(page); 618 619 if (!test_clear_page_writeback(page)) 620 BUG(); 621 622 smp_mb__after_clear_bit(); 623 wake_up_page(page, PG_writeback); 624 } 625 EXPORT_SYMBOL(end_page_writeback); 626 627 /** 628 * __lock_page - get a lock on the page, assuming we need to sleep to get it 629 * @page: the page to lock 630 */ 631 void __lock_page(struct page *page) 632 { 633 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); 634 635 __wait_on_bit_lock(page_waitqueue(page), &wait, sleep_on_page, 636 TASK_UNINTERRUPTIBLE); 637 } 638 EXPORT_SYMBOL(__lock_page); 639 640 int __lock_page_killable(struct page *page) 641 { 642 DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); 643 644 return __wait_on_bit_lock(page_waitqueue(page), &wait, 645 sleep_on_page_killable, TASK_KILLABLE); 646 } 647 EXPORT_SYMBOL_GPL(__lock_page_killable); 648 649 int __lock_page_or_retry(struct page *page, struct mm_struct *mm, 650 unsigned int flags) 651 { 652 if (flags & FAULT_FLAG_ALLOW_RETRY) { 653 /* 654 * CAUTION! In this case, mmap_sem is not released 655 * even though return 0. 656 */ 657 if (flags & FAULT_FLAG_RETRY_NOWAIT) 658 return 0; 659 660 up_read(&mm->mmap_sem); 661 if (flags & FAULT_FLAG_KILLABLE) 662 wait_on_page_locked_killable(page); 663 else 664 wait_on_page_locked(page); 665 return 0; 666 } else { 667 if (flags & FAULT_FLAG_KILLABLE) { 668 int ret; 669 670 ret = __lock_page_killable(page); 671 if (ret) { 672 up_read(&mm->mmap_sem); 673 return 0; 674 } 675 } else 676 __lock_page(page); 677 return 1; 678 } 679 } 680 681 /** 682 * find_get_page - find and get a page reference 683 * @mapping: the address_space to search 684 * @offset: the page index 685 * 686 * Is there a pagecache struct page at the given (mapping, offset) tuple? 687 * If yes, increment its refcount and return it; if no, return NULL. 688 */ 689 struct page *find_get_page(struct address_space *mapping, pgoff_t offset) 690 { 691 void **pagep; 692 struct page *page; 693 694 rcu_read_lock(); 695 repeat: 696 page = NULL; 697 pagep = radix_tree_lookup_slot(&mapping->page_tree, offset); 698 if (pagep) { 699 page = radix_tree_deref_slot(pagep); 700 if (unlikely(!page)) 701 goto out; 702 if (radix_tree_exception(page)) { 703 if (radix_tree_deref_retry(page)) 704 goto repeat; 705 /* 706 * Otherwise, shmem/tmpfs must be storing a swap entry 707 * here as an exceptional entry: so return it without 708 * attempting to raise page count. 709 */ 710 goto out; 711 } 712 if (!page_cache_get_speculative(page)) 713 goto repeat; 714 715 /* 716 * Has the page moved? 717 * This is part of the lockless pagecache protocol. See 718 * include/linux/pagemap.h for details. 719 */ 720 if (unlikely(page != *pagep)) { 721 page_cache_release(page); 722 goto repeat; 723 } 724 } 725 out: 726 rcu_read_unlock(); 727 728 return page; 729 } 730 EXPORT_SYMBOL(find_get_page); 731 732 /** 733 * find_lock_page - locate, pin and lock a pagecache page 734 * @mapping: the address_space to search 735 * @offset: the page index 736 * 737 * Locates the desired pagecache page, locks it, increments its reference 738 * count and returns its address. 739 * 740 * Returns zero if the page was not present. find_lock_page() may sleep. 741 */ 742 struct page *find_lock_page(struct address_space *mapping, pgoff_t offset) 743 { 744 struct page *page; 745 746 repeat: 747 page = find_get_page(mapping, offset); 748 if (page && !radix_tree_exception(page)) { 749 lock_page(page); 750 /* Has the page been truncated? */ 751 if (unlikely(page->mapping != mapping)) { 752 unlock_page(page); 753 page_cache_release(page); 754 goto repeat; 755 } 756 VM_BUG_ON(page->index != offset); 757 } 758 return page; 759 } 760 EXPORT_SYMBOL(find_lock_page); 761 762 /** 763 * find_or_create_page - locate or add a pagecache page 764 * @mapping: the page's address_space 765 * @index: the page's index into the mapping 766 * @gfp_mask: page allocation mode 767 * 768 * Locates a page in the pagecache. If the page is not present, a new page 769 * is allocated using @gfp_mask and is added to the pagecache and to the VM's 770 * LRU list. The returned page is locked and has its reference count 771 * incremented. 772 * 773 * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic 774 * allocation! 775 * 776 * find_or_create_page() returns the desired page's address, or zero on 777 * memory exhaustion. 778 */ 779 struct page *find_or_create_page(struct address_space *mapping, 780 pgoff_t index, gfp_t gfp_mask) 781 { 782 struct page *page; 783 int err; 784 repeat: 785 page = find_lock_page(mapping, index); 786 if (!page) { 787 page = __page_cache_alloc(gfp_mask); 788 if (!page) 789 return NULL; 790 /* 791 * We want a regular kernel memory (not highmem or DMA etc) 792 * allocation for the radix tree nodes, but we need to honour 793 * the context-specific requirements the caller has asked for. 794 * GFP_RECLAIM_MASK collects those requirements. 795 */ 796 err = add_to_page_cache_lru(page, mapping, index, 797 (gfp_mask & GFP_RECLAIM_MASK)); 798 if (unlikely(err)) { 799 page_cache_release(page); 800 page = NULL; 801 if (err == -EEXIST) 802 goto repeat; 803 } 804 } 805 return page; 806 } 807 EXPORT_SYMBOL(find_or_create_page); 808 809 /** 810 * find_get_pages - gang pagecache lookup 811 * @mapping: The address_space to search 812 * @start: The starting page index 813 * @nr_pages: The maximum number of pages 814 * @pages: Where the resulting pages are placed 815 * 816 * find_get_pages() will search for and return a group of up to 817 * @nr_pages pages in the mapping. The pages are placed at @pages. 818 * find_get_pages() takes a reference against the returned pages. 819 * 820 * The search returns a group of mapping-contiguous pages with ascending 821 * indexes. There may be holes in the indices due to not-present pages. 822 * 823 * find_get_pages() returns the number of pages which were found. 824 */ 825 unsigned find_get_pages(struct address_space *mapping, pgoff_t start, 826 unsigned int nr_pages, struct page **pages) 827 { 828 unsigned int i; 829 unsigned int ret; 830 unsigned int nr_found; 831 832 rcu_read_lock(); 833 restart: 834 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree, 835 (void ***)pages, NULL, start, nr_pages); 836 ret = 0; 837 for (i = 0; i < nr_found; i++) { 838 struct page *page; 839 repeat: 840 page = radix_tree_deref_slot((void **)pages[i]); 841 if (unlikely(!page)) 842 continue; 843 844 if (radix_tree_exception(page)) { 845 if (radix_tree_deref_retry(page)) { 846 /* 847 * Transient condition which can only trigger 848 * when entry at index 0 moves out of or back 849 * to root: none yet gotten, safe to restart. 850 */ 851 WARN_ON(start | i); 852 goto restart; 853 } 854 /* 855 * Otherwise, shmem/tmpfs must be storing a swap entry 856 * here as an exceptional entry: so skip over it - 857 * we only reach this from invalidate_mapping_pages(). 858 */ 859 continue; 860 } 861 862 if (!page_cache_get_speculative(page)) 863 goto repeat; 864 865 /* Has the page moved? */ 866 if (unlikely(page != *((void **)pages[i]))) { 867 page_cache_release(page); 868 goto repeat; 869 } 870 871 pages[ret] = page; 872 ret++; 873 } 874 875 /* 876 * If all entries were removed before we could secure them, 877 * try again, because callers stop trying once 0 is returned. 878 */ 879 if (unlikely(!ret && nr_found)) 880 goto restart; 881 rcu_read_unlock(); 882 return ret; 883 } 884 885 /** 886 * find_get_pages_contig - gang contiguous pagecache lookup 887 * @mapping: The address_space to search 888 * @index: The starting page index 889 * @nr_pages: The maximum number of pages 890 * @pages: Where the resulting pages are placed 891 * 892 * find_get_pages_contig() works exactly like find_get_pages(), except 893 * that the returned number of pages are guaranteed to be contiguous. 894 * 895 * find_get_pages_contig() returns the number of pages which were found. 896 */ 897 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, 898 unsigned int nr_pages, struct page **pages) 899 { 900 unsigned int i; 901 unsigned int ret; 902 unsigned int nr_found; 903 904 rcu_read_lock(); 905 restart: 906 nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree, 907 (void ***)pages, NULL, index, nr_pages); 908 ret = 0; 909 for (i = 0; i < nr_found; i++) { 910 struct page *page; 911 repeat: 912 page = radix_tree_deref_slot((void **)pages[i]); 913 if (unlikely(!page)) 914 continue; 915 916 if (radix_tree_exception(page)) { 917 if (radix_tree_deref_retry(page)) { 918 /* 919 * Transient condition which can only trigger 920 * when entry at index 0 moves out of or back 921 * to root: none yet gotten, safe to restart. 922 */ 923 goto restart; 924 } 925 /* 926 * Otherwise, shmem/tmpfs must be storing a swap entry 927 * here as an exceptional entry: so stop looking for 928 * contiguous pages. 929 */ 930 break; 931 } 932 933 if (!page_cache_get_speculative(page)) 934 goto repeat; 935 936 /* Has the page moved? */ 937 if (unlikely(page != *((void **)pages[i]))) { 938 page_cache_release(page); 939 goto repeat; 940 } 941 942 /* 943 * must check mapping and index after taking the ref. 944 * otherwise we can get both false positives and false 945 * negatives, which is just confusing to the caller. 946 */ 947 if (page->mapping == NULL || page->index != index) { 948 page_cache_release(page); 949 break; 950 } 951 952 pages[ret] = page; 953 ret++; 954 index++; 955 } 956 rcu_read_unlock(); 957 return ret; 958 } 959 EXPORT_SYMBOL(find_get_pages_contig); 960 961 /** 962 * find_get_pages_tag - find and return pages that match @tag 963 * @mapping: the address_space to search 964 * @index: the starting page index 965 * @tag: the tag index 966 * @nr_pages: the maximum number of pages 967 * @pages: where the resulting pages are placed 968 * 969 * Like find_get_pages, except we only return pages which are tagged with 970 * @tag. We update @index to index the next page for the traversal. 971 */ 972 unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index, 973 int tag, unsigned int nr_pages, struct page **pages) 974 { 975 unsigned int i; 976 unsigned int ret; 977 unsigned int nr_found; 978 979 rcu_read_lock(); 980 restart: 981 nr_found = radix_tree_gang_lookup_tag_slot(&mapping->page_tree, 982 (void ***)pages, *index, nr_pages, tag); 983 ret = 0; 984 for (i = 0; i < nr_found; i++) { 985 struct page *page; 986 repeat: 987 page = radix_tree_deref_slot((void **)pages[i]); 988 if (unlikely(!page)) 989 continue; 990 991 if (radix_tree_exception(page)) { 992 if (radix_tree_deref_retry(page)) { 993 /* 994 * Transient condition which can only trigger 995 * when entry at index 0 moves out of or back 996 * to root: none yet gotten, safe to restart. 997 */ 998 goto restart; 999 } 1000 /* 1001 * This function is never used on a shmem/tmpfs 1002 * mapping, so a swap entry won't be found here. 1003 */ 1004 BUG(); 1005 } 1006 1007 if (!page_cache_get_speculative(page)) 1008 goto repeat; 1009 1010 /* Has the page moved? */ 1011 if (unlikely(page != *((void **)pages[i]))) { 1012 page_cache_release(page); 1013 goto repeat; 1014 } 1015 1016 pages[ret] = page; 1017 ret++; 1018 } 1019 1020 /* 1021 * If all entries were removed before we could secure them, 1022 * try again, because callers stop trying once 0 is returned. 1023 */ 1024 if (unlikely(!ret && nr_found)) 1025 goto restart; 1026 rcu_read_unlock(); 1027 1028 if (ret) 1029 *index = pages[ret - 1]->index + 1; 1030 1031 return ret; 1032 } 1033 EXPORT_SYMBOL(find_get_pages_tag); 1034 1035 /** 1036 * grab_cache_page_nowait - returns locked page at given index in given cache 1037 * @mapping: target address_space 1038 * @index: the page index 1039 * 1040 * Same as grab_cache_page(), but do not wait if the page is unavailable. 1041 * This is intended for speculative data generators, where the data can 1042 * be regenerated if the page couldn't be grabbed. This routine should 1043 * be safe to call while holding the lock for another page. 1044 * 1045 * Clear __GFP_FS when allocating the page to avoid recursion into the fs 1046 * and deadlock against the caller's locked page. 1047 */ 1048 struct page * 1049 grab_cache_page_nowait(struct address_space *mapping, pgoff_t index) 1050 { 1051 struct page *page = find_get_page(mapping, index); 1052 1053 if (page) { 1054 if (trylock_page(page)) 1055 return page; 1056 page_cache_release(page); 1057 return NULL; 1058 } 1059 page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~__GFP_FS); 1060 if (page && add_to_page_cache_lru(page, mapping, index, GFP_NOFS)) { 1061 page_cache_release(page); 1062 page = NULL; 1063 } 1064 return page; 1065 } 1066 EXPORT_SYMBOL(grab_cache_page_nowait); 1067 1068 /* 1069 * CD/DVDs are error prone. When a medium error occurs, the driver may fail 1070 * a _large_ part of the i/o request. Imagine the worst scenario: 1071 * 1072 * ---R__________________________________________B__________ 1073 * ^ reading here ^ bad block(assume 4k) 1074 * 1075 * read(R) => miss => readahead(R...B) => media error => frustrating retries 1076 * => failing the whole request => read(R) => read(R+1) => 1077 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) => 1078 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) => 1079 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ...... 1080 * 1081 * It is going insane. Fix it by quickly scaling down the readahead size. 1082 */ 1083 static void shrink_readahead_size_eio(struct file *filp, 1084 struct file_ra_state *ra) 1085 { 1086 ra->ra_pages /= 4; 1087 } 1088 1089 /** 1090 * do_generic_file_read - generic file read routine 1091 * @filp: the file to read 1092 * @ppos: current file position 1093 * @desc: read_descriptor 1094 * @actor: read method 1095 * 1096 * This is a generic file read routine, and uses the 1097 * mapping->a_ops->readpage() function for the actual low-level stuff. 1098 * 1099 * This is really ugly. But the goto's actually try to clarify some 1100 * of the logic when it comes to error handling etc. 1101 */ 1102 static void do_generic_file_read(struct file *filp, loff_t *ppos, 1103 read_descriptor_t *desc, read_actor_t actor) 1104 { 1105 struct address_space *mapping = filp->f_mapping; 1106 struct inode *inode = mapping->host; 1107 struct file_ra_state *ra = &filp->f_ra; 1108 pgoff_t index; 1109 pgoff_t last_index; 1110 pgoff_t prev_index; 1111 unsigned long offset; /* offset into pagecache page */ 1112 unsigned int prev_offset; 1113 int error; 1114 1115 index = *ppos >> PAGE_CACHE_SHIFT; 1116 prev_index = ra->prev_pos >> PAGE_CACHE_SHIFT; 1117 prev_offset = ra->prev_pos & (PAGE_CACHE_SIZE-1); 1118 last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT; 1119 offset = *ppos & ~PAGE_CACHE_MASK; 1120 1121 for (;;) { 1122 struct page *page; 1123 pgoff_t end_index; 1124 loff_t isize; 1125 unsigned long nr, ret; 1126 1127 cond_resched(); 1128 find_page: 1129 page = find_get_page(mapping, index); 1130 if (!page) { 1131 page_cache_sync_readahead(mapping, 1132 ra, filp, 1133 index, last_index - index); 1134 page = find_get_page(mapping, index); 1135 if (unlikely(page == NULL)) 1136 goto no_cached_page; 1137 } 1138 if (PageReadahead(page)) { 1139 page_cache_async_readahead(mapping, 1140 ra, filp, page, 1141 index, last_index - index); 1142 } 1143 if (!PageUptodate(page)) { 1144 if (inode->i_blkbits == PAGE_CACHE_SHIFT || 1145 !mapping->a_ops->is_partially_uptodate) 1146 goto page_not_up_to_date; 1147 if (!trylock_page(page)) 1148 goto page_not_up_to_date; 1149 /* Did it get truncated before we got the lock? */ 1150 if (!page->mapping) 1151 goto page_not_up_to_date_locked; 1152 if (!mapping->a_ops->is_partially_uptodate(page, 1153 desc, offset)) 1154 goto page_not_up_to_date_locked; 1155 unlock_page(page); 1156 } 1157 page_ok: 1158 /* 1159 * i_size must be checked after we know the page is Uptodate. 1160 * 1161 * Checking i_size after the check allows us to calculate 1162 * the correct value for "nr", which means the zero-filled 1163 * part of the page is not copied back to userspace (unless 1164 * another truncate extends the file - this is desired though). 1165 */ 1166 1167 isize = i_size_read(inode); 1168 end_index = (isize - 1) >> PAGE_CACHE_SHIFT; 1169 if (unlikely(!isize || index > end_index)) { 1170 page_cache_release(page); 1171 goto out; 1172 } 1173 1174 /* nr is the maximum number of bytes to copy from this page */ 1175 nr = PAGE_CACHE_SIZE; 1176 if (index == end_index) { 1177 nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1; 1178 if (nr <= offset) { 1179 page_cache_release(page); 1180 goto out; 1181 } 1182 } 1183 nr = nr - offset; 1184 1185 /* If users can be writing to this page using arbitrary 1186 * virtual addresses, take care about potential aliasing 1187 * before reading the page on the kernel side. 1188 */ 1189 if (mapping_writably_mapped(mapping)) 1190 flush_dcache_page(page); 1191 1192 /* 1193 * When a sequential read accesses a page several times, 1194 * only mark it as accessed the first time. 1195 */ 1196 if (prev_index != index || offset != prev_offset) 1197 mark_page_accessed(page); 1198 prev_index = index; 1199 1200 /* 1201 * Ok, we have the page, and it's up-to-date, so 1202 * now we can copy it to user space... 1203 * 1204 * The actor routine returns how many bytes were actually used.. 1205 * NOTE! This may not be the same as how much of a user buffer 1206 * we filled up (we may be padding etc), so we can only update 1207 * "pos" here (the actor routine has to update the user buffer 1208 * pointers and the remaining count). 1209 */ 1210 ret = actor(desc, page, offset, nr); 1211 offset += ret; 1212 index += offset >> PAGE_CACHE_SHIFT; 1213 offset &= ~PAGE_CACHE_MASK; 1214 prev_offset = offset; 1215 1216 page_cache_release(page); 1217 if (ret == nr && desc->count) 1218 continue; 1219 goto out; 1220 1221 page_not_up_to_date: 1222 /* Get exclusive access to the page ... */ 1223 error = lock_page_killable(page); 1224 if (unlikely(error)) 1225 goto readpage_error; 1226 1227 page_not_up_to_date_locked: 1228 /* Did it get truncated before we got the lock? */ 1229 if (!page->mapping) { 1230 unlock_page(page); 1231 page_cache_release(page); 1232 continue; 1233 } 1234 1235 /* Did somebody else fill it already? */ 1236 if (PageUptodate(page)) { 1237 unlock_page(page); 1238 goto page_ok; 1239 } 1240 1241 readpage: 1242 /* 1243 * A previous I/O error may have been due to temporary 1244 * failures, eg. multipath errors. 1245 * PG_error will be set again if readpage fails. 1246 */ 1247 ClearPageError(page); 1248 /* Start the actual read. The read will unlock the page. */ 1249 error = mapping->a_ops->readpage(filp, page); 1250 1251 if (unlikely(error)) { 1252 if (error == AOP_TRUNCATED_PAGE) { 1253 page_cache_release(page); 1254 goto find_page; 1255 } 1256 goto readpage_error; 1257 } 1258 1259 if (!PageUptodate(page)) { 1260 error = lock_page_killable(page); 1261 if (unlikely(error)) 1262 goto readpage_error; 1263 if (!PageUptodate(page)) { 1264 if (page->mapping == NULL) { 1265 /* 1266 * invalidate_mapping_pages got it 1267 */ 1268 unlock_page(page); 1269 page_cache_release(page); 1270 goto find_page; 1271 } 1272 unlock_page(page); 1273 shrink_readahead_size_eio(filp, ra); 1274 error = -EIO; 1275 goto readpage_error; 1276 } 1277 unlock_page(page); 1278 } 1279 1280 goto page_ok; 1281 1282 readpage_error: 1283 /* UHHUH! A synchronous read error occurred. Report it */ 1284 desc->error = error; 1285 page_cache_release(page); 1286 goto out; 1287 1288 no_cached_page: 1289 /* 1290 * Ok, it wasn't cached, so we need to create a new 1291 * page.. 1292 */ 1293 page = page_cache_alloc_cold(mapping); 1294 if (!page) { 1295 desc->error = -ENOMEM; 1296 goto out; 1297 } 1298 error = add_to_page_cache_lru(page, mapping, 1299 index, GFP_KERNEL); 1300 if (error) { 1301 page_cache_release(page); 1302 if (error == -EEXIST) 1303 goto find_page; 1304 desc->error = error; 1305 goto out; 1306 } 1307 goto readpage; 1308 } 1309 1310 out: 1311 ra->prev_pos = prev_index; 1312 ra->prev_pos <<= PAGE_CACHE_SHIFT; 1313 ra->prev_pos |= prev_offset; 1314 1315 *ppos = ((loff_t)index << PAGE_CACHE_SHIFT) + offset; 1316 file_accessed(filp); 1317 } 1318 1319 int file_read_actor(read_descriptor_t *desc, struct page *page, 1320 unsigned long offset, unsigned long size) 1321 { 1322 char *kaddr; 1323 unsigned long left, count = desc->count; 1324 1325 if (size > count) 1326 size = count; 1327 1328 /* 1329 * Faults on the destination of a read are common, so do it before 1330 * taking the kmap. 1331 */ 1332 if (!fault_in_pages_writeable(desc->arg.buf, size)) { 1333 kaddr = kmap_atomic(page, KM_USER0); 1334 left = __copy_to_user_inatomic(desc->arg.buf, 1335 kaddr + offset, size); 1336 kunmap_atomic(kaddr, KM_USER0); 1337 if (left == 0) 1338 goto success; 1339 } 1340 1341 /* Do it the slow way */ 1342 kaddr = kmap(page); 1343 left = __copy_to_user(desc->arg.buf, kaddr + offset, size); 1344 kunmap(page); 1345 1346 if (left) { 1347 size -= left; 1348 desc->error = -EFAULT; 1349 } 1350 success: 1351 desc->count = count - size; 1352 desc->written += size; 1353 desc->arg.buf += size; 1354 return size; 1355 } 1356 1357 /* 1358 * Performs necessary checks before doing a write 1359 * @iov: io vector request 1360 * @nr_segs: number of segments in the iovec 1361 * @count: number of bytes to write 1362 * @access_flags: type of access: %VERIFY_READ or %VERIFY_WRITE 1363 * 1364 * Adjust number of segments and amount of bytes to write (nr_segs should be 1365 * properly initialized first). Returns appropriate error code that caller 1366 * should return or zero in case that write should be allowed. 1367 */ 1368 int generic_segment_checks(const struct iovec *iov, 1369 unsigned long *nr_segs, size_t *count, int access_flags) 1370 { 1371 unsigned long seg; 1372 size_t cnt = 0; 1373 for (seg = 0; seg < *nr_segs; seg++) { 1374 const struct iovec *iv = &iov[seg]; 1375 1376 /* 1377 * If any segment has a negative length, or the cumulative 1378 * length ever wraps negative then return -EINVAL. 1379 */ 1380 cnt += iv->iov_len; 1381 if (unlikely((ssize_t)(cnt|iv->iov_len) < 0)) 1382 return -EINVAL; 1383 if (access_ok(access_flags, iv->iov_base, iv->iov_len)) 1384 continue; 1385 if (seg == 0) 1386 return -EFAULT; 1387 *nr_segs = seg; 1388 cnt -= iv->iov_len; /* This segment is no good */ 1389 break; 1390 } 1391 *count = cnt; 1392 return 0; 1393 } 1394 EXPORT_SYMBOL(generic_segment_checks); 1395 1396 /** 1397 * generic_file_aio_read - generic filesystem read routine 1398 * @iocb: kernel I/O control block 1399 * @iov: io vector request 1400 * @nr_segs: number of segments in the iovec 1401 * @pos: current file position 1402 * 1403 * This is the "read()" routine for all filesystems 1404 * that can use the page cache directly. 1405 */ 1406 ssize_t 1407 generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, 1408 unsigned long nr_segs, loff_t pos) 1409 { 1410 struct file *filp = iocb->ki_filp; 1411 ssize_t retval; 1412 unsigned long seg = 0; 1413 size_t count; 1414 loff_t *ppos = &iocb->ki_pos; 1415 struct blk_plug plug; 1416 1417 count = 0; 1418 retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE); 1419 if (retval) 1420 return retval; 1421 1422 blk_start_plug(&plug); 1423 1424 /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ 1425 if (filp->f_flags & O_DIRECT) { 1426 loff_t size; 1427 struct address_space *mapping; 1428 struct inode *inode; 1429 1430 mapping = filp->f_mapping; 1431 inode = mapping->host; 1432 if (!count) 1433 goto out; /* skip atime */ 1434 size = i_size_read(inode); 1435 if (pos < size) { 1436 retval = filemap_write_and_wait_range(mapping, pos, 1437 pos + iov_length(iov, nr_segs) - 1); 1438 if (!retval) { 1439 retval = mapping->a_ops->direct_IO(READ, iocb, 1440 iov, pos, nr_segs); 1441 } 1442 if (retval > 0) { 1443 *ppos = pos + retval; 1444 count -= retval; 1445 } 1446 1447 /* 1448 * Btrfs can have a short DIO read if we encounter 1449 * compressed extents, so if there was an error, or if 1450 * we've already read everything we wanted to, or if 1451 * there was a short read because we hit EOF, go ahead 1452 * and return. Otherwise fallthrough to buffered io for 1453 * the rest of the read. 1454 */ 1455 if (retval < 0 || !count || *ppos >= size) { 1456 file_accessed(filp); 1457 goto out; 1458 } 1459 } 1460 } 1461 1462 count = retval; 1463 for (seg = 0; seg < nr_segs; seg++) { 1464 read_descriptor_t desc; 1465 loff_t offset = 0; 1466 1467 /* 1468 * If we did a short DIO read we need to skip the section of the 1469 * iov that we've already read data into. 1470 */ 1471 if (count) { 1472 if (count > iov[seg].iov_len) { 1473 count -= iov[seg].iov_len; 1474 continue; 1475 } 1476 offset = count; 1477 count = 0; 1478 } 1479 1480 desc.written = 0; 1481 desc.arg.buf = iov[seg].iov_base + offset; 1482 desc.count = iov[seg].iov_len - offset; 1483 if (desc.count == 0) 1484 continue; 1485 desc.error = 0; 1486 do_generic_file_read(filp, ppos, &desc, file_read_actor); 1487 retval += desc.written; 1488 if (desc.error) { 1489 retval = retval ?: desc.error; 1490 break; 1491 } 1492 if (desc.count > 0) 1493 break; 1494 } 1495 out: 1496 blk_finish_plug(&plug); 1497 return retval; 1498 } 1499 EXPORT_SYMBOL(generic_file_aio_read); 1500 1501 static ssize_t 1502 do_readahead(struct address_space *mapping, struct file *filp, 1503 pgoff_t index, unsigned long nr) 1504 { 1505 if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage) 1506 return -EINVAL; 1507 1508 force_page_cache_readahead(mapping, filp, index, nr); 1509 return 0; 1510 } 1511 1512 SYSCALL_DEFINE(readahead)(int fd, loff_t offset, size_t count) 1513 { 1514 ssize_t ret; 1515 struct file *file; 1516 1517 ret = -EBADF; 1518 file = fget(fd); 1519 if (file) { 1520 if (file->f_mode & FMODE_READ) { 1521 struct address_space *mapping = file->f_mapping; 1522 pgoff_t start = offset >> PAGE_CACHE_SHIFT; 1523 pgoff_t end = (offset + count - 1) >> PAGE_CACHE_SHIFT; 1524 unsigned long len = end - start + 1; 1525 ret = do_readahead(mapping, file, start, len); 1526 } 1527 fput(file); 1528 } 1529 return ret; 1530 } 1531 #ifdef CONFIG_HAVE_SYSCALL_WRAPPERS 1532 asmlinkage long SyS_readahead(long fd, loff_t offset, long count) 1533 { 1534 return SYSC_readahead((int) fd, offset, (size_t) count); 1535 } 1536 SYSCALL_ALIAS(sys_readahead, SyS_readahead); 1537 #endif 1538 1539 #ifdef CONFIG_MMU 1540 /** 1541 * page_cache_read - adds requested page to the page cache if not already there 1542 * @file: file to read 1543 * @offset: page index 1544 * 1545 * This adds the requested page to the page cache if it isn't already there, 1546 * and schedules an I/O to read in its contents from disk. 1547 */ 1548 static int page_cache_read(struct file *file, pgoff_t offset) 1549 { 1550 struct address_space *mapping = file->f_mapping; 1551 struct page *page; 1552 int ret; 1553 1554 do { 1555 page = page_cache_alloc_cold(mapping); 1556 if (!page) 1557 return -ENOMEM; 1558 1559 ret = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL); 1560 if (ret == 0) 1561 ret = mapping->a_ops->readpage(file, page); 1562 else if (ret == -EEXIST) 1563 ret = 0; /* losing race to add is OK */ 1564 1565 page_cache_release(page); 1566 1567 } while (ret == AOP_TRUNCATED_PAGE); 1568 1569 return ret; 1570 } 1571 1572 #define MMAP_LOTSAMISS (100) 1573 1574 /* 1575 * Synchronous readahead happens when we don't even find 1576 * a page in the page cache at all. 1577 */ 1578 static void do_sync_mmap_readahead(struct vm_area_struct *vma, 1579 struct file_ra_state *ra, 1580 struct file *file, 1581 pgoff_t offset) 1582 { 1583 unsigned long ra_pages; 1584 struct address_space *mapping = file->f_mapping; 1585 1586 /* If we don't want any read-ahead, don't bother */ 1587 if (VM_RandomReadHint(vma)) 1588 return; 1589 if (!ra->ra_pages) 1590 return; 1591 1592 if (VM_SequentialReadHint(vma)) { 1593 page_cache_sync_readahead(mapping, ra, file, offset, 1594 ra->ra_pages); 1595 return; 1596 } 1597 1598 /* Avoid banging the cache line if not needed */ 1599 if (ra->mmap_miss < MMAP_LOTSAMISS * 10) 1600 ra->mmap_miss++; 1601 1602 /* 1603 * Do we miss much more than hit in this file? If so, 1604 * stop bothering with read-ahead. It will only hurt. 1605 */ 1606 if (ra->mmap_miss > MMAP_LOTSAMISS) 1607 return; 1608 1609 /* 1610 * mmap read-around 1611 */ 1612 ra_pages = max_sane_readahead(ra->ra_pages); 1613 ra->start = max_t(long, 0, offset - ra_pages / 2); 1614 ra->size = ra_pages; 1615 ra->async_size = ra_pages / 4; 1616 ra_submit(ra, mapping, file); 1617 } 1618 1619 /* 1620 * Asynchronous readahead happens when we find the page and PG_readahead, 1621 * so we want to possibly extend the readahead further.. 1622 */ 1623 static void do_async_mmap_readahead(struct vm_area_struct *vma, 1624 struct file_ra_state *ra, 1625 struct file *file, 1626 struct page *page, 1627 pgoff_t offset) 1628 { 1629 struct address_space *mapping = file->f_mapping; 1630 1631 /* If we don't want any read-ahead, don't bother */ 1632 if (VM_RandomReadHint(vma)) 1633 return; 1634 if (ra->mmap_miss > 0) 1635 ra->mmap_miss--; 1636 if (PageReadahead(page)) 1637 page_cache_async_readahead(mapping, ra, file, 1638 page, offset, ra->ra_pages); 1639 } 1640 1641 /** 1642 * filemap_fault - read in file data for page fault handling 1643 * @vma: vma in which the fault was taken 1644 * @vmf: struct vm_fault containing details of the fault 1645 * 1646 * filemap_fault() is invoked via the vma operations vector for a 1647 * mapped memory region to read in file data during a page fault. 1648 * 1649 * The goto's are kind of ugly, but this streamlines the normal case of having 1650 * it in the page cache, and handles the special cases reasonably without 1651 * having a lot of duplicated code. 1652 */ 1653 int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1654 { 1655 int error; 1656 struct file *file = vma->vm_file; 1657 struct address_space *mapping = file->f_mapping; 1658 struct file_ra_state *ra = &file->f_ra; 1659 struct inode *inode = mapping->host; 1660 pgoff_t offset = vmf->pgoff; 1661 struct page *page; 1662 pgoff_t size; 1663 int ret = 0; 1664 1665 size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 1666 if (offset >= size) 1667 return VM_FAULT_SIGBUS; 1668 1669 /* 1670 * Do we have something in the page cache already? 1671 */ 1672 page = find_get_page(mapping, offset); 1673 if (likely(page)) { 1674 /* 1675 * We found the page, so try async readahead before 1676 * waiting for the lock. 1677 */ 1678 do_async_mmap_readahead(vma, ra, file, page, offset); 1679 } else { 1680 /* No page in the page cache at all */ 1681 do_sync_mmap_readahead(vma, ra, file, offset); 1682 count_vm_event(PGMAJFAULT); 1683 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); 1684 ret = VM_FAULT_MAJOR; 1685 retry_find: 1686 page = find_get_page(mapping, offset); 1687 if (!page) 1688 goto no_cached_page; 1689 } 1690 1691 if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags)) { 1692 page_cache_release(page); 1693 return ret | VM_FAULT_RETRY; 1694 } 1695 1696 /* Did it get truncated? */ 1697 if (unlikely(page->mapping != mapping)) { 1698 unlock_page(page); 1699 put_page(page); 1700 goto retry_find; 1701 } 1702 VM_BUG_ON(page->index != offset); 1703 1704 /* 1705 * We have a locked page in the page cache, now we need to check 1706 * that it's up-to-date. If not, it is going to be due to an error. 1707 */ 1708 if (unlikely(!PageUptodate(page))) 1709 goto page_not_uptodate; 1710 1711 /* 1712 * Found the page and have a reference on it. 1713 * We must recheck i_size under page lock. 1714 */ 1715 size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 1716 if (unlikely(offset >= size)) { 1717 unlock_page(page); 1718 page_cache_release(page); 1719 return VM_FAULT_SIGBUS; 1720 } 1721 1722 vmf->page = page; 1723 return ret | VM_FAULT_LOCKED; 1724 1725 no_cached_page: 1726 /* 1727 * We're only likely to ever get here if MADV_RANDOM is in 1728 * effect. 1729 */ 1730 error = page_cache_read(file, offset); 1731 1732 /* 1733 * The page we want has now been added to the page cache. 1734 * In the unlikely event that someone removed it in the 1735 * meantime, we'll just come back here and read it again. 1736 */ 1737 if (error >= 0) 1738 goto retry_find; 1739 1740 /* 1741 * An error return from page_cache_read can result if the 1742 * system is low on memory, or a problem occurs while trying 1743 * to schedule I/O. 1744 */ 1745 if (error == -ENOMEM) 1746 return VM_FAULT_OOM; 1747 return VM_FAULT_SIGBUS; 1748 1749 page_not_uptodate: 1750 /* 1751 * Umm, take care of errors if the page isn't up-to-date. 1752 * Try to re-read it _once_. We do this synchronously, 1753 * because there really aren't any performance issues here 1754 * and we need to check for errors. 1755 */ 1756 ClearPageError(page); 1757 error = mapping->a_ops->readpage(file, page); 1758 if (!error) { 1759 wait_on_page_locked(page); 1760 if (!PageUptodate(page)) 1761 error = -EIO; 1762 } 1763 page_cache_release(page); 1764 1765 if (!error || error == AOP_TRUNCATED_PAGE) 1766 goto retry_find; 1767 1768 /* Things didn't work out. Return zero to tell the mm layer so. */ 1769 shrink_readahead_size_eio(file, ra); 1770 return VM_FAULT_SIGBUS; 1771 } 1772 EXPORT_SYMBOL(filemap_fault); 1773 1774 const struct vm_operations_struct generic_file_vm_ops = { 1775 .fault = filemap_fault, 1776 }; 1777 1778 /* This is used for a general mmap of a disk file */ 1779 1780 int generic_file_mmap(struct file * file, struct vm_area_struct * vma) 1781 { 1782 struct address_space *mapping = file->f_mapping; 1783 1784 if (!mapping->a_ops->readpage) 1785 return -ENOEXEC; 1786 file_accessed(file); 1787 vma->vm_ops = &generic_file_vm_ops; 1788 vma->vm_flags |= VM_CAN_NONLINEAR; 1789 return 0; 1790 } 1791 1792 /* 1793 * This is for filesystems which do not implement ->writepage. 1794 */ 1795 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) 1796 { 1797 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) 1798 return -EINVAL; 1799 return generic_file_mmap(file, vma); 1800 } 1801 #else 1802 int generic_file_mmap(struct file * file, struct vm_area_struct * vma) 1803 { 1804 return -ENOSYS; 1805 } 1806 int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma) 1807 { 1808 return -ENOSYS; 1809 } 1810 #endif /* CONFIG_MMU */ 1811 1812 EXPORT_SYMBOL(generic_file_mmap); 1813 EXPORT_SYMBOL(generic_file_readonly_mmap); 1814 1815 static struct page *__read_cache_page(struct address_space *mapping, 1816 pgoff_t index, 1817 int (*filler)(void *, struct page *), 1818 void *data, 1819 gfp_t gfp) 1820 { 1821 struct page *page; 1822 int err; 1823 repeat: 1824 page = find_get_page(mapping, index); 1825 if (!page) { 1826 page = __page_cache_alloc(gfp | __GFP_COLD); 1827 if (!page) 1828 return ERR_PTR(-ENOMEM); 1829 err = add_to_page_cache_lru(page, mapping, index, GFP_KERNEL); 1830 if (unlikely(err)) { 1831 page_cache_release(page); 1832 if (err == -EEXIST) 1833 goto repeat; 1834 /* Presumably ENOMEM for radix tree node */ 1835 return ERR_PTR(err); 1836 } 1837 err = filler(data, page); 1838 if (err < 0) { 1839 page_cache_release(page); 1840 page = ERR_PTR(err); 1841 } 1842 } 1843 return page; 1844 } 1845 1846 static struct page *do_read_cache_page(struct address_space *mapping, 1847 pgoff_t index, 1848 int (*filler)(void *, struct page *), 1849 void *data, 1850 gfp_t gfp) 1851 1852 { 1853 struct page *page; 1854 int err; 1855 1856 retry: 1857 page = __read_cache_page(mapping, index, filler, data, gfp); 1858 if (IS_ERR(page)) 1859 return page; 1860 if (PageUptodate(page)) 1861 goto out; 1862 1863 lock_page(page); 1864 if (!page->mapping) { 1865 unlock_page(page); 1866 page_cache_release(page); 1867 goto retry; 1868 } 1869 if (PageUptodate(page)) { 1870 unlock_page(page); 1871 goto out; 1872 } 1873 err = filler(data, page); 1874 if (err < 0) { 1875 page_cache_release(page); 1876 return ERR_PTR(err); 1877 } 1878 out: 1879 mark_page_accessed(page); 1880 return page; 1881 } 1882 1883 /** 1884 * read_cache_page_async - read into page cache, fill it if needed 1885 * @mapping: the page's address_space 1886 * @index: the page index 1887 * @filler: function to perform the read 1888 * @data: first arg to filler(data, page) function, often left as NULL 1889 * 1890 * Same as read_cache_page, but don't wait for page to become unlocked 1891 * after submitting it to the filler. 1892 * 1893 * Read into the page cache. If a page already exists, and PageUptodate() is 1894 * not set, try to fill the page but don't wait for it to become unlocked. 1895 * 1896 * If the page does not get brought uptodate, return -EIO. 1897 */ 1898 struct page *read_cache_page_async(struct address_space *mapping, 1899 pgoff_t index, 1900 int (*filler)(void *, struct page *), 1901 void *data) 1902 { 1903 return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping)); 1904 } 1905 EXPORT_SYMBOL(read_cache_page_async); 1906 1907 static struct page *wait_on_page_read(struct page *page) 1908 { 1909 if (!IS_ERR(page)) { 1910 wait_on_page_locked(page); 1911 if (!PageUptodate(page)) { 1912 page_cache_release(page); 1913 page = ERR_PTR(-EIO); 1914 } 1915 } 1916 return page; 1917 } 1918 1919 /** 1920 * read_cache_page_gfp - read into page cache, using specified page allocation flags. 1921 * @mapping: the page's address_space 1922 * @index: the page index 1923 * @gfp: the page allocator flags to use if allocating 1924 * 1925 * This is the same as "read_mapping_page(mapping, index, NULL)", but with 1926 * any new page allocations done using the specified allocation flags. Note 1927 * that the Radix tree operations will still use GFP_KERNEL, so you can't 1928 * expect to do this atomically or anything like that - but you can pass in 1929 * other page requirements. 1930 * 1931 * If the page does not get brought uptodate, return -EIO. 1932 */ 1933 struct page *read_cache_page_gfp(struct address_space *mapping, 1934 pgoff_t index, 1935 gfp_t gfp) 1936 { 1937 filler_t *filler = (filler_t *)mapping->a_ops->readpage; 1938 1939 return wait_on_page_read(do_read_cache_page(mapping, index, filler, NULL, gfp)); 1940 } 1941 EXPORT_SYMBOL(read_cache_page_gfp); 1942 1943 /** 1944 * read_cache_page - read into page cache, fill it if needed 1945 * @mapping: the page's address_space 1946 * @index: the page index 1947 * @filler: function to perform the read 1948 * @data: first arg to filler(data, page) function, often left as NULL 1949 * 1950 * Read into the page cache. If a page already exists, and PageUptodate() is 1951 * not set, try to fill the page then wait for it to become unlocked. 1952 * 1953 * If the page does not get brought uptodate, return -EIO. 1954 */ 1955 struct page *read_cache_page(struct address_space *mapping, 1956 pgoff_t index, 1957 int (*filler)(void *, struct page *), 1958 void *data) 1959 { 1960 return wait_on_page_read(read_cache_page_async(mapping, index, filler, data)); 1961 } 1962 EXPORT_SYMBOL(read_cache_page); 1963 1964 /* 1965 * The logic we want is 1966 * 1967 * if suid or (sgid and xgrp) 1968 * remove privs 1969 */ 1970 int should_remove_suid(struct dentry *dentry) 1971 { 1972 mode_t mode = dentry->d_inode->i_mode; 1973 int kill = 0; 1974 1975 /* suid always must be killed */ 1976 if (unlikely(mode & S_ISUID)) 1977 kill = ATTR_KILL_SUID; 1978 1979 /* 1980 * sgid without any exec bits is just a mandatory locking mark; leave 1981 * it alone. If some exec bits are set, it's a real sgid; kill it. 1982 */ 1983 if (unlikely((mode & S_ISGID) && (mode & S_IXGRP))) 1984 kill |= ATTR_KILL_SGID; 1985 1986 if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode))) 1987 return kill; 1988 1989 return 0; 1990 } 1991 EXPORT_SYMBOL(should_remove_suid); 1992 1993 static int __remove_suid(struct dentry *dentry, int kill) 1994 { 1995 struct iattr newattrs; 1996 1997 newattrs.ia_valid = ATTR_FORCE | kill; 1998 return notify_change(dentry, &newattrs); 1999 } 2000 2001 int file_remove_suid(struct file *file) 2002 { 2003 struct dentry *dentry = file->f_path.dentry; 2004 struct inode *inode = dentry->d_inode; 2005 int killsuid; 2006 int killpriv; 2007 int error = 0; 2008 2009 /* Fast path for nothing security related */ 2010 if (IS_NOSEC(inode)) 2011 return 0; 2012 2013 killsuid = should_remove_suid(dentry); 2014 killpriv = security_inode_need_killpriv(dentry); 2015 2016 if (killpriv < 0) 2017 return killpriv; 2018 if (killpriv) 2019 error = security_inode_killpriv(dentry); 2020 if (!error && killsuid) 2021 error = __remove_suid(dentry, killsuid); 2022 if (!error && (inode->i_sb->s_flags & MS_NOSEC)) 2023 inode->i_flags |= S_NOSEC; 2024 2025 return error; 2026 } 2027 EXPORT_SYMBOL(file_remove_suid); 2028 2029 static size_t __iovec_copy_from_user_inatomic(char *vaddr, 2030 const struct iovec *iov, size_t base, size_t bytes) 2031 { 2032 size_t copied = 0, left = 0; 2033 2034 while (bytes) { 2035 char __user *buf = iov->iov_base + base; 2036 int copy = min(bytes, iov->iov_len - base); 2037 2038 base = 0; 2039 left = __copy_from_user_inatomic(vaddr, buf, copy); 2040 copied += copy; 2041 bytes -= copy; 2042 vaddr += copy; 2043 iov++; 2044 2045 if (unlikely(left)) 2046 break; 2047 } 2048 return copied - left; 2049 } 2050 2051 /* 2052 * Copy as much as we can into the page and return the number of bytes which 2053 * were successfully copied. If a fault is encountered then return the number of 2054 * bytes which were copied. 2055 */ 2056 size_t iov_iter_copy_from_user_atomic(struct page *page, 2057 struct iov_iter *i, unsigned long offset, size_t bytes) 2058 { 2059 char *kaddr; 2060 size_t copied; 2061 2062 BUG_ON(!in_atomic()); 2063 kaddr = kmap_atomic(page, KM_USER0); 2064 if (likely(i->nr_segs == 1)) { 2065 int left; 2066 char __user *buf = i->iov->iov_base + i->iov_offset; 2067 left = __copy_from_user_inatomic(kaddr + offset, buf, bytes); 2068 copied = bytes - left; 2069 } else { 2070 copied = __iovec_copy_from_user_inatomic(kaddr + offset, 2071 i->iov, i->iov_offset, bytes); 2072 } 2073 kunmap_atomic(kaddr, KM_USER0); 2074 2075 return copied; 2076 } 2077 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); 2078 2079 /* 2080 * This has the same sideeffects and return value as 2081 * iov_iter_copy_from_user_atomic(). 2082 * The difference is that it attempts to resolve faults. 2083 * Page must not be locked. 2084 */ 2085 size_t iov_iter_copy_from_user(struct page *page, 2086 struct iov_iter *i, unsigned long offset, size_t bytes) 2087 { 2088 char *kaddr; 2089 size_t copied; 2090 2091 kaddr = kmap(page); 2092 if (likely(i->nr_segs == 1)) { 2093 int left; 2094 char __user *buf = i->iov->iov_base + i->iov_offset; 2095 left = __copy_from_user(kaddr + offset, buf, bytes); 2096 copied = bytes - left; 2097 } else { 2098 copied = __iovec_copy_from_user_inatomic(kaddr + offset, 2099 i->iov, i->iov_offset, bytes); 2100 } 2101 kunmap(page); 2102 return copied; 2103 } 2104 EXPORT_SYMBOL(iov_iter_copy_from_user); 2105 2106 void iov_iter_advance(struct iov_iter *i, size_t bytes) 2107 { 2108 BUG_ON(i->count < bytes); 2109 2110 if (likely(i->nr_segs == 1)) { 2111 i->iov_offset += bytes; 2112 i->count -= bytes; 2113 } else { 2114 const struct iovec *iov = i->iov; 2115 size_t base = i->iov_offset; 2116 2117 /* 2118 * The !iov->iov_len check ensures we skip over unlikely 2119 * zero-length segments (without overruning the iovec). 2120 */ 2121 while (bytes || unlikely(i->count && !iov->iov_len)) { 2122 int copy; 2123 2124 copy = min(bytes, iov->iov_len - base); 2125 BUG_ON(!i->count || i->count < copy); 2126 i->count -= copy; 2127 bytes -= copy; 2128 base += copy; 2129 if (iov->iov_len == base) { 2130 iov++; 2131 base = 0; 2132 } 2133 } 2134 i->iov = iov; 2135 i->iov_offset = base; 2136 } 2137 } 2138 EXPORT_SYMBOL(iov_iter_advance); 2139 2140 /* 2141 * Fault in the first iovec of the given iov_iter, to a maximum length 2142 * of bytes. Returns 0 on success, or non-zero if the memory could not be 2143 * accessed (ie. because it is an invalid address). 2144 * 2145 * writev-intensive code may want this to prefault several iovecs -- that 2146 * would be possible (callers must not rely on the fact that _only_ the 2147 * first iovec will be faulted with the current implementation). 2148 */ 2149 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) 2150 { 2151 char __user *buf = i->iov->iov_base + i->iov_offset; 2152 bytes = min(bytes, i->iov->iov_len - i->iov_offset); 2153 return fault_in_pages_readable(buf, bytes); 2154 } 2155 EXPORT_SYMBOL(iov_iter_fault_in_readable); 2156 2157 /* 2158 * Return the count of just the current iov_iter segment. 2159 */ 2160 size_t iov_iter_single_seg_count(struct iov_iter *i) 2161 { 2162 const struct iovec *iov = i->iov; 2163 if (i->nr_segs == 1) 2164 return i->count; 2165 else 2166 return min(i->count, iov->iov_len - i->iov_offset); 2167 } 2168 EXPORT_SYMBOL(iov_iter_single_seg_count); 2169 2170 /* 2171 * Performs necessary checks before doing a write 2172 * 2173 * Can adjust writing position or amount of bytes to write. 2174 * Returns appropriate error code that caller should return or 2175 * zero in case that write should be allowed. 2176 */ 2177 inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk) 2178 { 2179 struct inode *inode = file->f_mapping->host; 2180 unsigned long limit = rlimit(RLIMIT_FSIZE); 2181 2182 if (unlikely(*pos < 0)) 2183 return -EINVAL; 2184 2185 if (!isblk) { 2186 /* FIXME: this is for backwards compatibility with 2.4 */ 2187 if (file->f_flags & O_APPEND) 2188 *pos = i_size_read(inode); 2189 2190 if (limit != RLIM_INFINITY) { 2191 if (*pos >= limit) { 2192 send_sig(SIGXFSZ, current, 0); 2193 return -EFBIG; 2194 } 2195 if (*count > limit - (typeof(limit))*pos) { 2196 *count = limit - (typeof(limit))*pos; 2197 } 2198 } 2199 } 2200 2201 /* 2202 * LFS rule 2203 */ 2204 if (unlikely(*pos + *count > MAX_NON_LFS && 2205 !(file->f_flags & O_LARGEFILE))) { 2206 if (*pos >= MAX_NON_LFS) { 2207 return -EFBIG; 2208 } 2209 if (*count > MAX_NON_LFS - (unsigned long)*pos) { 2210 *count = MAX_NON_LFS - (unsigned long)*pos; 2211 } 2212 } 2213 2214 /* 2215 * Are we about to exceed the fs block limit ? 2216 * 2217 * If we have written data it becomes a short write. If we have 2218 * exceeded without writing data we send a signal and return EFBIG. 2219 * Linus frestrict idea will clean these up nicely.. 2220 */ 2221 if (likely(!isblk)) { 2222 if (unlikely(*pos >= inode->i_sb->s_maxbytes)) { 2223 if (*count || *pos > inode->i_sb->s_maxbytes) { 2224 return -EFBIG; 2225 } 2226 /* zero-length writes at ->s_maxbytes are OK */ 2227 } 2228 2229 if (unlikely(*pos + *count > inode->i_sb->s_maxbytes)) 2230 *count = inode->i_sb->s_maxbytes - *pos; 2231 } else { 2232 #ifdef CONFIG_BLOCK 2233 loff_t isize; 2234 if (bdev_read_only(I_BDEV(inode))) 2235 return -EPERM; 2236 isize = i_size_read(inode); 2237 if (*pos >= isize) { 2238 if (*count || *pos > isize) 2239 return -ENOSPC; 2240 } 2241 2242 if (*pos + *count > isize) 2243 *count = isize - *pos; 2244 #else 2245 return -EPERM; 2246 #endif 2247 } 2248 return 0; 2249 } 2250 EXPORT_SYMBOL(generic_write_checks); 2251 2252 int pagecache_write_begin(struct file *file, struct address_space *mapping, 2253 loff_t pos, unsigned len, unsigned flags, 2254 struct page **pagep, void **fsdata) 2255 { 2256 const struct address_space_operations *aops = mapping->a_ops; 2257 2258 return aops->write_begin(file, mapping, pos, len, flags, 2259 pagep, fsdata); 2260 } 2261 EXPORT_SYMBOL(pagecache_write_begin); 2262 2263 int pagecache_write_end(struct file *file, struct address_space *mapping, 2264 loff_t pos, unsigned len, unsigned copied, 2265 struct page *page, void *fsdata) 2266 { 2267 const struct address_space_operations *aops = mapping->a_ops; 2268 2269 mark_page_accessed(page); 2270 return aops->write_end(file, mapping, pos, len, copied, page, fsdata); 2271 } 2272 EXPORT_SYMBOL(pagecache_write_end); 2273 2274 ssize_t 2275 generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov, 2276 unsigned long *nr_segs, loff_t pos, loff_t *ppos, 2277 size_t count, size_t ocount) 2278 { 2279 struct file *file = iocb->ki_filp; 2280 struct address_space *mapping = file->f_mapping; 2281 struct inode *inode = mapping->host; 2282 ssize_t written; 2283 size_t write_len; 2284 pgoff_t end; 2285 2286 if (count != ocount) 2287 *nr_segs = iov_shorten((struct iovec *)iov, *nr_segs, count); 2288 2289 write_len = iov_length(iov, *nr_segs); 2290 end = (pos + write_len - 1) >> PAGE_CACHE_SHIFT; 2291 2292 written = filemap_write_and_wait_range(mapping, pos, pos + write_len - 1); 2293 if (written) 2294 goto out; 2295 2296 /* 2297 * After a write we want buffered reads to be sure to go to disk to get 2298 * the new data. We invalidate clean cached page from the region we're 2299 * about to write. We do this *before* the write so that we can return 2300 * without clobbering -EIOCBQUEUED from ->direct_IO(). 2301 */ 2302 if (mapping->nrpages) { 2303 written = invalidate_inode_pages2_range(mapping, 2304 pos >> PAGE_CACHE_SHIFT, end); 2305 /* 2306 * If a page can not be invalidated, return 0 to fall back 2307 * to buffered write. 2308 */ 2309 if (written) { 2310 if (written == -EBUSY) 2311 return 0; 2312 goto out; 2313 } 2314 } 2315 2316 written = mapping->a_ops->direct_IO(WRITE, iocb, iov, pos, *nr_segs); 2317 2318 /* 2319 * Finally, try again to invalidate clean pages which might have been 2320 * cached by non-direct readahead, or faulted in by get_user_pages() 2321 * if the source of the write was an mmap'ed region of the file 2322 * we're writing. Either one is a pretty crazy thing to do, 2323 * so we don't support it 100%. If this invalidation 2324 * fails, tough, the write still worked... 2325 */ 2326 if (mapping->nrpages) { 2327 invalidate_inode_pages2_range(mapping, 2328 pos >> PAGE_CACHE_SHIFT, end); 2329 } 2330 2331 if (written > 0) { 2332 pos += written; 2333 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { 2334 i_size_write(inode, pos); 2335 mark_inode_dirty(inode); 2336 } 2337 *ppos = pos; 2338 } 2339 out: 2340 return written; 2341 } 2342 EXPORT_SYMBOL(generic_file_direct_write); 2343 2344 /* 2345 * Find or create a page at the given pagecache position. Return the locked 2346 * page. This function is specifically for buffered writes. 2347 */ 2348 struct page *grab_cache_page_write_begin(struct address_space *mapping, 2349 pgoff_t index, unsigned flags) 2350 { 2351 int status; 2352 struct page *page; 2353 gfp_t gfp_notmask = 0; 2354 if (flags & AOP_FLAG_NOFS) 2355 gfp_notmask = __GFP_FS; 2356 repeat: 2357 page = find_lock_page(mapping, index); 2358 if (page) 2359 goto found; 2360 2361 page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~gfp_notmask); 2362 if (!page) 2363 return NULL; 2364 status = add_to_page_cache_lru(page, mapping, index, 2365 GFP_KERNEL & ~gfp_notmask); 2366 if (unlikely(status)) { 2367 page_cache_release(page); 2368 if (status == -EEXIST) 2369 goto repeat; 2370 return NULL; 2371 } 2372 found: 2373 wait_on_page_writeback(page); 2374 return page; 2375 } 2376 EXPORT_SYMBOL(grab_cache_page_write_begin); 2377 2378 static ssize_t generic_perform_write(struct file *file, 2379 struct iov_iter *i, loff_t pos) 2380 { 2381 struct address_space *mapping = file->f_mapping; 2382 const struct address_space_operations *a_ops = mapping->a_ops; 2383 long status = 0; 2384 ssize_t written = 0; 2385 unsigned int flags = 0; 2386 2387 /* 2388 * Copies from kernel address space cannot fail (NFSD is a big user). 2389 */ 2390 if (segment_eq(get_fs(), KERNEL_DS)) 2391 flags |= AOP_FLAG_UNINTERRUPTIBLE; 2392 2393 do { 2394 struct page *page; 2395 unsigned long offset; /* Offset into pagecache page */ 2396 unsigned long bytes; /* Bytes to write to page */ 2397 size_t copied; /* Bytes copied from user */ 2398 void *fsdata; 2399 2400 offset = (pos & (PAGE_CACHE_SIZE - 1)); 2401 bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset, 2402 iov_iter_count(i)); 2403 2404 again: 2405 2406 /* 2407 * Bring in the user page that we will copy from _first_. 2408 * Otherwise there's a nasty deadlock on copying from the 2409 * same page as we're writing to, without it being marked 2410 * up-to-date. 2411 * 2412 * Not only is this an optimisation, but it is also required 2413 * to check that the address is actually valid, when atomic 2414 * usercopies are used, below. 2415 */ 2416 if (unlikely(iov_iter_fault_in_readable(i, bytes))) { 2417 status = -EFAULT; 2418 break; 2419 } 2420 2421 status = a_ops->write_begin(file, mapping, pos, bytes, flags, 2422 &page, &fsdata); 2423 if (unlikely(status)) 2424 break; 2425 2426 if (mapping_writably_mapped(mapping)) 2427 flush_dcache_page(page); 2428 2429 pagefault_disable(); 2430 copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes); 2431 pagefault_enable(); 2432 flush_dcache_page(page); 2433 2434 mark_page_accessed(page); 2435 status = a_ops->write_end(file, mapping, pos, bytes, copied, 2436 page, fsdata); 2437 if (unlikely(status < 0)) 2438 break; 2439 copied = status; 2440 2441 cond_resched(); 2442 2443 iov_iter_advance(i, copied); 2444 if (unlikely(copied == 0)) { 2445 /* 2446 * If we were unable to copy any data at all, we must 2447 * fall back to a single segment length write. 2448 * 2449 * If we didn't fallback here, we could livelock 2450 * because not all segments in the iov can be copied at 2451 * once without a pagefault. 2452 */ 2453 bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset, 2454 iov_iter_single_seg_count(i)); 2455 goto again; 2456 } 2457 pos += copied; 2458 written += copied; 2459 2460 balance_dirty_pages_ratelimited(mapping); 2461 2462 } while (iov_iter_count(i)); 2463 2464 return written ? written : status; 2465 } 2466 2467 ssize_t 2468 generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov, 2469 unsigned long nr_segs, loff_t pos, loff_t *ppos, 2470 size_t count, ssize_t written) 2471 { 2472 struct file *file = iocb->ki_filp; 2473 ssize_t status; 2474 struct iov_iter i; 2475 2476 iov_iter_init(&i, iov, nr_segs, count, written); 2477 status = generic_perform_write(file, &i, pos); 2478 2479 if (likely(status >= 0)) { 2480 written += status; 2481 *ppos = pos + status; 2482 } 2483 2484 return written ? written : status; 2485 } 2486 EXPORT_SYMBOL(generic_file_buffered_write); 2487 2488 /** 2489 * __generic_file_aio_write - write data to a file 2490 * @iocb: IO state structure (file, offset, etc.) 2491 * @iov: vector with data to write 2492 * @nr_segs: number of segments in the vector 2493 * @ppos: position where to write 2494 * 2495 * This function does all the work needed for actually writing data to a 2496 * file. It does all basic checks, removes SUID from the file, updates 2497 * modification times and calls proper subroutines depending on whether we 2498 * do direct IO or a standard buffered write. 2499 * 2500 * It expects i_mutex to be grabbed unless we work on a block device or similar 2501 * object which does not need locking at all. 2502 * 2503 * This function does *not* take care of syncing data in case of O_SYNC write. 2504 * A caller has to handle it. This is mainly due to the fact that we want to 2505 * avoid syncing under i_mutex. 2506 */ 2507 ssize_t __generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, 2508 unsigned long nr_segs, loff_t *ppos) 2509 { 2510 struct file *file = iocb->ki_filp; 2511 struct address_space * mapping = file->f_mapping; 2512 size_t ocount; /* original count */ 2513 size_t count; /* after file limit checks */ 2514 struct inode *inode = mapping->host; 2515 loff_t pos; 2516 ssize_t written; 2517 ssize_t err; 2518 2519 ocount = 0; 2520 err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ); 2521 if (err) 2522 return err; 2523 2524 count = ocount; 2525 pos = *ppos; 2526 2527 vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE); 2528 2529 /* We can write back this queue in page reclaim */ 2530 current->backing_dev_info = mapping->backing_dev_info; 2531 written = 0; 2532 2533 err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode)); 2534 if (err) 2535 goto out; 2536 2537 if (count == 0) 2538 goto out; 2539 2540 err = file_remove_suid(file); 2541 if (err) 2542 goto out; 2543 2544 file_update_time(file); 2545 2546 /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ 2547 if (unlikely(file->f_flags & O_DIRECT)) { 2548 loff_t endbyte; 2549 ssize_t written_buffered; 2550 2551 written = generic_file_direct_write(iocb, iov, &nr_segs, pos, 2552 ppos, count, ocount); 2553 if (written < 0 || written == count) 2554 goto out; 2555 /* 2556 * direct-io write to a hole: fall through to buffered I/O 2557 * for completing the rest of the request. 2558 */ 2559 pos += written; 2560 count -= written; 2561 written_buffered = generic_file_buffered_write(iocb, iov, 2562 nr_segs, pos, ppos, count, 2563 written); 2564 /* 2565 * If generic_file_buffered_write() retuned a synchronous error 2566 * then we want to return the number of bytes which were 2567 * direct-written, or the error code if that was zero. Note 2568 * that this differs from normal direct-io semantics, which 2569 * will return -EFOO even if some bytes were written. 2570 */ 2571 if (written_buffered < 0) { 2572 err = written_buffered; 2573 goto out; 2574 } 2575 2576 /* 2577 * We need to ensure that the page cache pages are written to 2578 * disk and invalidated to preserve the expected O_DIRECT 2579 * semantics. 2580 */ 2581 endbyte = pos + written_buffered - written - 1; 2582 err = filemap_write_and_wait_range(file->f_mapping, pos, endbyte); 2583 if (err == 0) { 2584 written = written_buffered; 2585 invalidate_mapping_pages(mapping, 2586 pos >> PAGE_CACHE_SHIFT, 2587 endbyte >> PAGE_CACHE_SHIFT); 2588 } else { 2589 /* 2590 * We don't know how much we wrote, so just return 2591 * the number of bytes which were direct-written 2592 */ 2593 } 2594 } else { 2595 written = generic_file_buffered_write(iocb, iov, nr_segs, 2596 pos, ppos, count, written); 2597 } 2598 out: 2599 current->backing_dev_info = NULL; 2600 return written ? written : err; 2601 } 2602 EXPORT_SYMBOL(__generic_file_aio_write); 2603 2604 /** 2605 * generic_file_aio_write - write data to a file 2606 * @iocb: IO state structure 2607 * @iov: vector with data to write 2608 * @nr_segs: number of segments in the vector 2609 * @pos: position in file where to write 2610 * 2611 * This is a wrapper around __generic_file_aio_write() to be used by most 2612 * filesystems. It takes care of syncing the file in case of O_SYNC file 2613 * and acquires i_mutex as needed. 2614 */ 2615 ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, 2616 unsigned long nr_segs, loff_t pos) 2617 { 2618 struct file *file = iocb->ki_filp; 2619 struct inode *inode = file->f_mapping->host; 2620 struct blk_plug plug; 2621 ssize_t ret; 2622 2623 BUG_ON(iocb->ki_pos != pos); 2624 2625 mutex_lock(&inode->i_mutex); 2626 blk_start_plug(&plug); 2627 ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); 2628 mutex_unlock(&inode->i_mutex); 2629 2630 if (ret > 0 || ret == -EIOCBQUEUED) { 2631 ssize_t err; 2632 2633 err = generic_write_sync(file, pos, ret); 2634 if (err < 0 && ret > 0) 2635 ret = err; 2636 } 2637 blk_finish_plug(&plug); 2638 return ret; 2639 } 2640 EXPORT_SYMBOL(generic_file_aio_write); 2641 2642 /** 2643 * try_to_release_page() - release old fs-specific metadata on a page 2644 * 2645 * @page: the page which the kernel is trying to free 2646 * @gfp_mask: memory allocation flags (and I/O mode) 2647 * 2648 * The address_space is to try to release any data against the page 2649 * (presumably at page->private). If the release was successful, return `1'. 2650 * Otherwise return zero. 2651 * 2652 * This may also be called if PG_fscache is set on a page, indicating that the 2653 * page is known to the local caching routines. 2654 * 2655 * The @gfp_mask argument specifies whether I/O may be performed to release 2656 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT & __GFP_FS). 2657 * 2658 */ 2659 int try_to_release_page(struct page *page, gfp_t gfp_mask) 2660 { 2661 struct address_space * const mapping = page->mapping; 2662 2663 BUG_ON(!PageLocked(page)); 2664 if (PageWriteback(page)) 2665 return 0; 2666 2667 if (mapping && mapping->a_ops->releasepage) 2668 return mapping->a_ops->releasepage(page, gfp_mask); 2669 return try_to_free_buffers(page); 2670 } 2671 2672 EXPORT_SYMBOL(try_to_release_page); 2673