1 /* 2 * linux/mm/filemap.c 3 * 4 * Copyright (C) 1994-1999 Linus Torvalds 5 */ 6 7 /* 8 * This file handles the generic file mmap semantics used by 9 * most "normal" filesystems (but you don't /have/ to use this: 10 * the NFS filesystem used to do this differently, for example) 11 */ 12 #include <linux/export.h> 13 #include <linux/compiler.h> 14 #include <linux/dax.h> 15 #include <linux/fs.h> 16 #include <linux/uaccess.h> 17 #include <linux/capability.h> 18 #include <linux/kernel_stat.h> 19 #include <linux/gfp.h> 20 #include <linux/mm.h> 21 #include <linux/swap.h> 22 #include <linux/mman.h> 23 #include <linux/pagemap.h> 24 #include <linux/file.h> 25 #include <linux/uio.h> 26 #include <linux/hash.h> 27 #include <linux/writeback.h> 28 #include <linux/backing-dev.h> 29 #include <linux/pagevec.h> 30 #include <linux/blkdev.h> 31 #include <linux/security.h> 32 #include <linux/cpuset.h> 33 #include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */ 34 #include <linux/hugetlb.h> 35 #include <linux/memcontrol.h> 36 #include <linux/cleancache.h> 37 #include <linux/rmap.h> 38 #include "internal.h" 39 40 #define CREATE_TRACE_POINTS 41 #include <trace/events/filemap.h> 42 43 /* 44 * FIXME: remove all knowledge of the buffer layer from the core VM 45 */ 46 #include <linux/buffer_head.h> /* for try_to_free_buffers */ 47 48 #include <asm/mman.h> 49 50 /* 51 * Shared mappings implemented 30.11.1994. It's not fully working yet, 52 * though. 53 * 54 * Shared mappings now work. 15.8.1995 Bruno. 55 * 56 * finished 'unifying' the page and buffer cache and SMP-threaded the 57 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com> 58 * 59 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de> 60 */ 61 62 /* 63 * Lock ordering: 64 * 65 * ->i_mmap_rwsem (truncate_pagecache) 66 * ->private_lock (__free_pte->__set_page_dirty_buffers) 67 * ->swap_lock (exclusive_swap_page, others) 68 * ->mapping->tree_lock 69 * 70 * ->i_mutex 71 * ->i_mmap_rwsem (truncate->unmap_mapping_range) 72 * 73 * ->mmap_sem 74 * ->i_mmap_rwsem 75 * ->page_table_lock or pte_lock (various, mainly in memory.c) 76 * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock) 77 * 78 * ->mmap_sem 79 * ->lock_page (access_process_vm) 80 * 81 * ->i_mutex (generic_perform_write) 82 * ->mmap_sem (fault_in_pages_readable->do_page_fault) 83 * 84 * bdi->wb.list_lock 85 * sb_lock (fs/fs-writeback.c) 86 * ->mapping->tree_lock (__sync_single_inode) 87 * 88 * ->i_mmap_rwsem 89 * ->anon_vma.lock (vma_adjust) 90 * 91 * ->anon_vma.lock 92 * ->page_table_lock or pte_lock (anon_vma_prepare and various) 93 * 94 * ->page_table_lock or pte_lock 95 * ->swap_lock (try_to_unmap_one) 96 * ->private_lock (try_to_unmap_one) 97 * ->tree_lock (try_to_unmap_one) 98 * ->zone_lru_lock(zone) (follow_page->mark_page_accessed) 99 * ->zone_lru_lock(zone) (check_pte_range->isolate_lru_page) 100 * ->private_lock (page_remove_rmap->set_page_dirty) 101 * ->tree_lock (page_remove_rmap->set_page_dirty) 102 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) 103 * ->inode->i_lock (page_remove_rmap->set_page_dirty) 104 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg) 105 * bdi.wb->list_lock (zap_pte_range->set_page_dirty) 106 * ->inode->i_lock (zap_pte_range->set_page_dirty) 107 * ->private_lock (zap_pte_range->__set_page_dirty_buffers) 108 * 109 * ->i_mmap_rwsem 110 * ->tasklist_lock (memory_failure, collect_procs_ao) 111 */ 112 113 static int page_cache_tree_insert(struct address_space *mapping, 114 struct page *page, void **shadowp) 115 { 116 struct radix_tree_node *node; 117 void **slot; 118 int error; 119 120 error = __radix_tree_create(&mapping->page_tree, page->index, 0, 121 &node, &slot); 122 if (error) 123 return error; 124 if (*slot) { 125 void *p; 126 127 p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock); 128 if (!radix_tree_exceptional_entry(p)) 129 return -EEXIST; 130 131 mapping->nrexceptional--; 132 if (!dax_mapping(mapping)) { 133 if (shadowp) 134 *shadowp = p; 135 } else { 136 /* DAX can replace empty locked entry with a hole */ 137 WARN_ON_ONCE(p != 138 dax_radix_locked_entry(0, RADIX_DAX_EMPTY)); 139 /* Wakeup waiters for exceptional entry lock */ 140 dax_wake_mapping_entry_waiter(mapping, page->index, p, 141 false); 142 } 143 } 144 __radix_tree_replace(&mapping->page_tree, node, slot, page, 145 workingset_update_node, mapping); 146 mapping->nrpages++; 147 return 0; 148 } 149 150 static void page_cache_tree_delete(struct address_space *mapping, 151 struct page *page, void *shadow) 152 { 153 int i, nr; 154 155 /* hugetlb pages are represented by one entry in the radix tree */ 156 nr = PageHuge(page) ? 1 : hpage_nr_pages(page); 157 158 VM_BUG_ON_PAGE(!PageLocked(page), page); 159 VM_BUG_ON_PAGE(PageTail(page), page); 160 VM_BUG_ON_PAGE(nr != 1 && shadow, page); 161 162 for (i = 0; i < nr; i++) { 163 struct radix_tree_node *node; 164 void **slot; 165 166 __radix_tree_lookup(&mapping->page_tree, page->index + i, 167 &node, &slot); 168 169 VM_BUG_ON_PAGE(!node && nr != 1, page); 170 171 radix_tree_clear_tags(&mapping->page_tree, node, slot); 172 __radix_tree_replace(&mapping->page_tree, node, slot, shadow, 173 workingset_update_node, mapping); 174 } 175 176 if (shadow) { 177 mapping->nrexceptional += nr; 178 /* 179 * Make sure the nrexceptional update is committed before 180 * the nrpages update so that final truncate racing 181 * with reclaim does not see both counters 0 at the 182 * same time and miss a shadow entry. 183 */ 184 smp_wmb(); 185 } 186 mapping->nrpages -= nr; 187 } 188 189 /* 190 * Delete a page from the page cache and free it. Caller has to make 191 * sure the page is locked and that nobody else uses it - or that usage 192 * is safe. The caller must hold the mapping's tree_lock. 193 */ 194 void __delete_from_page_cache(struct page *page, void *shadow) 195 { 196 struct address_space *mapping = page->mapping; 197 int nr = hpage_nr_pages(page); 198 199 trace_mm_filemap_delete_from_page_cache(page); 200 /* 201 * if we're uptodate, flush out into the cleancache, otherwise 202 * invalidate any existing cleancache entries. We can't leave 203 * stale data around in the cleancache once our page is gone 204 */ 205 if (PageUptodate(page) && PageMappedToDisk(page)) 206 cleancache_put_page(page); 207 else 208 cleancache_invalidate_page(mapping, page); 209 210 VM_BUG_ON_PAGE(PageTail(page), page); 211 VM_BUG_ON_PAGE(page_mapped(page), page); 212 if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) { 213 int mapcount; 214 215 pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n", 216 current->comm, page_to_pfn(page)); 217 dump_page(page, "still mapped when deleted"); 218 dump_stack(); 219 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 220 221 mapcount = page_mapcount(page); 222 if (mapping_exiting(mapping) && 223 page_count(page) >= mapcount + 2) { 224 /* 225 * All vmas have already been torn down, so it's 226 * a good bet that actually the page is unmapped, 227 * and we'd prefer not to leak it: if we're wrong, 228 * some other bad page check should catch it later. 229 */ 230 page_mapcount_reset(page); 231 page_ref_sub(page, mapcount); 232 } 233 } 234 235 page_cache_tree_delete(mapping, page, shadow); 236 237 page->mapping = NULL; 238 /* Leave page->index set: truncation lookup relies upon it */ 239 240 /* hugetlb pages do not participate in page cache accounting. */ 241 if (!PageHuge(page)) 242 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr); 243 if (PageSwapBacked(page)) { 244 __mod_node_page_state(page_pgdat(page), NR_SHMEM, -nr); 245 if (PageTransHuge(page)) 246 __dec_node_page_state(page, NR_SHMEM_THPS); 247 } else { 248 VM_BUG_ON_PAGE(PageTransHuge(page) && !PageHuge(page), page); 249 } 250 251 /* 252 * At this point page must be either written or cleaned by truncate. 253 * Dirty page here signals a bug and loss of unwritten data. 254 * 255 * This fixes dirty accounting after removing the page entirely but 256 * leaves PageDirty set: it has no effect for truncated page and 257 * anyway will be cleared before returning page into buddy allocator. 258 */ 259 if (WARN_ON_ONCE(PageDirty(page))) 260 account_page_cleaned(page, mapping, inode_to_wb(mapping->host)); 261 } 262 263 /** 264 * delete_from_page_cache - delete page from page cache 265 * @page: the page which the kernel is trying to remove from page cache 266 * 267 * This must be called only on pages that have been verified to be in the page 268 * cache and locked. It will never put the page into the free list, the caller 269 * has a reference on the page. 270 */ 271 void delete_from_page_cache(struct page *page) 272 { 273 struct address_space *mapping = page_mapping(page); 274 unsigned long flags; 275 void (*freepage)(struct page *); 276 277 BUG_ON(!PageLocked(page)); 278 279 freepage = mapping->a_ops->freepage; 280 281 spin_lock_irqsave(&mapping->tree_lock, flags); 282 __delete_from_page_cache(page, NULL); 283 spin_unlock_irqrestore(&mapping->tree_lock, flags); 284 285 if (freepage) 286 freepage(page); 287 288 if (PageTransHuge(page) && !PageHuge(page)) { 289 page_ref_sub(page, HPAGE_PMD_NR); 290 VM_BUG_ON_PAGE(page_count(page) <= 0, page); 291 } else { 292 put_page(page); 293 } 294 } 295 EXPORT_SYMBOL(delete_from_page_cache); 296 297 int filemap_check_errors(struct address_space *mapping) 298 { 299 int ret = 0; 300 /* Check for outstanding write errors */ 301 if (test_bit(AS_ENOSPC, &mapping->flags) && 302 test_and_clear_bit(AS_ENOSPC, &mapping->flags)) 303 ret = -ENOSPC; 304 if (test_bit(AS_EIO, &mapping->flags) && 305 test_and_clear_bit(AS_EIO, &mapping->flags)) 306 ret = -EIO; 307 return ret; 308 } 309 EXPORT_SYMBOL(filemap_check_errors); 310 311 /** 312 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range 313 * @mapping: address space structure to write 314 * @start: offset in bytes where the range starts 315 * @end: offset in bytes where the range ends (inclusive) 316 * @sync_mode: enable synchronous operation 317 * 318 * Start writeback against all of a mapping's dirty pages that lie 319 * within the byte offsets <start, end> inclusive. 320 * 321 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as 322 * opposed to a regular memory cleansing writeback. The difference between 323 * these two operations is that if a dirty page/buffer is encountered, it must 324 * be waited upon, and not just skipped over. 325 */ 326 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 327 loff_t end, int sync_mode) 328 { 329 int ret; 330 struct writeback_control wbc = { 331 .sync_mode = sync_mode, 332 .nr_to_write = LONG_MAX, 333 .range_start = start, 334 .range_end = end, 335 }; 336 337 if (!mapping_cap_writeback_dirty(mapping)) 338 return 0; 339 340 wbc_attach_fdatawrite_inode(&wbc, mapping->host); 341 ret = do_writepages(mapping, &wbc); 342 wbc_detach_inode(&wbc); 343 return ret; 344 } 345 346 static inline int __filemap_fdatawrite(struct address_space *mapping, 347 int sync_mode) 348 { 349 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode); 350 } 351 352 int filemap_fdatawrite(struct address_space *mapping) 353 { 354 return __filemap_fdatawrite(mapping, WB_SYNC_ALL); 355 } 356 EXPORT_SYMBOL(filemap_fdatawrite); 357 358 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 359 loff_t end) 360 { 361 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); 362 } 363 EXPORT_SYMBOL(filemap_fdatawrite_range); 364 365 /** 366 * filemap_flush - mostly a non-blocking flush 367 * @mapping: target address_space 368 * 369 * This is a mostly non-blocking flush. Not suitable for data-integrity 370 * purposes - I/O may not be started against all dirty pages. 371 */ 372 int filemap_flush(struct address_space *mapping) 373 { 374 return __filemap_fdatawrite(mapping, WB_SYNC_NONE); 375 } 376 EXPORT_SYMBOL(filemap_flush); 377 378 static int __filemap_fdatawait_range(struct address_space *mapping, 379 loff_t start_byte, loff_t end_byte) 380 { 381 pgoff_t index = start_byte >> PAGE_SHIFT; 382 pgoff_t end = end_byte >> PAGE_SHIFT; 383 struct pagevec pvec; 384 int nr_pages; 385 int ret = 0; 386 387 if (end_byte < start_byte) 388 goto out; 389 390 pagevec_init(&pvec, 0); 391 while ((index <= end) && 392 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, 393 PAGECACHE_TAG_WRITEBACK, 394 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) { 395 unsigned i; 396 397 for (i = 0; i < nr_pages; i++) { 398 struct page *page = pvec.pages[i]; 399 400 /* until radix tree lookup accepts end_index */ 401 if (page->index > end) 402 continue; 403 404 wait_on_page_writeback(page); 405 if (TestClearPageError(page)) 406 ret = -EIO; 407 } 408 pagevec_release(&pvec); 409 cond_resched(); 410 } 411 out: 412 return ret; 413 } 414 415 /** 416 * filemap_fdatawait_range - wait for writeback to complete 417 * @mapping: address space structure to wait for 418 * @start_byte: offset in bytes where the range starts 419 * @end_byte: offset in bytes where the range ends (inclusive) 420 * 421 * Walk the list of under-writeback pages of the given address space 422 * in the given range and wait for all of them. Check error status of 423 * the address space and return it. 424 * 425 * Since the error status of the address space is cleared by this function, 426 * callers are responsible for checking the return value and handling and/or 427 * reporting the error. 428 */ 429 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte, 430 loff_t end_byte) 431 { 432 int ret, ret2; 433 434 ret = __filemap_fdatawait_range(mapping, start_byte, end_byte); 435 ret2 = filemap_check_errors(mapping); 436 if (!ret) 437 ret = ret2; 438 439 return ret; 440 } 441 EXPORT_SYMBOL(filemap_fdatawait_range); 442 443 /** 444 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors 445 * @mapping: address space structure to wait for 446 * 447 * Walk the list of under-writeback pages of the given address space 448 * and wait for all of them. Unlike filemap_fdatawait(), this function 449 * does not clear error status of the address space. 450 * 451 * Use this function if callers don't handle errors themselves. Expected 452 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2), 453 * fsfreeze(8) 454 */ 455 void filemap_fdatawait_keep_errors(struct address_space *mapping) 456 { 457 loff_t i_size = i_size_read(mapping->host); 458 459 if (i_size == 0) 460 return; 461 462 __filemap_fdatawait_range(mapping, 0, i_size - 1); 463 } 464 465 /** 466 * filemap_fdatawait - wait for all under-writeback pages to complete 467 * @mapping: address space structure to wait for 468 * 469 * Walk the list of under-writeback pages of the given address space 470 * and wait for all of them. Check error status of the address space 471 * and return it. 472 * 473 * Since the error status of the address space is cleared by this function, 474 * callers are responsible for checking the return value and handling and/or 475 * reporting the error. 476 */ 477 int filemap_fdatawait(struct address_space *mapping) 478 { 479 loff_t i_size = i_size_read(mapping->host); 480 481 if (i_size == 0) 482 return 0; 483 484 return filemap_fdatawait_range(mapping, 0, i_size - 1); 485 } 486 EXPORT_SYMBOL(filemap_fdatawait); 487 488 int filemap_write_and_wait(struct address_space *mapping) 489 { 490 int err = 0; 491 492 if ((!dax_mapping(mapping) && mapping->nrpages) || 493 (dax_mapping(mapping) && mapping->nrexceptional)) { 494 err = filemap_fdatawrite(mapping); 495 /* 496 * Even if the above returned error, the pages may be 497 * written partially (e.g. -ENOSPC), so we wait for it. 498 * But the -EIO is special case, it may indicate the worst 499 * thing (e.g. bug) happened, so we avoid waiting for it. 500 */ 501 if (err != -EIO) { 502 int err2 = filemap_fdatawait(mapping); 503 if (!err) 504 err = err2; 505 } 506 } else { 507 err = filemap_check_errors(mapping); 508 } 509 return err; 510 } 511 EXPORT_SYMBOL(filemap_write_and_wait); 512 513 /** 514 * filemap_write_and_wait_range - write out & wait on a file range 515 * @mapping: the address_space for the pages 516 * @lstart: offset in bytes where the range starts 517 * @lend: offset in bytes where the range ends (inclusive) 518 * 519 * Write out and wait upon file offsets lstart->lend, inclusive. 520 * 521 * Note that `lend' is inclusive (describes the last byte to be written) so 522 * that this function can be used to write to the very end-of-file (end = -1). 523 */ 524 int filemap_write_and_wait_range(struct address_space *mapping, 525 loff_t lstart, loff_t lend) 526 { 527 int err = 0; 528 529 if ((!dax_mapping(mapping) && mapping->nrpages) || 530 (dax_mapping(mapping) && mapping->nrexceptional)) { 531 err = __filemap_fdatawrite_range(mapping, lstart, lend, 532 WB_SYNC_ALL); 533 /* See comment of filemap_write_and_wait() */ 534 if (err != -EIO) { 535 int err2 = filemap_fdatawait_range(mapping, 536 lstart, lend); 537 if (!err) 538 err = err2; 539 } 540 } else { 541 err = filemap_check_errors(mapping); 542 } 543 return err; 544 } 545 EXPORT_SYMBOL(filemap_write_and_wait_range); 546 547 /** 548 * replace_page_cache_page - replace a pagecache page with a new one 549 * @old: page to be replaced 550 * @new: page to replace with 551 * @gfp_mask: allocation mode 552 * 553 * This function replaces a page in the pagecache with a new one. On 554 * success it acquires the pagecache reference for the new page and 555 * drops it for the old page. Both the old and new pages must be 556 * locked. This function does not add the new page to the LRU, the 557 * caller must do that. 558 * 559 * The remove + add is atomic. The only way this function can fail is 560 * memory allocation failure. 561 */ 562 int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) 563 { 564 int error; 565 566 VM_BUG_ON_PAGE(!PageLocked(old), old); 567 VM_BUG_ON_PAGE(!PageLocked(new), new); 568 VM_BUG_ON_PAGE(new->mapping, new); 569 570 error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM); 571 if (!error) { 572 struct address_space *mapping = old->mapping; 573 void (*freepage)(struct page *); 574 unsigned long flags; 575 576 pgoff_t offset = old->index; 577 freepage = mapping->a_ops->freepage; 578 579 get_page(new); 580 new->mapping = mapping; 581 new->index = offset; 582 583 spin_lock_irqsave(&mapping->tree_lock, flags); 584 __delete_from_page_cache(old, NULL); 585 error = page_cache_tree_insert(mapping, new, NULL); 586 BUG_ON(error); 587 588 /* 589 * hugetlb pages do not participate in page cache accounting. 590 */ 591 if (!PageHuge(new)) 592 __inc_node_page_state(new, NR_FILE_PAGES); 593 if (PageSwapBacked(new)) 594 __inc_node_page_state(new, NR_SHMEM); 595 spin_unlock_irqrestore(&mapping->tree_lock, flags); 596 mem_cgroup_migrate(old, new); 597 radix_tree_preload_end(); 598 if (freepage) 599 freepage(old); 600 put_page(old); 601 } 602 603 return error; 604 } 605 EXPORT_SYMBOL_GPL(replace_page_cache_page); 606 607 static int __add_to_page_cache_locked(struct page *page, 608 struct address_space *mapping, 609 pgoff_t offset, gfp_t gfp_mask, 610 void **shadowp) 611 { 612 int huge = PageHuge(page); 613 struct mem_cgroup *memcg; 614 int error; 615 616 VM_BUG_ON_PAGE(!PageLocked(page), page); 617 VM_BUG_ON_PAGE(PageSwapBacked(page), page); 618 619 if (!huge) { 620 error = mem_cgroup_try_charge(page, current->mm, 621 gfp_mask, &memcg, false); 622 if (error) 623 return error; 624 } 625 626 error = radix_tree_maybe_preload(gfp_mask & ~__GFP_HIGHMEM); 627 if (error) { 628 if (!huge) 629 mem_cgroup_cancel_charge(page, memcg, false); 630 return error; 631 } 632 633 get_page(page); 634 page->mapping = mapping; 635 page->index = offset; 636 637 spin_lock_irq(&mapping->tree_lock); 638 error = page_cache_tree_insert(mapping, page, shadowp); 639 radix_tree_preload_end(); 640 if (unlikely(error)) 641 goto err_insert; 642 643 /* hugetlb pages do not participate in page cache accounting. */ 644 if (!huge) 645 __inc_node_page_state(page, NR_FILE_PAGES); 646 spin_unlock_irq(&mapping->tree_lock); 647 if (!huge) 648 mem_cgroup_commit_charge(page, memcg, false, false); 649 trace_mm_filemap_add_to_page_cache(page); 650 return 0; 651 err_insert: 652 page->mapping = NULL; 653 /* Leave page->index set: truncation relies upon it */ 654 spin_unlock_irq(&mapping->tree_lock); 655 if (!huge) 656 mem_cgroup_cancel_charge(page, memcg, false); 657 put_page(page); 658 return error; 659 } 660 661 /** 662 * add_to_page_cache_locked - add a locked page to the pagecache 663 * @page: page to add 664 * @mapping: the page's address_space 665 * @offset: page index 666 * @gfp_mask: page allocation mode 667 * 668 * This function is used to add a page to the pagecache. It must be locked. 669 * This function does not add the page to the LRU. The caller must do that. 670 */ 671 int add_to_page_cache_locked(struct page *page, struct address_space *mapping, 672 pgoff_t offset, gfp_t gfp_mask) 673 { 674 return __add_to_page_cache_locked(page, mapping, offset, 675 gfp_mask, NULL); 676 } 677 EXPORT_SYMBOL(add_to_page_cache_locked); 678 679 int add_to_page_cache_lru(struct page *page, struct address_space *mapping, 680 pgoff_t offset, gfp_t gfp_mask) 681 { 682 void *shadow = NULL; 683 int ret; 684 685 __SetPageLocked(page); 686 ret = __add_to_page_cache_locked(page, mapping, offset, 687 gfp_mask, &shadow); 688 if (unlikely(ret)) 689 __ClearPageLocked(page); 690 else { 691 /* 692 * The page might have been evicted from cache only 693 * recently, in which case it should be activated like 694 * any other repeatedly accessed page. 695 * The exception is pages getting rewritten; evicting other 696 * data from the working set, only to cache data that will 697 * get overwritten with something else, is a waste of memory. 698 */ 699 if (!(gfp_mask & __GFP_WRITE) && 700 shadow && workingset_refault(shadow)) { 701 SetPageActive(page); 702 workingset_activation(page); 703 } else 704 ClearPageActive(page); 705 lru_cache_add(page); 706 } 707 return ret; 708 } 709 EXPORT_SYMBOL_GPL(add_to_page_cache_lru); 710 711 #ifdef CONFIG_NUMA 712 struct page *__page_cache_alloc(gfp_t gfp) 713 { 714 int n; 715 struct page *page; 716 717 if (cpuset_do_page_mem_spread()) { 718 unsigned int cpuset_mems_cookie; 719 do { 720 cpuset_mems_cookie = read_mems_allowed_begin(); 721 n = cpuset_mem_spread_node(); 722 page = __alloc_pages_node(n, gfp, 0); 723 } while (!page && read_mems_allowed_retry(cpuset_mems_cookie)); 724 725 return page; 726 } 727 return alloc_pages(gfp, 0); 728 } 729 EXPORT_SYMBOL(__page_cache_alloc); 730 #endif 731 732 /* 733 * In order to wait for pages to become available there must be 734 * waitqueues associated with pages. By using a hash table of 735 * waitqueues where the bucket discipline is to maintain all 736 * waiters on the same queue and wake all when any of the pages 737 * become available, and for the woken contexts to check to be 738 * sure the appropriate page became available, this saves space 739 * at a cost of "thundering herd" phenomena during rare hash 740 * collisions. 741 */ 742 #define PAGE_WAIT_TABLE_BITS 8 743 #define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS) 744 static wait_queue_head_t page_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned; 745 746 static wait_queue_head_t *page_waitqueue(struct page *page) 747 { 748 return &page_wait_table[hash_ptr(page, PAGE_WAIT_TABLE_BITS)]; 749 } 750 751 void __init pagecache_init(void) 752 { 753 int i; 754 755 for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++) 756 init_waitqueue_head(&page_wait_table[i]); 757 758 page_writeback_init(); 759 } 760 761 struct wait_page_key { 762 struct page *page; 763 int bit_nr; 764 int page_match; 765 }; 766 767 struct wait_page_queue { 768 struct page *page; 769 int bit_nr; 770 wait_queue_t wait; 771 }; 772 773 static int wake_page_function(wait_queue_t *wait, unsigned mode, int sync, void *arg) 774 { 775 struct wait_page_key *key = arg; 776 struct wait_page_queue *wait_page 777 = container_of(wait, struct wait_page_queue, wait); 778 779 if (wait_page->page != key->page) 780 return 0; 781 key->page_match = 1; 782 783 if (wait_page->bit_nr != key->bit_nr) 784 return 0; 785 if (test_bit(key->bit_nr, &key->page->flags)) 786 return 0; 787 788 return autoremove_wake_function(wait, mode, sync, key); 789 } 790 791 void wake_up_page_bit(struct page *page, int bit_nr) 792 { 793 wait_queue_head_t *q = page_waitqueue(page); 794 struct wait_page_key key; 795 unsigned long flags; 796 797 key.page = page; 798 key.bit_nr = bit_nr; 799 key.page_match = 0; 800 801 spin_lock_irqsave(&q->lock, flags); 802 __wake_up_locked_key(q, TASK_NORMAL, &key); 803 /* 804 * It is possible for other pages to have collided on the waitqueue 805 * hash, so in that case check for a page match. That prevents a long- 806 * term waiter 807 * 808 * It is still possible to miss a case here, when we woke page waiters 809 * and removed them from the waitqueue, but there are still other 810 * page waiters. 811 */ 812 if (!waitqueue_active(q) || !key.page_match) { 813 ClearPageWaiters(page); 814 /* 815 * It's possible to miss clearing Waiters here, when we woke 816 * our page waiters, but the hashed waitqueue has waiters for 817 * other pages on it. 818 * 819 * That's okay, it's a rare case. The next waker will clear it. 820 */ 821 } 822 spin_unlock_irqrestore(&q->lock, flags); 823 } 824 EXPORT_SYMBOL(wake_up_page_bit); 825 826 static inline int wait_on_page_bit_common(wait_queue_head_t *q, 827 struct page *page, int bit_nr, int state, bool lock) 828 { 829 struct wait_page_queue wait_page; 830 wait_queue_t *wait = &wait_page.wait; 831 int ret = 0; 832 833 init_wait(wait); 834 wait->func = wake_page_function; 835 wait_page.page = page; 836 wait_page.bit_nr = bit_nr; 837 838 for (;;) { 839 spin_lock_irq(&q->lock); 840 841 if (likely(list_empty(&wait->task_list))) { 842 if (lock) 843 __add_wait_queue_tail_exclusive(q, wait); 844 else 845 __add_wait_queue(q, wait); 846 SetPageWaiters(page); 847 } 848 849 set_current_state(state); 850 851 spin_unlock_irq(&q->lock); 852 853 if (likely(test_bit(bit_nr, &page->flags))) { 854 io_schedule(); 855 if (unlikely(signal_pending_state(state, current))) { 856 ret = -EINTR; 857 break; 858 } 859 } 860 861 if (lock) { 862 if (!test_and_set_bit_lock(bit_nr, &page->flags)) 863 break; 864 } else { 865 if (!test_bit(bit_nr, &page->flags)) 866 break; 867 } 868 } 869 870 finish_wait(q, wait); 871 872 /* 873 * A signal could leave PageWaiters set. Clearing it here if 874 * !waitqueue_active would be possible (by open-coding finish_wait), 875 * but still fail to catch it in the case of wait hash collision. We 876 * already can fail to clear wait hash collision cases, so don't 877 * bother with signals either. 878 */ 879 880 return ret; 881 } 882 883 void wait_on_page_bit(struct page *page, int bit_nr) 884 { 885 wait_queue_head_t *q = page_waitqueue(page); 886 wait_on_page_bit_common(q, page, bit_nr, TASK_UNINTERRUPTIBLE, false); 887 } 888 EXPORT_SYMBOL(wait_on_page_bit); 889 890 int wait_on_page_bit_killable(struct page *page, int bit_nr) 891 { 892 wait_queue_head_t *q = page_waitqueue(page); 893 return wait_on_page_bit_common(q, page, bit_nr, TASK_KILLABLE, false); 894 } 895 896 /** 897 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue 898 * @page: Page defining the wait queue of interest 899 * @waiter: Waiter to add to the queue 900 * 901 * Add an arbitrary @waiter to the wait queue for the nominated @page. 902 */ 903 void add_page_wait_queue(struct page *page, wait_queue_t *waiter) 904 { 905 wait_queue_head_t *q = page_waitqueue(page); 906 unsigned long flags; 907 908 spin_lock_irqsave(&q->lock, flags); 909 __add_wait_queue(q, waiter); 910 SetPageWaiters(page); 911 spin_unlock_irqrestore(&q->lock, flags); 912 } 913 EXPORT_SYMBOL_GPL(add_page_wait_queue); 914 915 /** 916 * unlock_page - unlock a locked page 917 * @page: the page 918 * 919 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked(). 920 * Also wakes sleepers in wait_on_page_writeback() because the wakeup 921 * mechanism between PageLocked pages and PageWriteback pages is shared. 922 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep. 923 * 924 * The mb is necessary to enforce ordering between the clear_bit and the read 925 * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()). 926 */ 927 void unlock_page(struct page *page) 928 { 929 page = compound_head(page); 930 VM_BUG_ON_PAGE(!PageLocked(page), page); 931 clear_bit_unlock(PG_locked, &page->flags); 932 smp_mb__after_atomic(); 933 wake_up_page(page, PG_locked); 934 } 935 EXPORT_SYMBOL(unlock_page); 936 937 /** 938 * end_page_writeback - end writeback against a page 939 * @page: the page 940 */ 941 void end_page_writeback(struct page *page) 942 { 943 /* 944 * TestClearPageReclaim could be used here but it is an atomic 945 * operation and overkill in this particular case. Failing to 946 * shuffle a page marked for immediate reclaim is too mild to 947 * justify taking an atomic operation penalty at the end of 948 * ever page writeback. 949 */ 950 if (PageReclaim(page)) { 951 ClearPageReclaim(page); 952 rotate_reclaimable_page(page); 953 } 954 955 if (!test_clear_page_writeback(page)) 956 BUG(); 957 958 smp_mb__after_atomic(); 959 wake_up_page(page, PG_writeback); 960 } 961 EXPORT_SYMBOL(end_page_writeback); 962 963 /* 964 * After completing I/O on a page, call this routine to update the page 965 * flags appropriately 966 */ 967 void page_endio(struct page *page, bool is_write, int err) 968 { 969 if (!is_write) { 970 if (!err) { 971 SetPageUptodate(page); 972 } else { 973 ClearPageUptodate(page); 974 SetPageError(page); 975 } 976 unlock_page(page); 977 } else { 978 if (err) { 979 SetPageError(page); 980 if (page->mapping) 981 mapping_set_error(page->mapping, err); 982 } 983 end_page_writeback(page); 984 } 985 } 986 EXPORT_SYMBOL_GPL(page_endio); 987 988 /** 989 * __lock_page - get a lock on the page, assuming we need to sleep to get it 990 * @page: the page to lock 991 */ 992 void __lock_page(struct page *__page) 993 { 994 struct page *page = compound_head(__page); 995 wait_queue_head_t *q = page_waitqueue(page); 996 wait_on_page_bit_common(q, page, PG_locked, TASK_UNINTERRUPTIBLE, true); 997 } 998 EXPORT_SYMBOL(__lock_page); 999 1000 int __lock_page_killable(struct page *__page) 1001 { 1002 struct page *page = compound_head(__page); 1003 wait_queue_head_t *q = page_waitqueue(page); 1004 return wait_on_page_bit_common(q, page, PG_locked, TASK_KILLABLE, true); 1005 } 1006 EXPORT_SYMBOL_GPL(__lock_page_killable); 1007 1008 /* 1009 * Return values: 1010 * 1 - page is locked; mmap_sem is still held. 1011 * 0 - page is not locked. 1012 * mmap_sem has been released (up_read()), unless flags had both 1013 * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in 1014 * which case mmap_sem is still held. 1015 * 1016 * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1 1017 * with the page locked and the mmap_sem unperturbed. 1018 */ 1019 int __lock_page_or_retry(struct page *page, struct mm_struct *mm, 1020 unsigned int flags) 1021 { 1022 if (flags & FAULT_FLAG_ALLOW_RETRY) { 1023 /* 1024 * CAUTION! In this case, mmap_sem is not released 1025 * even though return 0. 1026 */ 1027 if (flags & FAULT_FLAG_RETRY_NOWAIT) 1028 return 0; 1029 1030 up_read(&mm->mmap_sem); 1031 if (flags & FAULT_FLAG_KILLABLE) 1032 wait_on_page_locked_killable(page); 1033 else 1034 wait_on_page_locked(page); 1035 return 0; 1036 } else { 1037 if (flags & FAULT_FLAG_KILLABLE) { 1038 int ret; 1039 1040 ret = __lock_page_killable(page); 1041 if (ret) { 1042 up_read(&mm->mmap_sem); 1043 return 0; 1044 } 1045 } else 1046 __lock_page(page); 1047 return 1; 1048 } 1049 } 1050 1051 /** 1052 * page_cache_next_hole - find the next hole (not-present entry) 1053 * @mapping: mapping 1054 * @index: index 1055 * @max_scan: maximum range to search 1056 * 1057 * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the 1058 * lowest indexed hole. 1059 * 1060 * Returns: the index of the hole if found, otherwise returns an index 1061 * outside of the set specified (in which case 'return - index >= 1062 * max_scan' will be true). In rare cases of index wrap-around, 0 will 1063 * be returned. 1064 * 1065 * page_cache_next_hole may be called under rcu_read_lock. However, 1066 * like radix_tree_gang_lookup, this will not atomically search a 1067 * snapshot of the tree at a single point in time. For example, if a 1068 * hole is created at index 5, then subsequently a hole is created at 1069 * index 10, page_cache_next_hole covering both indexes may return 10 1070 * if called under rcu_read_lock. 1071 */ 1072 pgoff_t page_cache_next_hole(struct address_space *mapping, 1073 pgoff_t index, unsigned long max_scan) 1074 { 1075 unsigned long i; 1076 1077 for (i = 0; i < max_scan; i++) { 1078 struct page *page; 1079 1080 page = radix_tree_lookup(&mapping->page_tree, index); 1081 if (!page || radix_tree_exceptional_entry(page)) 1082 break; 1083 index++; 1084 if (index == 0) 1085 break; 1086 } 1087 1088 return index; 1089 } 1090 EXPORT_SYMBOL(page_cache_next_hole); 1091 1092 /** 1093 * page_cache_prev_hole - find the prev hole (not-present entry) 1094 * @mapping: mapping 1095 * @index: index 1096 * @max_scan: maximum range to search 1097 * 1098 * Search backwards in the range [max(index-max_scan+1, 0), index] for 1099 * the first hole. 1100 * 1101 * Returns: the index of the hole if found, otherwise returns an index 1102 * outside of the set specified (in which case 'index - return >= 1103 * max_scan' will be true). In rare cases of wrap-around, ULONG_MAX 1104 * will be returned. 1105 * 1106 * page_cache_prev_hole may be called under rcu_read_lock. However, 1107 * like radix_tree_gang_lookup, this will not atomically search a 1108 * snapshot of the tree at a single point in time. For example, if a 1109 * hole is created at index 10, then subsequently a hole is created at 1110 * index 5, page_cache_prev_hole covering both indexes may return 5 if 1111 * called under rcu_read_lock. 1112 */ 1113 pgoff_t page_cache_prev_hole(struct address_space *mapping, 1114 pgoff_t index, unsigned long max_scan) 1115 { 1116 unsigned long i; 1117 1118 for (i = 0; i < max_scan; i++) { 1119 struct page *page; 1120 1121 page = radix_tree_lookup(&mapping->page_tree, index); 1122 if (!page || radix_tree_exceptional_entry(page)) 1123 break; 1124 index--; 1125 if (index == ULONG_MAX) 1126 break; 1127 } 1128 1129 return index; 1130 } 1131 EXPORT_SYMBOL(page_cache_prev_hole); 1132 1133 /** 1134 * find_get_entry - find and get a page cache entry 1135 * @mapping: the address_space to search 1136 * @offset: the page cache index 1137 * 1138 * Looks up the page cache slot at @mapping & @offset. If there is a 1139 * page cache page, it is returned with an increased refcount. 1140 * 1141 * If the slot holds a shadow entry of a previously evicted page, or a 1142 * swap entry from shmem/tmpfs, it is returned. 1143 * 1144 * Otherwise, %NULL is returned. 1145 */ 1146 struct page *find_get_entry(struct address_space *mapping, pgoff_t offset) 1147 { 1148 void **pagep; 1149 struct page *head, *page; 1150 1151 rcu_read_lock(); 1152 repeat: 1153 page = NULL; 1154 pagep = radix_tree_lookup_slot(&mapping->page_tree, offset); 1155 if (pagep) { 1156 page = radix_tree_deref_slot(pagep); 1157 if (unlikely(!page)) 1158 goto out; 1159 if (radix_tree_exception(page)) { 1160 if (radix_tree_deref_retry(page)) 1161 goto repeat; 1162 /* 1163 * A shadow entry of a recently evicted page, 1164 * or a swap entry from shmem/tmpfs. Return 1165 * it without attempting to raise page count. 1166 */ 1167 goto out; 1168 } 1169 1170 head = compound_head(page); 1171 if (!page_cache_get_speculative(head)) 1172 goto repeat; 1173 1174 /* The page was split under us? */ 1175 if (compound_head(page) != head) { 1176 put_page(head); 1177 goto repeat; 1178 } 1179 1180 /* 1181 * Has the page moved? 1182 * This is part of the lockless pagecache protocol. See 1183 * include/linux/pagemap.h for details. 1184 */ 1185 if (unlikely(page != *pagep)) { 1186 put_page(head); 1187 goto repeat; 1188 } 1189 } 1190 out: 1191 rcu_read_unlock(); 1192 1193 return page; 1194 } 1195 EXPORT_SYMBOL(find_get_entry); 1196 1197 /** 1198 * find_lock_entry - locate, pin and lock a page cache entry 1199 * @mapping: the address_space to search 1200 * @offset: the page cache index 1201 * 1202 * Looks up the page cache slot at @mapping & @offset. If there is a 1203 * page cache page, it is returned locked and with an increased 1204 * refcount. 1205 * 1206 * If the slot holds a shadow entry of a previously evicted page, or a 1207 * swap entry from shmem/tmpfs, it is returned. 1208 * 1209 * Otherwise, %NULL is returned. 1210 * 1211 * find_lock_entry() may sleep. 1212 */ 1213 struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset) 1214 { 1215 struct page *page; 1216 1217 repeat: 1218 page = find_get_entry(mapping, offset); 1219 if (page && !radix_tree_exception(page)) { 1220 lock_page(page); 1221 /* Has the page been truncated? */ 1222 if (unlikely(page_mapping(page) != mapping)) { 1223 unlock_page(page); 1224 put_page(page); 1225 goto repeat; 1226 } 1227 VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page); 1228 } 1229 return page; 1230 } 1231 EXPORT_SYMBOL(find_lock_entry); 1232 1233 /** 1234 * pagecache_get_page - find and get a page reference 1235 * @mapping: the address_space to search 1236 * @offset: the page index 1237 * @fgp_flags: PCG flags 1238 * @gfp_mask: gfp mask to use for the page cache data page allocation 1239 * 1240 * Looks up the page cache slot at @mapping & @offset. 1241 * 1242 * PCG flags modify how the page is returned. 1243 * 1244 * FGP_ACCESSED: the page will be marked accessed 1245 * FGP_LOCK: Page is return locked 1246 * FGP_CREAT: If page is not present then a new page is allocated using 1247 * @gfp_mask and added to the page cache and the VM's LRU 1248 * list. The page is returned locked and with an increased 1249 * refcount. Otherwise, %NULL is returned. 1250 * 1251 * If FGP_LOCK or FGP_CREAT are specified then the function may sleep even 1252 * if the GFP flags specified for FGP_CREAT are atomic. 1253 * 1254 * If there is a page cache page, it is returned with an increased refcount. 1255 */ 1256 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset, 1257 int fgp_flags, gfp_t gfp_mask) 1258 { 1259 struct page *page; 1260 1261 repeat: 1262 page = find_get_entry(mapping, offset); 1263 if (radix_tree_exceptional_entry(page)) 1264 page = NULL; 1265 if (!page) 1266 goto no_page; 1267 1268 if (fgp_flags & FGP_LOCK) { 1269 if (fgp_flags & FGP_NOWAIT) { 1270 if (!trylock_page(page)) { 1271 put_page(page); 1272 return NULL; 1273 } 1274 } else { 1275 lock_page(page); 1276 } 1277 1278 /* Has the page been truncated? */ 1279 if (unlikely(page->mapping != mapping)) { 1280 unlock_page(page); 1281 put_page(page); 1282 goto repeat; 1283 } 1284 VM_BUG_ON_PAGE(page->index != offset, page); 1285 } 1286 1287 if (page && (fgp_flags & FGP_ACCESSED)) 1288 mark_page_accessed(page); 1289 1290 no_page: 1291 if (!page && (fgp_flags & FGP_CREAT)) { 1292 int err; 1293 if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping)) 1294 gfp_mask |= __GFP_WRITE; 1295 if (fgp_flags & FGP_NOFS) 1296 gfp_mask &= ~__GFP_FS; 1297 1298 page = __page_cache_alloc(gfp_mask); 1299 if (!page) 1300 return NULL; 1301 1302 if (WARN_ON_ONCE(!(fgp_flags & FGP_LOCK))) 1303 fgp_flags |= FGP_LOCK; 1304 1305 /* Init accessed so avoid atomic mark_page_accessed later */ 1306 if (fgp_flags & FGP_ACCESSED) 1307 __SetPageReferenced(page); 1308 1309 err = add_to_page_cache_lru(page, mapping, offset, 1310 gfp_mask & GFP_RECLAIM_MASK); 1311 if (unlikely(err)) { 1312 put_page(page); 1313 page = NULL; 1314 if (err == -EEXIST) 1315 goto repeat; 1316 } 1317 } 1318 1319 return page; 1320 } 1321 EXPORT_SYMBOL(pagecache_get_page); 1322 1323 /** 1324 * find_get_entries - gang pagecache lookup 1325 * @mapping: The address_space to search 1326 * @start: The starting page cache index 1327 * @nr_entries: The maximum number of entries 1328 * @entries: Where the resulting entries are placed 1329 * @indices: The cache indices corresponding to the entries in @entries 1330 * 1331 * find_get_entries() will search for and return a group of up to 1332 * @nr_entries entries in the mapping. The entries are placed at 1333 * @entries. find_get_entries() takes a reference against any actual 1334 * pages it returns. 1335 * 1336 * The search returns a group of mapping-contiguous page cache entries 1337 * with ascending indexes. There may be holes in the indices due to 1338 * not-present pages. 1339 * 1340 * Any shadow entries of evicted pages, or swap entries from 1341 * shmem/tmpfs, are included in the returned array. 1342 * 1343 * find_get_entries() returns the number of pages and shadow entries 1344 * which were found. 1345 */ 1346 unsigned find_get_entries(struct address_space *mapping, 1347 pgoff_t start, unsigned int nr_entries, 1348 struct page **entries, pgoff_t *indices) 1349 { 1350 void **slot; 1351 unsigned int ret = 0; 1352 struct radix_tree_iter iter; 1353 1354 if (!nr_entries) 1355 return 0; 1356 1357 rcu_read_lock(); 1358 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 1359 struct page *head, *page; 1360 repeat: 1361 page = radix_tree_deref_slot(slot); 1362 if (unlikely(!page)) 1363 continue; 1364 if (radix_tree_exception(page)) { 1365 if (radix_tree_deref_retry(page)) { 1366 slot = radix_tree_iter_retry(&iter); 1367 continue; 1368 } 1369 /* 1370 * A shadow entry of a recently evicted page, a swap 1371 * entry from shmem/tmpfs or a DAX entry. Return it 1372 * without attempting to raise page count. 1373 */ 1374 goto export; 1375 } 1376 1377 head = compound_head(page); 1378 if (!page_cache_get_speculative(head)) 1379 goto repeat; 1380 1381 /* The page was split under us? */ 1382 if (compound_head(page) != head) { 1383 put_page(head); 1384 goto repeat; 1385 } 1386 1387 /* Has the page moved? */ 1388 if (unlikely(page != *slot)) { 1389 put_page(head); 1390 goto repeat; 1391 } 1392 export: 1393 indices[ret] = iter.index; 1394 entries[ret] = page; 1395 if (++ret == nr_entries) 1396 break; 1397 } 1398 rcu_read_unlock(); 1399 return ret; 1400 } 1401 1402 /** 1403 * find_get_pages - gang pagecache lookup 1404 * @mapping: The address_space to search 1405 * @start: The starting page index 1406 * @nr_pages: The maximum number of pages 1407 * @pages: Where the resulting pages are placed 1408 * 1409 * find_get_pages() will search for and return a group of up to 1410 * @nr_pages pages in the mapping. The pages are placed at @pages. 1411 * find_get_pages() takes a reference against the returned pages. 1412 * 1413 * The search returns a group of mapping-contiguous pages with ascending 1414 * indexes. There may be holes in the indices due to not-present pages. 1415 * 1416 * find_get_pages() returns the number of pages which were found. 1417 */ 1418 unsigned find_get_pages(struct address_space *mapping, pgoff_t start, 1419 unsigned int nr_pages, struct page **pages) 1420 { 1421 struct radix_tree_iter iter; 1422 void **slot; 1423 unsigned ret = 0; 1424 1425 if (unlikely(!nr_pages)) 1426 return 0; 1427 1428 rcu_read_lock(); 1429 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 1430 struct page *head, *page; 1431 repeat: 1432 page = radix_tree_deref_slot(slot); 1433 if (unlikely(!page)) 1434 continue; 1435 1436 if (radix_tree_exception(page)) { 1437 if (radix_tree_deref_retry(page)) { 1438 slot = radix_tree_iter_retry(&iter); 1439 continue; 1440 } 1441 /* 1442 * A shadow entry of a recently evicted page, 1443 * or a swap entry from shmem/tmpfs. Skip 1444 * over it. 1445 */ 1446 continue; 1447 } 1448 1449 head = compound_head(page); 1450 if (!page_cache_get_speculative(head)) 1451 goto repeat; 1452 1453 /* The page was split under us? */ 1454 if (compound_head(page) != head) { 1455 put_page(head); 1456 goto repeat; 1457 } 1458 1459 /* Has the page moved? */ 1460 if (unlikely(page != *slot)) { 1461 put_page(head); 1462 goto repeat; 1463 } 1464 1465 pages[ret] = page; 1466 if (++ret == nr_pages) 1467 break; 1468 } 1469 1470 rcu_read_unlock(); 1471 return ret; 1472 } 1473 1474 /** 1475 * find_get_pages_contig - gang contiguous pagecache lookup 1476 * @mapping: The address_space to search 1477 * @index: The starting page index 1478 * @nr_pages: The maximum number of pages 1479 * @pages: Where the resulting pages are placed 1480 * 1481 * find_get_pages_contig() works exactly like find_get_pages(), except 1482 * that the returned number of pages are guaranteed to be contiguous. 1483 * 1484 * find_get_pages_contig() returns the number of pages which were found. 1485 */ 1486 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, 1487 unsigned int nr_pages, struct page **pages) 1488 { 1489 struct radix_tree_iter iter; 1490 void **slot; 1491 unsigned int ret = 0; 1492 1493 if (unlikely(!nr_pages)) 1494 return 0; 1495 1496 rcu_read_lock(); 1497 radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) { 1498 struct page *head, *page; 1499 repeat: 1500 page = radix_tree_deref_slot(slot); 1501 /* The hole, there no reason to continue */ 1502 if (unlikely(!page)) 1503 break; 1504 1505 if (radix_tree_exception(page)) { 1506 if (radix_tree_deref_retry(page)) { 1507 slot = radix_tree_iter_retry(&iter); 1508 continue; 1509 } 1510 /* 1511 * A shadow entry of a recently evicted page, 1512 * or a swap entry from shmem/tmpfs. Stop 1513 * looking for contiguous pages. 1514 */ 1515 break; 1516 } 1517 1518 head = compound_head(page); 1519 if (!page_cache_get_speculative(head)) 1520 goto repeat; 1521 1522 /* The page was split under us? */ 1523 if (compound_head(page) != head) { 1524 put_page(head); 1525 goto repeat; 1526 } 1527 1528 /* Has the page moved? */ 1529 if (unlikely(page != *slot)) { 1530 put_page(head); 1531 goto repeat; 1532 } 1533 1534 /* 1535 * must check mapping and index after taking the ref. 1536 * otherwise we can get both false positives and false 1537 * negatives, which is just confusing to the caller. 1538 */ 1539 if (page->mapping == NULL || page_to_pgoff(page) != iter.index) { 1540 put_page(page); 1541 break; 1542 } 1543 1544 pages[ret] = page; 1545 if (++ret == nr_pages) 1546 break; 1547 } 1548 rcu_read_unlock(); 1549 return ret; 1550 } 1551 EXPORT_SYMBOL(find_get_pages_contig); 1552 1553 /** 1554 * find_get_pages_tag - find and return pages that match @tag 1555 * @mapping: the address_space to search 1556 * @index: the starting page index 1557 * @tag: the tag index 1558 * @nr_pages: the maximum number of pages 1559 * @pages: where the resulting pages are placed 1560 * 1561 * Like find_get_pages, except we only return pages which are tagged with 1562 * @tag. We update @index to index the next page for the traversal. 1563 */ 1564 unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index, 1565 int tag, unsigned int nr_pages, struct page **pages) 1566 { 1567 struct radix_tree_iter iter; 1568 void **slot; 1569 unsigned ret = 0; 1570 1571 if (unlikely(!nr_pages)) 1572 return 0; 1573 1574 rcu_read_lock(); 1575 radix_tree_for_each_tagged(slot, &mapping->page_tree, 1576 &iter, *index, tag) { 1577 struct page *head, *page; 1578 repeat: 1579 page = radix_tree_deref_slot(slot); 1580 if (unlikely(!page)) 1581 continue; 1582 1583 if (radix_tree_exception(page)) { 1584 if (radix_tree_deref_retry(page)) { 1585 slot = radix_tree_iter_retry(&iter); 1586 continue; 1587 } 1588 /* 1589 * A shadow entry of a recently evicted page. 1590 * 1591 * Those entries should never be tagged, but 1592 * this tree walk is lockless and the tags are 1593 * looked up in bulk, one radix tree node at a 1594 * time, so there is a sizable window for page 1595 * reclaim to evict a page we saw tagged. 1596 * 1597 * Skip over it. 1598 */ 1599 continue; 1600 } 1601 1602 head = compound_head(page); 1603 if (!page_cache_get_speculative(head)) 1604 goto repeat; 1605 1606 /* The page was split under us? */ 1607 if (compound_head(page) != head) { 1608 put_page(head); 1609 goto repeat; 1610 } 1611 1612 /* Has the page moved? */ 1613 if (unlikely(page != *slot)) { 1614 put_page(head); 1615 goto repeat; 1616 } 1617 1618 pages[ret] = page; 1619 if (++ret == nr_pages) 1620 break; 1621 } 1622 1623 rcu_read_unlock(); 1624 1625 if (ret) 1626 *index = pages[ret - 1]->index + 1; 1627 1628 return ret; 1629 } 1630 EXPORT_SYMBOL(find_get_pages_tag); 1631 1632 /** 1633 * find_get_entries_tag - find and return entries that match @tag 1634 * @mapping: the address_space to search 1635 * @start: the starting page cache index 1636 * @tag: the tag index 1637 * @nr_entries: the maximum number of entries 1638 * @entries: where the resulting entries are placed 1639 * @indices: the cache indices corresponding to the entries in @entries 1640 * 1641 * Like find_get_entries, except we only return entries which are tagged with 1642 * @tag. 1643 */ 1644 unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start, 1645 int tag, unsigned int nr_entries, 1646 struct page **entries, pgoff_t *indices) 1647 { 1648 void **slot; 1649 unsigned int ret = 0; 1650 struct radix_tree_iter iter; 1651 1652 if (!nr_entries) 1653 return 0; 1654 1655 rcu_read_lock(); 1656 radix_tree_for_each_tagged(slot, &mapping->page_tree, 1657 &iter, start, tag) { 1658 struct page *head, *page; 1659 repeat: 1660 page = radix_tree_deref_slot(slot); 1661 if (unlikely(!page)) 1662 continue; 1663 if (radix_tree_exception(page)) { 1664 if (radix_tree_deref_retry(page)) { 1665 slot = radix_tree_iter_retry(&iter); 1666 continue; 1667 } 1668 1669 /* 1670 * A shadow entry of a recently evicted page, a swap 1671 * entry from shmem/tmpfs or a DAX entry. Return it 1672 * without attempting to raise page count. 1673 */ 1674 goto export; 1675 } 1676 1677 head = compound_head(page); 1678 if (!page_cache_get_speculative(head)) 1679 goto repeat; 1680 1681 /* The page was split under us? */ 1682 if (compound_head(page) != head) { 1683 put_page(head); 1684 goto repeat; 1685 } 1686 1687 /* Has the page moved? */ 1688 if (unlikely(page != *slot)) { 1689 put_page(head); 1690 goto repeat; 1691 } 1692 export: 1693 indices[ret] = iter.index; 1694 entries[ret] = page; 1695 if (++ret == nr_entries) 1696 break; 1697 } 1698 rcu_read_unlock(); 1699 return ret; 1700 } 1701 EXPORT_SYMBOL(find_get_entries_tag); 1702 1703 /* 1704 * CD/DVDs are error prone. When a medium error occurs, the driver may fail 1705 * a _large_ part of the i/o request. Imagine the worst scenario: 1706 * 1707 * ---R__________________________________________B__________ 1708 * ^ reading here ^ bad block(assume 4k) 1709 * 1710 * read(R) => miss => readahead(R...B) => media error => frustrating retries 1711 * => failing the whole request => read(R) => read(R+1) => 1712 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) => 1713 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) => 1714 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ...... 1715 * 1716 * It is going insane. Fix it by quickly scaling down the readahead size. 1717 */ 1718 static void shrink_readahead_size_eio(struct file *filp, 1719 struct file_ra_state *ra) 1720 { 1721 ra->ra_pages /= 4; 1722 } 1723 1724 /** 1725 * do_generic_file_read - generic file read routine 1726 * @filp: the file to read 1727 * @ppos: current file position 1728 * @iter: data destination 1729 * @written: already copied 1730 * 1731 * This is a generic file read routine, and uses the 1732 * mapping->a_ops->readpage() function for the actual low-level stuff. 1733 * 1734 * This is really ugly. But the goto's actually try to clarify some 1735 * of the logic when it comes to error handling etc. 1736 */ 1737 static ssize_t do_generic_file_read(struct file *filp, loff_t *ppos, 1738 struct iov_iter *iter, ssize_t written) 1739 { 1740 struct address_space *mapping = filp->f_mapping; 1741 struct inode *inode = mapping->host; 1742 struct file_ra_state *ra = &filp->f_ra; 1743 pgoff_t index; 1744 pgoff_t last_index; 1745 pgoff_t prev_index; 1746 unsigned long offset; /* offset into pagecache page */ 1747 unsigned int prev_offset; 1748 int error = 0; 1749 1750 if (unlikely(*ppos >= inode->i_sb->s_maxbytes)) 1751 return 0; 1752 iov_iter_truncate(iter, inode->i_sb->s_maxbytes); 1753 1754 index = *ppos >> PAGE_SHIFT; 1755 prev_index = ra->prev_pos >> PAGE_SHIFT; 1756 prev_offset = ra->prev_pos & (PAGE_SIZE-1); 1757 last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT; 1758 offset = *ppos & ~PAGE_MASK; 1759 1760 for (;;) { 1761 struct page *page; 1762 pgoff_t end_index; 1763 loff_t isize; 1764 unsigned long nr, ret; 1765 1766 cond_resched(); 1767 find_page: 1768 page = find_get_page(mapping, index); 1769 if (!page) { 1770 page_cache_sync_readahead(mapping, 1771 ra, filp, 1772 index, last_index - index); 1773 page = find_get_page(mapping, index); 1774 if (unlikely(page == NULL)) 1775 goto no_cached_page; 1776 } 1777 if (PageReadahead(page)) { 1778 page_cache_async_readahead(mapping, 1779 ra, filp, page, 1780 index, last_index - index); 1781 } 1782 if (!PageUptodate(page)) { 1783 /* 1784 * See comment in do_read_cache_page on why 1785 * wait_on_page_locked is used to avoid unnecessarily 1786 * serialisations and why it's safe. 1787 */ 1788 error = wait_on_page_locked_killable(page); 1789 if (unlikely(error)) 1790 goto readpage_error; 1791 if (PageUptodate(page)) 1792 goto page_ok; 1793 1794 if (inode->i_blkbits == PAGE_SHIFT || 1795 !mapping->a_ops->is_partially_uptodate) 1796 goto page_not_up_to_date; 1797 /* pipes can't handle partially uptodate pages */ 1798 if (unlikely(iter->type & ITER_PIPE)) 1799 goto page_not_up_to_date; 1800 if (!trylock_page(page)) 1801 goto page_not_up_to_date; 1802 /* Did it get truncated before we got the lock? */ 1803 if (!page->mapping) 1804 goto page_not_up_to_date_locked; 1805 if (!mapping->a_ops->is_partially_uptodate(page, 1806 offset, iter->count)) 1807 goto page_not_up_to_date_locked; 1808 unlock_page(page); 1809 } 1810 page_ok: 1811 /* 1812 * i_size must be checked after we know the page is Uptodate. 1813 * 1814 * Checking i_size after the check allows us to calculate 1815 * the correct value for "nr", which means the zero-filled 1816 * part of the page is not copied back to userspace (unless 1817 * another truncate extends the file - this is desired though). 1818 */ 1819 1820 isize = i_size_read(inode); 1821 end_index = (isize - 1) >> PAGE_SHIFT; 1822 if (unlikely(!isize || index > end_index)) { 1823 put_page(page); 1824 goto out; 1825 } 1826 1827 /* nr is the maximum number of bytes to copy from this page */ 1828 nr = PAGE_SIZE; 1829 if (index == end_index) { 1830 nr = ((isize - 1) & ~PAGE_MASK) + 1; 1831 if (nr <= offset) { 1832 put_page(page); 1833 goto out; 1834 } 1835 } 1836 nr = nr - offset; 1837 1838 /* If users can be writing to this page using arbitrary 1839 * virtual addresses, take care about potential aliasing 1840 * before reading the page on the kernel side. 1841 */ 1842 if (mapping_writably_mapped(mapping)) 1843 flush_dcache_page(page); 1844 1845 /* 1846 * When a sequential read accesses a page several times, 1847 * only mark it as accessed the first time. 1848 */ 1849 if (prev_index != index || offset != prev_offset) 1850 mark_page_accessed(page); 1851 prev_index = index; 1852 1853 /* 1854 * Ok, we have the page, and it's up-to-date, so 1855 * now we can copy it to user space... 1856 */ 1857 1858 ret = copy_page_to_iter(page, offset, nr, iter); 1859 offset += ret; 1860 index += offset >> PAGE_SHIFT; 1861 offset &= ~PAGE_MASK; 1862 prev_offset = offset; 1863 1864 put_page(page); 1865 written += ret; 1866 if (!iov_iter_count(iter)) 1867 goto out; 1868 if (ret < nr) { 1869 error = -EFAULT; 1870 goto out; 1871 } 1872 continue; 1873 1874 page_not_up_to_date: 1875 /* Get exclusive access to the page ... */ 1876 error = lock_page_killable(page); 1877 if (unlikely(error)) 1878 goto readpage_error; 1879 1880 page_not_up_to_date_locked: 1881 /* Did it get truncated before we got the lock? */ 1882 if (!page->mapping) { 1883 unlock_page(page); 1884 put_page(page); 1885 continue; 1886 } 1887 1888 /* Did somebody else fill it already? */ 1889 if (PageUptodate(page)) { 1890 unlock_page(page); 1891 goto page_ok; 1892 } 1893 1894 readpage: 1895 /* 1896 * A previous I/O error may have been due to temporary 1897 * failures, eg. multipath errors. 1898 * PG_error will be set again if readpage fails. 1899 */ 1900 ClearPageError(page); 1901 /* Start the actual read. The read will unlock the page. */ 1902 error = mapping->a_ops->readpage(filp, page); 1903 1904 if (unlikely(error)) { 1905 if (error == AOP_TRUNCATED_PAGE) { 1906 put_page(page); 1907 error = 0; 1908 goto find_page; 1909 } 1910 goto readpage_error; 1911 } 1912 1913 if (!PageUptodate(page)) { 1914 error = lock_page_killable(page); 1915 if (unlikely(error)) 1916 goto readpage_error; 1917 if (!PageUptodate(page)) { 1918 if (page->mapping == NULL) { 1919 /* 1920 * invalidate_mapping_pages got it 1921 */ 1922 unlock_page(page); 1923 put_page(page); 1924 goto find_page; 1925 } 1926 unlock_page(page); 1927 shrink_readahead_size_eio(filp, ra); 1928 error = -EIO; 1929 goto readpage_error; 1930 } 1931 unlock_page(page); 1932 } 1933 1934 goto page_ok; 1935 1936 readpage_error: 1937 /* UHHUH! A synchronous read error occurred. Report it */ 1938 put_page(page); 1939 goto out; 1940 1941 no_cached_page: 1942 /* 1943 * Ok, it wasn't cached, so we need to create a new 1944 * page.. 1945 */ 1946 page = page_cache_alloc_cold(mapping); 1947 if (!page) { 1948 error = -ENOMEM; 1949 goto out; 1950 } 1951 error = add_to_page_cache_lru(page, mapping, index, 1952 mapping_gfp_constraint(mapping, GFP_KERNEL)); 1953 if (error) { 1954 put_page(page); 1955 if (error == -EEXIST) { 1956 error = 0; 1957 goto find_page; 1958 } 1959 goto out; 1960 } 1961 goto readpage; 1962 } 1963 1964 out: 1965 ra->prev_pos = prev_index; 1966 ra->prev_pos <<= PAGE_SHIFT; 1967 ra->prev_pos |= prev_offset; 1968 1969 *ppos = ((loff_t)index << PAGE_SHIFT) + offset; 1970 file_accessed(filp); 1971 return written ? written : error; 1972 } 1973 1974 /** 1975 * generic_file_read_iter - generic filesystem read routine 1976 * @iocb: kernel I/O control block 1977 * @iter: destination for the data read 1978 * 1979 * This is the "read_iter()" routine for all filesystems 1980 * that can use the page cache directly. 1981 */ 1982 ssize_t 1983 generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) 1984 { 1985 struct file *file = iocb->ki_filp; 1986 ssize_t retval = 0; 1987 size_t count = iov_iter_count(iter); 1988 1989 if (!count) 1990 goto out; /* skip atime */ 1991 1992 if (iocb->ki_flags & IOCB_DIRECT) { 1993 struct address_space *mapping = file->f_mapping; 1994 struct inode *inode = mapping->host; 1995 struct iov_iter data = *iter; 1996 loff_t size; 1997 1998 size = i_size_read(inode); 1999 retval = filemap_write_and_wait_range(mapping, iocb->ki_pos, 2000 iocb->ki_pos + count - 1); 2001 if (retval < 0) 2002 goto out; 2003 2004 file_accessed(file); 2005 2006 retval = mapping->a_ops->direct_IO(iocb, &data); 2007 if (retval >= 0) { 2008 iocb->ki_pos += retval; 2009 iov_iter_advance(iter, retval); 2010 } 2011 2012 /* 2013 * Btrfs can have a short DIO read if we encounter 2014 * compressed extents, so if there was an error, or if 2015 * we've already read everything we wanted to, or if 2016 * there was a short read because we hit EOF, go ahead 2017 * and return. Otherwise fallthrough to buffered io for 2018 * the rest of the read. Buffered reads will not work for 2019 * DAX files, so don't bother trying. 2020 */ 2021 if (retval < 0 || !iov_iter_count(iter) || iocb->ki_pos >= size || 2022 IS_DAX(inode)) 2023 goto out; 2024 } 2025 2026 retval = do_generic_file_read(file, &iocb->ki_pos, iter, retval); 2027 out: 2028 return retval; 2029 } 2030 EXPORT_SYMBOL(generic_file_read_iter); 2031 2032 #ifdef CONFIG_MMU 2033 /** 2034 * page_cache_read - adds requested page to the page cache if not already there 2035 * @file: file to read 2036 * @offset: page index 2037 * @gfp_mask: memory allocation flags 2038 * 2039 * This adds the requested page to the page cache if it isn't already there, 2040 * and schedules an I/O to read in its contents from disk. 2041 */ 2042 static int page_cache_read(struct file *file, pgoff_t offset, gfp_t gfp_mask) 2043 { 2044 struct address_space *mapping = file->f_mapping; 2045 struct page *page; 2046 int ret; 2047 2048 do { 2049 page = __page_cache_alloc(gfp_mask|__GFP_COLD); 2050 if (!page) 2051 return -ENOMEM; 2052 2053 ret = add_to_page_cache_lru(page, mapping, offset, gfp_mask & GFP_KERNEL); 2054 if (ret == 0) 2055 ret = mapping->a_ops->readpage(file, page); 2056 else if (ret == -EEXIST) 2057 ret = 0; /* losing race to add is OK */ 2058 2059 put_page(page); 2060 2061 } while (ret == AOP_TRUNCATED_PAGE); 2062 2063 return ret; 2064 } 2065 2066 #define MMAP_LOTSAMISS (100) 2067 2068 /* 2069 * Synchronous readahead happens when we don't even find 2070 * a page in the page cache at all. 2071 */ 2072 static void do_sync_mmap_readahead(struct vm_area_struct *vma, 2073 struct file_ra_state *ra, 2074 struct file *file, 2075 pgoff_t offset) 2076 { 2077 struct address_space *mapping = file->f_mapping; 2078 2079 /* If we don't want any read-ahead, don't bother */ 2080 if (vma->vm_flags & VM_RAND_READ) 2081 return; 2082 if (!ra->ra_pages) 2083 return; 2084 2085 if (vma->vm_flags & VM_SEQ_READ) { 2086 page_cache_sync_readahead(mapping, ra, file, offset, 2087 ra->ra_pages); 2088 return; 2089 } 2090 2091 /* Avoid banging the cache line if not needed */ 2092 if (ra->mmap_miss < MMAP_LOTSAMISS * 10) 2093 ra->mmap_miss++; 2094 2095 /* 2096 * Do we miss much more than hit in this file? If so, 2097 * stop bothering with read-ahead. It will only hurt. 2098 */ 2099 if (ra->mmap_miss > MMAP_LOTSAMISS) 2100 return; 2101 2102 /* 2103 * mmap read-around 2104 */ 2105 ra->start = max_t(long, 0, offset - ra->ra_pages / 2); 2106 ra->size = ra->ra_pages; 2107 ra->async_size = ra->ra_pages / 4; 2108 ra_submit(ra, mapping, file); 2109 } 2110 2111 /* 2112 * Asynchronous readahead happens when we find the page and PG_readahead, 2113 * so we want to possibly extend the readahead further.. 2114 */ 2115 static void do_async_mmap_readahead(struct vm_area_struct *vma, 2116 struct file_ra_state *ra, 2117 struct file *file, 2118 struct page *page, 2119 pgoff_t offset) 2120 { 2121 struct address_space *mapping = file->f_mapping; 2122 2123 /* If we don't want any read-ahead, don't bother */ 2124 if (vma->vm_flags & VM_RAND_READ) 2125 return; 2126 if (ra->mmap_miss > 0) 2127 ra->mmap_miss--; 2128 if (PageReadahead(page)) 2129 page_cache_async_readahead(mapping, ra, file, 2130 page, offset, ra->ra_pages); 2131 } 2132 2133 /** 2134 * filemap_fault - read in file data for page fault handling 2135 * @vma: vma in which the fault was taken 2136 * @vmf: struct vm_fault containing details of the fault 2137 * 2138 * filemap_fault() is invoked via the vma operations vector for a 2139 * mapped memory region to read in file data during a page fault. 2140 * 2141 * The goto's are kind of ugly, but this streamlines the normal case of having 2142 * it in the page cache, and handles the special cases reasonably without 2143 * having a lot of duplicated code. 2144 * 2145 * vma->vm_mm->mmap_sem must be held on entry. 2146 * 2147 * If our return value has VM_FAULT_RETRY set, it's because 2148 * lock_page_or_retry() returned 0. 2149 * The mmap_sem has usually been released in this case. 2150 * See __lock_page_or_retry() for the exception. 2151 * 2152 * If our return value does not have VM_FAULT_RETRY set, the mmap_sem 2153 * has not been released. 2154 * 2155 * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set. 2156 */ 2157 int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 2158 { 2159 int error; 2160 struct file *file = vma->vm_file; 2161 struct address_space *mapping = file->f_mapping; 2162 struct file_ra_state *ra = &file->f_ra; 2163 struct inode *inode = mapping->host; 2164 pgoff_t offset = vmf->pgoff; 2165 struct page *page; 2166 loff_t size; 2167 int ret = 0; 2168 2169 size = round_up(i_size_read(inode), PAGE_SIZE); 2170 if (offset >= size >> PAGE_SHIFT) 2171 return VM_FAULT_SIGBUS; 2172 2173 /* 2174 * Do we have something in the page cache already? 2175 */ 2176 page = find_get_page(mapping, offset); 2177 if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) { 2178 /* 2179 * We found the page, so try async readahead before 2180 * waiting for the lock. 2181 */ 2182 do_async_mmap_readahead(vma, ra, file, page, offset); 2183 } else if (!page) { 2184 /* No page in the page cache at all */ 2185 do_sync_mmap_readahead(vma, ra, file, offset); 2186 count_vm_event(PGMAJFAULT); 2187 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); 2188 ret = VM_FAULT_MAJOR; 2189 retry_find: 2190 page = find_get_page(mapping, offset); 2191 if (!page) 2192 goto no_cached_page; 2193 } 2194 2195 if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags)) { 2196 put_page(page); 2197 return ret | VM_FAULT_RETRY; 2198 } 2199 2200 /* Did it get truncated? */ 2201 if (unlikely(page->mapping != mapping)) { 2202 unlock_page(page); 2203 put_page(page); 2204 goto retry_find; 2205 } 2206 VM_BUG_ON_PAGE(page->index != offset, page); 2207 2208 /* 2209 * We have a locked page in the page cache, now we need to check 2210 * that it's up-to-date. If not, it is going to be due to an error. 2211 */ 2212 if (unlikely(!PageUptodate(page))) 2213 goto page_not_uptodate; 2214 2215 /* 2216 * Found the page and have a reference on it. 2217 * We must recheck i_size under page lock. 2218 */ 2219 size = round_up(i_size_read(inode), PAGE_SIZE); 2220 if (unlikely(offset >= size >> PAGE_SHIFT)) { 2221 unlock_page(page); 2222 put_page(page); 2223 return VM_FAULT_SIGBUS; 2224 } 2225 2226 vmf->page = page; 2227 return ret | VM_FAULT_LOCKED; 2228 2229 no_cached_page: 2230 /* 2231 * We're only likely to ever get here if MADV_RANDOM is in 2232 * effect. 2233 */ 2234 error = page_cache_read(file, offset, vmf->gfp_mask); 2235 2236 /* 2237 * The page we want has now been added to the page cache. 2238 * In the unlikely event that someone removed it in the 2239 * meantime, we'll just come back here and read it again. 2240 */ 2241 if (error >= 0) 2242 goto retry_find; 2243 2244 /* 2245 * An error return from page_cache_read can result if the 2246 * system is low on memory, or a problem occurs while trying 2247 * to schedule I/O. 2248 */ 2249 if (error == -ENOMEM) 2250 return VM_FAULT_OOM; 2251 return VM_FAULT_SIGBUS; 2252 2253 page_not_uptodate: 2254 /* 2255 * Umm, take care of errors if the page isn't up-to-date. 2256 * Try to re-read it _once_. We do this synchronously, 2257 * because there really aren't any performance issues here 2258 * and we need to check for errors. 2259 */ 2260 ClearPageError(page); 2261 error = mapping->a_ops->readpage(file, page); 2262 if (!error) { 2263 wait_on_page_locked(page); 2264 if (!PageUptodate(page)) 2265 error = -EIO; 2266 } 2267 put_page(page); 2268 2269 if (!error || error == AOP_TRUNCATED_PAGE) 2270 goto retry_find; 2271 2272 /* Things didn't work out. Return zero to tell the mm layer so. */ 2273 shrink_readahead_size_eio(file, ra); 2274 return VM_FAULT_SIGBUS; 2275 } 2276 EXPORT_SYMBOL(filemap_fault); 2277 2278 void filemap_map_pages(struct vm_fault *vmf, 2279 pgoff_t start_pgoff, pgoff_t end_pgoff) 2280 { 2281 struct radix_tree_iter iter; 2282 void **slot; 2283 struct file *file = vmf->vma->vm_file; 2284 struct address_space *mapping = file->f_mapping; 2285 pgoff_t last_pgoff = start_pgoff; 2286 loff_t size; 2287 struct page *head, *page; 2288 2289 rcu_read_lock(); 2290 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, 2291 start_pgoff) { 2292 if (iter.index > end_pgoff) 2293 break; 2294 repeat: 2295 page = radix_tree_deref_slot(slot); 2296 if (unlikely(!page)) 2297 goto next; 2298 if (radix_tree_exception(page)) { 2299 if (radix_tree_deref_retry(page)) { 2300 slot = radix_tree_iter_retry(&iter); 2301 continue; 2302 } 2303 goto next; 2304 } 2305 2306 head = compound_head(page); 2307 if (!page_cache_get_speculative(head)) 2308 goto repeat; 2309 2310 /* The page was split under us? */ 2311 if (compound_head(page) != head) { 2312 put_page(head); 2313 goto repeat; 2314 } 2315 2316 /* Has the page moved? */ 2317 if (unlikely(page != *slot)) { 2318 put_page(head); 2319 goto repeat; 2320 } 2321 2322 if (!PageUptodate(page) || 2323 PageReadahead(page) || 2324 PageHWPoison(page)) 2325 goto skip; 2326 if (!trylock_page(page)) 2327 goto skip; 2328 2329 if (page->mapping != mapping || !PageUptodate(page)) 2330 goto unlock; 2331 2332 size = round_up(i_size_read(mapping->host), PAGE_SIZE); 2333 if (page->index >= size >> PAGE_SHIFT) 2334 goto unlock; 2335 2336 if (file->f_ra.mmap_miss > 0) 2337 file->f_ra.mmap_miss--; 2338 2339 vmf->address += (iter.index - last_pgoff) << PAGE_SHIFT; 2340 if (vmf->pte) 2341 vmf->pte += iter.index - last_pgoff; 2342 last_pgoff = iter.index; 2343 if (alloc_set_pte(vmf, NULL, page)) 2344 goto unlock; 2345 unlock_page(page); 2346 goto next; 2347 unlock: 2348 unlock_page(page); 2349 skip: 2350 put_page(page); 2351 next: 2352 /* Huge page is mapped? No need to proceed. */ 2353 if (pmd_trans_huge(*vmf->pmd)) 2354 break; 2355 if (iter.index == end_pgoff) 2356 break; 2357 } 2358 rcu_read_unlock(); 2359 } 2360 EXPORT_SYMBOL(filemap_map_pages); 2361 2362 int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) 2363 { 2364 struct page *page = vmf->page; 2365 struct inode *inode = file_inode(vma->vm_file); 2366 int ret = VM_FAULT_LOCKED; 2367 2368 sb_start_pagefault(inode->i_sb); 2369 file_update_time(vma->vm_file); 2370 lock_page(page); 2371 if (page->mapping != inode->i_mapping) { 2372 unlock_page(page); 2373 ret = VM_FAULT_NOPAGE; 2374 goto out; 2375 } 2376 /* 2377 * We mark the page dirty already here so that when freeze is in 2378 * progress, we are guaranteed that writeback during freezing will 2379 * see the dirty page and writeprotect it again. 2380 */ 2381 set_page_dirty(page); 2382 wait_for_stable_page(page); 2383 out: 2384 sb_end_pagefault(inode->i_sb); 2385 return ret; 2386 } 2387 EXPORT_SYMBOL(filemap_page_mkwrite); 2388 2389 const struct vm_operations_struct generic_file_vm_ops = { 2390 .fault = filemap_fault, 2391 .map_pages = filemap_map_pages, 2392 .page_mkwrite = filemap_page_mkwrite, 2393 }; 2394 2395 /* This is used for a general mmap of a disk file */ 2396 2397 int generic_file_mmap(struct file * file, struct vm_area_struct * vma) 2398 { 2399 struct address_space *mapping = file->f_mapping; 2400 2401 if (!mapping->a_ops->readpage) 2402 return -ENOEXEC; 2403 file_accessed(file); 2404 vma->vm_ops = &generic_file_vm_ops; 2405 return 0; 2406 } 2407 2408 /* 2409 * This is for filesystems which do not implement ->writepage. 2410 */ 2411 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) 2412 { 2413 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) 2414 return -EINVAL; 2415 return generic_file_mmap(file, vma); 2416 } 2417 #else 2418 int generic_file_mmap(struct file * file, struct vm_area_struct * vma) 2419 { 2420 return -ENOSYS; 2421 } 2422 int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma) 2423 { 2424 return -ENOSYS; 2425 } 2426 #endif /* CONFIG_MMU */ 2427 2428 EXPORT_SYMBOL(generic_file_mmap); 2429 EXPORT_SYMBOL(generic_file_readonly_mmap); 2430 2431 static struct page *wait_on_page_read(struct page *page) 2432 { 2433 if (!IS_ERR(page)) { 2434 wait_on_page_locked(page); 2435 if (!PageUptodate(page)) { 2436 put_page(page); 2437 page = ERR_PTR(-EIO); 2438 } 2439 } 2440 return page; 2441 } 2442 2443 static struct page *do_read_cache_page(struct address_space *mapping, 2444 pgoff_t index, 2445 int (*filler)(void *, struct page *), 2446 void *data, 2447 gfp_t gfp) 2448 { 2449 struct page *page; 2450 int err; 2451 repeat: 2452 page = find_get_page(mapping, index); 2453 if (!page) { 2454 page = __page_cache_alloc(gfp | __GFP_COLD); 2455 if (!page) 2456 return ERR_PTR(-ENOMEM); 2457 err = add_to_page_cache_lru(page, mapping, index, gfp); 2458 if (unlikely(err)) { 2459 put_page(page); 2460 if (err == -EEXIST) 2461 goto repeat; 2462 /* Presumably ENOMEM for radix tree node */ 2463 return ERR_PTR(err); 2464 } 2465 2466 filler: 2467 err = filler(data, page); 2468 if (err < 0) { 2469 put_page(page); 2470 return ERR_PTR(err); 2471 } 2472 2473 page = wait_on_page_read(page); 2474 if (IS_ERR(page)) 2475 return page; 2476 goto out; 2477 } 2478 if (PageUptodate(page)) 2479 goto out; 2480 2481 /* 2482 * Page is not up to date and may be locked due one of the following 2483 * case a: Page is being filled and the page lock is held 2484 * case b: Read/write error clearing the page uptodate status 2485 * case c: Truncation in progress (page locked) 2486 * case d: Reclaim in progress 2487 * 2488 * Case a, the page will be up to date when the page is unlocked. 2489 * There is no need to serialise on the page lock here as the page 2490 * is pinned so the lock gives no additional protection. Even if the 2491 * the page is truncated, the data is still valid if PageUptodate as 2492 * it's a race vs truncate race. 2493 * Case b, the page will not be up to date 2494 * Case c, the page may be truncated but in itself, the data may still 2495 * be valid after IO completes as it's a read vs truncate race. The 2496 * operation must restart if the page is not uptodate on unlock but 2497 * otherwise serialising on page lock to stabilise the mapping gives 2498 * no additional guarantees to the caller as the page lock is 2499 * released before return. 2500 * Case d, similar to truncation. If reclaim holds the page lock, it 2501 * will be a race with remove_mapping that determines if the mapping 2502 * is valid on unlock but otherwise the data is valid and there is 2503 * no need to serialise with page lock. 2504 * 2505 * As the page lock gives no additional guarantee, we optimistically 2506 * wait on the page to be unlocked and check if it's up to date and 2507 * use the page if it is. Otherwise, the page lock is required to 2508 * distinguish between the different cases. The motivation is that we 2509 * avoid spurious serialisations and wakeups when multiple processes 2510 * wait on the same page for IO to complete. 2511 */ 2512 wait_on_page_locked(page); 2513 if (PageUptodate(page)) 2514 goto out; 2515 2516 /* Distinguish between all the cases under the safety of the lock */ 2517 lock_page(page); 2518 2519 /* Case c or d, restart the operation */ 2520 if (!page->mapping) { 2521 unlock_page(page); 2522 put_page(page); 2523 goto repeat; 2524 } 2525 2526 /* Someone else locked and filled the page in a very small window */ 2527 if (PageUptodate(page)) { 2528 unlock_page(page); 2529 goto out; 2530 } 2531 goto filler; 2532 2533 out: 2534 mark_page_accessed(page); 2535 return page; 2536 } 2537 2538 /** 2539 * read_cache_page - read into page cache, fill it if needed 2540 * @mapping: the page's address_space 2541 * @index: the page index 2542 * @filler: function to perform the read 2543 * @data: first arg to filler(data, page) function, often left as NULL 2544 * 2545 * Read into the page cache. If a page already exists, and PageUptodate() is 2546 * not set, try to fill the page and wait for it to become unlocked. 2547 * 2548 * If the page does not get brought uptodate, return -EIO. 2549 */ 2550 struct page *read_cache_page(struct address_space *mapping, 2551 pgoff_t index, 2552 int (*filler)(void *, struct page *), 2553 void *data) 2554 { 2555 return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping)); 2556 } 2557 EXPORT_SYMBOL(read_cache_page); 2558 2559 /** 2560 * read_cache_page_gfp - read into page cache, using specified page allocation flags. 2561 * @mapping: the page's address_space 2562 * @index: the page index 2563 * @gfp: the page allocator flags to use if allocating 2564 * 2565 * This is the same as "read_mapping_page(mapping, index, NULL)", but with 2566 * any new page allocations done using the specified allocation flags. 2567 * 2568 * If the page does not get brought uptodate, return -EIO. 2569 */ 2570 struct page *read_cache_page_gfp(struct address_space *mapping, 2571 pgoff_t index, 2572 gfp_t gfp) 2573 { 2574 filler_t *filler = (filler_t *)mapping->a_ops->readpage; 2575 2576 return do_read_cache_page(mapping, index, filler, NULL, gfp); 2577 } 2578 EXPORT_SYMBOL(read_cache_page_gfp); 2579 2580 /* 2581 * Performs necessary checks before doing a write 2582 * 2583 * Can adjust writing position or amount of bytes to write. 2584 * Returns appropriate error code that caller should return or 2585 * zero in case that write should be allowed. 2586 */ 2587 inline ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from) 2588 { 2589 struct file *file = iocb->ki_filp; 2590 struct inode *inode = file->f_mapping->host; 2591 unsigned long limit = rlimit(RLIMIT_FSIZE); 2592 loff_t pos; 2593 2594 if (!iov_iter_count(from)) 2595 return 0; 2596 2597 /* FIXME: this is for backwards compatibility with 2.4 */ 2598 if (iocb->ki_flags & IOCB_APPEND) 2599 iocb->ki_pos = i_size_read(inode); 2600 2601 pos = iocb->ki_pos; 2602 2603 if (limit != RLIM_INFINITY) { 2604 if (iocb->ki_pos >= limit) { 2605 send_sig(SIGXFSZ, current, 0); 2606 return -EFBIG; 2607 } 2608 iov_iter_truncate(from, limit - (unsigned long)pos); 2609 } 2610 2611 /* 2612 * LFS rule 2613 */ 2614 if (unlikely(pos + iov_iter_count(from) > MAX_NON_LFS && 2615 !(file->f_flags & O_LARGEFILE))) { 2616 if (pos >= MAX_NON_LFS) 2617 return -EFBIG; 2618 iov_iter_truncate(from, MAX_NON_LFS - (unsigned long)pos); 2619 } 2620 2621 /* 2622 * Are we about to exceed the fs block limit ? 2623 * 2624 * If we have written data it becomes a short write. If we have 2625 * exceeded without writing data we send a signal and return EFBIG. 2626 * Linus frestrict idea will clean these up nicely.. 2627 */ 2628 if (unlikely(pos >= inode->i_sb->s_maxbytes)) 2629 return -EFBIG; 2630 2631 iov_iter_truncate(from, inode->i_sb->s_maxbytes - pos); 2632 return iov_iter_count(from); 2633 } 2634 EXPORT_SYMBOL(generic_write_checks); 2635 2636 int pagecache_write_begin(struct file *file, struct address_space *mapping, 2637 loff_t pos, unsigned len, unsigned flags, 2638 struct page **pagep, void **fsdata) 2639 { 2640 const struct address_space_operations *aops = mapping->a_ops; 2641 2642 return aops->write_begin(file, mapping, pos, len, flags, 2643 pagep, fsdata); 2644 } 2645 EXPORT_SYMBOL(pagecache_write_begin); 2646 2647 int pagecache_write_end(struct file *file, struct address_space *mapping, 2648 loff_t pos, unsigned len, unsigned copied, 2649 struct page *page, void *fsdata) 2650 { 2651 const struct address_space_operations *aops = mapping->a_ops; 2652 2653 return aops->write_end(file, mapping, pos, len, copied, page, fsdata); 2654 } 2655 EXPORT_SYMBOL(pagecache_write_end); 2656 2657 ssize_t 2658 generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from) 2659 { 2660 struct file *file = iocb->ki_filp; 2661 struct address_space *mapping = file->f_mapping; 2662 struct inode *inode = mapping->host; 2663 loff_t pos = iocb->ki_pos; 2664 ssize_t written; 2665 size_t write_len; 2666 pgoff_t end; 2667 struct iov_iter data; 2668 2669 write_len = iov_iter_count(from); 2670 end = (pos + write_len - 1) >> PAGE_SHIFT; 2671 2672 written = filemap_write_and_wait_range(mapping, pos, pos + write_len - 1); 2673 if (written) 2674 goto out; 2675 2676 /* 2677 * After a write we want buffered reads to be sure to go to disk to get 2678 * the new data. We invalidate clean cached page from the region we're 2679 * about to write. We do this *before* the write so that we can return 2680 * without clobbering -EIOCBQUEUED from ->direct_IO(). 2681 */ 2682 if (mapping->nrpages) { 2683 written = invalidate_inode_pages2_range(mapping, 2684 pos >> PAGE_SHIFT, end); 2685 /* 2686 * If a page can not be invalidated, return 0 to fall back 2687 * to buffered write. 2688 */ 2689 if (written) { 2690 if (written == -EBUSY) 2691 return 0; 2692 goto out; 2693 } 2694 } 2695 2696 data = *from; 2697 written = mapping->a_ops->direct_IO(iocb, &data); 2698 2699 /* 2700 * Finally, try again to invalidate clean pages which might have been 2701 * cached by non-direct readahead, or faulted in by get_user_pages() 2702 * if the source of the write was an mmap'ed region of the file 2703 * we're writing. Either one is a pretty crazy thing to do, 2704 * so we don't support it 100%. If this invalidation 2705 * fails, tough, the write still worked... 2706 */ 2707 if (mapping->nrpages) { 2708 invalidate_inode_pages2_range(mapping, 2709 pos >> PAGE_SHIFT, end); 2710 } 2711 2712 if (written > 0) { 2713 pos += written; 2714 iov_iter_advance(from, written); 2715 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { 2716 i_size_write(inode, pos); 2717 mark_inode_dirty(inode); 2718 } 2719 iocb->ki_pos = pos; 2720 } 2721 out: 2722 return written; 2723 } 2724 EXPORT_SYMBOL(generic_file_direct_write); 2725 2726 /* 2727 * Find or create a page at the given pagecache position. Return the locked 2728 * page. This function is specifically for buffered writes. 2729 */ 2730 struct page *grab_cache_page_write_begin(struct address_space *mapping, 2731 pgoff_t index, unsigned flags) 2732 { 2733 struct page *page; 2734 int fgp_flags = FGP_LOCK|FGP_WRITE|FGP_CREAT; 2735 2736 if (flags & AOP_FLAG_NOFS) 2737 fgp_flags |= FGP_NOFS; 2738 2739 page = pagecache_get_page(mapping, index, fgp_flags, 2740 mapping_gfp_mask(mapping)); 2741 if (page) 2742 wait_for_stable_page(page); 2743 2744 return page; 2745 } 2746 EXPORT_SYMBOL(grab_cache_page_write_begin); 2747 2748 ssize_t generic_perform_write(struct file *file, 2749 struct iov_iter *i, loff_t pos) 2750 { 2751 struct address_space *mapping = file->f_mapping; 2752 const struct address_space_operations *a_ops = mapping->a_ops; 2753 long status = 0; 2754 ssize_t written = 0; 2755 unsigned int flags = 0; 2756 2757 /* 2758 * Copies from kernel address space cannot fail (NFSD is a big user). 2759 */ 2760 if (!iter_is_iovec(i)) 2761 flags |= AOP_FLAG_UNINTERRUPTIBLE; 2762 2763 do { 2764 struct page *page; 2765 unsigned long offset; /* Offset into pagecache page */ 2766 unsigned long bytes; /* Bytes to write to page */ 2767 size_t copied; /* Bytes copied from user */ 2768 void *fsdata; 2769 2770 offset = (pos & (PAGE_SIZE - 1)); 2771 bytes = min_t(unsigned long, PAGE_SIZE - offset, 2772 iov_iter_count(i)); 2773 2774 again: 2775 /* 2776 * Bring in the user page that we will copy from _first_. 2777 * Otherwise there's a nasty deadlock on copying from the 2778 * same page as we're writing to, without it being marked 2779 * up-to-date. 2780 * 2781 * Not only is this an optimisation, but it is also required 2782 * to check that the address is actually valid, when atomic 2783 * usercopies are used, below. 2784 */ 2785 if (unlikely(iov_iter_fault_in_readable(i, bytes))) { 2786 status = -EFAULT; 2787 break; 2788 } 2789 2790 if (fatal_signal_pending(current)) { 2791 status = -EINTR; 2792 break; 2793 } 2794 2795 status = a_ops->write_begin(file, mapping, pos, bytes, flags, 2796 &page, &fsdata); 2797 if (unlikely(status < 0)) 2798 break; 2799 2800 if (mapping_writably_mapped(mapping)) 2801 flush_dcache_page(page); 2802 2803 copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes); 2804 flush_dcache_page(page); 2805 2806 status = a_ops->write_end(file, mapping, pos, bytes, copied, 2807 page, fsdata); 2808 if (unlikely(status < 0)) 2809 break; 2810 copied = status; 2811 2812 cond_resched(); 2813 2814 iov_iter_advance(i, copied); 2815 if (unlikely(copied == 0)) { 2816 /* 2817 * If we were unable to copy any data at all, we must 2818 * fall back to a single segment length write. 2819 * 2820 * If we didn't fallback here, we could livelock 2821 * because not all segments in the iov can be copied at 2822 * once without a pagefault. 2823 */ 2824 bytes = min_t(unsigned long, PAGE_SIZE - offset, 2825 iov_iter_single_seg_count(i)); 2826 goto again; 2827 } 2828 pos += copied; 2829 written += copied; 2830 2831 balance_dirty_pages_ratelimited(mapping); 2832 } while (iov_iter_count(i)); 2833 2834 return written ? written : status; 2835 } 2836 EXPORT_SYMBOL(generic_perform_write); 2837 2838 /** 2839 * __generic_file_write_iter - write data to a file 2840 * @iocb: IO state structure (file, offset, etc.) 2841 * @from: iov_iter with data to write 2842 * 2843 * This function does all the work needed for actually writing data to a 2844 * file. It does all basic checks, removes SUID from the file, updates 2845 * modification times and calls proper subroutines depending on whether we 2846 * do direct IO or a standard buffered write. 2847 * 2848 * It expects i_mutex to be grabbed unless we work on a block device or similar 2849 * object which does not need locking at all. 2850 * 2851 * This function does *not* take care of syncing data in case of O_SYNC write. 2852 * A caller has to handle it. This is mainly due to the fact that we want to 2853 * avoid syncing under i_mutex. 2854 */ 2855 ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 2856 { 2857 struct file *file = iocb->ki_filp; 2858 struct address_space * mapping = file->f_mapping; 2859 struct inode *inode = mapping->host; 2860 ssize_t written = 0; 2861 ssize_t err; 2862 ssize_t status; 2863 2864 /* We can write back this queue in page reclaim */ 2865 current->backing_dev_info = inode_to_bdi(inode); 2866 err = file_remove_privs(file); 2867 if (err) 2868 goto out; 2869 2870 err = file_update_time(file); 2871 if (err) 2872 goto out; 2873 2874 if (iocb->ki_flags & IOCB_DIRECT) { 2875 loff_t pos, endbyte; 2876 2877 written = generic_file_direct_write(iocb, from); 2878 /* 2879 * If the write stopped short of completing, fall back to 2880 * buffered writes. Some filesystems do this for writes to 2881 * holes, for example. For DAX files, a buffered write will 2882 * not succeed (even if it did, DAX does not handle dirty 2883 * page-cache pages correctly). 2884 */ 2885 if (written < 0 || !iov_iter_count(from) || IS_DAX(inode)) 2886 goto out; 2887 2888 status = generic_perform_write(file, from, pos = iocb->ki_pos); 2889 /* 2890 * If generic_perform_write() returned a synchronous error 2891 * then we want to return the number of bytes which were 2892 * direct-written, or the error code if that was zero. Note 2893 * that this differs from normal direct-io semantics, which 2894 * will return -EFOO even if some bytes were written. 2895 */ 2896 if (unlikely(status < 0)) { 2897 err = status; 2898 goto out; 2899 } 2900 /* 2901 * We need to ensure that the page cache pages are written to 2902 * disk and invalidated to preserve the expected O_DIRECT 2903 * semantics. 2904 */ 2905 endbyte = pos + status - 1; 2906 err = filemap_write_and_wait_range(mapping, pos, endbyte); 2907 if (err == 0) { 2908 iocb->ki_pos = endbyte + 1; 2909 written += status; 2910 invalidate_mapping_pages(mapping, 2911 pos >> PAGE_SHIFT, 2912 endbyte >> PAGE_SHIFT); 2913 } else { 2914 /* 2915 * We don't know how much we wrote, so just return 2916 * the number of bytes which were direct-written 2917 */ 2918 } 2919 } else { 2920 written = generic_perform_write(file, from, iocb->ki_pos); 2921 if (likely(written > 0)) 2922 iocb->ki_pos += written; 2923 } 2924 out: 2925 current->backing_dev_info = NULL; 2926 return written ? written : err; 2927 } 2928 EXPORT_SYMBOL(__generic_file_write_iter); 2929 2930 /** 2931 * generic_file_write_iter - write data to a file 2932 * @iocb: IO state structure 2933 * @from: iov_iter with data to write 2934 * 2935 * This is a wrapper around __generic_file_write_iter() to be used by most 2936 * filesystems. It takes care of syncing the file in case of O_SYNC file 2937 * and acquires i_mutex as needed. 2938 */ 2939 ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 2940 { 2941 struct file *file = iocb->ki_filp; 2942 struct inode *inode = file->f_mapping->host; 2943 ssize_t ret; 2944 2945 inode_lock(inode); 2946 ret = generic_write_checks(iocb, from); 2947 if (ret > 0) 2948 ret = __generic_file_write_iter(iocb, from); 2949 inode_unlock(inode); 2950 2951 if (ret > 0) 2952 ret = generic_write_sync(iocb, ret); 2953 return ret; 2954 } 2955 EXPORT_SYMBOL(generic_file_write_iter); 2956 2957 /** 2958 * try_to_release_page() - release old fs-specific metadata on a page 2959 * 2960 * @page: the page which the kernel is trying to free 2961 * @gfp_mask: memory allocation flags (and I/O mode) 2962 * 2963 * The address_space is to try to release any data against the page 2964 * (presumably at page->private). If the release was successful, return `1'. 2965 * Otherwise return zero. 2966 * 2967 * This may also be called if PG_fscache is set on a page, indicating that the 2968 * page is known to the local caching routines. 2969 * 2970 * The @gfp_mask argument specifies whether I/O may be performed to release 2971 * this page (__GFP_IO), and whether the call may block (__GFP_RECLAIM & __GFP_FS). 2972 * 2973 */ 2974 int try_to_release_page(struct page *page, gfp_t gfp_mask) 2975 { 2976 struct address_space * const mapping = page->mapping; 2977 2978 BUG_ON(!PageLocked(page)); 2979 if (PageWriteback(page)) 2980 return 0; 2981 2982 if (mapping && mapping->a_ops->releasepage) 2983 return mapping->a_ops->releasepage(page, gfp_mask); 2984 return try_to_free_buffers(page); 2985 } 2986 2987 EXPORT_SYMBOL(try_to_release_page); 2988