1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/filemap.c 4 * 5 * Copyright (C) 1994-1999 Linus Torvalds 6 */ 7 8 /* 9 * This file handles the generic file mmap semantics used by 10 * most "normal" filesystems (but you don't /have/ to use this: 11 * the NFS filesystem used to do this differently, for example) 12 */ 13 #include <linux/export.h> 14 #include <linux/compiler.h> 15 #include <linux/dax.h> 16 #include <linux/fs.h> 17 #include <linux/sched/signal.h> 18 #include <linux/uaccess.h> 19 #include <linux/capability.h> 20 #include <linux/kernel_stat.h> 21 #include <linux/gfp.h> 22 #include <linux/mm.h> 23 #include <linux/swap.h> 24 #include <linux/mman.h> 25 #include <linux/pagemap.h> 26 #include <linux/file.h> 27 #include <linux/uio.h> 28 #include <linux/error-injection.h> 29 #include <linux/hash.h> 30 #include <linux/writeback.h> 31 #include <linux/backing-dev.h> 32 #include <linux/pagevec.h> 33 #include <linux/blkdev.h> 34 #include <linux/security.h> 35 #include <linux/cpuset.h> 36 #include <linux/hugetlb.h> 37 #include <linux/memcontrol.h> 38 #include <linux/cleancache.h> 39 #include <linux/shmem_fs.h> 40 #include <linux/rmap.h> 41 #include <linux/delayacct.h> 42 #include <linux/psi.h> 43 #include <linux/ramfs.h> 44 #include <linux/page_idle.h> 45 #include "internal.h" 46 47 #define CREATE_TRACE_POINTS 48 #include <trace/events/filemap.h> 49 50 /* 51 * FIXME: remove all knowledge of the buffer layer from the core VM 52 */ 53 #include <linux/buffer_head.h> /* for try_to_free_buffers */ 54 55 #include <asm/mman.h> 56 57 /* 58 * Shared mappings implemented 30.11.1994. It's not fully working yet, 59 * though. 60 * 61 * Shared mappings now work. 15.8.1995 Bruno. 62 * 63 * finished 'unifying' the page and buffer cache and SMP-threaded the 64 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com> 65 * 66 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de> 67 */ 68 69 /* 70 * Lock ordering: 71 * 72 * ->i_mmap_rwsem (truncate_pagecache) 73 * ->private_lock (__free_pte->__set_page_dirty_buffers) 74 * ->swap_lock (exclusive_swap_page, others) 75 * ->i_pages lock 76 * 77 * ->i_mutex 78 * ->i_mmap_rwsem (truncate->unmap_mapping_range) 79 * 80 * ->mmap_lock 81 * ->i_mmap_rwsem 82 * ->page_table_lock or pte_lock (various, mainly in memory.c) 83 * ->i_pages lock (arch-dependent flush_dcache_mmap_lock) 84 * 85 * ->mmap_lock 86 * ->lock_page (access_process_vm) 87 * 88 * ->i_mutex (generic_perform_write) 89 * ->mmap_lock (fault_in_pages_readable->do_page_fault) 90 * 91 * bdi->wb.list_lock 92 * sb_lock (fs/fs-writeback.c) 93 * ->i_pages lock (__sync_single_inode) 94 * 95 * ->i_mmap_rwsem 96 * ->anon_vma.lock (vma_adjust) 97 * 98 * ->anon_vma.lock 99 * ->page_table_lock or pte_lock (anon_vma_prepare and various) 100 * 101 * ->page_table_lock or pte_lock 102 * ->swap_lock (try_to_unmap_one) 103 * ->private_lock (try_to_unmap_one) 104 * ->i_pages lock (try_to_unmap_one) 105 * ->pgdat->lru_lock (follow_page->mark_page_accessed) 106 * ->pgdat->lru_lock (check_pte_range->isolate_lru_page) 107 * ->private_lock (page_remove_rmap->set_page_dirty) 108 * ->i_pages lock (page_remove_rmap->set_page_dirty) 109 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) 110 * ->inode->i_lock (page_remove_rmap->set_page_dirty) 111 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg) 112 * bdi.wb->list_lock (zap_pte_range->set_page_dirty) 113 * ->inode->i_lock (zap_pte_range->set_page_dirty) 114 * ->private_lock (zap_pte_range->__set_page_dirty_buffers) 115 * 116 * ->i_mmap_rwsem 117 * ->tasklist_lock (memory_failure, collect_procs_ao) 118 */ 119 120 static void page_cache_delete(struct address_space *mapping, 121 struct page *page, void *shadow) 122 { 123 XA_STATE(xas, &mapping->i_pages, page->index); 124 unsigned int nr = 1; 125 126 mapping_set_update(&xas, mapping); 127 128 /* hugetlb pages are represented by a single entry in the xarray */ 129 if (!PageHuge(page)) { 130 xas_set_order(&xas, page->index, compound_order(page)); 131 nr = compound_nr(page); 132 } 133 134 VM_BUG_ON_PAGE(!PageLocked(page), page); 135 VM_BUG_ON_PAGE(PageTail(page), page); 136 VM_BUG_ON_PAGE(nr != 1 && shadow, page); 137 138 xas_store(&xas, shadow); 139 xas_init_marks(&xas); 140 141 page->mapping = NULL; 142 /* Leave page->index set: truncation lookup relies upon it */ 143 144 if (shadow) { 145 mapping->nrexceptional += nr; 146 /* 147 * Make sure the nrexceptional update is committed before 148 * the nrpages update so that final truncate racing 149 * with reclaim does not see both counters 0 at the 150 * same time and miss a shadow entry. 151 */ 152 smp_wmb(); 153 } 154 mapping->nrpages -= nr; 155 } 156 157 static void unaccount_page_cache_page(struct address_space *mapping, 158 struct page *page) 159 { 160 int nr; 161 162 /* 163 * if we're uptodate, flush out into the cleancache, otherwise 164 * invalidate any existing cleancache entries. We can't leave 165 * stale data around in the cleancache once our page is gone 166 */ 167 if (PageUptodate(page) && PageMappedToDisk(page)) 168 cleancache_put_page(page); 169 else 170 cleancache_invalidate_page(mapping, page); 171 172 VM_BUG_ON_PAGE(PageTail(page), page); 173 VM_BUG_ON_PAGE(page_mapped(page), page); 174 if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) { 175 int mapcount; 176 177 pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n", 178 current->comm, page_to_pfn(page)); 179 dump_page(page, "still mapped when deleted"); 180 dump_stack(); 181 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 182 183 mapcount = page_mapcount(page); 184 if (mapping_exiting(mapping) && 185 page_count(page) >= mapcount + 2) { 186 /* 187 * All vmas have already been torn down, so it's 188 * a good bet that actually the page is unmapped, 189 * and we'd prefer not to leak it: if we're wrong, 190 * some other bad page check should catch it later. 191 */ 192 page_mapcount_reset(page); 193 page_ref_sub(page, mapcount); 194 } 195 } 196 197 /* hugetlb pages do not participate in page cache accounting. */ 198 if (PageHuge(page)) 199 return; 200 201 nr = hpage_nr_pages(page); 202 203 __mod_lruvec_page_state(page, NR_FILE_PAGES, -nr); 204 if (PageSwapBacked(page)) { 205 __mod_lruvec_page_state(page, NR_SHMEM, -nr); 206 if (PageTransHuge(page)) 207 __dec_node_page_state(page, NR_SHMEM_THPS); 208 } else if (PageTransHuge(page)) { 209 __dec_node_page_state(page, NR_FILE_THPS); 210 filemap_nr_thps_dec(mapping); 211 } 212 213 /* 214 * At this point page must be either written or cleaned by 215 * truncate. Dirty page here signals a bug and loss of 216 * unwritten data. 217 * 218 * This fixes dirty accounting after removing the page entirely 219 * but leaves PageDirty set: it has no effect for truncated 220 * page and anyway will be cleared before returning page into 221 * buddy allocator. 222 */ 223 if (WARN_ON_ONCE(PageDirty(page))) 224 account_page_cleaned(page, mapping, inode_to_wb(mapping->host)); 225 } 226 227 /* 228 * Delete a page from the page cache and free it. Caller has to make 229 * sure the page is locked and that nobody else uses it - or that usage 230 * is safe. The caller must hold the i_pages lock. 231 */ 232 void __delete_from_page_cache(struct page *page, void *shadow) 233 { 234 struct address_space *mapping = page->mapping; 235 236 trace_mm_filemap_delete_from_page_cache(page); 237 238 unaccount_page_cache_page(mapping, page); 239 page_cache_delete(mapping, page, shadow); 240 } 241 242 static void page_cache_free_page(struct address_space *mapping, 243 struct page *page) 244 { 245 void (*freepage)(struct page *); 246 247 freepage = mapping->a_ops->freepage; 248 if (freepage) 249 freepage(page); 250 251 if (PageTransHuge(page) && !PageHuge(page)) { 252 page_ref_sub(page, HPAGE_PMD_NR); 253 VM_BUG_ON_PAGE(page_count(page) <= 0, page); 254 } else { 255 put_page(page); 256 } 257 } 258 259 /** 260 * delete_from_page_cache - delete page from page cache 261 * @page: the page which the kernel is trying to remove from page cache 262 * 263 * This must be called only on pages that have been verified to be in the page 264 * cache and locked. It will never put the page into the free list, the caller 265 * has a reference on the page. 266 */ 267 void delete_from_page_cache(struct page *page) 268 { 269 struct address_space *mapping = page_mapping(page); 270 unsigned long flags; 271 272 BUG_ON(!PageLocked(page)); 273 xa_lock_irqsave(&mapping->i_pages, flags); 274 __delete_from_page_cache(page, NULL); 275 xa_unlock_irqrestore(&mapping->i_pages, flags); 276 277 page_cache_free_page(mapping, page); 278 } 279 EXPORT_SYMBOL(delete_from_page_cache); 280 281 /* 282 * page_cache_delete_batch - delete several pages from page cache 283 * @mapping: the mapping to which pages belong 284 * @pvec: pagevec with pages to delete 285 * 286 * The function walks over mapping->i_pages and removes pages passed in @pvec 287 * from the mapping. The function expects @pvec to be sorted by page index 288 * and is optimised for it to be dense. 289 * It tolerates holes in @pvec (mapping entries at those indices are not 290 * modified). The function expects only THP head pages to be present in the 291 * @pvec. 292 * 293 * The function expects the i_pages lock to be held. 294 */ 295 static void page_cache_delete_batch(struct address_space *mapping, 296 struct pagevec *pvec) 297 { 298 XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index); 299 int total_pages = 0; 300 int i = 0; 301 struct page *page; 302 303 mapping_set_update(&xas, mapping); 304 xas_for_each(&xas, page, ULONG_MAX) { 305 if (i >= pagevec_count(pvec)) 306 break; 307 308 /* A swap/dax/shadow entry got inserted? Skip it. */ 309 if (xa_is_value(page)) 310 continue; 311 /* 312 * A page got inserted in our range? Skip it. We have our 313 * pages locked so they are protected from being removed. 314 * If we see a page whose index is higher than ours, it 315 * means our page has been removed, which shouldn't be 316 * possible because we're holding the PageLock. 317 */ 318 if (page != pvec->pages[i]) { 319 VM_BUG_ON_PAGE(page->index > pvec->pages[i]->index, 320 page); 321 continue; 322 } 323 324 WARN_ON_ONCE(!PageLocked(page)); 325 326 if (page->index == xas.xa_index) 327 page->mapping = NULL; 328 /* Leave page->index set: truncation lookup relies on it */ 329 330 /* 331 * Move to the next page in the vector if this is a regular 332 * page or the index is of the last sub-page of this compound 333 * page. 334 */ 335 if (page->index + compound_nr(page) - 1 == xas.xa_index) 336 i++; 337 xas_store(&xas, NULL); 338 total_pages++; 339 } 340 mapping->nrpages -= total_pages; 341 } 342 343 void delete_from_page_cache_batch(struct address_space *mapping, 344 struct pagevec *pvec) 345 { 346 int i; 347 unsigned long flags; 348 349 if (!pagevec_count(pvec)) 350 return; 351 352 xa_lock_irqsave(&mapping->i_pages, flags); 353 for (i = 0; i < pagevec_count(pvec); i++) { 354 trace_mm_filemap_delete_from_page_cache(pvec->pages[i]); 355 356 unaccount_page_cache_page(mapping, pvec->pages[i]); 357 } 358 page_cache_delete_batch(mapping, pvec); 359 xa_unlock_irqrestore(&mapping->i_pages, flags); 360 361 for (i = 0; i < pagevec_count(pvec); i++) 362 page_cache_free_page(mapping, pvec->pages[i]); 363 } 364 365 int filemap_check_errors(struct address_space *mapping) 366 { 367 int ret = 0; 368 /* Check for outstanding write errors */ 369 if (test_bit(AS_ENOSPC, &mapping->flags) && 370 test_and_clear_bit(AS_ENOSPC, &mapping->flags)) 371 ret = -ENOSPC; 372 if (test_bit(AS_EIO, &mapping->flags) && 373 test_and_clear_bit(AS_EIO, &mapping->flags)) 374 ret = -EIO; 375 return ret; 376 } 377 EXPORT_SYMBOL(filemap_check_errors); 378 379 static int filemap_check_and_keep_errors(struct address_space *mapping) 380 { 381 /* Check for outstanding write errors */ 382 if (test_bit(AS_EIO, &mapping->flags)) 383 return -EIO; 384 if (test_bit(AS_ENOSPC, &mapping->flags)) 385 return -ENOSPC; 386 return 0; 387 } 388 389 /** 390 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range 391 * @mapping: address space structure to write 392 * @start: offset in bytes where the range starts 393 * @end: offset in bytes where the range ends (inclusive) 394 * @sync_mode: enable synchronous operation 395 * 396 * Start writeback against all of a mapping's dirty pages that lie 397 * within the byte offsets <start, end> inclusive. 398 * 399 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as 400 * opposed to a regular memory cleansing writeback. The difference between 401 * these two operations is that if a dirty page/buffer is encountered, it must 402 * be waited upon, and not just skipped over. 403 * 404 * Return: %0 on success, negative error code otherwise. 405 */ 406 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 407 loff_t end, int sync_mode) 408 { 409 int ret; 410 struct writeback_control wbc = { 411 .sync_mode = sync_mode, 412 .nr_to_write = LONG_MAX, 413 .range_start = start, 414 .range_end = end, 415 }; 416 417 if (!mapping_cap_writeback_dirty(mapping) || 418 !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 419 return 0; 420 421 wbc_attach_fdatawrite_inode(&wbc, mapping->host); 422 ret = do_writepages(mapping, &wbc); 423 wbc_detach_inode(&wbc); 424 return ret; 425 } 426 427 static inline int __filemap_fdatawrite(struct address_space *mapping, 428 int sync_mode) 429 { 430 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode); 431 } 432 433 int filemap_fdatawrite(struct address_space *mapping) 434 { 435 return __filemap_fdatawrite(mapping, WB_SYNC_ALL); 436 } 437 EXPORT_SYMBOL(filemap_fdatawrite); 438 439 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 440 loff_t end) 441 { 442 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); 443 } 444 EXPORT_SYMBOL(filemap_fdatawrite_range); 445 446 /** 447 * filemap_flush - mostly a non-blocking flush 448 * @mapping: target address_space 449 * 450 * This is a mostly non-blocking flush. Not suitable for data-integrity 451 * purposes - I/O may not be started against all dirty pages. 452 * 453 * Return: %0 on success, negative error code otherwise. 454 */ 455 int filemap_flush(struct address_space *mapping) 456 { 457 return __filemap_fdatawrite(mapping, WB_SYNC_NONE); 458 } 459 EXPORT_SYMBOL(filemap_flush); 460 461 /** 462 * filemap_range_has_page - check if a page exists in range. 463 * @mapping: address space within which to check 464 * @start_byte: offset in bytes where the range starts 465 * @end_byte: offset in bytes where the range ends (inclusive) 466 * 467 * Find at least one page in the range supplied, usually used to check if 468 * direct writing in this range will trigger a writeback. 469 * 470 * Return: %true if at least one page exists in the specified range, 471 * %false otherwise. 472 */ 473 bool filemap_range_has_page(struct address_space *mapping, 474 loff_t start_byte, loff_t end_byte) 475 { 476 struct page *page; 477 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); 478 pgoff_t max = end_byte >> PAGE_SHIFT; 479 480 if (end_byte < start_byte) 481 return false; 482 483 rcu_read_lock(); 484 for (;;) { 485 page = xas_find(&xas, max); 486 if (xas_retry(&xas, page)) 487 continue; 488 /* Shadow entries don't count */ 489 if (xa_is_value(page)) 490 continue; 491 /* 492 * We don't need to try to pin this page; we're about to 493 * release the RCU lock anyway. It is enough to know that 494 * there was a page here recently. 495 */ 496 break; 497 } 498 rcu_read_unlock(); 499 500 return page != NULL; 501 } 502 EXPORT_SYMBOL(filemap_range_has_page); 503 504 static void __filemap_fdatawait_range(struct address_space *mapping, 505 loff_t start_byte, loff_t end_byte) 506 { 507 pgoff_t index = start_byte >> PAGE_SHIFT; 508 pgoff_t end = end_byte >> PAGE_SHIFT; 509 struct pagevec pvec; 510 int nr_pages; 511 512 if (end_byte < start_byte) 513 return; 514 515 pagevec_init(&pvec); 516 while (index <= end) { 517 unsigned i; 518 519 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, 520 end, PAGECACHE_TAG_WRITEBACK); 521 if (!nr_pages) 522 break; 523 524 for (i = 0; i < nr_pages; i++) { 525 struct page *page = pvec.pages[i]; 526 527 wait_on_page_writeback(page); 528 ClearPageError(page); 529 } 530 pagevec_release(&pvec); 531 cond_resched(); 532 } 533 } 534 535 /** 536 * filemap_fdatawait_range - wait for writeback to complete 537 * @mapping: address space structure to wait for 538 * @start_byte: offset in bytes where the range starts 539 * @end_byte: offset in bytes where the range ends (inclusive) 540 * 541 * Walk the list of under-writeback pages of the given address space 542 * in the given range and wait for all of them. Check error status of 543 * the address space and return it. 544 * 545 * Since the error status of the address space is cleared by this function, 546 * callers are responsible for checking the return value and handling and/or 547 * reporting the error. 548 * 549 * Return: error status of the address space. 550 */ 551 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte, 552 loff_t end_byte) 553 { 554 __filemap_fdatawait_range(mapping, start_byte, end_byte); 555 return filemap_check_errors(mapping); 556 } 557 EXPORT_SYMBOL(filemap_fdatawait_range); 558 559 /** 560 * filemap_fdatawait_range_keep_errors - wait for writeback to complete 561 * @mapping: address space structure to wait for 562 * @start_byte: offset in bytes where the range starts 563 * @end_byte: offset in bytes where the range ends (inclusive) 564 * 565 * Walk the list of under-writeback pages of the given address space in the 566 * given range and wait for all of them. Unlike filemap_fdatawait_range(), 567 * this function does not clear error status of the address space. 568 * 569 * Use this function if callers don't handle errors themselves. Expected 570 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2), 571 * fsfreeze(8) 572 */ 573 int filemap_fdatawait_range_keep_errors(struct address_space *mapping, 574 loff_t start_byte, loff_t end_byte) 575 { 576 __filemap_fdatawait_range(mapping, start_byte, end_byte); 577 return filemap_check_and_keep_errors(mapping); 578 } 579 EXPORT_SYMBOL(filemap_fdatawait_range_keep_errors); 580 581 /** 582 * file_fdatawait_range - wait for writeback to complete 583 * @file: file pointing to address space structure to wait for 584 * @start_byte: offset in bytes where the range starts 585 * @end_byte: offset in bytes where the range ends (inclusive) 586 * 587 * Walk the list of under-writeback pages of the address space that file 588 * refers to, in the given range and wait for all of them. Check error 589 * status of the address space vs. the file->f_wb_err cursor and return it. 590 * 591 * Since the error status of the file is advanced by this function, 592 * callers are responsible for checking the return value and handling and/or 593 * reporting the error. 594 * 595 * Return: error status of the address space vs. the file->f_wb_err cursor. 596 */ 597 int file_fdatawait_range(struct file *file, loff_t start_byte, loff_t end_byte) 598 { 599 struct address_space *mapping = file->f_mapping; 600 601 __filemap_fdatawait_range(mapping, start_byte, end_byte); 602 return file_check_and_advance_wb_err(file); 603 } 604 EXPORT_SYMBOL(file_fdatawait_range); 605 606 /** 607 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors 608 * @mapping: address space structure to wait for 609 * 610 * Walk the list of under-writeback pages of the given address space 611 * and wait for all of them. Unlike filemap_fdatawait(), this function 612 * does not clear error status of the address space. 613 * 614 * Use this function if callers don't handle errors themselves. Expected 615 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2), 616 * fsfreeze(8) 617 * 618 * Return: error status of the address space. 619 */ 620 int filemap_fdatawait_keep_errors(struct address_space *mapping) 621 { 622 __filemap_fdatawait_range(mapping, 0, LLONG_MAX); 623 return filemap_check_and_keep_errors(mapping); 624 } 625 EXPORT_SYMBOL(filemap_fdatawait_keep_errors); 626 627 /* Returns true if writeback might be needed or already in progress. */ 628 static bool mapping_needs_writeback(struct address_space *mapping) 629 { 630 if (dax_mapping(mapping)) 631 return mapping->nrexceptional; 632 633 return mapping->nrpages; 634 } 635 636 /** 637 * filemap_write_and_wait_range - write out & wait on a file range 638 * @mapping: the address_space for the pages 639 * @lstart: offset in bytes where the range starts 640 * @lend: offset in bytes where the range ends (inclusive) 641 * 642 * Write out and wait upon file offsets lstart->lend, inclusive. 643 * 644 * Note that @lend is inclusive (describes the last byte to be written) so 645 * that this function can be used to write to the very end-of-file (end = -1). 646 * 647 * Return: error status of the address space. 648 */ 649 int filemap_write_and_wait_range(struct address_space *mapping, 650 loff_t lstart, loff_t lend) 651 { 652 int err = 0; 653 654 if (mapping_needs_writeback(mapping)) { 655 err = __filemap_fdatawrite_range(mapping, lstart, lend, 656 WB_SYNC_ALL); 657 /* 658 * Even if the above returned error, the pages may be 659 * written partially (e.g. -ENOSPC), so we wait for it. 660 * But the -EIO is special case, it may indicate the worst 661 * thing (e.g. bug) happened, so we avoid waiting for it. 662 */ 663 if (err != -EIO) { 664 int err2 = filemap_fdatawait_range(mapping, 665 lstart, lend); 666 if (!err) 667 err = err2; 668 } else { 669 /* Clear any previously stored errors */ 670 filemap_check_errors(mapping); 671 } 672 } else { 673 err = filemap_check_errors(mapping); 674 } 675 return err; 676 } 677 EXPORT_SYMBOL(filemap_write_and_wait_range); 678 679 void __filemap_set_wb_err(struct address_space *mapping, int err) 680 { 681 errseq_t eseq = errseq_set(&mapping->wb_err, err); 682 683 trace_filemap_set_wb_err(mapping, eseq); 684 } 685 EXPORT_SYMBOL(__filemap_set_wb_err); 686 687 /** 688 * file_check_and_advance_wb_err - report wb error (if any) that was previously 689 * and advance wb_err to current one 690 * @file: struct file on which the error is being reported 691 * 692 * When userland calls fsync (or something like nfsd does the equivalent), we 693 * want to report any writeback errors that occurred since the last fsync (or 694 * since the file was opened if there haven't been any). 695 * 696 * Grab the wb_err from the mapping. If it matches what we have in the file, 697 * then just quickly return 0. The file is all caught up. 698 * 699 * If it doesn't match, then take the mapping value, set the "seen" flag in 700 * it and try to swap it into place. If it works, or another task beat us 701 * to it with the new value, then update the f_wb_err and return the error 702 * portion. The error at this point must be reported via proper channels 703 * (a'la fsync, or NFS COMMIT operation, etc.). 704 * 705 * While we handle mapping->wb_err with atomic operations, the f_wb_err 706 * value is protected by the f_lock since we must ensure that it reflects 707 * the latest value swapped in for this file descriptor. 708 * 709 * Return: %0 on success, negative error code otherwise. 710 */ 711 int file_check_and_advance_wb_err(struct file *file) 712 { 713 int err = 0; 714 errseq_t old = READ_ONCE(file->f_wb_err); 715 struct address_space *mapping = file->f_mapping; 716 717 /* Locklessly handle the common case where nothing has changed */ 718 if (errseq_check(&mapping->wb_err, old)) { 719 /* Something changed, must use slow path */ 720 spin_lock(&file->f_lock); 721 old = file->f_wb_err; 722 err = errseq_check_and_advance(&mapping->wb_err, 723 &file->f_wb_err); 724 trace_file_check_and_advance_wb_err(file, old); 725 spin_unlock(&file->f_lock); 726 } 727 728 /* 729 * We're mostly using this function as a drop in replacement for 730 * filemap_check_errors. Clear AS_EIO/AS_ENOSPC to emulate the effect 731 * that the legacy code would have had on these flags. 732 */ 733 clear_bit(AS_EIO, &mapping->flags); 734 clear_bit(AS_ENOSPC, &mapping->flags); 735 return err; 736 } 737 EXPORT_SYMBOL(file_check_and_advance_wb_err); 738 739 /** 740 * file_write_and_wait_range - write out & wait on a file range 741 * @file: file pointing to address_space with pages 742 * @lstart: offset in bytes where the range starts 743 * @lend: offset in bytes where the range ends (inclusive) 744 * 745 * Write out and wait upon file offsets lstart->lend, inclusive. 746 * 747 * Note that @lend is inclusive (describes the last byte to be written) so 748 * that this function can be used to write to the very end-of-file (end = -1). 749 * 750 * After writing out and waiting on the data, we check and advance the 751 * f_wb_err cursor to the latest value, and return any errors detected there. 752 * 753 * Return: %0 on success, negative error code otherwise. 754 */ 755 int file_write_and_wait_range(struct file *file, loff_t lstart, loff_t lend) 756 { 757 int err = 0, err2; 758 struct address_space *mapping = file->f_mapping; 759 760 if (mapping_needs_writeback(mapping)) { 761 err = __filemap_fdatawrite_range(mapping, lstart, lend, 762 WB_SYNC_ALL); 763 /* See comment of filemap_write_and_wait() */ 764 if (err != -EIO) 765 __filemap_fdatawait_range(mapping, lstart, lend); 766 } 767 err2 = file_check_and_advance_wb_err(file); 768 if (!err) 769 err = err2; 770 return err; 771 } 772 EXPORT_SYMBOL(file_write_and_wait_range); 773 774 /** 775 * replace_page_cache_page - replace a pagecache page with a new one 776 * @old: page to be replaced 777 * @new: page to replace with 778 * @gfp_mask: allocation mode 779 * 780 * This function replaces a page in the pagecache with a new one. On 781 * success it acquires the pagecache reference for the new page and 782 * drops it for the old page. Both the old and new pages must be 783 * locked. This function does not add the new page to the LRU, the 784 * caller must do that. 785 * 786 * The remove + add is atomic. This function cannot fail. 787 * 788 * Return: %0 789 */ 790 int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) 791 { 792 struct address_space *mapping = old->mapping; 793 void (*freepage)(struct page *) = mapping->a_ops->freepage; 794 pgoff_t offset = old->index; 795 XA_STATE(xas, &mapping->i_pages, offset); 796 unsigned long flags; 797 798 VM_BUG_ON_PAGE(!PageLocked(old), old); 799 VM_BUG_ON_PAGE(!PageLocked(new), new); 800 VM_BUG_ON_PAGE(new->mapping, new); 801 802 get_page(new); 803 new->mapping = mapping; 804 new->index = offset; 805 806 mem_cgroup_migrate(old, new); 807 808 xas_lock_irqsave(&xas, flags); 809 xas_store(&xas, new); 810 811 old->mapping = NULL; 812 /* hugetlb pages do not participate in page cache accounting. */ 813 if (!PageHuge(old)) 814 __dec_lruvec_page_state(old, NR_FILE_PAGES); 815 if (!PageHuge(new)) 816 __inc_lruvec_page_state(new, NR_FILE_PAGES); 817 if (PageSwapBacked(old)) 818 __dec_lruvec_page_state(old, NR_SHMEM); 819 if (PageSwapBacked(new)) 820 __inc_lruvec_page_state(new, NR_SHMEM); 821 xas_unlock_irqrestore(&xas, flags); 822 if (freepage) 823 freepage(old); 824 put_page(old); 825 826 return 0; 827 } 828 EXPORT_SYMBOL_GPL(replace_page_cache_page); 829 830 static int __add_to_page_cache_locked(struct page *page, 831 struct address_space *mapping, 832 pgoff_t offset, gfp_t gfp_mask, 833 void **shadowp) 834 { 835 XA_STATE(xas, &mapping->i_pages, offset); 836 int huge = PageHuge(page); 837 int error; 838 void *old; 839 840 VM_BUG_ON_PAGE(!PageLocked(page), page); 841 VM_BUG_ON_PAGE(PageSwapBacked(page), page); 842 mapping_set_update(&xas, mapping); 843 844 get_page(page); 845 page->mapping = mapping; 846 page->index = offset; 847 848 if (!huge) { 849 error = mem_cgroup_charge(page, current->mm, gfp_mask); 850 if (error) 851 goto error; 852 } 853 854 do { 855 xas_lock_irq(&xas); 856 old = xas_load(&xas); 857 if (old && !xa_is_value(old)) 858 xas_set_err(&xas, -EEXIST); 859 xas_store(&xas, page); 860 if (xas_error(&xas)) 861 goto unlock; 862 863 if (xa_is_value(old)) { 864 mapping->nrexceptional--; 865 if (shadowp) 866 *shadowp = old; 867 } 868 mapping->nrpages++; 869 870 /* hugetlb pages do not participate in page cache accounting */ 871 if (!huge) 872 __inc_lruvec_page_state(page, NR_FILE_PAGES); 873 unlock: 874 xas_unlock_irq(&xas); 875 } while (xas_nomem(&xas, gfp_mask & GFP_RECLAIM_MASK)); 876 877 if (xas_error(&xas)) { 878 error = xas_error(&xas); 879 goto error; 880 } 881 882 trace_mm_filemap_add_to_page_cache(page); 883 return 0; 884 error: 885 page->mapping = NULL; 886 /* Leave page->index set: truncation relies upon it */ 887 put_page(page); 888 return error; 889 } 890 ALLOW_ERROR_INJECTION(__add_to_page_cache_locked, ERRNO); 891 892 /** 893 * add_to_page_cache_locked - add a locked page to the pagecache 894 * @page: page to add 895 * @mapping: the page's address_space 896 * @offset: page index 897 * @gfp_mask: page allocation mode 898 * 899 * This function is used to add a page to the pagecache. It must be locked. 900 * This function does not add the page to the LRU. The caller must do that. 901 * 902 * Return: %0 on success, negative error code otherwise. 903 */ 904 int add_to_page_cache_locked(struct page *page, struct address_space *mapping, 905 pgoff_t offset, gfp_t gfp_mask) 906 { 907 return __add_to_page_cache_locked(page, mapping, offset, 908 gfp_mask, NULL); 909 } 910 EXPORT_SYMBOL(add_to_page_cache_locked); 911 912 int add_to_page_cache_lru(struct page *page, struct address_space *mapping, 913 pgoff_t offset, gfp_t gfp_mask) 914 { 915 void *shadow = NULL; 916 int ret; 917 918 __SetPageLocked(page); 919 ret = __add_to_page_cache_locked(page, mapping, offset, 920 gfp_mask, &shadow); 921 if (unlikely(ret)) 922 __ClearPageLocked(page); 923 else { 924 /* 925 * The page might have been evicted from cache only 926 * recently, in which case it should be activated like 927 * any other repeatedly accessed page. 928 * The exception is pages getting rewritten; evicting other 929 * data from the working set, only to cache data that will 930 * get overwritten with something else, is a waste of memory. 931 */ 932 WARN_ON_ONCE(PageActive(page)); 933 if (!(gfp_mask & __GFP_WRITE) && shadow) 934 workingset_refault(page, shadow); 935 lru_cache_add(page); 936 } 937 return ret; 938 } 939 EXPORT_SYMBOL_GPL(add_to_page_cache_lru); 940 941 #ifdef CONFIG_NUMA 942 struct page *__page_cache_alloc(gfp_t gfp) 943 { 944 int n; 945 struct page *page; 946 947 if (cpuset_do_page_mem_spread()) { 948 unsigned int cpuset_mems_cookie; 949 do { 950 cpuset_mems_cookie = read_mems_allowed_begin(); 951 n = cpuset_mem_spread_node(); 952 page = __alloc_pages_node(n, gfp, 0); 953 } while (!page && read_mems_allowed_retry(cpuset_mems_cookie)); 954 955 return page; 956 } 957 return alloc_pages(gfp, 0); 958 } 959 EXPORT_SYMBOL(__page_cache_alloc); 960 #endif 961 962 /* 963 * In order to wait for pages to become available there must be 964 * waitqueues associated with pages. By using a hash table of 965 * waitqueues where the bucket discipline is to maintain all 966 * waiters on the same queue and wake all when any of the pages 967 * become available, and for the woken contexts to check to be 968 * sure the appropriate page became available, this saves space 969 * at a cost of "thundering herd" phenomena during rare hash 970 * collisions. 971 */ 972 #define PAGE_WAIT_TABLE_BITS 8 973 #define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS) 974 static wait_queue_head_t page_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned; 975 976 static wait_queue_head_t *page_waitqueue(struct page *page) 977 { 978 return &page_wait_table[hash_ptr(page, PAGE_WAIT_TABLE_BITS)]; 979 } 980 981 void __init pagecache_init(void) 982 { 983 int i; 984 985 for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++) 986 init_waitqueue_head(&page_wait_table[i]); 987 988 page_writeback_init(); 989 } 990 991 static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg) 992 { 993 int ret; 994 struct wait_page_key *key = arg; 995 struct wait_page_queue *wait_page 996 = container_of(wait, struct wait_page_queue, wait); 997 998 if (!wake_page_match(wait_page, key)) 999 return 0; 1000 1001 /* 1002 * If it's an exclusive wait, we get the bit for it, and 1003 * stop walking if we can't. 1004 * 1005 * If it's a non-exclusive wait, then the fact that this 1006 * wake function was called means that the bit already 1007 * was cleared, and we don't care if somebody then 1008 * re-took it. 1009 */ 1010 ret = 0; 1011 if (wait->flags & WQ_FLAG_EXCLUSIVE) { 1012 if (test_and_set_bit(key->bit_nr, &key->page->flags)) 1013 return -1; 1014 ret = 1; 1015 } 1016 wait->flags |= WQ_FLAG_WOKEN; 1017 1018 wake_up_state(wait->private, mode); 1019 1020 /* 1021 * Ok, we have successfully done what we're waiting for, 1022 * and we can unconditionally remove the wait entry. 1023 * 1024 * Note that this has to be the absolute last thing we do, 1025 * since after list_del_init(&wait->entry) the wait entry 1026 * might be de-allocated and the process might even have 1027 * exited. 1028 */ 1029 list_del_init_careful(&wait->entry); 1030 return ret; 1031 } 1032 1033 static void wake_up_page_bit(struct page *page, int bit_nr) 1034 { 1035 wait_queue_head_t *q = page_waitqueue(page); 1036 struct wait_page_key key; 1037 unsigned long flags; 1038 wait_queue_entry_t bookmark; 1039 1040 key.page = page; 1041 key.bit_nr = bit_nr; 1042 key.page_match = 0; 1043 1044 bookmark.flags = 0; 1045 bookmark.private = NULL; 1046 bookmark.func = NULL; 1047 INIT_LIST_HEAD(&bookmark.entry); 1048 1049 spin_lock_irqsave(&q->lock, flags); 1050 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark); 1051 1052 while (bookmark.flags & WQ_FLAG_BOOKMARK) { 1053 /* 1054 * Take a breather from holding the lock, 1055 * allow pages that finish wake up asynchronously 1056 * to acquire the lock and remove themselves 1057 * from wait queue 1058 */ 1059 spin_unlock_irqrestore(&q->lock, flags); 1060 cpu_relax(); 1061 spin_lock_irqsave(&q->lock, flags); 1062 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark); 1063 } 1064 1065 /* 1066 * It is possible for other pages to have collided on the waitqueue 1067 * hash, so in that case check for a page match. That prevents a long- 1068 * term waiter 1069 * 1070 * It is still possible to miss a case here, when we woke page waiters 1071 * and removed them from the waitqueue, but there are still other 1072 * page waiters. 1073 */ 1074 if (!waitqueue_active(q) || !key.page_match) { 1075 ClearPageWaiters(page); 1076 /* 1077 * It's possible to miss clearing Waiters here, when we woke 1078 * our page waiters, but the hashed waitqueue has waiters for 1079 * other pages on it. 1080 * 1081 * That's okay, it's a rare case. The next waker will clear it. 1082 */ 1083 } 1084 spin_unlock_irqrestore(&q->lock, flags); 1085 } 1086 1087 static void wake_up_page(struct page *page, int bit) 1088 { 1089 if (!PageWaiters(page)) 1090 return; 1091 wake_up_page_bit(page, bit); 1092 } 1093 1094 /* 1095 * A choice of three behaviors for wait_on_page_bit_common(): 1096 */ 1097 enum behavior { 1098 EXCLUSIVE, /* Hold ref to page and take the bit when woken, like 1099 * __lock_page() waiting on then setting PG_locked. 1100 */ 1101 SHARED, /* Hold ref to page and check the bit when woken, like 1102 * wait_on_page_writeback() waiting on PG_writeback. 1103 */ 1104 DROP, /* Drop ref to page before wait, no check when woken, 1105 * like put_and_wait_on_page_locked() on PG_locked. 1106 */ 1107 }; 1108 1109 /* 1110 * Attempt to check (or get) the page bit, and mark the 1111 * waiter woken if successful. 1112 */ 1113 static inline bool trylock_page_bit_common(struct page *page, int bit_nr, 1114 struct wait_queue_entry *wait) 1115 { 1116 if (wait->flags & WQ_FLAG_EXCLUSIVE) { 1117 if (test_and_set_bit(bit_nr, &page->flags)) 1118 return false; 1119 } else if (test_bit(bit_nr, &page->flags)) 1120 return false; 1121 1122 wait->flags |= WQ_FLAG_WOKEN; 1123 return true; 1124 } 1125 1126 static inline int wait_on_page_bit_common(wait_queue_head_t *q, 1127 struct page *page, int bit_nr, int state, enum behavior behavior) 1128 { 1129 struct wait_page_queue wait_page; 1130 wait_queue_entry_t *wait = &wait_page.wait; 1131 bool thrashing = false; 1132 bool delayacct = false; 1133 unsigned long pflags; 1134 1135 if (bit_nr == PG_locked && 1136 !PageUptodate(page) && PageWorkingset(page)) { 1137 if (!PageSwapBacked(page)) { 1138 delayacct_thrashing_start(); 1139 delayacct = true; 1140 } 1141 psi_memstall_enter(&pflags); 1142 thrashing = true; 1143 } 1144 1145 init_wait(wait); 1146 wait->flags = behavior == EXCLUSIVE ? WQ_FLAG_EXCLUSIVE : 0; 1147 wait->func = wake_page_function; 1148 wait_page.page = page; 1149 wait_page.bit_nr = bit_nr; 1150 1151 /* 1152 * Do one last check whether we can get the 1153 * page bit synchronously. 1154 * 1155 * Do the SetPageWaiters() marking before that 1156 * to let any waker we _just_ missed know they 1157 * need to wake us up (otherwise they'll never 1158 * even go to the slow case that looks at the 1159 * page queue), and add ourselves to the wait 1160 * queue if we need to sleep. 1161 * 1162 * This part needs to be done under the queue 1163 * lock to avoid races. 1164 */ 1165 spin_lock_irq(&q->lock); 1166 SetPageWaiters(page); 1167 if (!trylock_page_bit_common(page, bit_nr, wait)) 1168 __add_wait_queue_entry_tail(q, wait); 1169 spin_unlock_irq(&q->lock); 1170 1171 /* 1172 * From now on, all the logic will be based on 1173 * the WQ_FLAG_WOKEN flag, and the and the page 1174 * bit testing (and setting) will be - or has 1175 * already been - done by the wake function. 1176 * 1177 * We can drop our reference to the page. 1178 */ 1179 if (behavior == DROP) 1180 put_page(page); 1181 1182 for (;;) { 1183 set_current_state(state); 1184 1185 if (signal_pending_state(state, current)) 1186 break; 1187 1188 if (wait->flags & WQ_FLAG_WOKEN) 1189 break; 1190 1191 io_schedule(); 1192 } 1193 1194 finish_wait(q, wait); 1195 1196 if (thrashing) { 1197 if (delayacct) 1198 delayacct_thrashing_end(); 1199 psi_memstall_leave(&pflags); 1200 } 1201 1202 /* 1203 * A signal could leave PageWaiters set. Clearing it here if 1204 * !waitqueue_active would be possible (by open-coding finish_wait), 1205 * but still fail to catch it in the case of wait hash collision. We 1206 * already can fail to clear wait hash collision cases, so don't 1207 * bother with signals either. 1208 */ 1209 1210 return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR; 1211 } 1212 1213 void wait_on_page_bit(struct page *page, int bit_nr) 1214 { 1215 wait_queue_head_t *q = page_waitqueue(page); 1216 wait_on_page_bit_common(q, page, bit_nr, TASK_UNINTERRUPTIBLE, SHARED); 1217 } 1218 EXPORT_SYMBOL(wait_on_page_bit); 1219 1220 int wait_on_page_bit_killable(struct page *page, int bit_nr) 1221 { 1222 wait_queue_head_t *q = page_waitqueue(page); 1223 return wait_on_page_bit_common(q, page, bit_nr, TASK_KILLABLE, SHARED); 1224 } 1225 EXPORT_SYMBOL(wait_on_page_bit_killable); 1226 1227 static int __wait_on_page_locked_async(struct page *page, 1228 struct wait_page_queue *wait, bool set) 1229 { 1230 struct wait_queue_head *q = page_waitqueue(page); 1231 int ret = 0; 1232 1233 wait->page = page; 1234 wait->bit_nr = PG_locked; 1235 1236 spin_lock_irq(&q->lock); 1237 __add_wait_queue_entry_tail(q, &wait->wait); 1238 SetPageWaiters(page); 1239 if (set) 1240 ret = !trylock_page(page); 1241 else 1242 ret = PageLocked(page); 1243 /* 1244 * If we were succesful now, we know we're still on the 1245 * waitqueue as we're still under the lock. This means it's 1246 * safe to remove and return success, we know the callback 1247 * isn't going to trigger. 1248 */ 1249 if (!ret) 1250 __remove_wait_queue(q, &wait->wait); 1251 else 1252 ret = -EIOCBQUEUED; 1253 spin_unlock_irq(&q->lock); 1254 return ret; 1255 } 1256 1257 static int wait_on_page_locked_async(struct page *page, 1258 struct wait_page_queue *wait) 1259 { 1260 if (!PageLocked(page)) 1261 return 0; 1262 return __wait_on_page_locked_async(compound_head(page), wait, false); 1263 } 1264 1265 /** 1266 * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked 1267 * @page: The page to wait for. 1268 * 1269 * The caller should hold a reference on @page. They expect the page to 1270 * become unlocked relatively soon, but do not wish to hold up migration 1271 * (for example) by holding the reference while waiting for the page to 1272 * come unlocked. After this function returns, the caller should not 1273 * dereference @page. 1274 */ 1275 void put_and_wait_on_page_locked(struct page *page) 1276 { 1277 wait_queue_head_t *q; 1278 1279 page = compound_head(page); 1280 q = page_waitqueue(page); 1281 wait_on_page_bit_common(q, page, PG_locked, TASK_UNINTERRUPTIBLE, DROP); 1282 } 1283 1284 /** 1285 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue 1286 * @page: Page defining the wait queue of interest 1287 * @waiter: Waiter to add to the queue 1288 * 1289 * Add an arbitrary @waiter to the wait queue for the nominated @page. 1290 */ 1291 void add_page_wait_queue(struct page *page, wait_queue_entry_t *waiter) 1292 { 1293 wait_queue_head_t *q = page_waitqueue(page); 1294 unsigned long flags; 1295 1296 spin_lock_irqsave(&q->lock, flags); 1297 __add_wait_queue_entry_tail(q, waiter); 1298 SetPageWaiters(page); 1299 spin_unlock_irqrestore(&q->lock, flags); 1300 } 1301 EXPORT_SYMBOL_GPL(add_page_wait_queue); 1302 1303 #ifndef clear_bit_unlock_is_negative_byte 1304 1305 /* 1306 * PG_waiters is the high bit in the same byte as PG_lock. 1307 * 1308 * On x86 (and on many other architectures), we can clear PG_lock and 1309 * test the sign bit at the same time. But if the architecture does 1310 * not support that special operation, we just do this all by hand 1311 * instead. 1312 * 1313 * The read of PG_waiters has to be after (or concurrently with) PG_locked 1314 * being cleared, but a memory barrier should be unnecessary since it is 1315 * in the same byte as PG_locked. 1316 */ 1317 static inline bool clear_bit_unlock_is_negative_byte(long nr, volatile void *mem) 1318 { 1319 clear_bit_unlock(nr, mem); 1320 /* smp_mb__after_atomic(); */ 1321 return test_bit(PG_waiters, mem); 1322 } 1323 1324 #endif 1325 1326 /** 1327 * unlock_page - unlock a locked page 1328 * @page: the page 1329 * 1330 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked(). 1331 * Also wakes sleepers in wait_on_page_writeback() because the wakeup 1332 * mechanism between PageLocked pages and PageWriteback pages is shared. 1333 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep. 1334 * 1335 * Note that this depends on PG_waiters being the sign bit in the byte 1336 * that contains PG_locked - thus the BUILD_BUG_ON(). That allows us to 1337 * clear the PG_locked bit and test PG_waiters at the same time fairly 1338 * portably (architectures that do LL/SC can test any bit, while x86 can 1339 * test the sign bit). 1340 */ 1341 void unlock_page(struct page *page) 1342 { 1343 BUILD_BUG_ON(PG_waiters != 7); 1344 page = compound_head(page); 1345 VM_BUG_ON_PAGE(!PageLocked(page), page); 1346 if (clear_bit_unlock_is_negative_byte(PG_locked, &page->flags)) 1347 wake_up_page_bit(page, PG_locked); 1348 } 1349 EXPORT_SYMBOL(unlock_page); 1350 1351 /** 1352 * end_page_writeback - end writeback against a page 1353 * @page: the page 1354 */ 1355 void end_page_writeback(struct page *page) 1356 { 1357 /* 1358 * TestClearPageReclaim could be used here but it is an atomic 1359 * operation and overkill in this particular case. Failing to 1360 * shuffle a page marked for immediate reclaim is too mild to 1361 * justify taking an atomic operation penalty at the end of 1362 * ever page writeback. 1363 */ 1364 if (PageReclaim(page)) { 1365 ClearPageReclaim(page); 1366 rotate_reclaimable_page(page); 1367 } 1368 1369 if (!test_clear_page_writeback(page)) 1370 BUG(); 1371 1372 smp_mb__after_atomic(); 1373 wake_up_page(page, PG_writeback); 1374 } 1375 EXPORT_SYMBOL(end_page_writeback); 1376 1377 /* 1378 * After completing I/O on a page, call this routine to update the page 1379 * flags appropriately 1380 */ 1381 void page_endio(struct page *page, bool is_write, int err) 1382 { 1383 if (!is_write) { 1384 if (!err) { 1385 SetPageUptodate(page); 1386 } else { 1387 ClearPageUptodate(page); 1388 SetPageError(page); 1389 } 1390 unlock_page(page); 1391 } else { 1392 if (err) { 1393 struct address_space *mapping; 1394 1395 SetPageError(page); 1396 mapping = page_mapping(page); 1397 if (mapping) 1398 mapping_set_error(mapping, err); 1399 } 1400 end_page_writeback(page); 1401 } 1402 } 1403 EXPORT_SYMBOL_GPL(page_endio); 1404 1405 /** 1406 * __lock_page - get a lock on the page, assuming we need to sleep to get it 1407 * @__page: the page to lock 1408 */ 1409 void __lock_page(struct page *__page) 1410 { 1411 struct page *page = compound_head(__page); 1412 wait_queue_head_t *q = page_waitqueue(page); 1413 wait_on_page_bit_common(q, page, PG_locked, TASK_UNINTERRUPTIBLE, 1414 EXCLUSIVE); 1415 } 1416 EXPORT_SYMBOL(__lock_page); 1417 1418 int __lock_page_killable(struct page *__page) 1419 { 1420 struct page *page = compound_head(__page); 1421 wait_queue_head_t *q = page_waitqueue(page); 1422 return wait_on_page_bit_common(q, page, PG_locked, TASK_KILLABLE, 1423 EXCLUSIVE); 1424 } 1425 EXPORT_SYMBOL_GPL(__lock_page_killable); 1426 1427 int __lock_page_async(struct page *page, struct wait_page_queue *wait) 1428 { 1429 return __wait_on_page_locked_async(page, wait, true); 1430 } 1431 1432 /* 1433 * Return values: 1434 * 1 - page is locked; mmap_lock is still held. 1435 * 0 - page is not locked. 1436 * mmap_lock has been released (mmap_read_unlock(), unless flags had both 1437 * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in 1438 * which case mmap_lock is still held. 1439 * 1440 * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1 1441 * with the page locked and the mmap_lock unperturbed. 1442 */ 1443 int __lock_page_or_retry(struct page *page, struct mm_struct *mm, 1444 unsigned int flags) 1445 { 1446 if (fault_flag_allow_retry_first(flags)) { 1447 /* 1448 * CAUTION! In this case, mmap_lock is not released 1449 * even though return 0. 1450 */ 1451 if (flags & FAULT_FLAG_RETRY_NOWAIT) 1452 return 0; 1453 1454 mmap_read_unlock(mm); 1455 if (flags & FAULT_FLAG_KILLABLE) 1456 wait_on_page_locked_killable(page); 1457 else 1458 wait_on_page_locked(page); 1459 return 0; 1460 } else { 1461 if (flags & FAULT_FLAG_KILLABLE) { 1462 int ret; 1463 1464 ret = __lock_page_killable(page); 1465 if (ret) { 1466 mmap_read_unlock(mm); 1467 return 0; 1468 } 1469 } else 1470 __lock_page(page); 1471 return 1; 1472 } 1473 } 1474 1475 /** 1476 * page_cache_next_miss() - Find the next gap in the page cache. 1477 * @mapping: Mapping. 1478 * @index: Index. 1479 * @max_scan: Maximum range to search. 1480 * 1481 * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the 1482 * gap with the lowest index. 1483 * 1484 * This function may be called under the rcu_read_lock. However, this will 1485 * not atomically search a snapshot of the cache at a single point in time. 1486 * For example, if a gap is created at index 5, then subsequently a gap is 1487 * created at index 10, page_cache_next_miss covering both indices may 1488 * return 10 if called under the rcu_read_lock. 1489 * 1490 * Return: The index of the gap if found, otherwise an index outside the 1491 * range specified (in which case 'return - index >= max_scan' will be true). 1492 * In the rare case of index wrap-around, 0 will be returned. 1493 */ 1494 pgoff_t page_cache_next_miss(struct address_space *mapping, 1495 pgoff_t index, unsigned long max_scan) 1496 { 1497 XA_STATE(xas, &mapping->i_pages, index); 1498 1499 while (max_scan--) { 1500 void *entry = xas_next(&xas); 1501 if (!entry || xa_is_value(entry)) 1502 break; 1503 if (xas.xa_index == 0) 1504 break; 1505 } 1506 1507 return xas.xa_index; 1508 } 1509 EXPORT_SYMBOL(page_cache_next_miss); 1510 1511 /** 1512 * page_cache_prev_miss() - Find the previous gap in the page cache. 1513 * @mapping: Mapping. 1514 * @index: Index. 1515 * @max_scan: Maximum range to search. 1516 * 1517 * Search the range [max(index - max_scan + 1, 0), index] for the 1518 * gap with the highest index. 1519 * 1520 * This function may be called under the rcu_read_lock. However, this will 1521 * not atomically search a snapshot of the cache at a single point in time. 1522 * For example, if a gap is created at index 10, then subsequently a gap is 1523 * created at index 5, page_cache_prev_miss() covering both indices may 1524 * return 5 if called under the rcu_read_lock. 1525 * 1526 * Return: The index of the gap if found, otherwise an index outside the 1527 * range specified (in which case 'index - return >= max_scan' will be true). 1528 * In the rare case of wrap-around, ULONG_MAX will be returned. 1529 */ 1530 pgoff_t page_cache_prev_miss(struct address_space *mapping, 1531 pgoff_t index, unsigned long max_scan) 1532 { 1533 XA_STATE(xas, &mapping->i_pages, index); 1534 1535 while (max_scan--) { 1536 void *entry = xas_prev(&xas); 1537 if (!entry || xa_is_value(entry)) 1538 break; 1539 if (xas.xa_index == ULONG_MAX) 1540 break; 1541 } 1542 1543 return xas.xa_index; 1544 } 1545 EXPORT_SYMBOL(page_cache_prev_miss); 1546 1547 /** 1548 * find_get_entry - find and get a page cache entry 1549 * @mapping: the address_space to search 1550 * @offset: the page cache index 1551 * 1552 * Looks up the page cache slot at @mapping & @offset. If there is a 1553 * page cache page, it is returned with an increased refcount. 1554 * 1555 * If the slot holds a shadow entry of a previously evicted page, or a 1556 * swap entry from shmem/tmpfs, it is returned. 1557 * 1558 * Return: the found page or shadow entry, %NULL if nothing is found. 1559 */ 1560 struct page *find_get_entry(struct address_space *mapping, pgoff_t offset) 1561 { 1562 XA_STATE(xas, &mapping->i_pages, offset); 1563 struct page *page; 1564 1565 rcu_read_lock(); 1566 repeat: 1567 xas_reset(&xas); 1568 page = xas_load(&xas); 1569 if (xas_retry(&xas, page)) 1570 goto repeat; 1571 /* 1572 * A shadow entry of a recently evicted page, or a swap entry from 1573 * shmem/tmpfs. Return it without attempting to raise page count. 1574 */ 1575 if (!page || xa_is_value(page)) 1576 goto out; 1577 1578 if (!page_cache_get_speculative(page)) 1579 goto repeat; 1580 1581 /* 1582 * Has the page moved or been split? 1583 * This is part of the lockless pagecache protocol. See 1584 * include/linux/pagemap.h for details. 1585 */ 1586 if (unlikely(page != xas_reload(&xas))) { 1587 put_page(page); 1588 goto repeat; 1589 } 1590 page = find_subpage(page, offset); 1591 out: 1592 rcu_read_unlock(); 1593 1594 return page; 1595 } 1596 1597 /** 1598 * find_lock_entry - locate, pin and lock a page cache entry 1599 * @mapping: the address_space to search 1600 * @offset: the page cache index 1601 * 1602 * Looks up the page cache slot at @mapping & @offset. If there is a 1603 * page cache page, it is returned locked and with an increased 1604 * refcount. 1605 * 1606 * If the slot holds a shadow entry of a previously evicted page, or a 1607 * swap entry from shmem/tmpfs, it is returned. 1608 * 1609 * find_lock_entry() may sleep. 1610 * 1611 * Return: the found page or shadow entry, %NULL if nothing is found. 1612 */ 1613 struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset) 1614 { 1615 struct page *page; 1616 1617 repeat: 1618 page = find_get_entry(mapping, offset); 1619 if (page && !xa_is_value(page)) { 1620 lock_page(page); 1621 /* Has the page been truncated? */ 1622 if (unlikely(page_mapping(page) != mapping)) { 1623 unlock_page(page); 1624 put_page(page); 1625 goto repeat; 1626 } 1627 VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page); 1628 } 1629 return page; 1630 } 1631 EXPORT_SYMBOL(find_lock_entry); 1632 1633 /** 1634 * pagecache_get_page - Find and get a reference to a page. 1635 * @mapping: The address_space to search. 1636 * @index: The page index. 1637 * @fgp_flags: %FGP flags modify how the page is returned. 1638 * @gfp_mask: Memory allocation flags to use if %FGP_CREAT is specified. 1639 * 1640 * Looks up the page cache entry at @mapping & @index. 1641 * 1642 * @fgp_flags can be zero or more of these flags: 1643 * 1644 * * %FGP_ACCESSED - The page will be marked accessed. 1645 * * %FGP_LOCK - The page is returned locked. 1646 * * %FGP_CREAT - If no page is present then a new page is allocated using 1647 * @gfp_mask and added to the page cache and the VM's LRU list. 1648 * The page is returned locked and with an increased refcount. 1649 * * %FGP_FOR_MMAP - The caller wants to do its own locking dance if the 1650 * page is already in cache. If the page was allocated, unlock it before 1651 * returning so the caller can do the same dance. 1652 * * %FGP_WRITE - The page will be written 1653 * * %FGP_NOFS - __GFP_FS will get cleared in gfp mask 1654 * * %FGP_NOWAIT - Don't get blocked by page lock 1655 * 1656 * If %FGP_LOCK or %FGP_CREAT are specified then the function may sleep even 1657 * if the %GFP flags specified for %FGP_CREAT are atomic. 1658 * 1659 * If there is a page cache page, it is returned with an increased refcount. 1660 * 1661 * Return: The found page or %NULL otherwise. 1662 */ 1663 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index, 1664 int fgp_flags, gfp_t gfp_mask) 1665 { 1666 struct page *page; 1667 1668 repeat: 1669 page = find_get_entry(mapping, index); 1670 if (xa_is_value(page)) 1671 page = NULL; 1672 if (!page) 1673 goto no_page; 1674 1675 if (fgp_flags & FGP_LOCK) { 1676 if (fgp_flags & FGP_NOWAIT) { 1677 if (!trylock_page(page)) { 1678 put_page(page); 1679 return NULL; 1680 } 1681 } else { 1682 lock_page(page); 1683 } 1684 1685 /* Has the page been truncated? */ 1686 if (unlikely(compound_head(page)->mapping != mapping)) { 1687 unlock_page(page); 1688 put_page(page); 1689 goto repeat; 1690 } 1691 VM_BUG_ON_PAGE(page->index != index, page); 1692 } 1693 1694 if (fgp_flags & FGP_ACCESSED) 1695 mark_page_accessed(page); 1696 else if (fgp_flags & FGP_WRITE) { 1697 /* Clear idle flag for buffer write */ 1698 if (page_is_idle(page)) 1699 clear_page_idle(page); 1700 } 1701 1702 no_page: 1703 if (!page && (fgp_flags & FGP_CREAT)) { 1704 int err; 1705 if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping)) 1706 gfp_mask |= __GFP_WRITE; 1707 if (fgp_flags & FGP_NOFS) 1708 gfp_mask &= ~__GFP_FS; 1709 1710 page = __page_cache_alloc(gfp_mask); 1711 if (!page) 1712 return NULL; 1713 1714 if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP)))) 1715 fgp_flags |= FGP_LOCK; 1716 1717 /* Init accessed so avoid atomic mark_page_accessed later */ 1718 if (fgp_flags & FGP_ACCESSED) 1719 __SetPageReferenced(page); 1720 1721 err = add_to_page_cache_lru(page, mapping, index, gfp_mask); 1722 if (unlikely(err)) { 1723 put_page(page); 1724 page = NULL; 1725 if (err == -EEXIST) 1726 goto repeat; 1727 } 1728 1729 /* 1730 * add_to_page_cache_lru locks the page, and for mmap we expect 1731 * an unlocked page. 1732 */ 1733 if (page && (fgp_flags & FGP_FOR_MMAP)) 1734 unlock_page(page); 1735 } 1736 1737 return page; 1738 } 1739 EXPORT_SYMBOL(pagecache_get_page); 1740 1741 /** 1742 * find_get_entries - gang pagecache lookup 1743 * @mapping: The address_space to search 1744 * @start: The starting page cache index 1745 * @nr_entries: The maximum number of entries 1746 * @entries: Where the resulting entries are placed 1747 * @indices: The cache indices corresponding to the entries in @entries 1748 * 1749 * find_get_entries() will search for and return a group of up to 1750 * @nr_entries entries in the mapping. The entries are placed at 1751 * @entries. find_get_entries() takes a reference against any actual 1752 * pages it returns. 1753 * 1754 * The search returns a group of mapping-contiguous page cache entries 1755 * with ascending indexes. There may be holes in the indices due to 1756 * not-present pages. 1757 * 1758 * Any shadow entries of evicted pages, or swap entries from 1759 * shmem/tmpfs, are included in the returned array. 1760 * 1761 * If it finds a Transparent Huge Page, head or tail, find_get_entries() 1762 * stops at that page: the caller is likely to have a better way to handle 1763 * the compound page as a whole, and then skip its extent, than repeatedly 1764 * calling find_get_entries() to return all its tails. 1765 * 1766 * Return: the number of pages and shadow entries which were found. 1767 */ 1768 unsigned find_get_entries(struct address_space *mapping, 1769 pgoff_t start, unsigned int nr_entries, 1770 struct page **entries, pgoff_t *indices) 1771 { 1772 XA_STATE(xas, &mapping->i_pages, start); 1773 struct page *page; 1774 unsigned int ret = 0; 1775 1776 if (!nr_entries) 1777 return 0; 1778 1779 rcu_read_lock(); 1780 xas_for_each(&xas, page, ULONG_MAX) { 1781 if (xas_retry(&xas, page)) 1782 continue; 1783 /* 1784 * A shadow entry of a recently evicted page, a swap 1785 * entry from shmem/tmpfs or a DAX entry. Return it 1786 * without attempting to raise page count. 1787 */ 1788 if (xa_is_value(page)) 1789 goto export; 1790 1791 if (!page_cache_get_speculative(page)) 1792 goto retry; 1793 1794 /* Has the page moved or been split? */ 1795 if (unlikely(page != xas_reload(&xas))) 1796 goto put_page; 1797 1798 /* 1799 * Terminate early on finding a THP, to allow the caller to 1800 * handle it all at once; but continue if this is hugetlbfs. 1801 */ 1802 if (PageTransHuge(page) && !PageHuge(page)) { 1803 page = find_subpage(page, xas.xa_index); 1804 nr_entries = ret + 1; 1805 } 1806 export: 1807 indices[ret] = xas.xa_index; 1808 entries[ret] = page; 1809 if (++ret == nr_entries) 1810 break; 1811 continue; 1812 put_page: 1813 put_page(page); 1814 retry: 1815 xas_reset(&xas); 1816 } 1817 rcu_read_unlock(); 1818 return ret; 1819 } 1820 1821 /** 1822 * find_get_pages_range - gang pagecache lookup 1823 * @mapping: The address_space to search 1824 * @start: The starting page index 1825 * @end: The final page index (inclusive) 1826 * @nr_pages: The maximum number of pages 1827 * @pages: Where the resulting pages are placed 1828 * 1829 * find_get_pages_range() will search for and return a group of up to @nr_pages 1830 * pages in the mapping starting at index @start and up to index @end 1831 * (inclusive). The pages are placed at @pages. find_get_pages_range() takes 1832 * a reference against the returned pages. 1833 * 1834 * The search returns a group of mapping-contiguous pages with ascending 1835 * indexes. There may be holes in the indices due to not-present pages. 1836 * We also update @start to index the next page for the traversal. 1837 * 1838 * Return: the number of pages which were found. If this number is 1839 * smaller than @nr_pages, the end of specified range has been 1840 * reached. 1841 */ 1842 unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, 1843 pgoff_t end, unsigned int nr_pages, 1844 struct page **pages) 1845 { 1846 XA_STATE(xas, &mapping->i_pages, *start); 1847 struct page *page; 1848 unsigned ret = 0; 1849 1850 if (unlikely(!nr_pages)) 1851 return 0; 1852 1853 rcu_read_lock(); 1854 xas_for_each(&xas, page, end) { 1855 if (xas_retry(&xas, page)) 1856 continue; 1857 /* Skip over shadow, swap and DAX entries */ 1858 if (xa_is_value(page)) 1859 continue; 1860 1861 if (!page_cache_get_speculative(page)) 1862 goto retry; 1863 1864 /* Has the page moved or been split? */ 1865 if (unlikely(page != xas_reload(&xas))) 1866 goto put_page; 1867 1868 pages[ret] = find_subpage(page, xas.xa_index); 1869 if (++ret == nr_pages) { 1870 *start = xas.xa_index + 1; 1871 goto out; 1872 } 1873 continue; 1874 put_page: 1875 put_page(page); 1876 retry: 1877 xas_reset(&xas); 1878 } 1879 1880 /* 1881 * We come here when there is no page beyond @end. We take care to not 1882 * overflow the index @start as it confuses some of the callers. This 1883 * breaks the iteration when there is a page at index -1 but that is 1884 * already broken anyway. 1885 */ 1886 if (end == (pgoff_t)-1) 1887 *start = (pgoff_t)-1; 1888 else 1889 *start = end + 1; 1890 out: 1891 rcu_read_unlock(); 1892 1893 return ret; 1894 } 1895 1896 /** 1897 * find_get_pages_contig - gang contiguous pagecache lookup 1898 * @mapping: The address_space to search 1899 * @index: The starting page index 1900 * @nr_pages: The maximum number of pages 1901 * @pages: Where the resulting pages are placed 1902 * 1903 * find_get_pages_contig() works exactly like find_get_pages(), except 1904 * that the returned number of pages are guaranteed to be contiguous. 1905 * 1906 * Return: the number of pages which were found. 1907 */ 1908 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, 1909 unsigned int nr_pages, struct page **pages) 1910 { 1911 XA_STATE(xas, &mapping->i_pages, index); 1912 struct page *page; 1913 unsigned int ret = 0; 1914 1915 if (unlikely(!nr_pages)) 1916 return 0; 1917 1918 rcu_read_lock(); 1919 for (page = xas_load(&xas); page; page = xas_next(&xas)) { 1920 if (xas_retry(&xas, page)) 1921 continue; 1922 /* 1923 * If the entry has been swapped out, we can stop looking. 1924 * No current caller is looking for DAX entries. 1925 */ 1926 if (xa_is_value(page)) 1927 break; 1928 1929 if (!page_cache_get_speculative(page)) 1930 goto retry; 1931 1932 /* Has the page moved or been split? */ 1933 if (unlikely(page != xas_reload(&xas))) 1934 goto put_page; 1935 1936 pages[ret] = find_subpage(page, xas.xa_index); 1937 if (++ret == nr_pages) 1938 break; 1939 continue; 1940 put_page: 1941 put_page(page); 1942 retry: 1943 xas_reset(&xas); 1944 } 1945 rcu_read_unlock(); 1946 return ret; 1947 } 1948 EXPORT_SYMBOL(find_get_pages_contig); 1949 1950 /** 1951 * find_get_pages_range_tag - find and return pages in given range matching @tag 1952 * @mapping: the address_space to search 1953 * @index: the starting page index 1954 * @end: The final page index (inclusive) 1955 * @tag: the tag index 1956 * @nr_pages: the maximum number of pages 1957 * @pages: where the resulting pages are placed 1958 * 1959 * Like find_get_pages, except we only return pages which are tagged with 1960 * @tag. We update @index to index the next page for the traversal. 1961 * 1962 * Return: the number of pages which were found. 1963 */ 1964 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, 1965 pgoff_t end, xa_mark_t tag, unsigned int nr_pages, 1966 struct page **pages) 1967 { 1968 XA_STATE(xas, &mapping->i_pages, *index); 1969 struct page *page; 1970 unsigned ret = 0; 1971 1972 if (unlikely(!nr_pages)) 1973 return 0; 1974 1975 rcu_read_lock(); 1976 xas_for_each_marked(&xas, page, end, tag) { 1977 if (xas_retry(&xas, page)) 1978 continue; 1979 /* 1980 * Shadow entries should never be tagged, but this iteration 1981 * is lockless so there is a window for page reclaim to evict 1982 * a page we saw tagged. Skip over it. 1983 */ 1984 if (xa_is_value(page)) 1985 continue; 1986 1987 if (!page_cache_get_speculative(page)) 1988 goto retry; 1989 1990 /* Has the page moved or been split? */ 1991 if (unlikely(page != xas_reload(&xas))) 1992 goto put_page; 1993 1994 pages[ret] = find_subpage(page, xas.xa_index); 1995 if (++ret == nr_pages) { 1996 *index = xas.xa_index + 1; 1997 goto out; 1998 } 1999 continue; 2000 put_page: 2001 put_page(page); 2002 retry: 2003 xas_reset(&xas); 2004 } 2005 2006 /* 2007 * We come here when we got to @end. We take care to not overflow the 2008 * index @index as it confuses some of the callers. This breaks the 2009 * iteration when there is a page at index -1 but that is already 2010 * broken anyway. 2011 */ 2012 if (end == (pgoff_t)-1) 2013 *index = (pgoff_t)-1; 2014 else 2015 *index = end + 1; 2016 out: 2017 rcu_read_unlock(); 2018 2019 return ret; 2020 } 2021 EXPORT_SYMBOL(find_get_pages_range_tag); 2022 2023 /* 2024 * CD/DVDs are error prone. When a medium error occurs, the driver may fail 2025 * a _large_ part of the i/o request. Imagine the worst scenario: 2026 * 2027 * ---R__________________________________________B__________ 2028 * ^ reading here ^ bad block(assume 4k) 2029 * 2030 * read(R) => miss => readahead(R...B) => media error => frustrating retries 2031 * => failing the whole request => read(R) => read(R+1) => 2032 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) => 2033 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) => 2034 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ...... 2035 * 2036 * It is going insane. Fix it by quickly scaling down the readahead size. 2037 */ 2038 static void shrink_readahead_size_eio(struct file_ra_state *ra) 2039 { 2040 ra->ra_pages /= 4; 2041 } 2042 2043 /** 2044 * generic_file_buffered_read - generic file read routine 2045 * @iocb: the iocb to read 2046 * @iter: data destination 2047 * @written: already copied 2048 * 2049 * This is a generic file read routine, and uses the 2050 * mapping->a_ops->readpage() function for the actual low-level stuff. 2051 * 2052 * This is really ugly. But the goto's actually try to clarify some 2053 * of the logic when it comes to error handling etc. 2054 * 2055 * Return: 2056 * * total number of bytes copied, including those the were already @written 2057 * * negative error code if nothing was copied 2058 */ 2059 ssize_t generic_file_buffered_read(struct kiocb *iocb, 2060 struct iov_iter *iter, ssize_t written) 2061 { 2062 struct file *filp = iocb->ki_filp; 2063 struct address_space *mapping = filp->f_mapping; 2064 struct inode *inode = mapping->host; 2065 struct file_ra_state *ra = &filp->f_ra; 2066 loff_t *ppos = &iocb->ki_pos; 2067 pgoff_t index; 2068 pgoff_t last_index; 2069 pgoff_t prev_index; 2070 unsigned long offset; /* offset into pagecache page */ 2071 unsigned int prev_offset; 2072 int error = 0; 2073 2074 if (unlikely(*ppos >= inode->i_sb->s_maxbytes)) 2075 return 0; 2076 iov_iter_truncate(iter, inode->i_sb->s_maxbytes); 2077 2078 index = *ppos >> PAGE_SHIFT; 2079 prev_index = ra->prev_pos >> PAGE_SHIFT; 2080 prev_offset = ra->prev_pos & (PAGE_SIZE-1); 2081 last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT; 2082 offset = *ppos & ~PAGE_MASK; 2083 2084 for (;;) { 2085 struct page *page; 2086 pgoff_t end_index; 2087 loff_t isize; 2088 unsigned long nr, ret; 2089 2090 cond_resched(); 2091 find_page: 2092 if (fatal_signal_pending(current)) { 2093 error = -EINTR; 2094 goto out; 2095 } 2096 2097 page = find_get_page(mapping, index); 2098 if (!page) { 2099 if (iocb->ki_flags & IOCB_NOIO) 2100 goto would_block; 2101 page_cache_sync_readahead(mapping, 2102 ra, filp, 2103 index, last_index - index); 2104 page = find_get_page(mapping, index); 2105 if (unlikely(page == NULL)) 2106 goto no_cached_page; 2107 } 2108 if (PageReadahead(page)) { 2109 if (iocb->ki_flags & IOCB_NOIO) { 2110 put_page(page); 2111 goto out; 2112 } 2113 page_cache_async_readahead(mapping, 2114 ra, filp, page, 2115 index, last_index - index); 2116 } 2117 if (!PageUptodate(page)) { 2118 /* 2119 * See comment in do_read_cache_page on why 2120 * wait_on_page_locked is used to avoid unnecessarily 2121 * serialisations and why it's safe. 2122 */ 2123 if (iocb->ki_flags & IOCB_WAITQ) { 2124 if (written) { 2125 put_page(page); 2126 goto out; 2127 } 2128 error = wait_on_page_locked_async(page, 2129 iocb->ki_waitq); 2130 } else { 2131 if (iocb->ki_flags & IOCB_NOWAIT) { 2132 put_page(page); 2133 goto would_block; 2134 } 2135 error = wait_on_page_locked_killable(page); 2136 } 2137 if (unlikely(error)) 2138 goto readpage_error; 2139 if (PageUptodate(page)) 2140 goto page_ok; 2141 2142 if (inode->i_blkbits == PAGE_SHIFT || 2143 !mapping->a_ops->is_partially_uptodate) 2144 goto page_not_up_to_date; 2145 /* pipes can't handle partially uptodate pages */ 2146 if (unlikely(iov_iter_is_pipe(iter))) 2147 goto page_not_up_to_date; 2148 if (!trylock_page(page)) 2149 goto page_not_up_to_date; 2150 /* Did it get truncated before we got the lock? */ 2151 if (!page->mapping) 2152 goto page_not_up_to_date_locked; 2153 if (!mapping->a_ops->is_partially_uptodate(page, 2154 offset, iter->count)) 2155 goto page_not_up_to_date_locked; 2156 unlock_page(page); 2157 } 2158 page_ok: 2159 /* 2160 * i_size must be checked after we know the page is Uptodate. 2161 * 2162 * Checking i_size after the check allows us to calculate 2163 * the correct value for "nr", which means the zero-filled 2164 * part of the page is not copied back to userspace (unless 2165 * another truncate extends the file - this is desired though). 2166 */ 2167 2168 isize = i_size_read(inode); 2169 end_index = (isize - 1) >> PAGE_SHIFT; 2170 if (unlikely(!isize || index > end_index)) { 2171 put_page(page); 2172 goto out; 2173 } 2174 2175 /* nr is the maximum number of bytes to copy from this page */ 2176 nr = PAGE_SIZE; 2177 if (index == end_index) { 2178 nr = ((isize - 1) & ~PAGE_MASK) + 1; 2179 if (nr <= offset) { 2180 put_page(page); 2181 goto out; 2182 } 2183 } 2184 nr = nr - offset; 2185 2186 /* If users can be writing to this page using arbitrary 2187 * virtual addresses, take care about potential aliasing 2188 * before reading the page on the kernel side. 2189 */ 2190 if (mapping_writably_mapped(mapping)) 2191 flush_dcache_page(page); 2192 2193 /* 2194 * When a sequential read accesses a page several times, 2195 * only mark it as accessed the first time. 2196 */ 2197 if (prev_index != index || offset != prev_offset) 2198 mark_page_accessed(page); 2199 prev_index = index; 2200 2201 /* 2202 * Ok, we have the page, and it's up-to-date, so 2203 * now we can copy it to user space... 2204 */ 2205 2206 ret = copy_page_to_iter(page, offset, nr, iter); 2207 offset += ret; 2208 index += offset >> PAGE_SHIFT; 2209 offset &= ~PAGE_MASK; 2210 prev_offset = offset; 2211 2212 put_page(page); 2213 written += ret; 2214 if (!iov_iter_count(iter)) 2215 goto out; 2216 if (ret < nr) { 2217 error = -EFAULT; 2218 goto out; 2219 } 2220 continue; 2221 2222 page_not_up_to_date: 2223 /* Get exclusive access to the page ... */ 2224 if (iocb->ki_flags & IOCB_WAITQ) 2225 error = lock_page_async(page, iocb->ki_waitq); 2226 else 2227 error = lock_page_killable(page); 2228 if (unlikely(error)) 2229 goto readpage_error; 2230 2231 page_not_up_to_date_locked: 2232 /* Did it get truncated before we got the lock? */ 2233 if (!page->mapping) { 2234 unlock_page(page); 2235 put_page(page); 2236 continue; 2237 } 2238 2239 /* Did somebody else fill it already? */ 2240 if (PageUptodate(page)) { 2241 unlock_page(page); 2242 goto page_ok; 2243 } 2244 2245 readpage: 2246 if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT)) { 2247 unlock_page(page); 2248 put_page(page); 2249 goto would_block; 2250 } 2251 /* 2252 * A previous I/O error may have been due to temporary 2253 * failures, eg. multipath errors. 2254 * PG_error will be set again if readpage fails. 2255 */ 2256 ClearPageError(page); 2257 /* Start the actual read. The read will unlock the page. */ 2258 error = mapping->a_ops->readpage(filp, page); 2259 2260 if (unlikely(error)) { 2261 if (error == AOP_TRUNCATED_PAGE) { 2262 put_page(page); 2263 error = 0; 2264 goto find_page; 2265 } 2266 goto readpage_error; 2267 } 2268 2269 if (!PageUptodate(page)) { 2270 error = lock_page_killable(page); 2271 if (unlikely(error)) 2272 goto readpage_error; 2273 if (!PageUptodate(page)) { 2274 if (page->mapping == NULL) { 2275 /* 2276 * invalidate_mapping_pages got it 2277 */ 2278 unlock_page(page); 2279 put_page(page); 2280 goto find_page; 2281 } 2282 unlock_page(page); 2283 shrink_readahead_size_eio(ra); 2284 error = -EIO; 2285 goto readpage_error; 2286 } 2287 unlock_page(page); 2288 } 2289 2290 goto page_ok; 2291 2292 readpage_error: 2293 /* UHHUH! A synchronous read error occurred. Report it */ 2294 put_page(page); 2295 goto out; 2296 2297 no_cached_page: 2298 /* 2299 * Ok, it wasn't cached, so we need to create a new 2300 * page.. 2301 */ 2302 page = page_cache_alloc(mapping); 2303 if (!page) { 2304 error = -ENOMEM; 2305 goto out; 2306 } 2307 error = add_to_page_cache_lru(page, mapping, index, 2308 mapping_gfp_constraint(mapping, GFP_KERNEL)); 2309 if (error) { 2310 put_page(page); 2311 if (error == -EEXIST) { 2312 error = 0; 2313 goto find_page; 2314 } 2315 goto out; 2316 } 2317 goto readpage; 2318 } 2319 2320 would_block: 2321 error = -EAGAIN; 2322 out: 2323 ra->prev_pos = prev_index; 2324 ra->prev_pos <<= PAGE_SHIFT; 2325 ra->prev_pos |= prev_offset; 2326 2327 *ppos = ((loff_t)index << PAGE_SHIFT) + offset; 2328 file_accessed(filp); 2329 return written ? written : error; 2330 } 2331 EXPORT_SYMBOL_GPL(generic_file_buffered_read); 2332 2333 /** 2334 * generic_file_read_iter - generic filesystem read routine 2335 * @iocb: kernel I/O control block 2336 * @iter: destination for the data read 2337 * 2338 * This is the "read_iter()" routine for all filesystems 2339 * that can use the page cache directly. 2340 * 2341 * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall 2342 * be returned when no data can be read without waiting for I/O requests 2343 * to complete; it doesn't prevent readahead. 2344 * 2345 * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O 2346 * requests shall be made for the read or for readahead. When no data 2347 * can be read, -EAGAIN shall be returned. When readahead would be 2348 * triggered, a partial, possibly empty read shall be returned. 2349 * 2350 * Return: 2351 * * number of bytes copied, even for partial reads 2352 * * negative error code (or 0 if IOCB_NOIO) if nothing was read 2353 */ 2354 ssize_t 2355 generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) 2356 { 2357 size_t count = iov_iter_count(iter); 2358 ssize_t retval = 0; 2359 2360 if (!count) 2361 goto out; /* skip atime */ 2362 2363 if (iocb->ki_flags & IOCB_DIRECT) { 2364 struct file *file = iocb->ki_filp; 2365 struct address_space *mapping = file->f_mapping; 2366 struct inode *inode = mapping->host; 2367 loff_t size; 2368 2369 size = i_size_read(inode); 2370 if (iocb->ki_flags & IOCB_NOWAIT) { 2371 if (filemap_range_has_page(mapping, iocb->ki_pos, 2372 iocb->ki_pos + count - 1)) 2373 return -EAGAIN; 2374 } else { 2375 retval = filemap_write_and_wait_range(mapping, 2376 iocb->ki_pos, 2377 iocb->ki_pos + count - 1); 2378 if (retval < 0) 2379 goto out; 2380 } 2381 2382 file_accessed(file); 2383 2384 retval = mapping->a_ops->direct_IO(iocb, iter); 2385 if (retval >= 0) { 2386 iocb->ki_pos += retval; 2387 count -= retval; 2388 } 2389 iov_iter_revert(iter, count - iov_iter_count(iter)); 2390 2391 /* 2392 * Btrfs can have a short DIO read if we encounter 2393 * compressed extents, so if there was an error, or if 2394 * we've already read everything we wanted to, or if 2395 * there was a short read because we hit EOF, go ahead 2396 * and return. Otherwise fallthrough to buffered io for 2397 * the rest of the read. Buffered reads will not work for 2398 * DAX files, so don't bother trying. 2399 */ 2400 if (retval < 0 || !count || iocb->ki_pos >= size || 2401 IS_DAX(inode)) 2402 goto out; 2403 } 2404 2405 retval = generic_file_buffered_read(iocb, iter, retval); 2406 out: 2407 return retval; 2408 } 2409 EXPORT_SYMBOL(generic_file_read_iter); 2410 2411 #ifdef CONFIG_MMU 2412 #define MMAP_LOTSAMISS (100) 2413 /* 2414 * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock 2415 * @vmf - the vm_fault for this fault. 2416 * @page - the page to lock. 2417 * @fpin - the pointer to the file we may pin (or is already pinned). 2418 * 2419 * This works similar to lock_page_or_retry in that it can drop the mmap_lock. 2420 * It differs in that it actually returns the page locked if it returns 1 and 0 2421 * if it couldn't lock the page. If we did have to drop the mmap_lock then fpin 2422 * will point to the pinned file and needs to be fput()'ed at a later point. 2423 */ 2424 static int lock_page_maybe_drop_mmap(struct vm_fault *vmf, struct page *page, 2425 struct file **fpin) 2426 { 2427 if (trylock_page(page)) 2428 return 1; 2429 2430 /* 2431 * NOTE! This will make us return with VM_FAULT_RETRY, but with 2432 * the mmap_lock still held. That's how FAULT_FLAG_RETRY_NOWAIT 2433 * is supposed to work. We have way too many special cases.. 2434 */ 2435 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) 2436 return 0; 2437 2438 *fpin = maybe_unlock_mmap_for_io(vmf, *fpin); 2439 if (vmf->flags & FAULT_FLAG_KILLABLE) { 2440 if (__lock_page_killable(page)) { 2441 /* 2442 * We didn't have the right flags to drop the mmap_lock, 2443 * but all fault_handlers only check for fatal signals 2444 * if we return VM_FAULT_RETRY, so we need to drop the 2445 * mmap_lock here and return 0 if we don't have a fpin. 2446 */ 2447 if (*fpin == NULL) 2448 mmap_read_unlock(vmf->vma->vm_mm); 2449 return 0; 2450 } 2451 } else 2452 __lock_page(page); 2453 return 1; 2454 } 2455 2456 2457 /* 2458 * Synchronous readahead happens when we don't even find a page in the page 2459 * cache at all. We don't want to perform IO under the mmap sem, so if we have 2460 * to drop the mmap sem we return the file that was pinned in order for us to do 2461 * that. If we didn't pin a file then we return NULL. The file that is 2462 * returned needs to be fput()'ed when we're done with it. 2463 */ 2464 static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) 2465 { 2466 struct file *file = vmf->vma->vm_file; 2467 struct file_ra_state *ra = &file->f_ra; 2468 struct address_space *mapping = file->f_mapping; 2469 struct file *fpin = NULL; 2470 pgoff_t offset = vmf->pgoff; 2471 2472 /* If we don't want any read-ahead, don't bother */ 2473 if (vmf->vma->vm_flags & VM_RAND_READ) 2474 return fpin; 2475 if (!ra->ra_pages) 2476 return fpin; 2477 2478 if (vmf->vma->vm_flags & VM_SEQ_READ) { 2479 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 2480 page_cache_sync_readahead(mapping, ra, file, offset, 2481 ra->ra_pages); 2482 return fpin; 2483 } 2484 2485 /* Avoid banging the cache line if not needed */ 2486 if (ra->mmap_miss < MMAP_LOTSAMISS * 10) 2487 ra->mmap_miss++; 2488 2489 /* 2490 * Do we miss much more than hit in this file? If so, 2491 * stop bothering with read-ahead. It will only hurt. 2492 */ 2493 if (ra->mmap_miss > MMAP_LOTSAMISS) 2494 return fpin; 2495 2496 /* 2497 * mmap read-around 2498 */ 2499 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 2500 ra->start = max_t(long, 0, offset - ra->ra_pages / 2); 2501 ra->size = ra->ra_pages; 2502 ra->async_size = ra->ra_pages / 4; 2503 ra_submit(ra, mapping, file); 2504 return fpin; 2505 } 2506 2507 /* 2508 * Asynchronous readahead happens when we find the page and PG_readahead, 2509 * so we want to possibly extend the readahead further. We return the file that 2510 * was pinned if we have to drop the mmap_lock in order to do IO. 2511 */ 2512 static struct file *do_async_mmap_readahead(struct vm_fault *vmf, 2513 struct page *page) 2514 { 2515 struct file *file = vmf->vma->vm_file; 2516 struct file_ra_state *ra = &file->f_ra; 2517 struct address_space *mapping = file->f_mapping; 2518 struct file *fpin = NULL; 2519 pgoff_t offset = vmf->pgoff; 2520 2521 /* If we don't want any read-ahead, don't bother */ 2522 if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages) 2523 return fpin; 2524 if (ra->mmap_miss > 0) 2525 ra->mmap_miss--; 2526 if (PageReadahead(page)) { 2527 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 2528 page_cache_async_readahead(mapping, ra, file, 2529 page, offset, ra->ra_pages); 2530 } 2531 return fpin; 2532 } 2533 2534 /** 2535 * filemap_fault - read in file data for page fault handling 2536 * @vmf: struct vm_fault containing details of the fault 2537 * 2538 * filemap_fault() is invoked via the vma operations vector for a 2539 * mapped memory region to read in file data during a page fault. 2540 * 2541 * The goto's are kind of ugly, but this streamlines the normal case of having 2542 * it in the page cache, and handles the special cases reasonably without 2543 * having a lot of duplicated code. 2544 * 2545 * vma->vm_mm->mmap_lock must be held on entry. 2546 * 2547 * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock 2548 * may be dropped before doing I/O or by lock_page_maybe_drop_mmap(). 2549 * 2550 * If our return value does not have VM_FAULT_RETRY set, the mmap_lock 2551 * has not been released. 2552 * 2553 * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set. 2554 * 2555 * Return: bitwise-OR of %VM_FAULT_ codes. 2556 */ 2557 vm_fault_t filemap_fault(struct vm_fault *vmf) 2558 { 2559 int error; 2560 struct file *file = vmf->vma->vm_file; 2561 struct file *fpin = NULL; 2562 struct address_space *mapping = file->f_mapping; 2563 struct file_ra_state *ra = &file->f_ra; 2564 struct inode *inode = mapping->host; 2565 pgoff_t offset = vmf->pgoff; 2566 pgoff_t max_off; 2567 struct page *page; 2568 vm_fault_t ret = 0; 2569 2570 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 2571 if (unlikely(offset >= max_off)) 2572 return VM_FAULT_SIGBUS; 2573 2574 /* 2575 * Do we have something in the page cache already? 2576 */ 2577 page = find_get_page(mapping, offset); 2578 if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) { 2579 /* 2580 * We found the page, so try async readahead before 2581 * waiting for the lock. 2582 */ 2583 fpin = do_async_mmap_readahead(vmf, page); 2584 } else if (!page) { 2585 /* No page in the page cache at all */ 2586 count_vm_event(PGMAJFAULT); 2587 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT); 2588 ret = VM_FAULT_MAJOR; 2589 fpin = do_sync_mmap_readahead(vmf); 2590 retry_find: 2591 page = pagecache_get_page(mapping, offset, 2592 FGP_CREAT|FGP_FOR_MMAP, 2593 vmf->gfp_mask); 2594 if (!page) { 2595 if (fpin) 2596 goto out_retry; 2597 return VM_FAULT_OOM; 2598 } 2599 } 2600 2601 if (!lock_page_maybe_drop_mmap(vmf, page, &fpin)) 2602 goto out_retry; 2603 2604 /* Did it get truncated? */ 2605 if (unlikely(compound_head(page)->mapping != mapping)) { 2606 unlock_page(page); 2607 put_page(page); 2608 goto retry_find; 2609 } 2610 VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page); 2611 2612 /* 2613 * We have a locked page in the page cache, now we need to check 2614 * that it's up-to-date. If not, it is going to be due to an error. 2615 */ 2616 if (unlikely(!PageUptodate(page))) 2617 goto page_not_uptodate; 2618 2619 /* 2620 * We've made it this far and we had to drop our mmap_lock, now is the 2621 * time to return to the upper layer and have it re-find the vma and 2622 * redo the fault. 2623 */ 2624 if (fpin) { 2625 unlock_page(page); 2626 goto out_retry; 2627 } 2628 2629 /* 2630 * Found the page and have a reference on it. 2631 * We must recheck i_size under page lock. 2632 */ 2633 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 2634 if (unlikely(offset >= max_off)) { 2635 unlock_page(page); 2636 put_page(page); 2637 return VM_FAULT_SIGBUS; 2638 } 2639 2640 vmf->page = page; 2641 return ret | VM_FAULT_LOCKED; 2642 2643 page_not_uptodate: 2644 /* 2645 * Umm, take care of errors if the page isn't up-to-date. 2646 * Try to re-read it _once_. We do this synchronously, 2647 * because there really aren't any performance issues here 2648 * and we need to check for errors. 2649 */ 2650 ClearPageError(page); 2651 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 2652 error = mapping->a_ops->readpage(file, page); 2653 if (!error) { 2654 wait_on_page_locked(page); 2655 if (!PageUptodate(page)) 2656 error = -EIO; 2657 } 2658 if (fpin) 2659 goto out_retry; 2660 put_page(page); 2661 2662 if (!error || error == AOP_TRUNCATED_PAGE) 2663 goto retry_find; 2664 2665 shrink_readahead_size_eio(ra); 2666 return VM_FAULT_SIGBUS; 2667 2668 out_retry: 2669 /* 2670 * We dropped the mmap_lock, we need to return to the fault handler to 2671 * re-find the vma and come back and find our hopefully still populated 2672 * page. 2673 */ 2674 if (page) 2675 put_page(page); 2676 if (fpin) 2677 fput(fpin); 2678 return ret | VM_FAULT_RETRY; 2679 } 2680 EXPORT_SYMBOL(filemap_fault); 2681 2682 void filemap_map_pages(struct vm_fault *vmf, 2683 pgoff_t start_pgoff, pgoff_t end_pgoff) 2684 { 2685 struct file *file = vmf->vma->vm_file; 2686 struct address_space *mapping = file->f_mapping; 2687 pgoff_t last_pgoff = start_pgoff; 2688 unsigned long max_idx; 2689 XA_STATE(xas, &mapping->i_pages, start_pgoff); 2690 struct page *page; 2691 2692 rcu_read_lock(); 2693 xas_for_each(&xas, page, end_pgoff) { 2694 if (xas_retry(&xas, page)) 2695 continue; 2696 if (xa_is_value(page)) 2697 goto next; 2698 2699 /* 2700 * Check for a locked page first, as a speculative 2701 * reference may adversely influence page migration. 2702 */ 2703 if (PageLocked(page)) 2704 goto next; 2705 if (!page_cache_get_speculative(page)) 2706 goto next; 2707 2708 /* Has the page moved or been split? */ 2709 if (unlikely(page != xas_reload(&xas))) 2710 goto skip; 2711 page = find_subpage(page, xas.xa_index); 2712 2713 if (!PageUptodate(page) || 2714 PageReadahead(page) || 2715 PageHWPoison(page)) 2716 goto skip; 2717 if (!trylock_page(page)) 2718 goto skip; 2719 2720 if (page->mapping != mapping || !PageUptodate(page)) 2721 goto unlock; 2722 2723 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 2724 if (page->index >= max_idx) 2725 goto unlock; 2726 2727 if (file->f_ra.mmap_miss > 0) 2728 file->f_ra.mmap_miss--; 2729 2730 vmf->address += (xas.xa_index - last_pgoff) << PAGE_SHIFT; 2731 if (vmf->pte) 2732 vmf->pte += xas.xa_index - last_pgoff; 2733 last_pgoff = xas.xa_index; 2734 if (alloc_set_pte(vmf, page)) 2735 goto unlock; 2736 unlock_page(page); 2737 goto next; 2738 unlock: 2739 unlock_page(page); 2740 skip: 2741 put_page(page); 2742 next: 2743 /* Huge page is mapped? No need to proceed. */ 2744 if (pmd_trans_huge(*vmf->pmd)) 2745 break; 2746 } 2747 rcu_read_unlock(); 2748 } 2749 EXPORT_SYMBOL(filemap_map_pages); 2750 2751 vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf) 2752 { 2753 struct page *page = vmf->page; 2754 struct inode *inode = file_inode(vmf->vma->vm_file); 2755 vm_fault_t ret = VM_FAULT_LOCKED; 2756 2757 sb_start_pagefault(inode->i_sb); 2758 file_update_time(vmf->vma->vm_file); 2759 lock_page(page); 2760 if (page->mapping != inode->i_mapping) { 2761 unlock_page(page); 2762 ret = VM_FAULT_NOPAGE; 2763 goto out; 2764 } 2765 /* 2766 * We mark the page dirty already here so that when freeze is in 2767 * progress, we are guaranteed that writeback during freezing will 2768 * see the dirty page and writeprotect it again. 2769 */ 2770 set_page_dirty(page); 2771 wait_for_stable_page(page); 2772 out: 2773 sb_end_pagefault(inode->i_sb); 2774 return ret; 2775 } 2776 2777 const struct vm_operations_struct generic_file_vm_ops = { 2778 .fault = filemap_fault, 2779 .map_pages = filemap_map_pages, 2780 .page_mkwrite = filemap_page_mkwrite, 2781 }; 2782 2783 /* This is used for a general mmap of a disk file */ 2784 2785 int generic_file_mmap(struct file * file, struct vm_area_struct * vma) 2786 { 2787 struct address_space *mapping = file->f_mapping; 2788 2789 if (!mapping->a_ops->readpage) 2790 return -ENOEXEC; 2791 file_accessed(file); 2792 vma->vm_ops = &generic_file_vm_ops; 2793 return 0; 2794 } 2795 2796 /* 2797 * This is for filesystems which do not implement ->writepage. 2798 */ 2799 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) 2800 { 2801 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) 2802 return -EINVAL; 2803 return generic_file_mmap(file, vma); 2804 } 2805 #else 2806 vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf) 2807 { 2808 return VM_FAULT_SIGBUS; 2809 } 2810 int generic_file_mmap(struct file * file, struct vm_area_struct * vma) 2811 { 2812 return -ENOSYS; 2813 } 2814 int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma) 2815 { 2816 return -ENOSYS; 2817 } 2818 #endif /* CONFIG_MMU */ 2819 2820 EXPORT_SYMBOL(filemap_page_mkwrite); 2821 EXPORT_SYMBOL(generic_file_mmap); 2822 EXPORT_SYMBOL(generic_file_readonly_mmap); 2823 2824 static struct page *wait_on_page_read(struct page *page) 2825 { 2826 if (!IS_ERR(page)) { 2827 wait_on_page_locked(page); 2828 if (!PageUptodate(page)) { 2829 put_page(page); 2830 page = ERR_PTR(-EIO); 2831 } 2832 } 2833 return page; 2834 } 2835 2836 static struct page *do_read_cache_page(struct address_space *mapping, 2837 pgoff_t index, 2838 int (*filler)(void *, struct page *), 2839 void *data, 2840 gfp_t gfp) 2841 { 2842 struct page *page; 2843 int err; 2844 repeat: 2845 page = find_get_page(mapping, index); 2846 if (!page) { 2847 page = __page_cache_alloc(gfp); 2848 if (!page) 2849 return ERR_PTR(-ENOMEM); 2850 err = add_to_page_cache_lru(page, mapping, index, gfp); 2851 if (unlikely(err)) { 2852 put_page(page); 2853 if (err == -EEXIST) 2854 goto repeat; 2855 /* Presumably ENOMEM for xarray node */ 2856 return ERR_PTR(err); 2857 } 2858 2859 filler: 2860 if (filler) 2861 err = filler(data, page); 2862 else 2863 err = mapping->a_ops->readpage(data, page); 2864 2865 if (err < 0) { 2866 put_page(page); 2867 return ERR_PTR(err); 2868 } 2869 2870 page = wait_on_page_read(page); 2871 if (IS_ERR(page)) 2872 return page; 2873 goto out; 2874 } 2875 if (PageUptodate(page)) 2876 goto out; 2877 2878 /* 2879 * Page is not up to date and may be locked due one of the following 2880 * case a: Page is being filled and the page lock is held 2881 * case b: Read/write error clearing the page uptodate status 2882 * case c: Truncation in progress (page locked) 2883 * case d: Reclaim in progress 2884 * 2885 * Case a, the page will be up to date when the page is unlocked. 2886 * There is no need to serialise on the page lock here as the page 2887 * is pinned so the lock gives no additional protection. Even if the 2888 * page is truncated, the data is still valid if PageUptodate as 2889 * it's a race vs truncate race. 2890 * Case b, the page will not be up to date 2891 * Case c, the page may be truncated but in itself, the data may still 2892 * be valid after IO completes as it's a read vs truncate race. The 2893 * operation must restart if the page is not uptodate on unlock but 2894 * otherwise serialising on page lock to stabilise the mapping gives 2895 * no additional guarantees to the caller as the page lock is 2896 * released before return. 2897 * Case d, similar to truncation. If reclaim holds the page lock, it 2898 * will be a race with remove_mapping that determines if the mapping 2899 * is valid on unlock but otherwise the data is valid and there is 2900 * no need to serialise with page lock. 2901 * 2902 * As the page lock gives no additional guarantee, we optimistically 2903 * wait on the page to be unlocked and check if it's up to date and 2904 * use the page if it is. Otherwise, the page lock is required to 2905 * distinguish between the different cases. The motivation is that we 2906 * avoid spurious serialisations and wakeups when multiple processes 2907 * wait on the same page for IO to complete. 2908 */ 2909 wait_on_page_locked(page); 2910 if (PageUptodate(page)) 2911 goto out; 2912 2913 /* Distinguish between all the cases under the safety of the lock */ 2914 lock_page(page); 2915 2916 /* Case c or d, restart the operation */ 2917 if (!page->mapping) { 2918 unlock_page(page); 2919 put_page(page); 2920 goto repeat; 2921 } 2922 2923 /* Someone else locked and filled the page in a very small window */ 2924 if (PageUptodate(page)) { 2925 unlock_page(page); 2926 goto out; 2927 } 2928 2929 /* 2930 * A previous I/O error may have been due to temporary 2931 * failures. 2932 * Clear page error before actual read, PG_error will be 2933 * set again if read page fails. 2934 */ 2935 ClearPageError(page); 2936 goto filler; 2937 2938 out: 2939 mark_page_accessed(page); 2940 return page; 2941 } 2942 2943 /** 2944 * read_cache_page - read into page cache, fill it if needed 2945 * @mapping: the page's address_space 2946 * @index: the page index 2947 * @filler: function to perform the read 2948 * @data: first arg to filler(data, page) function, often left as NULL 2949 * 2950 * Read into the page cache. If a page already exists, and PageUptodate() is 2951 * not set, try to fill the page and wait for it to become unlocked. 2952 * 2953 * If the page does not get brought uptodate, return -EIO. 2954 * 2955 * Return: up to date page on success, ERR_PTR() on failure. 2956 */ 2957 struct page *read_cache_page(struct address_space *mapping, 2958 pgoff_t index, 2959 int (*filler)(void *, struct page *), 2960 void *data) 2961 { 2962 return do_read_cache_page(mapping, index, filler, data, 2963 mapping_gfp_mask(mapping)); 2964 } 2965 EXPORT_SYMBOL(read_cache_page); 2966 2967 /** 2968 * read_cache_page_gfp - read into page cache, using specified page allocation flags. 2969 * @mapping: the page's address_space 2970 * @index: the page index 2971 * @gfp: the page allocator flags to use if allocating 2972 * 2973 * This is the same as "read_mapping_page(mapping, index, NULL)", but with 2974 * any new page allocations done using the specified allocation flags. 2975 * 2976 * If the page does not get brought uptodate, return -EIO. 2977 * 2978 * Return: up to date page on success, ERR_PTR() on failure. 2979 */ 2980 struct page *read_cache_page_gfp(struct address_space *mapping, 2981 pgoff_t index, 2982 gfp_t gfp) 2983 { 2984 return do_read_cache_page(mapping, index, NULL, NULL, gfp); 2985 } 2986 EXPORT_SYMBOL(read_cache_page_gfp); 2987 2988 /* 2989 * Don't operate on ranges the page cache doesn't support, and don't exceed the 2990 * LFS limits. If pos is under the limit it becomes a short access. If it 2991 * exceeds the limit we return -EFBIG. 2992 */ 2993 static int generic_write_check_limits(struct file *file, loff_t pos, 2994 loff_t *count) 2995 { 2996 struct inode *inode = file->f_mapping->host; 2997 loff_t max_size = inode->i_sb->s_maxbytes; 2998 loff_t limit = rlimit(RLIMIT_FSIZE); 2999 3000 if (limit != RLIM_INFINITY) { 3001 if (pos >= limit) { 3002 send_sig(SIGXFSZ, current, 0); 3003 return -EFBIG; 3004 } 3005 *count = min(*count, limit - pos); 3006 } 3007 3008 if (!(file->f_flags & O_LARGEFILE)) 3009 max_size = MAX_NON_LFS; 3010 3011 if (unlikely(pos >= max_size)) 3012 return -EFBIG; 3013 3014 *count = min(*count, max_size - pos); 3015 3016 return 0; 3017 } 3018 3019 /* 3020 * Performs necessary checks before doing a write 3021 * 3022 * Can adjust writing position or amount of bytes to write. 3023 * Returns appropriate error code that caller should return or 3024 * zero in case that write should be allowed. 3025 */ 3026 inline ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from) 3027 { 3028 struct file *file = iocb->ki_filp; 3029 struct inode *inode = file->f_mapping->host; 3030 loff_t count; 3031 int ret; 3032 3033 if (IS_SWAPFILE(inode)) 3034 return -ETXTBSY; 3035 3036 if (!iov_iter_count(from)) 3037 return 0; 3038 3039 /* FIXME: this is for backwards compatibility with 2.4 */ 3040 if (iocb->ki_flags & IOCB_APPEND) 3041 iocb->ki_pos = i_size_read(inode); 3042 3043 if ((iocb->ki_flags & IOCB_NOWAIT) && !(iocb->ki_flags & IOCB_DIRECT)) 3044 return -EINVAL; 3045 3046 count = iov_iter_count(from); 3047 ret = generic_write_check_limits(file, iocb->ki_pos, &count); 3048 if (ret) 3049 return ret; 3050 3051 iov_iter_truncate(from, count); 3052 return iov_iter_count(from); 3053 } 3054 EXPORT_SYMBOL(generic_write_checks); 3055 3056 /* 3057 * Performs necessary checks before doing a clone. 3058 * 3059 * Can adjust amount of bytes to clone via @req_count argument. 3060 * Returns appropriate error code that caller should return or 3061 * zero in case the clone should be allowed. 3062 */ 3063 int generic_remap_checks(struct file *file_in, loff_t pos_in, 3064 struct file *file_out, loff_t pos_out, 3065 loff_t *req_count, unsigned int remap_flags) 3066 { 3067 struct inode *inode_in = file_in->f_mapping->host; 3068 struct inode *inode_out = file_out->f_mapping->host; 3069 uint64_t count = *req_count; 3070 uint64_t bcount; 3071 loff_t size_in, size_out; 3072 loff_t bs = inode_out->i_sb->s_blocksize; 3073 int ret; 3074 3075 /* The start of both ranges must be aligned to an fs block. */ 3076 if (!IS_ALIGNED(pos_in, bs) || !IS_ALIGNED(pos_out, bs)) 3077 return -EINVAL; 3078 3079 /* Ensure offsets don't wrap. */ 3080 if (pos_in + count < pos_in || pos_out + count < pos_out) 3081 return -EINVAL; 3082 3083 size_in = i_size_read(inode_in); 3084 size_out = i_size_read(inode_out); 3085 3086 /* Dedupe requires both ranges to be within EOF. */ 3087 if ((remap_flags & REMAP_FILE_DEDUP) && 3088 (pos_in >= size_in || pos_in + count > size_in || 3089 pos_out >= size_out || pos_out + count > size_out)) 3090 return -EINVAL; 3091 3092 /* Ensure the infile range is within the infile. */ 3093 if (pos_in >= size_in) 3094 return -EINVAL; 3095 count = min(count, size_in - (uint64_t)pos_in); 3096 3097 ret = generic_write_check_limits(file_out, pos_out, &count); 3098 if (ret) 3099 return ret; 3100 3101 /* 3102 * If the user wanted us to link to the infile's EOF, round up to the 3103 * next block boundary for this check. 3104 * 3105 * Otherwise, make sure the count is also block-aligned, having 3106 * already confirmed the starting offsets' block alignment. 3107 */ 3108 if (pos_in + count == size_in) { 3109 bcount = ALIGN(size_in, bs) - pos_in; 3110 } else { 3111 if (!IS_ALIGNED(count, bs)) 3112 count = ALIGN_DOWN(count, bs); 3113 bcount = count; 3114 } 3115 3116 /* Don't allow overlapped cloning within the same file. */ 3117 if (inode_in == inode_out && 3118 pos_out + bcount > pos_in && 3119 pos_out < pos_in + bcount) 3120 return -EINVAL; 3121 3122 /* 3123 * We shortened the request but the caller can't deal with that, so 3124 * bounce the request back to userspace. 3125 */ 3126 if (*req_count != count && !(remap_flags & REMAP_FILE_CAN_SHORTEN)) 3127 return -EINVAL; 3128 3129 *req_count = count; 3130 return 0; 3131 } 3132 3133 3134 /* 3135 * Performs common checks before doing a file copy/clone 3136 * from @file_in to @file_out. 3137 */ 3138 int generic_file_rw_checks(struct file *file_in, struct file *file_out) 3139 { 3140 struct inode *inode_in = file_inode(file_in); 3141 struct inode *inode_out = file_inode(file_out); 3142 3143 /* Don't copy dirs, pipes, sockets... */ 3144 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode)) 3145 return -EISDIR; 3146 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode)) 3147 return -EINVAL; 3148 3149 if (!(file_in->f_mode & FMODE_READ) || 3150 !(file_out->f_mode & FMODE_WRITE) || 3151 (file_out->f_flags & O_APPEND)) 3152 return -EBADF; 3153 3154 return 0; 3155 } 3156 3157 /* 3158 * Performs necessary checks before doing a file copy 3159 * 3160 * Can adjust amount of bytes to copy via @req_count argument. 3161 * Returns appropriate error code that caller should return or 3162 * zero in case the copy should be allowed. 3163 */ 3164 int generic_copy_file_checks(struct file *file_in, loff_t pos_in, 3165 struct file *file_out, loff_t pos_out, 3166 size_t *req_count, unsigned int flags) 3167 { 3168 struct inode *inode_in = file_inode(file_in); 3169 struct inode *inode_out = file_inode(file_out); 3170 uint64_t count = *req_count; 3171 loff_t size_in; 3172 int ret; 3173 3174 ret = generic_file_rw_checks(file_in, file_out); 3175 if (ret) 3176 return ret; 3177 3178 /* Don't touch certain kinds of inodes */ 3179 if (IS_IMMUTABLE(inode_out)) 3180 return -EPERM; 3181 3182 if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out)) 3183 return -ETXTBSY; 3184 3185 /* Ensure offsets don't wrap. */ 3186 if (pos_in + count < pos_in || pos_out + count < pos_out) 3187 return -EOVERFLOW; 3188 3189 /* Shorten the copy to EOF */ 3190 size_in = i_size_read(inode_in); 3191 if (pos_in >= size_in) 3192 count = 0; 3193 else 3194 count = min(count, size_in - (uint64_t)pos_in); 3195 3196 ret = generic_write_check_limits(file_out, pos_out, &count); 3197 if (ret) 3198 return ret; 3199 3200 /* Don't allow overlapped copying within the same file. */ 3201 if (inode_in == inode_out && 3202 pos_out + count > pos_in && 3203 pos_out < pos_in + count) 3204 return -EINVAL; 3205 3206 *req_count = count; 3207 return 0; 3208 } 3209 3210 int pagecache_write_begin(struct file *file, struct address_space *mapping, 3211 loff_t pos, unsigned len, unsigned flags, 3212 struct page **pagep, void **fsdata) 3213 { 3214 const struct address_space_operations *aops = mapping->a_ops; 3215 3216 return aops->write_begin(file, mapping, pos, len, flags, 3217 pagep, fsdata); 3218 } 3219 EXPORT_SYMBOL(pagecache_write_begin); 3220 3221 int pagecache_write_end(struct file *file, struct address_space *mapping, 3222 loff_t pos, unsigned len, unsigned copied, 3223 struct page *page, void *fsdata) 3224 { 3225 const struct address_space_operations *aops = mapping->a_ops; 3226 3227 return aops->write_end(file, mapping, pos, len, copied, page, fsdata); 3228 } 3229 EXPORT_SYMBOL(pagecache_write_end); 3230 3231 /* 3232 * Warn about a page cache invalidation failure during a direct I/O write. 3233 */ 3234 void dio_warn_stale_pagecache(struct file *filp) 3235 { 3236 static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST); 3237 char pathname[128]; 3238 struct inode *inode = file_inode(filp); 3239 char *path; 3240 3241 errseq_set(&inode->i_mapping->wb_err, -EIO); 3242 if (__ratelimit(&_rs)) { 3243 path = file_path(filp, pathname, sizeof(pathname)); 3244 if (IS_ERR(path)) 3245 path = "(unknown)"; 3246 pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n"); 3247 pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid, 3248 current->comm); 3249 } 3250 } 3251 3252 ssize_t 3253 generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from) 3254 { 3255 struct file *file = iocb->ki_filp; 3256 struct address_space *mapping = file->f_mapping; 3257 struct inode *inode = mapping->host; 3258 loff_t pos = iocb->ki_pos; 3259 ssize_t written; 3260 size_t write_len; 3261 pgoff_t end; 3262 3263 write_len = iov_iter_count(from); 3264 end = (pos + write_len - 1) >> PAGE_SHIFT; 3265 3266 if (iocb->ki_flags & IOCB_NOWAIT) { 3267 /* If there are pages to writeback, return */ 3268 if (filemap_range_has_page(inode->i_mapping, pos, 3269 pos + write_len - 1)) 3270 return -EAGAIN; 3271 } else { 3272 written = filemap_write_and_wait_range(mapping, pos, 3273 pos + write_len - 1); 3274 if (written) 3275 goto out; 3276 } 3277 3278 /* 3279 * After a write we want buffered reads to be sure to go to disk to get 3280 * the new data. We invalidate clean cached page from the region we're 3281 * about to write. We do this *before* the write so that we can return 3282 * without clobbering -EIOCBQUEUED from ->direct_IO(). 3283 */ 3284 written = invalidate_inode_pages2_range(mapping, 3285 pos >> PAGE_SHIFT, end); 3286 /* 3287 * If a page can not be invalidated, return 0 to fall back 3288 * to buffered write. 3289 */ 3290 if (written) { 3291 if (written == -EBUSY) 3292 return 0; 3293 goto out; 3294 } 3295 3296 written = mapping->a_ops->direct_IO(iocb, from); 3297 3298 /* 3299 * Finally, try again to invalidate clean pages which might have been 3300 * cached by non-direct readahead, or faulted in by get_user_pages() 3301 * if the source of the write was an mmap'ed region of the file 3302 * we're writing. Either one is a pretty crazy thing to do, 3303 * so we don't support it 100%. If this invalidation 3304 * fails, tough, the write still worked... 3305 * 3306 * Most of the time we do not need this since dio_complete() will do 3307 * the invalidation for us. However there are some file systems that 3308 * do not end up with dio_complete() being called, so let's not break 3309 * them by removing it completely. 3310 * 3311 * Noticeable example is a blkdev_direct_IO(). 3312 * 3313 * Skip invalidation for async writes or if mapping has no pages. 3314 */ 3315 if (written > 0 && mapping->nrpages && 3316 invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, end)) 3317 dio_warn_stale_pagecache(file); 3318 3319 if (written > 0) { 3320 pos += written; 3321 write_len -= written; 3322 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { 3323 i_size_write(inode, pos); 3324 mark_inode_dirty(inode); 3325 } 3326 iocb->ki_pos = pos; 3327 } 3328 iov_iter_revert(from, write_len - iov_iter_count(from)); 3329 out: 3330 return written; 3331 } 3332 EXPORT_SYMBOL(generic_file_direct_write); 3333 3334 /* 3335 * Find or create a page at the given pagecache position. Return the locked 3336 * page. This function is specifically for buffered writes. 3337 */ 3338 struct page *grab_cache_page_write_begin(struct address_space *mapping, 3339 pgoff_t index, unsigned flags) 3340 { 3341 struct page *page; 3342 int fgp_flags = FGP_LOCK|FGP_WRITE|FGP_CREAT; 3343 3344 if (flags & AOP_FLAG_NOFS) 3345 fgp_flags |= FGP_NOFS; 3346 3347 page = pagecache_get_page(mapping, index, fgp_flags, 3348 mapping_gfp_mask(mapping)); 3349 if (page) 3350 wait_for_stable_page(page); 3351 3352 return page; 3353 } 3354 EXPORT_SYMBOL(grab_cache_page_write_begin); 3355 3356 ssize_t generic_perform_write(struct file *file, 3357 struct iov_iter *i, loff_t pos) 3358 { 3359 struct address_space *mapping = file->f_mapping; 3360 const struct address_space_operations *a_ops = mapping->a_ops; 3361 long status = 0; 3362 ssize_t written = 0; 3363 unsigned int flags = 0; 3364 3365 do { 3366 struct page *page; 3367 unsigned long offset; /* Offset into pagecache page */ 3368 unsigned long bytes; /* Bytes to write to page */ 3369 size_t copied; /* Bytes copied from user */ 3370 void *fsdata; 3371 3372 offset = (pos & (PAGE_SIZE - 1)); 3373 bytes = min_t(unsigned long, PAGE_SIZE - offset, 3374 iov_iter_count(i)); 3375 3376 again: 3377 /* 3378 * Bring in the user page that we will copy from _first_. 3379 * Otherwise there's a nasty deadlock on copying from the 3380 * same page as we're writing to, without it being marked 3381 * up-to-date. 3382 * 3383 * Not only is this an optimisation, but it is also required 3384 * to check that the address is actually valid, when atomic 3385 * usercopies are used, below. 3386 */ 3387 if (unlikely(iov_iter_fault_in_readable(i, bytes))) { 3388 status = -EFAULT; 3389 break; 3390 } 3391 3392 if (fatal_signal_pending(current)) { 3393 status = -EINTR; 3394 break; 3395 } 3396 3397 status = a_ops->write_begin(file, mapping, pos, bytes, flags, 3398 &page, &fsdata); 3399 if (unlikely(status < 0)) 3400 break; 3401 3402 if (mapping_writably_mapped(mapping)) 3403 flush_dcache_page(page); 3404 3405 copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes); 3406 flush_dcache_page(page); 3407 3408 status = a_ops->write_end(file, mapping, pos, bytes, copied, 3409 page, fsdata); 3410 if (unlikely(status < 0)) 3411 break; 3412 copied = status; 3413 3414 cond_resched(); 3415 3416 iov_iter_advance(i, copied); 3417 if (unlikely(copied == 0)) { 3418 /* 3419 * If we were unable to copy any data at all, we must 3420 * fall back to a single segment length write. 3421 * 3422 * If we didn't fallback here, we could livelock 3423 * because not all segments in the iov can be copied at 3424 * once without a pagefault. 3425 */ 3426 bytes = min_t(unsigned long, PAGE_SIZE - offset, 3427 iov_iter_single_seg_count(i)); 3428 goto again; 3429 } 3430 pos += copied; 3431 written += copied; 3432 3433 balance_dirty_pages_ratelimited(mapping); 3434 } while (iov_iter_count(i)); 3435 3436 return written ? written : status; 3437 } 3438 EXPORT_SYMBOL(generic_perform_write); 3439 3440 /** 3441 * __generic_file_write_iter - write data to a file 3442 * @iocb: IO state structure (file, offset, etc.) 3443 * @from: iov_iter with data to write 3444 * 3445 * This function does all the work needed for actually writing data to a 3446 * file. It does all basic checks, removes SUID from the file, updates 3447 * modification times and calls proper subroutines depending on whether we 3448 * do direct IO or a standard buffered write. 3449 * 3450 * It expects i_mutex to be grabbed unless we work on a block device or similar 3451 * object which does not need locking at all. 3452 * 3453 * This function does *not* take care of syncing data in case of O_SYNC write. 3454 * A caller has to handle it. This is mainly due to the fact that we want to 3455 * avoid syncing under i_mutex. 3456 * 3457 * Return: 3458 * * number of bytes written, even for truncated writes 3459 * * negative error code if no data has been written at all 3460 */ 3461 ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 3462 { 3463 struct file *file = iocb->ki_filp; 3464 struct address_space * mapping = file->f_mapping; 3465 struct inode *inode = mapping->host; 3466 ssize_t written = 0; 3467 ssize_t err; 3468 ssize_t status; 3469 3470 /* We can write back this queue in page reclaim */ 3471 current->backing_dev_info = inode_to_bdi(inode); 3472 err = file_remove_privs(file); 3473 if (err) 3474 goto out; 3475 3476 err = file_update_time(file); 3477 if (err) 3478 goto out; 3479 3480 if (iocb->ki_flags & IOCB_DIRECT) { 3481 loff_t pos, endbyte; 3482 3483 written = generic_file_direct_write(iocb, from); 3484 /* 3485 * If the write stopped short of completing, fall back to 3486 * buffered writes. Some filesystems do this for writes to 3487 * holes, for example. For DAX files, a buffered write will 3488 * not succeed (even if it did, DAX does not handle dirty 3489 * page-cache pages correctly). 3490 */ 3491 if (written < 0 || !iov_iter_count(from) || IS_DAX(inode)) 3492 goto out; 3493 3494 status = generic_perform_write(file, from, pos = iocb->ki_pos); 3495 /* 3496 * If generic_perform_write() returned a synchronous error 3497 * then we want to return the number of bytes which were 3498 * direct-written, or the error code if that was zero. Note 3499 * that this differs from normal direct-io semantics, which 3500 * will return -EFOO even if some bytes were written. 3501 */ 3502 if (unlikely(status < 0)) { 3503 err = status; 3504 goto out; 3505 } 3506 /* 3507 * We need to ensure that the page cache pages are written to 3508 * disk and invalidated to preserve the expected O_DIRECT 3509 * semantics. 3510 */ 3511 endbyte = pos + status - 1; 3512 err = filemap_write_and_wait_range(mapping, pos, endbyte); 3513 if (err == 0) { 3514 iocb->ki_pos = endbyte + 1; 3515 written += status; 3516 invalidate_mapping_pages(mapping, 3517 pos >> PAGE_SHIFT, 3518 endbyte >> PAGE_SHIFT); 3519 } else { 3520 /* 3521 * We don't know how much we wrote, so just return 3522 * the number of bytes which were direct-written 3523 */ 3524 } 3525 } else { 3526 written = generic_perform_write(file, from, iocb->ki_pos); 3527 if (likely(written > 0)) 3528 iocb->ki_pos += written; 3529 } 3530 out: 3531 current->backing_dev_info = NULL; 3532 return written ? written : err; 3533 } 3534 EXPORT_SYMBOL(__generic_file_write_iter); 3535 3536 /** 3537 * generic_file_write_iter - write data to a file 3538 * @iocb: IO state structure 3539 * @from: iov_iter with data to write 3540 * 3541 * This is a wrapper around __generic_file_write_iter() to be used by most 3542 * filesystems. It takes care of syncing the file in case of O_SYNC file 3543 * and acquires i_mutex as needed. 3544 * Return: 3545 * * negative error code if no data has been written at all of 3546 * vfs_fsync_range() failed for a synchronous write 3547 * * number of bytes written, even for truncated writes 3548 */ 3549 ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 3550 { 3551 struct file *file = iocb->ki_filp; 3552 struct inode *inode = file->f_mapping->host; 3553 ssize_t ret; 3554 3555 inode_lock(inode); 3556 ret = generic_write_checks(iocb, from); 3557 if (ret > 0) 3558 ret = __generic_file_write_iter(iocb, from); 3559 inode_unlock(inode); 3560 3561 if (ret > 0) 3562 ret = generic_write_sync(iocb, ret); 3563 return ret; 3564 } 3565 EXPORT_SYMBOL(generic_file_write_iter); 3566 3567 /** 3568 * try_to_release_page() - release old fs-specific metadata on a page 3569 * 3570 * @page: the page which the kernel is trying to free 3571 * @gfp_mask: memory allocation flags (and I/O mode) 3572 * 3573 * The address_space is to try to release any data against the page 3574 * (presumably at page->private). 3575 * 3576 * This may also be called if PG_fscache is set on a page, indicating that the 3577 * page is known to the local caching routines. 3578 * 3579 * The @gfp_mask argument specifies whether I/O may be performed to release 3580 * this page (__GFP_IO), and whether the call may block (__GFP_RECLAIM & __GFP_FS). 3581 * 3582 * Return: %1 if the release was successful, otherwise return zero. 3583 */ 3584 int try_to_release_page(struct page *page, gfp_t gfp_mask) 3585 { 3586 struct address_space * const mapping = page->mapping; 3587 3588 BUG_ON(!PageLocked(page)); 3589 if (PageWriteback(page)) 3590 return 0; 3591 3592 if (mapping && mapping->a_ops->releasepage) 3593 return mapping->a_ops->releasepage(page, gfp_mask); 3594 return try_to_free_buffers(page); 3595 } 3596 3597 EXPORT_SYMBOL(try_to_release_page); 3598