1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/mm/filemap.c 4 * 5 * Copyright (C) 1994-1999 Linus Torvalds 6 */ 7 8 /* 9 * This file handles the generic file mmap semantics used by 10 * most "normal" filesystems (but you don't /have/ to use this: 11 * the NFS filesystem used to do this differently, for example) 12 */ 13 #include <linux/export.h> 14 #include <linux/compiler.h> 15 #include <linux/dax.h> 16 #include <linux/fs.h> 17 #include <linux/sched/signal.h> 18 #include <linux/uaccess.h> 19 #include <linux/capability.h> 20 #include <linux/kernel_stat.h> 21 #include <linux/gfp.h> 22 #include <linux/mm.h> 23 #include <linux/swap.h> 24 #include <linux/swapops.h> 25 #include <linux/syscalls.h> 26 #include <linux/mman.h> 27 #include <linux/pagemap.h> 28 #include <linux/file.h> 29 #include <linux/uio.h> 30 #include <linux/error-injection.h> 31 #include <linux/hash.h> 32 #include <linux/writeback.h> 33 #include <linux/backing-dev.h> 34 #include <linux/pagevec.h> 35 #include <linux/security.h> 36 #include <linux/cpuset.h> 37 #include <linux/hugetlb.h> 38 #include <linux/memcontrol.h> 39 #include <linux/shmem_fs.h> 40 #include <linux/rmap.h> 41 #include <linux/delayacct.h> 42 #include <linux/psi.h> 43 #include <linux/ramfs.h> 44 #include <linux/page_idle.h> 45 #include <linux/migrate.h> 46 #include <linux/pipe_fs_i.h> 47 #include <linux/splice.h> 48 #include <asm/pgalloc.h> 49 #include <asm/tlbflush.h> 50 #include "internal.h" 51 52 #define CREATE_TRACE_POINTS 53 #include <trace/events/filemap.h> 54 55 /* 56 * FIXME: remove all knowledge of the buffer layer from the core VM 57 */ 58 #include <linux/buffer_head.h> /* for try_to_free_buffers */ 59 60 #include <asm/mman.h> 61 62 #include "swap.h" 63 64 /* 65 * Shared mappings implemented 30.11.1994. It's not fully working yet, 66 * though. 67 * 68 * Shared mappings now work. 15.8.1995 Bruno. 69 * 70 * finished 'unifying' the page and buffer cache and SMP-threaded the 71 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com> 72 * 73 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de> 74 */ 75 76 /* 77 * Lock ordering: 78 * 79 * ->i_mmap_rwsem (truncate_pagecache) 80 * ->private_lock (__free_pte->block_dirty_folio) 81 * ->swap_lock (exclusive_swap_page, others) 82 * ->i_pages lock 83 * 84 * ->i_rwsem 85 * ->invalidate_lock (acquired by fs in truncate path) 86 * ->i_mmap_rwsem (truncate->unmap_mapping_range) 87 * 88 * ->mmap_lock 89 * ->i_mmap_rwsem 90 * ->page_table_lock or pte_lock (various, mainly in memory.c) 91 * ->i_pages lock (arch-dependent flush_dcache_mmap_lock) 92 * 93 * ->mmap_lock 94 * ->invalidate_lock (filemap_fault) 95 * ->lock_page (filemap_fault, access_process_vm) 96 * 97 * ->i_rwsem (generic_perform_write) 98 * ->mmap_lock (fault_in_readable->do_page_fault) 99 * 100 * bdi->wb.list_lock 101 * sb_lock (fs/fs-writeback.c) 102 * ->i_pages lock (__sync_single_inode) 103 * 104 * ->i_mmap_rwsem 105 * ->anon_vma.lock (vma_merge) 106 * 107 * ->anon_vma.lock 108 * ->page_table_lock or pte_lock (anon_vma_prepare and various) 109 * 110 * ->page_table_lock or pte_lock 111 * ->swap_lock (try_to_unmap_one) 112 * ->private_lock (try_to_unmap_one) 113 * ->i_pages lock (try_to_unmap_one) 114 * ->lruvec->lru_lock (follow_page->mark_page_accessed) 115 * ->lruvec->lru_lock (check_pte_range->isolate_lru_page) 116 * ->private_lock (page_remove_rmap->set_page_dirty) 117 * ->i_pages lock (page_remove_rmap->set_page_dirty) 118 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) 119 * ->inode->i_lock (page_remove_rmap->set_page_dirty) 120 * ->memcg->move_lock (page_remove_rmap->folio_memcg_lock) 121 * bdi.wb->list_lock (zap_pte_range->set_page_dirty) 122 * ->inode->i_lock (zap_pte_range->set_page_dirty) 123 * ->private_lock (zap_pte_range->block_dirty_folio) 124 * 125 * ->i_mmap_rwsem 126 * ->tasklist_lock (memory_failure, collect_procs_ao) 127 */ 128 129 static void page_cache_delete(struct address_space *mapping, 130 struct folio *folio, void *shadow) 131 { 132 XA_STATE(xas, &mapping->i_pages, folio->index); 133 long nr = 1; 134 135 mapping_set_update(&xas, mapping); 136 137 /* hugetlb pages are represented by a single entry in the xarray */ 138 if (!folio_test_hugetlb(folio)) { 139 xas_set_order(&xas, folio->index, folio_order(folio)); 140 nr = folio_nr_pages(folio); 141 } 142 143 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 144 145 xas_store(&xas, shadow); 146 xas_init_marks(&xas); 147 148 folio->mapping = NULL; 149 /* Leave page->index set: truncation lookup relies upon it */ 150 mapping->nrpages -= nr; 151 } 152 153 static void filemap_unaccount_folio(struct address_space *mapping, 154 struct folio *folio) 155 { 156 long nr; 157 158 VM_BUG_ON_FOLIO(folio_mapped(folio), folio); 159 if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(folio_mapped(folio))) { 160 pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n", 161 current->comm, folio_pfn(folio)); 162 dump_page(&folio->page, "still mapped when deleted"); 163 dump_stack(); 164 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 165 166 if (mapping_exiting(mapping) && !folio_test_large(folio)) { 167 int mapcount = page_mapcount(&folio->page); 168 169 if (folio_ref_count(folio) >= mapcount + 2) { 170 /* 171 * All vmas have already been torn down, so it's 172 * a good bet that actually the page is unmapped 173 * and we'd rather not leak it: if we're wrong, 174 * another bad page check should catch it later. 175 */ 176 page_mapcount_reset(&folio->page); 177 folio_ref_sub(folio, mapcount); 178 } 179 } 180 } 181 182 /* hugetlb folios do not participate in page cache accounting. */ 183 if (folio_test_hugetlb(folio)) 184 return; 185 186 nr = folio_nr_pages(folio); 187 188 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr); 189 if (folio_test_swapbacked(folio)) { 190 __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr); 191 if (folio_test_pmd_mappable(folio)) 192 __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr); 193 } else if (folio_test_pmd_mappable(folio)) { 194 __lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr); 195 filemap_nr_thps_dec(mapping); 196 } 197 198 /* 199 * At this point folio must be either written or cleaned by 200 * truncate. Dirty folio here signals a bug and loss of 201 * unwritten data - on ordinary filesystems. 202 * 203 * But it's harmless on in-memory filesystems like tmpfs; and can 204 * occur when a driver which did get_user_pages() sets page dirty 205 * before putting it, while the inode is being finally evicted. 206 * 207 * Below fixes dirty accounting after removing the folio entirely 208 * but leaves the dirty flag set: it has no effect for truncated 209 * folio and anyway will be cleared before returning folio to 210 * buddy allocator. 211 */ 212 if (WARN_ON_ONCE(folio_test_dirty(folio) && 213 mapping_can_writeback(mapping))) 214 folio_account_cleaned(folio, inode_to_wb(mapping->host)); 215 } 216 217 /* 218 * Delete a page from the page cache and free it. Caller has to make 219 * sure the page is locked and that nobody else uses it - or that usage 220 * is safe. The caller must hold the i_pages lock. 221 */ 222 void __filemap_remove_folio(struct folio *folio, void *shadow) 223 { 224 struct address_space *mapping = folio->mapping; 225 226 trace_mm_filemap_delete_from_page_cache(folio); 227 filemap_unaccount_folio(mapping, folio); 228 page_cache_delete(mapping, folio, shadow); 229 } 230 231 void filemap_free_folio(struct address_space *mapping, struct folio *folio) 232 { 233 void (*free_folio)(struct folio *); 234 int refs = 1; 235 236 free_folio = mapping->a_ops->free_folio; 237 if (free_folio) 238 free_folio(folio); 239 240 if (folio_test_large(folio) && !folio_test_hugetlb(folio)) 241 refs = folio_nr_pages(folio); 242 folio_put_refs(folio, refs); 243 } 244 245 /** 246 * filemap_remove_folio - Remove folio from page cache. 247 * @folio: The folio. 248 * 249 * This must be called only on folios that are locked and have been 250 * verified to be in the page cache. It will never put the folio into 251 * the free list because the caller has a reference on the page. 252 */ 253 void filemap_remove_folio(struct folio *folio) 254 { 255 struct address_space *mapping = folio->mapping; 256 257 BUG_ON(!folio_test_locked(folio)); 258 spin_lock(&mapping->host->i_lock); 259 xa_lock_irq(&mapping->i_pages); 260 __filemap_remove_folio(folio, NULL); 261 xa_unlock_irq(&mapping->i_pages); 262 if (mapping_shrinkable(mapping)) 263 inode_add_lru(mapping->host); 264 spin_unlock(&mapping->host->i_lock); 265 266 filemap_free_folio(mapping, folio); 267 } 268 269 /* 270 * page_cache_delete_batch - delete several folios from page cache 271 * @mapping: the mapping to which folios belong 272 * @fbatch: batch of folios to delete 273 * 274 * The function walks over mapping->i_pages and removes folios passed in 275 * @fbatch from the mapping. The function expects @fbatch to be sorted 276 * by page index and is optimised for it to be dense. 277 * It tolerates holes in @fbatch (mapping entries at those indices are not 278 * modified). 279 * 280 * The function expects the i_pages lock to be held. 281 */ 282 static void page_cache_delete_batch(struct address_space *mapping, 283 struct folio_batch *fbatch) 284 { 285 XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index); 286 long total_pages = 0; 287 int i = 0; 288 struct folio *folio; 289 290 mapping_set_update(&xas, mapping); 291 xas_for_each(&xas, folio, ULONG_MAX) { 292 if (i >= folio_batch_count(fbatch)) 293 break; 294 295 /* A swap/dax/shadow entry got inserted? Skip it. */ 296 if (xa_is_value(folio)) 297 continue; 298 /* 299 * A page got inserted in our range? Skip it. We have our 300 * pages locked so they are protected from being removed. 301 * If we see a page whose index is higher than ours, it 302 * means our page has been removed, which shouldn't be 303 * possible because we're holding the PageLock. 304 */ 305 if (folio != fbatch->folios[i]) { 306 VM_BUG_ON_FOLIO(folio->index > 307 fbatch->folios[i]->index, folio); 308 continue; 309 } 310 311 WARN_ON_ONCE(!folio_test_locked(folio)); 312 313 folio->mapping = NULL; 314 /* Leave folio->index set: truncation lookup relies on it */ 315 316 i++; 317 xas_store(&xas, NULL); 318 total_pages += folio_nr_pages(folio); 319 } 320 mapping->nrpages -= total_pages; 321 } 322 323 void delete_from_page_cache_batch(struct address_space *mapping, 324 struct folio_batch *fbatch) 325 { 326 int i; 327 328 if (!folio_batch_count(fbatch)) 329 return; 330 331 spin_lock(&mapping->host->i_lock); 332 xa_lock_irq(&mapping->i_pages); 333 for (i = 0; i < folio_batch_count(fbatch); i++) { 334 struct folio *folio = fbatch->folios[i]; 335 336 trace_mm_filemap_delete_from_page_cache(folio); 337 filemap_unaccount_folio(mapping, folio); 338 } 339 page_cache_delete_batch(mapping, fbatch); 340 xa_unlock_irq(&mapping->i_pages); 341 if (mapping_shrinkable(mapping)) 342 inode_add_lru(mapping->host); 343 spin_unlock(&mapping->host->i_lock); 344 345 for (i = 0; i < folio_batch_count(fbatch); i++) 346 filemap_free_folio(mapping, fbatch->folios[i]); 347 } 348 349 int filemap_check_errors(struct address_space *mapping) 350 { 351 int ret = 0; 352 /* Check for outstanding write errors */ 353 if (test_bit(AS_ENOSPC, &mapping->flags) && 354 test_and_clear_bit(AS_ENOSPC, &mapping->flags)) 355 ret = -ENOSPC; 356 if (test_bit(AS_EIO, &mapping->flags) && 357 test_and_clear_bit(AS_EIO, &mapping->flags)) 358 ret = -EIO; 359 return ret; 360 } 361 EXPORT_SYMBOL(filemap_check_errors); 362 363 static int filemap_check_and_keep_errors(struct address_space *mapping) 364 { 365 /* Check for outstanding write errors */ 366 if (test_bit(AS_EIO, &mapping->flags)) 367 return -EIO; 368 if (test_bit(AS_ENOSPC, &mapping->flags)) 369 return -ENOSPC; 370 return 0; 371 } 372 373 /** 374 * filemap_fdatawrite_wbc - start writeback on mapping dirty pages in range 375 * @mapping: address space structure to write 376 * @wbc: the writeback_control controlling the writeout 377 * 378 * Call writepages on the mapping using the provided wbc to control the 379 * writeout. 380 * 381 * Return: %0 on success, negative error code otherwise. 382 */ 383 int filemap_fdatawrite_wbc(struct address_space *mapping, 384 struct writeback_control *wbc) 385 { 386 int ret; 387 388 if (!mapping_can_writeback(mapping) || 389 !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 390 return 0; 391 392 wbc_attach_fdatawrite_inode(wbc, mapping->host); 393 ret = do_writepages(mapping, wbc); 394 wbc_detach_inode(wbc); 395 return ret; 396 } 397 EXPORT_SYMBOL(filemap_fdatawrite_wbc); 398 399 /** 400 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range 401 * @mapping: address space structure to write 402 * @start: offset in bytes where the range starts 403 * @end: offset in bytes where the range ends (inclusive) 404 * @sync_mode: enable synchronous operation 405 * 406 * Start writeback against all of a mapping's dirty pages that lie 407 * within the byte offsets <start, end> inclusive. 408 * 409 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as 410 * opposed to a regular memory cleansing writeback. The difference between 411 * these two operations is that if a dirty page/buffer is encountered, it must 412 * be waited upon, and not just skipped over. 413 * 414 * Return: %0 on success, negative error code otherwise. 415 */ 416 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 417 loff_t end, int sync_mode) 418 { 419 struct writeback_control wbc = { 420 .sync_mode = sync_mode, 421 .nr_to_write = LONG_MAX, 422 .range_start = start, 423 .range_end = end, 424 }; 425 426 return filemap_fdatawrite_wbc(mapping, &wbc); 427 } 428 429 static inline int __filemap_fdatawrite(struct address_space *mapping, 430 int sync_mode) 431 { 432 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode); 433 } 434 435 int filemap_fdatawrite(struct address_space *mapping) 436 { 437 return __filemap_fdatawrite(mapping, WB_SYNC_ALL); 438 } 439 EXPORT_SYMBOL(filemap_fdatawrite); 440 441 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, 442 loff_t end) 443 { 444 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); 445 } 446 EXPORT_SYMBOL(filemap_fdatawrite_range); 447 448 /** 449 * filemap_flush - mostly a non-blocking flush 450 * @mapping: target address_space 451 * 452 * This is a mostly non-blocking flush. Not suitable for data-integrity 453 * purposes - I/O may not be started against all dirty pages. 454 * 455 * Return: %0 on success, negative error code otherwise. 456 */ 457 int filemap_flush(struct address_space *mapping) 458 { 459 return __filemap_fdatawrite(mapping, WB_SYNC_NONE); 460 } 461 EXPORT_SYMBOL(filemap_flush); 462 463 /** 464 * filemap_range_has_page - check if a page exists in range. 465 * @mapping: address space within which to check 466 * @start_byte: offset in bytes where the range starts 467 * @end_byte: offset in bytes where the range ends (inclusive) 468 * 469 * Find at least one page in the range supplied, usually used to check if 470 * direct writing in this range will trigger a writeback. 471 * 472 * Return: %true if at least one page exists in the specified range, 473 * %false otherwise. 474 */ 475 bool filemap_range_has_page(struct address_space *mapping, 476 loff_t start_byte, loff_t end_byte) 477 { 478 struct folio *folio; 479 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); 480 pgoff_t max = end_byte >> PAGE_SHIFT; 481 482 if (end_byte < start_byte) 483 return false; 484 485 rcu_read_lock(); 486 for (;;) { 487 folio = xas_find(&xas, max); 488 if (xas_retry(&xas, folio)) 489 continue; 490 /* Shadow entries don't count */ 491 if (xa_is_value(folio)) 492 continue; 493 /* 494 * We don't need to try to pin this page; we're about to 495 * release the RCU lock anyway. It is enough to know that 496 * there was a page here recently. 497 */ 498 break; 499 } 500 rcu_read_unlock(); 501 502 return folio != NULL; 503 } 504 EXPORT_SYMBOL(filemap_range_has_page); 505 506 static void __filemap_fdatawait_range(struct address_space *mapping, 507 loff_t start_byte, loff_t end_byte) 508 { 509 pgoff_t index = start_byte >> PAGE_SHIFT; 510 pgoff_t end = end_byte >> PAGE_SHIFT; 511 struct folio_batch fbatch; 512 unsigned nr_folios; 513 514 folio_batch_init(&fbatch); 515 516 while (index <= end) { 517 unsigned i; 518 519 nr_folios = filemap_get_folios_tag(mapping, &index, end, 520 PAGECACHE_TAG_WRITEBACK, &fbatch); 521 522 if (!nr_folios) 523 break; 524 525 for (i = 0; i < nr_folios; i++) { 526 struct folio *folio = fbatch.folios[i]; 527 528 folio_wait_writeback(folio); 529 folio_clear_error(folio); 530 } 531 folio_batch_release(&fbatch); 532 cond_resched(); 533 } 534 } 535 536 /** 537 * filemap_fdatawait_range - wait for writeback to complete 538 * @mapping: address space structure to wait for 539 * @start_byte: offset in bytes where the range starts 540 * @end_byte: offset in bytes where the range ends (inclusive) 541 * 542 * Walk the list of under-writeback pages of the given address space 543 * in the given range and wait for all of them. Check error status of 544 * the address space and return it. 545 * 546 * Since the error status of the address space is cleared by this function, 547 * callers are responsible for checking the return value and handling and/or 548 * reporting the error. 549 * 550 * Return: error status of the address space. 551 */ 552 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte, 553 loff_t end_byte) 554 { 555 __filemap_fdatawait_range(mapping, start_byte, end_byte); 556 return filemap_check_errors(mapping); 557 } 558 EXPORT_SYMBOL(filemap_fdatawait_range); 559 560 /** 561 * filemap_fdatawait_range_keep_errors - wait for writeback to complete 562 * @mapping: address space structure to wait for 563 * @start_byte: offset in bytes where the range starts 564 * @end_byte: offset in bytes where the range ends (inclusive) 565 * 566 * Walk the list of under-writeback pages of the given address space in the 567 * given range and wait for all of them. Unlike filemap_fdatawait_range(), 568 * this function does not clear error status of the address space. 569 * 570 * Use this function if callers don't handle errors themselves. Expected 571 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2), 572 * fsfreeze(8) 573 */ 574 int filemap_fdatawait_range_keep_errors(struct address_space *mapping, 575 loff_t start_byte, loff_t end_byte) 576 { 577 __filemap_fdatawait_range(mapping, start_byte, end_byte); 578 return filemap_check_and_keep_errors(mapping); 579 } 580 EXPORT_SYMBOL(filemap_fdatawait_range_keep_errors); 581 582 /** 583 * file_fdatawait_range - wait for writeback to complete 584 * @file: file pointing to address space structure to wait for 585 * @start_byte: offset in bytes where the range starts 586 * @end_byte: offset in bytes where the range ends (inclusive) 587 * 588 * Walk the list of under-writeback pages of the address space that file 589 * refers to, in the given range and wait for all of them. Check error 590 * status of the address space vs. the file->f_wb_err cursor and return it. 591 * 592 * Since the error status of the file is advanced by this function, 593 * callers are responsible for checking the return value and handling and/or 594 * reporting the error. 595 * 596 * Return: error status of the address space vs. the file->f_wb_err cursor. 597 */ 598 int file_fdatawait_range(struct file *file, loff_t start_byte, loff_t end_byte) 599 { 600 struct address_space *mapping = file->f_mapping; 601 602 __filemap_fdatawait_range(mapping, start_byte, end_byte); 603 return file_check_and_advance_wb_err(file); 604 } 605 EXPORT_SYMBOL(file_fdatawait_range); 606 607 /** 608 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors 609 * @mapping: address space structure to wait for 610 * 611 * Walk the list of under-writeback pages of the given address space 612 * and wait for all of them. Unlike filemap_fdatawait(), this function 613 * does not clear error status of the address space. 614 * 615 * Use this function if callers don't handle errors themselves. Expected 616 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2), 617 * fsfreeze(8) 618 * 619 * Return: error status of the address space. 620 */ 621 int filemap_fdatawait_keep_errors(struct address_space *mapping) 622 { 623 __filemap_fdatawait_range(mapping, 0, LLONG_MAX); 624 return filemap_check_and_keep_errors(mapping); 625 } 626 EXPORT_SYMBOL(filemap_fdatawait_keep_errors); 627 628 /* Returns true if writeback might be needed or already in progress. */ 629 static bool mapping_needs_writeback(struct address_space *mapping) 630 { 631 return mapping->nrpages; 632 } 633 634 bool filemap_range_has_writeback(struct address_space *mapping, 635 loff_t start_byte, loff_t end_byte) 636 { 637 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); 638 pgoff_t max = end_byte >> PAGE_SHIFT; 639 struct folio *folio; 640 641 if (end_byte < start_byte) 642 return false; 643 644 rcu_read_lock(); 645 xas_for_each(&xas, folio, max) { 646 if (xas_retry(&xas, folio)) 647 continue; 648 if (xa_is_value(folio)) 649 continue; 650 if (folio_test_dirty(folio) || folio_test_locked(folio) || 651 folio_test_writeback(folio)) 652 break; 653 } 654 rcu_read_unlock(); 655 return folio != NULL; 656 } 657 EXPORT_SYMBOL_GPL(filemap_range_has_writeback); 658 659 /** 660 * filemap_write_and_wait_range - write out & wait on a file range 661 * @mapping: the address_space for the pages 662 * @lstart: offset in bytes where the range starts 663 * @lend: offset in bytes where the range ends (inclusive) 664 * 665 * Write out and wait upon file offsets lstart->lend, inclusive. 666 * 667 * Note that @lend is inclusive (describes the last byte to be written) so 668 * that this function can be used to write to the very end-of-file (end = -1). 669 * 670 * Return: error status of the address space. 671 */ 672 int filemap_write_and_wait_range(struct address_space *mapping, 673 loff_t lstart, loff_t lend) 674 { 675 int err = 0, err2; 676 677 if (lend < lstart) 678 return 0; 679 680 if (mapping_needs_writeback(mapping)) { 681 err = __filemap_fdatawrite_range(mapping, lstart, lend, 682 WB_SYNC_ALL); 683 /* 684 * Even if the above returned error, the pages may be 685 * written partially (e.g. -ENOSPC), so we wait for it. 686 * But the -EIO is special case, it may indicate the worst 687 * thing (e.g. bug) happened, so we avoid waiting for it. 688 */ 689 if (err != -EIO) 690 __filemap_fdatawait_range(mapping, lstart, lend); 691 } 692 err2 = filemap_check_errors(mapping); 693 if (!err) 694 err = err2; 695 return err; 696 } 697 EXPORT_SYMBOL(filemap_write_and_wait_range); 698 699 void __filemap_set_wb_err(struct address_space *mapping, int err) 700 { 701 errseq_t eseq = errseq_set(&mapping->wb_err, err); 702 703 trace_filemap_set_wb_err(mapping, eseq); 704 } 705 EXPORT_SYMBOL(__filemap_set_wb_err); 706 707 /** 708 * file_check_and_advance_wb_err - report wb error (if any) that was previously 709 * and advance wb_err to current one 710 * @file: struct file on which the error is being reported 711 * 712 * When userland calls fsync (or something like nfsd does the equivalent), we 713 * want to report any writeback errors that occurred since the last fsync (or 714 * since the file was opened if there haven't been any). 715 * 716 * Grab the wb_err from the mapping. If it matches what we have in the file, 717 * then just quickly return 0. The file is all caught up. 718 * 719 * If it doesn't match, then take the mapping value, set the "seen" flag in 720 * it and try to swap it into place. If it works, or another task beat us 721 * to it with the new value, then update the f_wb_err and return the error 722 * portion. The error at this point must be reported via proper channels 723 * (a'la fsync, or NFS COMMIT operation, etc.). 724 * 725 * While we handle mapping->wb_err with atomic operations, the f_wb_err 726 * value is protected by the f_lock since we must ensure that it reflects 727 * the latest value swapped in for this file descriptor. 728 * 729 * Return: %0 on success, negative error code otherwise. 730 */ 731 int file_check_and_advance_wb_err(struct file *file) 732 { 733 int err = 0; 734 errseq_t old = READ_ONCE(file->f_wb_err); 735 struct address_space *mapping = file->f_mapping; 736 737 /* Locklessly handle the common case where nothing has changed */ 738 if (errseq_check(&mapping->wb_err, old)) { 739 /* Something changed, must use slow path */ 740 spin_lock(&file->f_lock); 741 old = file->f_wb_err; 742 err = errseq_check_and_advance(&mapping->wb_err, 743 &file->f_wb_err); 744 trace_file_check_and_advance_wb_err(file, old); 745 spin_unlock(&file->f_lock); 746 } 747 748 /* 749 * We're mostly using this function as a drop in replacement for 750 * filemap_check_errors. Clear AS_EIO/AS_ENOSPC to emulate the effect 751 * that the legacy code would have had on these flags. 752 */ 753 clear_bit(AS_EIO, &mapping->flags); 754 clear_bit(AS_ENOSPC, &mapping->flags); 755 return err; 756 } 757 EXPORT_SYMBOL(file_check_and_advance_wb_err); 758 759 /** 760 * file_write_and_wait_range - write out & wait on a file range 761 * @file: file pointing to address_space with pages 762 * @lstart: offset in bytes where the range starts 763 * @lend: offset in bytes where the range ends (inclusive) 764 * 765 * Write out and wait upon file offsets lstart->lend, inclusive. 766 * 767 * Note that @lend is inclusive (describes the last byte to be written) so 768 * that this function can be used to write to the very end-of-file (end = -1). 769 * 770 * After writing out and waiting on the data, we check and advance the 771 * f_wb_err cursor to the latest value, and return any errors detected there. 772 * 773 * Return: %0 on success, negative error code otherwise. 774 */ 775 int file_write_and_wait_range(struct file *file, loff_t lstart, loff_t lend) 776 { 777 int err = 0, err2; 778 struct address_space *mapping = file->f_mapping; 779 780 if (lend < lstart) 781 return 0; 782 783 if (mapping_needs_writeback(mapping)) { 784 err = __filemap_fdatawrite_range(mapping, lstart, lend, 785 WB_SYNC_ALL); 786 /* See comment of filemap_write_and_wait() */ 787 if (err != -EIO) 788 __filemap_fdatawait_range(mapping, lstart, lend); 789 } 790 err2 = file_check_and_advance_wb_err(file); 791 if (!err) 792 err = err2; 793 return err; 794 } 795 EXPORT_SYMBOL(file_write_and_wait_range); 796 797 /** 798 * replace_page_cache_folio - replace a pagecache folio with a new one 799 * @old: folio to be replaced 800 * @new: folio to replace with 801 * 802 * This function replaces a folio in the pagecache with a new one. On 803 * success it acquires the pagecache reference for the new folio and 804 * drops it for the old folio. Both the old and new folios must be 805 * locked. This function does not add the new folio to the LRU, the 806 * caller must do that. 807 * 808 * The remove + add is atomic. This function cannot fail. 809 */ 810 void replace_page_cache_folio(struct folio *old, struct folio *new) 811 { 812 struct address_space *mapping = old->mapping; 813 void (*free_folio)(struct folio *) = mapping->a_ops->free_folio; 814 pgoff_t offset = old->index; 815 XA_STATE(xas, &mapping->i_pages, offset); 816 817 VM_BUG_ON_FOLIO(!folio_test_locked(old), old); 818 VM_BUG_ON_FOLIO(!folio_test_locked(new), new); 819 VM_BUG_ON_FOLIO(new->mapping, new); 820 821 folio_get(new); 822 new->mapping = mapping; 823 new->index = offset; 824 825 mem_cgroup_migrate(old, new); 826 827 xas_lock_irq(&xas); 828 xas_store(&xas, new); 829 830 old->mapping = NULL; 831 /* hugetlb pages do not participate in page cache accounting. */ 832 if (!folio_test_hugetlb(old)) 833 __lruvec_stat_sub_folio(old, NR_FILE_PAGES); 834 if (!folio_test_hugetlb(new)) 835 __lruvec_stat_add_folio(new, NR_FILE_PAGES); 836 if (folio_test_swapbacked(old)) 837 __lruvec_stat_sub_folio(old, NR_SHMEM); 838 if (folio_test_swapbacked(new)) 839 __lruvec_stat_add_folio(new, NR_SHMEM); 840 xas_unlock_irq(&xas); 841 if (free_folio) 842 free_folio(old); 843 folio_put(old); 844 } 845 EXPORT_SYMBOL_GPL(replace_page_cache_folio); 846 847 noinline int __filemap_add_folio(struct address_space *mapping, 848 struct folio *folio, pgoff_t index, gfp_t gfp, void **shadowp) 849 { 850 XA_STATE(xas, &mapping->i_pages, index); 851 int huge = folio_test_hugetlb(folio); 852 bool charged = false; 853 long nr = 1; 854 855 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 856 VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio); 857 mapping_set_update(&xas, mapping); 858 859 if (!huge) { 860 int error = mem_cgroup_charge(folio, NULL, gfp); 861 VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio); 862 if (error) 863 return error; 864 charged = true; 865 xas_set_order(&xas, index, folio_order(folio)); 866 nr = folio_nr_pages(folio); 867 } 868 869 gfp &= GFP_RECLAIM_MASK; 870 folio_ref_add(folio, nr); 871 folio->mapping = mapping; 872 folio->index = xas.xa_index; 873 874 do { 875 unsigned int order = xa_get_order(xas.xa, xas.xa_index); 876 void *entry, *old = NULL; 877 878 if (order > folio_order(folio)) 879 xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index), 880 order, gfp); 881 xas_lock_irq(&xas); 882 xas_for_each_conflict(&xas, entry) { 883 old = entry; 884 if (!xa_is_value(entry)) { 885 xas_set_err(&xas, -EEXIST); 886 goto unlock; 887 } 888 } 889 890 if (old) { 891 if (shadowp) 892 *shadowp = old; 893 /* entry may have been split before we acquired lock */ 894 order = xa_get_order(xas.xa, xas.xa_index); 895 if (order > folio_order(folio)) { 896 /* How to handle large swap entries? */ 897 BUG_ON(shmem_mapping(mapping)); 898 xas_split(&xas, old, order); 899 xas_reset(&xas); 900 } 901 } 902 903 xas_store(&xas, folio); 904 if (xas_error(&xas)) 905 goto unlock; 906 907 mapping->nrpages += nr; 908 909 /* hugetlb pages do not participate in page cache accounting */ 910 if (!huge) { 911 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr); 912 if (folio_test_pmd_mappable(folio)) 913 __lruvec_stat_mod_folio(folio, 914 NR_FILE_THPS, nr); 915 } 916 unlock: 917 xas_unlock_irq(&xas); 918 } while (xas_nomem(&xas, gfp)); 919 920 if (xas_error(&xas)) 921 goto error; 922 923 trace_mm_filemap_add_to_page_cache(folio); 924 return 0; 925 error: 926 if (charged) 927 mem_cgroup_uncharge(folio); 928 folio->mapping = NULL; 929 /* Leave page->index set: truncation relies upon it */ 930 folio_put_refs(folio, nr); 931 return xas_error(&xas); 932 } 933 ALLOW_ERROR_INJECTION(__filemap_add_folio, ERRNO); 934 935 int filemap_add_folio(struct address_space *mapping, struct folio *folio, 936 pgoff_t index, gfp_t gfp) 937 { 938 void *shadow = NULL; 939 int ret; 940 941 __folio_set_locked(folio); 942 ret = __filemap_add_folio(mapping, folio, index, gfp, &shadow); 943 if (unlikely(ret)) 944 __folio_clear_locked(folio); 945 else { 946 /* 947 * The folio might have been evicted from cache only 948 * recently, in which case it should be activated like 949 * any other repeatedly accessed folio. 950 * The exception is folios getting rewritten; evicting other 951 * data from the working set, only to cache data that will 952 * get overwritten with something else, is a waste of memory. 953 */ 954 WARN_ON_ONCE(folio_test_active(folio)); 955 if (!(gfp & __GFP_WRITE) && shadow) 956 workingset_refault(folio, shadow); 957 folio_add_lru(folio); 958 } 959 return ret; 960 } 961 EXPORT_SYMBOL_GPL(filemap_add_folio); 962 963 #ifdef CONFIG_NUMA 964 struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order) 965 { 966 int n; 967 struct folio *folio; 968 969 if (cpuset_do_page_mem_spread()) { 970 unsigned int cpuset_mems_cookie; 971 do { 972 cpuset_mems_cookie = read_mems_allowed_begin(); 973 n = cpuset_mem_spread_node(); 974 folio = __folio_alloc_node(gfp, order, n); 975 } while (!folio && read_mems_allowed_retry(cpuset_mems_cookie)); 976 977 return folio; 978 } 979 return folio_alloc(gfp, order); 980 } 981 EXPORT_SYMBOL(filemap_alloc_folio); 982 #endif 983 984 /* 985 * filemap_invalidate_lock_two - lock invalidate_lock for two mappings 986 * 987 * Lock exclusively invalidate_lock of any passed mapping that is not NULL. 988 * 989 * @mapping1: the first mapping to lock 990 * @mapping2: the second mapping to lock 991 */ 992 void filemap_invalidate_lock_two(struct address_space *mapping1, 993 struct address_space *mapping2) 994 { 995 if (mapping1 > mapping2) 996 swap(mapping1, mapping2); 997 if (mapping1) 998 down_write(&mapping1->invalidate_lock); 999 if (mapping2 && mapping1 != mapping2) 1000 down_write_nested(&mapping2->invalidate_lock, 1); 1001 } 1002 EXPORT_SYMBOL(filemap_invalidate_lock_two); 1003 1004 /* 1005 * filemap_invalidate_unlock_two - unlock invalidate_lock for two mappings 1006 * 1007 * Unlock exclusive invalidate_lock of any passed mapping that is not NULL. 1008 * 1009 * @mapping1: the first mapping to unlock 1010 * @mapping2: the second mapping to unlock 1011 */ 1012 void filemap_invalidate_unlock_two(struct address_space *mapping1, 1013 struct address_space *mapping2) 1014 { 1015 if (mapping1) 1016 up_write(&mapping1->invalidate_lock); 1017 if (mapping2 && mapping1 != mapping2) 1018 up_write(&mapping2->invalidate_lock); 1019 } 1020 EXPORT_SYMBOL(filemap_invalidate_unlock_two); 1021 1022 /* 1023 * In order to wait for pages to become available there must be 1024 * waitqueues associated with pages. By using a hash table of 1025 * waitqueues where the bucket discipline is to maintain all 1026 * waiters on the same queue and wake all when any of the pages 1027 * become available, and for the woken contexts to check to be 1028 * sure the appropriate page became available, this saves space 1029 * at a cost of "thundering herd" phenomena during rare hash 1030 * collisions. 1031 */ 1032 #define PAGE_WAIT_TABLE_BITS 8 1033 #define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS) 1034 static wait_queue_head_t folio_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned; 1035 1036 static wait_queue_head_t *folio_waitqueue(struct folio *folio) 1037 { 1038 return &folio_wait_table[hash_ptr(folio, PAGE_WAIT_TABLE_BITS)]; 1039 } 1040 1041 void __init pagecache_init(void) 1042 { 1043 int i; 1044 1045 for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++) 1046 init_waitqueue_head(&folio_wait_table[i]); 1047 1048 page_writeback_init(); 1049 } 1050 1051 /* 1052 * The page wait code treats the "wait->flags" somewhat unusually, because 1053 * we have multiple different kinds of waits, not just the usual "exclusive" 1054 * one. 1055 * 1056 * We have: 1057 * 1058 * (a) no special bits set: 1059 * 1060 * We're just waiting for the bit to be released, and when a waker 1061 * calls the wakeup function, we set WQ_FLAG_WOKEN and wake it up, 1062 * and remove it from the wait queue. 1063 * 1064 * Simple and straightforward. 1065 * 1066 * (b) WQ_FLAG_EXCLUSIVE: 1067 * 1068 * The waiter is waiting to get the lock, and only one waiter should 1069 * be woken up to avoid any thundering herd behavior. We'll set the 1070 * WQ_FLAG_WOKEN bit, wake it up, and remove it from the wait queue. 1071 * 1072 * This is the traditional exclusive wait. 1073 * 1074 * (c) WQ_FLAG_EXCLUSIVE | WQ_FLAG_CUSTOM: 1075 * 1076 * The waiter is waiting to get the bit, and additionally wants the 1077 * lock to be transferred to it for fair lock behavior. If the lock 1078 * cannot be taken, we stop walking the wait queue without waking 1079 * the waiter. 1080 * 1081 * This is the "fair lock handoff" case, and in addition to setting 1082 * WQ_FLAG_WOKEN, we set WQ_FLAG_DONE to let the waiter easily see 1083 * that it now has the lock. 1084 */ 1085 static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg) 1086 { 1087 unsigned int flags; 1088 struct wait_page_key *key = arg; 1089 struct wait_page_queue *wait_page 1090 = container_of(wait, struct wait_page_queue, wait); 1091 1092 if (!wake_page_match(wait_page, key)) 1093 return 0; 1094 1095 /* 1096 * If it's a lock handoff wait, we get the bit for it, and 1097 * stop walking (and do not wake it up) if we can't. 1098 */ 1099 flags = wait->flags; 1100 if (flags & WQ_FLAG_EXCLUSIVE) { 1101 if (test_bit(key->bit_nr, &key->folio->flags)) 1102 return -1; 1103 if (flags & WQ_FLAG_CUSTOM) { 1104 if (test_and_set_bit(key->bit_nr, &key->folio->flags)) 1105 return -1; 1106 flags |= WQ_FLAG_DONE; 1107 } 1108 } 1109 1110 /* 1111 * We are holding the wait-queue lock, but the waiter that 1112 * is waiting for this will be checking the flags without 1113 * any locking. 1114 * 1115 * So update the flags atomically, and wake up the waiter 1116 * afterwards to avoid any races. This store-release pairs 1117 * with the load-acquire in folio_wait_bit_common(). 1118 */ 1119 smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN); 1120 wake_up_state(wait->private, mode); 1121 1122 /* 1123 * Ok, we have successfully done what we're waiting for, 1124 * and we can unconditionally remove the wait entry. 1125 * 1126 * Note that this pairs with the "finish_wait()" in the 1127 * waiter, and has to be the absolute last thing we do. 1128 * After this list_del_init(&wait->entry) the wait entry 1129 * might be de-allocated and the process might even have 1130 * exited. 1131 */ 1132 list_del_init_careful(&wait->entry); 1133 return (flags & WQ_FLAG_EXCLUSIVE) != 0; 1134 } 1135 1136 static void folio_wake_bit(struct folio *folio, int bit_nr) 1137 { 1138 wait_queue_head_t *q = folio_waitqueue(folio); 1139 struct wait_page_key key; 1140 unsigned long flags; 1141 wait_queue_entry_t bookmark; 1142 1143 key.folio = folio; 1144 key.bit_nr = bit_nr; 1145 key.page_match = 0; 1146 1147 bookmark.flags = 0; 1148 bookmark.private = NULL; 1149 bookmark.func = NULL; 1150 INIT_LIST_HEAD(&bookmark.entry); 1151 1152 spin_lock_irqsave(&q->lock, flags); 1153 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark); 1154 1155 while (bookmark.flags & WQ_FLAG_BOOKMARK) { 1156 /* 1157 * Take a breather from holding the lock, 1158 * allow pages that finish wake up asynchronously 1159 * to acquire the lock and remove themselves 1160 * from wait queue 1161 */ 1162 spin_unlock_irqrestore(&q->lock, flags); 1163 cpu_relax(); 1164 spin_lock_irqsave(&q->lock, flags); 1165 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark); 1166 } 1167 1168 /* 1169 * It's possible to miss clearing waiters here, when we woke our page 1170 * waiters, but the hashed waitqueue has waiters for other pages on it. 1171 * That's okay, it's a rare case. The next waker will clear it. 1172 * 1173 * Note that, depending on the page pool (buddy, hugetlb, ZONE_DEVICE, 1174 * other), the flag may be cleared in the course of freeing the page; 1175 * but that is not required for correctness. 1176 */ 1177 if (!waitqueue_active(q) || !key.page_match) 1178 folio_clear_waiters(folio); 1179 1180 spin_unlock_irqrestore(&q->lock, flags); 1181 } 1182 1183 static void folio_wake(struct folio *folio, int bit) 1184 { 1185 if (!folio_test_waiters(folio)) 1186 return; 1187 folio_wake_bit(folio, bit); 1188 } 1189 1190 /* 1191 * A choice of three behaviors for folio_wait_bit_common(): 1192 */ 1193 enum behavior { 1194 EXCLUSIVE, /* Hold ref to page and take the bit when woken, like 1195 * __folio_lock() waiting on then setting PG_locked. 1196 */ 1197 SHARED, /* Hold ref to page and check the bit when woken, like 1198 * folio_wait_writeback() waiting on PG_writeback. 1199 */ 1200 DROP, /* Drop ref to page before wait, no check when woken, 1201 * like folio_put_wait_locked() on PG_locked. 1202 */ 1203 }; 1204 1205 /* 1206 * Attempt to check (or get) the folio flag, and mark us done 1207 * if successful. 1208 */ 1209 static inline bool folio_trylock_flag(struct folio *folio, int bit_nr, 1210 struct wait_queue_entry *wait) 1211 { 1212 if (wait->flags & WQ_FLAG_EXCLUSIVE) { 1213 if (test_and_set_bit(bit_nr, &folio->flags)) 1214 return false; 1215 } else if (test_bit(bit_nr, &folio->flags)) 1216 return false; 1217 1218 wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE; 1219 return true; 1220 } 1221 1222 /* How many times do we accept lock stealing from under a waiter? */ 1223 int sysctl_page_lock_unfairness = 5; 1224 1225 static inline int folio_wait_bit_common(struct folio *folio, int bit_nr, 1226 int state, enum behavior behavior) 1227 { 1228 wait_queue_head_t *q = folio_waitqueue(folio); 1229 int unfairness = sysctl_page_lock_unfairness; 1230 struct wait_page_queue wait_page; 1231 wait_queue_entry_t *wait = &wait_page.wait; 1232 bool thrashing = false; 1233 unsigned long pflags; 1234 bool in_thrashing; 1235 1236 if (bit_nr == PG_locked && 1237 !folio_test_uptodate(folio) && folio_test_workingset(folio)) { 1238 delayacct_thrashing_start(&in_thrashing); 1239 psi_memstall_enter(&pflags); 1240 thrashing = true; 1241 } 1242 1243 init_wait(wait); 1244 wait->func = wake_page_function; 1245 wait_page.folio = folio; 1246 wait_page.bit_nr = bit_nr; 1247 1248 repeat: 1249 wait->flags = 0; 1250 if (behavior == EXCLUSIVE) { 1251 wait->flags = WQ_FLAG_EXCLUSIVE; 1252 if (--unfairness < 0) 1253 wait->flags |= WQ_FLAG_CUSTOM; 1254 } 1255 1256 /* 1257 * Do one last check whether we can get the 1258 * page bit synchronously. 1259 * 1260 * Do the folio_set_waiters() marking before that 1261 * to let any waker we _just_ missed know they 1262 * need to wake us up (otherwise they'll never 1263 * even go to the slow case that looks at the 1264 * page queue), and add ourselves to the wait 1265 * queue if we need to sleep. 1266 * 1267 * This part needs to be done under the queue 1268 * lock to avoid races. 1269 */ 1270 spin_lock_irq(&q->lock); 1271 folio_set_waiters(folio); 1272 if (!folio_trylock_flag(folio, bit_nr, wait)) 1273 __add_wait_queue_entry_tail(q, wait); 1274 spin_unlock_irq(&q->lock); 1275 1276 /* 1277 * From now on, all the logic will be based on 1278 * the WQ_FLAG_WOKEN and WQ_FLAG_DONE flag, to 1279 * see whether the page bit testing has already 1280 * been done by the wake function. 1281 * 1282 * We can drop our reference to the folio. 1283 */ 1284 if (behavior == DROP) 1285 folio_put(folio); 1286 1287 /* 1288 * Note that until the "finish_wait()", or until 1289 * we see the WQ_FLAG_WOKEN flag, we need to 1290 * be very careful with the 'wait->flags', because 1291 * we may race with a waker that sets them. 1292 */ 1293 for (;;) { 1294 unsigned int flags; 1295 1296 set_current_state(state); 1297 1298 /* Loop until we've been woken or interrupted */ 1299 flags = smp_load_acquire(&wait->flags); 1300 if (!(flags & WQ_FLAG_WOKEN)) { 1301 if (signal_pending_state(state, current)) 1302 break; 1303 1304 io_schedule(); 1305 continue; 1306 } 1307 1308 /* If we were non-exclusive, we're done */ 1309 if (behavior != EXCLUSIVE) 1310 break; 1311 1312 /* If the waker got the lock for us, we're done */ 1313 if (flags & WQ_FLAG_DONE) 1314 break; 1315 1316 /* 1317 * Otherwise, if we're getting the lock, we need to 1318 * try to get it ourselves. 1319 * 1320 * And if that fails, we'll have to retry this all. 1321 */ 1322 if (unlikely(test_and_set_bit(bit_nr, folio_flags(folio, 0)))) 1323 goto repeat; 1324 1325 wait->flags |= WQ_FLAG_DONE; 1326 break; 1327 } 1328 1329 /* 1330 * If a signal happened, this 'finish_wait()' may remove the last 1331 * waiter from the wait-queues, but the folio waiters bit will remain 1332 * set. That's ok. The next wakeup will take care of it, and trying 1333 * to do it here would be difficult and prone to races. 1334 */ 1335 finish_wait(q, wait); 1336 1337 if (thrashing) { 1338 delayacct_thrashing_end(&in_thrashing); 1339 psi_memstall_leave(&pflags); 1340 } 1341 1342 /* 1343 * NOTE! The wait->flags weren't stable until we've done the 1344 * 'finish_wait()', and we could have exited the loop above due 1345 * to a signal, and had a wakeup event happen after the signal 1346 * test but before the 'finish_wait()'. 1347 * 1348 * So only after the finish_wait() can we reliably determine 1349 * if we got woken up or not, so we can now figure out the final 1350 * return value based on that state without races. 1351 * 1352 * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive 1353 * waiter, but an exclusive one requires WQ_FLAG_DONE. 1354 */ 1355 if (behavior == EXCLUSIVE) 1356 return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR; 1357 1358 return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR; 1359 } 1360 1361 #ifdef CONFIG_MIGRATION 1362 /** 1363 * migration_entry_wait_on_locked - Wait for a migration entry to be removed 1364 * @entry: migration swap entry. 1365 * @ptl: already locked ptl. This function will drop the lock. 1366 * 1367 * Wait for a migration entry referencing the given page to be removed. This is 1368 * equivalent to put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE) except 1369 * this can be called without taking a reference on the page. Instead this 1370 * should be called while holding the ptl for the migration entry referencing 1371 * the page. 1372 * 1373 * Returns after unlocking the ptl. 1374 * 1375 * This follows the same logic as folio_wait_bit_common() so see the comments 1376 * there. 1377 */ 1378 void migration_entry_wait_on_locked(swp_entry_t entry, spinlock_t *ptl) 1379 __releases(ptl) 1380 { 1381 struct wait_page_queue wait_page; 1382 wait_queue_entry_t *wait = &wait_page.wait; 1383 bool thrashing = false; 1384 unsigned long pflags; 1385 bool in_thrashing; 1386 wait_queue_head_t *q; 1387 struct folio *folio = page_folio(pfn_swap_entry_to_page(entry)); 1388 1389 q = folio_waitqueue(folio); 1390 if (!folio_test_uptodate(folio) && folio_test_workingset(folio)) { 1391 delayacct_thrashing_start(&in_thrashing); 1392 psi_memstall_enter(&pflags); 1393 thrashing = true; 1394 } 1395 1396 init_wait(wait); 1397 wait->func = wake_page_function; 1398 wait_page.folio = folio; 1399 wait_page.bit_nr = PG_locked; 1400 wait->flags = 0; 1401 1402 spin_lock_irq(&q->lock); 1403 folio_set_waiters(folio); 1404 if (!folio_trylock_flag(folio, PG_locked, wait)) 1405 __add_wait_queue_entry_tail(q, wait); 1406 spin_unlock_irq(&q->lock); 1407 1408 /* 1409 * If a migration entry exists for the page the migration path must hold 1410 * a valid reference to the page, and it must take the ptl to remove the 1411 * migration entry. So the page is valid until the ptl is dropped. 1412 */ 1413 spin_unlock(ptl); 1414 1415 for (;;) { 1416 unsigned int flags; 1417 1418 set_current_state(TASK_UNINTERRUPTIBLE); 1419 1420 /* Loop until we've been woken or interrupted */ 1421 flags = smp_load_acquire(&wait->flags); 1422 if (!(flags & WQ_FLAG_WOKEN)) { 1423 if (signal_pending_state(TASK_UNINTERRUPTIBLE, current)) 1424 break; 1425 1426 io_schedule(); 1427 continue; 1428 } 1429 break; 1430 } 1431 1432 finish_wait(q, wait); 1433 1434 if (thrashing) { 1435 delayacct_thrashing_end(&in_thrashing); 1436 psi_memstall_leave(&pflags); 1437 } 1438 } 1439 #endif 1440 1441 void folio_wait_bit(struct folio *folio, int bit_nr) 1442 { 1443 folio_wait_bit_common(folio, bit_nr, TASK_UNINTERRUPTIBLE, SHARED); 1444 } 1445 EXPORT_SYMBOL(folio_wait_bit); 1446 1447 int folio_wait_bit_killable(struct folio *folio, int bit_nr) 1448 { 1449 return folio_wait_bit_common(folio, bit_nr, TASK_KILLABLE, SHARED); 1450 } 1451 EXPORT_SYMBOL(folio_wait_bit_killable); 1452 1453 /** 1454 * folio_put_wait_locked - Drop a reference and wait for it to be unlocked 1455 * @folio: The folio to wait for. 1456 * @state: The sleep state (TASK_KILLABLE, TASK_UNINTERRUPTIBLE, etc). 1457 * 1458 * The caller should hold a reference on @folio. They expect the page to 1459 * become unlocked relatively soon, but do not wish to hold up migration 1460 * (for example) by holding the reference while waiting for the folio to 1461 * come unlocked. After this function returns, the caller should not 1462 * dereference @folio. 1463 * 1464 * Return: 0 if the folio was unlocked or -EINTR if interrupted by a signal. 1465 */ 1466 static int folio_put_wait_locked(struct folio *folio, int state) 1467 { 1468 return folio_wait_bit_common(folio, PG_locked, state, DROP); 1469 } 1470 1471 /** 1472 * folio_add_wait_queue - Add an arbitrary waiter to a folio's wait queue 1473 * @folio: Folio defining the wait queue of interest 1474 * @waiter: Waiter to add to the queue 1475 * 1476 * Add an arbitrary @waiter to the wait queue for the nominated @folio. 1477 */ 1478 void folio_add_wait_queue(struct folio *folio, wait_queue_entry_t *waiter) 1479 { 1480 wait_queue_head_t *q = folio_waitqueue(folio); 1481 unsigned long flags; 1482 1483 spin_lock_irqsave(&q->lock, flags); 1484 __add_wait_queue_entry_tail(q, waiter); 1485 folio_set_waiters(folio); 1486 spin_unlock_irqrestore(&q->lock, flags); 1487 } 1488 EXPORT_SYMBOL_GPL(folio_add_wait_queue); 1489 1490 #ifndef clear_bit_unlock_is_negative_byte 1491 1492 /* 1493 * PG_waiters is the high bit in the same byte as PG_lock. 1494 * 1495 * On x86 (and on many other architectures), we can clear PG_lock and 1496 * test the sign bit at the same time. But if the architecture does 1497 * not support that special operation, we just do this all by hand 1498 * instead. 1499 * 1500 * The read of PG_waiters has to be after (or concurrently with) PG_locked 1501 * being cleared, but a memory barrier should be unnecessary since it is 1502 * in the same byte as PG_locked. 1503 */ 1504 static inline bool clear_bit_unlock_is_negative_byte(long nr, volatile void *mem) 1505 { 1506 clear_bit_unlock(nr, mem); 1507 /* smp_mb__after_atomic(); */ 1508 return test_bit(PG_waiters, mem); 1509 } 1510 1511 #endif 1512 1513 /** 1514 * folio_unlock - Unlock a locked folio. 1515 * @folio: The folio. 1516 * 1517 * Unlocks the folio and wakes up any thread sleeping on the page lock. 1518 * 1519 * Context: May be called from interrupt or process context. May not be 1520 * called from NMI context. 1521 */ 1522 void folio_unlock(struct folio *folio) 1523 { 1524 /* Bit 7 allows x86 to check the byte's sign bit */ 1525 BUILD_BUG_ON(PG_waiters != 7); 1526 BUILD_BUG_ON(PG_locked > 7); 1527 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 1528 if (clear_bit_unlock_is_negative_byte(PG_locked, folio_flags(folio, 0))) 1529 folio_wake_bit(folio, PG_locked); 1530 } 1531 EXPORT_SYMBOL(folio_unlock); 1532 1533 /** 1534 * folio_end_private_2 - Clear PG_private_2 and wake any waiters. 1535 * @folio: The folio. 1536 * 1537 * Clear the PG_private_2 bit on a folio and wake up any sleepers waiting for 1538 * it. The folio reference held for PG_private_2 being set is released. 1539 * 1540 * This is, for example, used when a netfs folio is being written to a local 1541 * disk cache, thereby allowing writes to the cache for the same folio to be 1542 * serialised. 1543 */ 1544 void folio_end_private_2(struct folio *folio) 1545 { 1546 VM_BUG_ON_FOLIO(!folio_test_private_2(folio), folio); 1547 clear_bit_unlock(PG_private_2, folio_flags(folio, 0)); 1548 folio_wake_bit(folio, PG_private_2); 1549 folio_put(folio); 1550 } 1551 EXPORT_SYMBOL(folio_end_private_2); 1552 1553 /** 1554 * folio_wait_private_2 - Wait for PG_private_2 to be cleared on a folio. 1555 * @folio: The folio to wait on. 1556 * 1557 * Wait for PG_private_2 (aka PG_fscache) to be cleared on a folio. 1558 */ 1559 void folio_wait_private_2(struct folio *folio) 1560 { 1561 while (folio_test_private_2(folio)) 1562 folio_wait_bit(folio, PG_private_2); 1563 } 1564 EXPORT_SYMBOL(folio_wait_private_2); 1565 1566 /** 1567 * folio_wait_private_2_killable - Wait for PG_private_2 to be cleared on a folio. 1568 * @folio: The folio to wait on. 1569 * 1570 * Wait for PG_private_2 (aka PG_fscache) to be cleared on a folio or until a 1571 * fatal signal is received by the calling task. 1572 * 1573 * Return: 1574 * - 0 if successful. 1575 * - -EINTR if a fatal signal was encountered. 1576 */ 1577 int folio_wait_private_2_killable(struct folio *folio) 1578 { 1579 int ret = 0; 1580 1581 while (folio_test_private_2(folio)) { 1582 ret = folio_wait_bit_killable(folio, PG_private_2); 1583 if (ret < 0) 1584 break; 1585 } 1586 1587 return ret; 1588 } 1589 EXPORT_SYMBOL(folio_wait_private_2_killable); 1590 1591 /** 1592 * folio_end_writeback - End writeback against a folio. 1593 * @folio: The folio. 1594 */ 1595 void folio_end_writeback(struct folio *folio) 1596 { 1597 /* 1598 * folio_test_clear_reclaim() could be used here but it is an 1599 * atomic operation and overkill in this particular case. Failing 1600 * to shuffle a folio marked for immediate reclaim is too mild 1601 * a gain to justify taking an atomic operation penalty at the 1602 * end of every folio writeback. 1603 */ 1604 if (folio_test_reclaim(folio)) { 1605 folio_clear_reclaim(folio); 1606 folio_rotate_reclaimable(folio); 1607 } 1608 1609 /* 1610 * Writeback does not hold a folio reference of its own, relying 1611 * on truncation to wait for the clearing of PG_writeback. 1612 * But here we must make sure that the folio is not freed and 1613 * reused before the folio_wake(). 1614 */ 1615 folio_get(folio); 1616 if (!__folio_end_writeback(folio)) 1617 BUG(); 1618 1619 smp_mb__after_atomic(); 1620 folio_wake(folio, PG_writeback); 1621 acct_reclaim_writeback(folio); 1622 folio_put(folio); 1623 } 1624 EXPORT_SYMBOL(folio_end_writeback); 1625 1626 /** 1627 * __folio_lock - Get a lock on the folio, assuming we need to sleep to get it. 1628 * @folio: The folio to lock 1629 */ 1630 void __folio_lock(struct folio *folio) 1631 { 1632 folio_wait_bit_common(folio, PG_locked, TASK_UNINTERRUPTIBLE, 1633 EXCLUSIVE); 1634 } 1635 EXPORT_SYMBOL(__folio_lock); 1636 1637 int __folio_lock_killable(struct folio *folio) 1638 { 1639 return folio_wait_bit_common(folio, PG_locked, TASK_KILLABLE, 1640 EXCLUSIVE); 1641 } 1642 EXPORT_SYMBOL_GPL(__folio_lock_killable); 1643 1644 static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait) 1645 { 1646 struct wait_queue_head *q = folio_waitqueue(folio); 1647 int ret = 0; 1648 1649 wait->folio = folio; 1650 wait->bit_nr = PG_locked; 1651 1652 spin_lock_irq(&q->lock); 1653 __add_wait_queue_entry_tail(q, &wait->wait); 1654 folio_set_waiters(folio); 1655 ret = !folio_trylock(folio); 1656 /* 1657 * If we were successful now, we know we're still on the 1658 * waitqueue as we're still under the lock. This means it's 1659 * safe to remove and return success, we know the callback 1660 * isn't going to trigger. 1661 */ 1662 if (!ret) 1663 __remove_wait_queue(q, &wait->wait); 1664 else 1665 ret = -EIOCBQUEUED; 1666 spin_unlock_irq(&q->lock); 1667 return ret; 1668 } 1669 1670 /* 1671 * Return values: 1672 * true - folio is locked; mmap_lock is still held. 1673 * false - folio is not locked. 1674 * mmap_lock has been released (mmap_read_unlock(), unless flags had both 1675 * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in 1676 * which case mmap_lock is still held. 1677 * 1678 * If neither ALLOW_RETRY nor KILLABLE are set, will always return true 1679 * with the folio locked and the mmap_lock unperturbed. 1680 */ 1681 bool __folio_lock_or_retry(struct folio *folio, struct mm_struct *mm, 1682 unsigned int flags) 1683 { 1684 if (fault_flag_allow_retry_first(flags)) { 1685 /* 1686 * CAUTION! In this case, mmap_lock is not released 1687 * even though return 0. 1688 */ 1689 if (flags & FAULT_FLAG_RETRY_NOWAIT) 1690 return false; 1691 1692 mmap_read_unlock(mm); 1693 if (flags & FAULT_FLAG_KILLABLE) 1694 folio_wait_locked_killable(folio); 1695 else 1696 folio_wait_locked(folio); 1697 return false; 1698 } 1699 if (flags & FAULT_FLAG_KILLABLE) { 1700 bool ret; 1701 1702 ret = __folio_lock_killable(folio); 1703 if (ret) { 1704 mmap_read_unlock(mm); 1705 return false; 1706 } 1707 } else { 1708 __folio_lock(folio); 1709 } 1710 1711 return true; 1712 } 1713 1714 /** 1715 * page_cache_next_miss() - Find the next gap in the page cache. 1716 * @mapping: Mapping. 1717 * @index: Index. 1718 * @max_scan: Maximum range to search. 1719 * 1720 * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the 1721 * gap with the lowest index. 1722 * 1723 * This function may be called under the rcu_read_lock. However, this will 1724 * not atomically search a snapshot of the cache at a single point in time. 1725 * For example, if a gap is created at index 5, then subsequently a gap is 1726 * created at index 10, page_cache_next_miss covering both indices may 1727 * return 10 if called under the rcu_read_lock. 1728 * 1729 * Return: The index of the gap if found, otherwise an index outside the 1730 * range specified (in which case 'return - index >= max_scan' will be true). 1731 * In the rare case of index wrap-around, 0 will be returned. 1732 */ 1733 pgoff_t page_cache_next_miss(struct address_space *mapping, 1734 pgoff_t index, unsigned long max_scan) 1735 { 1736 XA_STATE(xas, &mapping->i_pages, index); 1737 1738 while (max_scan--) { 1739 void *entry = xas_next(&xas); 1740 if (!entry || xa_is_value(entry)) 1741 break; 1742 if (xas.xa_index == 0) 1743 break; 1744 } 1745 1746 return xas.xa_index; 1747 } 1748 EXPORT_SYMBOL(page_cache_next_miss); 1749 1750 /** 1751 * page_cache_prev_miss() - Find the previous gap in the page cache. 1752 * @mapping: Mapping. 1753 * @index: Index. 1754 * @max_scan: Maximum range to search. 1755 * 1756 * Search the range [max(index - max_scan + 1, 0), index] for the 1757 * gap with the highest index. 1758 * 1759 * This function may be called under the rcu_read_lock. However, this will 1760 * not atomically search a snapshot of the cache at a single point in time. 1761 * For example, if a gap is created at index 10, then subsequently a gap is 1762 * created at index 5, page_cache_prev_miss() covering both indices may 1763 * return 5 if called under the rcu_read_lock. 1764 * 1765 * Return: The index of the gap if found, otherwise an index outside the 1766 * range specified (in which case 'index - return >= max_scan' will be true). 1767 * In the rare case of wrap-around, ULONG_MAX will be returned. 1768 */ 1769 pgoff_t page_cache_prev_miss(struct address_space *mapping, 1770 pgoff_t index, unsigned long max_scan) 1771 { 1772 XA_STATE(xas, &mapping->i_pages, index); 1773 1774 while (max_scan--) { 1775 void *entry = xas_prev(&xas); 1776 if (!entry || xa_is_value(entry)) 1777 break; 1778 if (xas.xa_index == ULONG_MAX) 1779 break; 1780 } 1781 1782 return xas.xa_index; 1783 } 1784 EXPORT_SYMBOL(page_cache_prev_miss); 1785 1786 /* 1787 * Lockless page cache protocol: 1788 * On the lookup side: 1789 * 1. Load the folio from i_pages 1790 * 2. Increment the refcount if it's not zero 1791 * 3. If the folio is not found by xas_reload(), put the refcount and retry 1792 * 1793 * On the removal side: 1794 * A. Freeze the page (by zeroing the refcount if nobody else has a reference) 1795 * B. Remove the page from i_pages 1796 * C. Return the page to the page allocator 1797 * 1798 * This means that any page may have its reference count temporarily 1799 * increased by a speculative page cache (or fast GUP) lookup as it can 1800 * be allocated by another user before the RCU grace period expires. 1801 * Because the refcount temporarily acquired here may end up being the 1802 * last refcount on the page, any page allocation must be freeable by 1803 * folio_put(). 1804 */ 1805 1806 /* 1807 * filemap_get_entry - Get a page cache entry. 1808 * @mapping: the address_space to search 1809 * @index: The page cache index. 1810 * 1811 * Looks up the page cache entry at @mapping & @index. If it is a folio, 1812 * it is returned with an increased refcount. If it is a shadow entry 1813 * of a previously evicted folio, or a swap entry from shmem/tmpfs, 1814 * it is returned without further action. 1815 * 1816 * Return: The folio, swap or shadow entry, %NULL if nothing is found. 1817 */ 1818 void *filemap_get_entry(struct address_space *mapping, pgoff_t index) 1819 { 1820 XA_STATE(xas, &mapping->i_pages, index); 1821 struct folio *folio; 1822 1823 rcu_read_lock(); 1824 repeat: 1825 xas_reset(&xas); 1826 folio = xas_load(&xas); 1827 if (xas_retry(&xas, folio)) 1828 goto repeat; 1829 /* 1830 * A shadow entry of a recently evicted page, or a swap entry from 1831 * shmem/tmpfs. Return it without attempting to raise page count. 1832 */ 1833 if (!folio || xa_is_value(folio)) 1834 goto out; 1835 1836 if (!folio_try_get_rcu(folio)) 1837 goto repeat; 1838 1839 if (unlikely(folio != xas_reload(&xas))) { 1840 folio_put(folio); 1841 goto repeat; 1842 } 1843 out: 1844 rcu_read_unlock(); 1845 1846 return folio; 1847 } 1848 1849 /** 1850 * __filemap_get_folio - Find and get a reference to a folio. 1851 * @mapping: The address_space to search. 1852 * @index: The page index. 1853 * @fgp_flags: %FGP flags modify how the folio is returned. 1854 * @gfp: Memory allocation flags to use if %FGP_CREAT is specified. 1855 * 1856 * Looks up the page cache entry at @mapping & @index. 1857 * 1858 * If %FGP_LOCK or %FGP_CREAT are specified then the function may sleep even 1859 * if the %GFP flags specified for %FGP_CREAT are atomic. 1860 * 1861 * If this function returns a folio, it is returned with an increased refcount. 1862 * 1863 * Return: The found folio or an ERR_PTR() otherwise. 1864 */ 1865 struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, 1866 fgf_t fgp_flags, gfp_t gfp) 1867 { 1868 struct folio *folio; 1869 1870 repeat: 1871 folio = filemap_get_entry(mapping, index); 1872 if (xa_is_value(folio)) 1873 folio = NULL; 1874 if (!folio) 1875 goto no_page; 1876 1877 if (fgp_flags & FGP_LOCK) { 1878 if (fgp_flags & FGP_NOWAIT) { 1879 if (!folio_trylock(folio)) { 1880 folio_put(folio); 1881 return ERR_PTR(-EAGAIN); 1882 } 1883 } else { 1884 folio_lock(folio); 1885 } 1886 1887 /* Has the page been truncated? */ 1888 if (unlikely(folio->mapping != mapping)) { 1889 folio_unlock(folio); 1890 folio_put(folio); 1891 goto repeat; 1892 } 1893 VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); 1894 } 1895 1896 if (fgp_flags & FGP_ACCESSED) 1897 folio_mark_accessed(folio); 1898 else if (fgp_flags & FGP_WRITE) { 1899 /* Clear idle flag for buffer write */ 1900 if (folio_test_idle(folio)) 1901 folio_clear_idle(folio); 1902 } 1903 1904 if (fgp_flags & FGP_STABLE) 1905 folio_wait_stable(folio); 1906 no_page: 1907 if (!folio && (fgp_flags & FGP_CREAT)) { 1908 int err; 1909 if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping)) 1910 gfp |= __GFP_WRITE; 1911 if (fgp_flags & FGP_NOFS) 1912 gfp &= ~__GFP_FS; 1913 if (fgp_flags & FGP_NOWAIT) { 1914 gfp &= ~GFP_KERNEL; 1915 gfp |= GFP_NOWAIT | __GFP_NOWARN; 1916 } 1917 1918 folio = filemap_alloc_folio(gfp, 0); 1919 if (!folio) 1920 return ERR_PTR(-ENOMEM); 1921 1922 if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP)))) 1923 fgp_flags |= FGP_LOCK; 1924 1925 /* Init accessed so avoid atomic mark_page_accessed later */ 1926 if (fgp_flags & FGP_ACCESSED) 1927 __folio_set_referenced(folio); 1928 1929 err = filemap_add_folio(mapping, folio, index, gfp); 1930 if (unlikely(err)) { 1931 folio_put(folio); 1932 folio = NULL; 1933 if (err == -EEXIST) 1934 goto repeat; 1935 } 1936 1937 /* 1938 * filemap_add_folio locks the page, and for mmap 1939 * we expect an unlocked page. 1940 */ 1941 if (folio && (fgp_flags & FGP_FOR_MMAP)) 1942 folio_unlock(folio); 1943 } 1944 1945 if (!folio) 1946 return ERR_PTR(-ENOENT); 1947 return folio; 1948 } 1949 EXPORT_SYMBOL(__filemap_get_folio); 1950 1951 static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max, 1952 xa_mark_t mark) 1953 { 1954 struct folio *folio; 1955 1956 retry: 1957 if (mark == XA_PRESENT) 1958 folio = xas_find(xas, max); 1959 else 1960 folio = xas_find_marked(xas, max, mark); 1961 1962 if (xas_retry(xas, folio)) 1963 goto retry; 1964 /* 1965 * A shadow entry of a recently evicted page, a swap 1966 * entry from shmem/tmpfs or a DAX entry. Return it 1967 * without attempting to raise page count. 1968 */ 1969 if (!folio || xa_is_value(folio)) 1970 return folio; 1971 1972 if (!folio_try_get_rcu(folio)) 1973 goto reset; 1974 1975 if (unlikely(folio != xas_reload(xas))) { 1976 folio_put(folio); 1977 goto reset; 1978 } 1979 1980 return folio; 1981 reset: 1982 xas_reset(xas); 1983 goto retry; 1984 } 1985 1986 /** 1987 * find_get_entries - gang pagecache lookup 1988 * @mapping: The address_space to search 1989 * @start: The starting page cache index 1990 * @end: The final page index (inclusive). 1991 * @fbatch: Where the resulting entries are placed. 1992 * @indices: The cache indices corresponding to the entries in @entries 1993 * 1994 * find_get_entries() will search for and return a batch of entries in 1995 * the mapping. The entries are placed in @fbatch. find_get_entries() 1996 * takes a reference on any actual folios it returns. 1997 * 1998 * The entries have ascending indexes. The indices may not be consecutive 1999 * due to not-present entries or large folios. 2000 * 2001 * Any shadow entries of evicted folios, or swap entries from 2002 * shmem/tmpfs, are included in the returned array. 2003 * 2004 * Return: The number of entries which were found. 2005 */ 2006 unsigned find_get_entries(struct address_space *mapping, pgoff_t *start, 2007 pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices) 2008 { 2009 XA_STATE(xas, &mapping->i_pages, *start); 2010 struct folio *folio; 2011 2012 rcu_read_lock(); 2013 while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) { 2014 indices[fbatch->nr] = xas.xa_index; 2015 if (!folio_batch_add(fbatch, folio)) 2016 break; 2017 } 2018 rcu_read_unlock(); 2019 2020 if (folio_batch_count(fbatch)) { 2021 unsigned long nr = 1; 2022 int idx = folio_batch_count(fbatch) - 1; 2023 2024 folio = fbatch->folios[idx]; 2025 if (!xa_is_value(folio) && !folio_test_hugetlb(folio)) 2026 nr = folio_nr_pages(folio); 2027 *start = indices[idx] + nr; 2028 } 2029 return folio_batch_count(fbatch); 2030 } 2031 2032 /** 2033 * find_lock_entries - Find a batch of pagecache entries. 2034 * @mapping: The address_space to search. 2035 * @start: The starting page cache index. 2036 * @end: The final page index (inclusive). 2037 * @fbatch: Where the resulting entries are placed. 2038 * @indices: The cache indices of the entries in @fbatch. 2039 * 2040 * find_lock_entries() will return a batch of entries from @mapping. 2041 * Swap, shadow and DAX entries are included. Folios are returned 2042 * locked and with an incremented refcount. Folios which are locked 2043 * by somebody else or under writeback are skipped. Folios which are 2044 * partially outside the range are not returned. 2045 * 2046 * The entries have ascending indexes. The indices may not be consecutive 2047 * due to not-present entries, large folios, folios which could not be 2048 * locked or folios under writeback. 2049 * 2050 * Return: The number of entries which were found. 2051 */ 2052 unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start, 2053 pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices) 2054 { 2055 XA_STATE(xas, &mapping->i_pages, *start); 2056 struct folio *folio; 2057 2058 rcu_read_lock(); 2059 while ((folio = find_get_entry(&xas, end, XA_PRESENT))) { 2060 if (!xa_is_value(folio)) { 2061 if (folio->index < *start) 2062 goto put; 2063 if (folio->index + folio_nr_pages(folio) - 1 > end) 2064 goto put; 2065 if (!folio_trylock(folio)) 2066 goto put; 2067 if (folio->mapping != mapping || 2068 folio_test_writeback(folio)) 2069 goto unlock; 2070 VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index), 2071 folio); 2072 } 2073 indices[fbatch->nr] = xas.xa_index; 2074 if (!folio_batch_add(fbatch, folio)) 2075 break; 2076 continue; 2077 unlock: 2078 folio_unlock(folio); 2079 put: 2080 folio_put(folio); 2081 } 2082 rcu_read_unlock(); 2083 2084 if (folio_batch_count(fbatch)) { 2085 unsigned long nr = 1; 2086 int idx = folio_batch_count(fbatch) - 1; 2087 2088 folio = fbatch->folios[idx]; 2089 if (!xa_is_value(folio) && !folio_test_hugetlb(folio)) 2090 nr = folio_nr_pages(folio); 2091 *start = indices[idx] + nr; 2092 } 2093 return folio_batch_count(fbatch); 2094 } 2095 2096 /** 2097 * filemap_get_folios - Get a batch of folios 2098 * @mapping: The address_space to search 2099 * @start: The starting page index 2100 * @end: The final page index (inclusive) 2101 * @fbatch: The batch to fill. 2102 * 2103 * Search for and return a batch of folios in the mapping starting at 2104 * index @start and up to index @end (inclusive). The folios are returned 2105 * in @fbatch with an elevated reference count. 2106 * 2107 * The first folio may start before @start; if it does, it will contain 2108 * @start. The final folio may extend beyond @end; if it does, it will 2109 * contain @end. The folios have ascending indices. There may be gaps 2110 * between the folios if there are indices which have no folio in the 2111 * page cache. If folios are added to or removed from the page cache 2112 * while this is running, they may or may not be found by this call. 2113 * 2114 * Return: The number of folios which were found. 2115 * We also update @start to index the next folio for the traversal. 2116 */ 2117 unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, 2118 pgoff_t end, struct folio_batch *fbatch) 2119 { 2120 XA_STATE(xas, &mapping->i_pages, *start); 2121 struct folio *folio; 2122 2123 rcu_read_lock(); 2124 while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) { 2125 /* Skip over shadow, swap and DAX entries */ 2126 if (xa_is_value(folio)) 2127 continue; 2128 if (!folio_batch_add(fbatch, folio)) { 2129 unsigned long nr = folio_nr_pages(folio); 2130 2131 if (folio_test_hugetlb(folio)) 2132 nr = 1; 2133 *start = folio->index + nr; 2134 goto out; 2135 } 2136 } 2137 2138 /* 2139 * We come here when there is no page beyond @end. We take care to not 2140 * overflow the index @start as it confuses some of the callers. This 2141 * breaks the iteration when there is a page at index -1 but that is 2142 * already broken anyway. 2143 */ 2144 if (end == (pgoff_t)-1) 2145 *start = (pgoff_t)-1; 2146 else 2147 *start = end + 1; 2148 out: 2149 rcu_read_unlock(); 2150 2151 return folio_batch_count(fbatch); 2152 } 2153 EXPORT_SYMBOL(filemap_get_folios); 2154 2155 static inline 2156 bool folio_more_pages(struct folio *folio, pgoff_t index, pgoff_t max) 2157 { 2158 if (!folio_test_large(folio) || folio_test_hugetlb(folio)) 2159 return false; 2160 if (index >= max) 2161 return false; 2162 return index < folio->index + folio_nr_pages(folio) - 1; 2163 } 2164 2165 /** 2166 * filemap_get_folios_contig - Get a batch of contiguous folios 2167 * @mapping: The address_space to search 2168 * @start: The starting page index 2169 * @end: The final page index (inclusive) 2170 * @fbatch: The batch to fill 2171 * 2172 * filemap_get_folios_contig() works exactly like filemap_get_folios(), 2173 * except the returned folios are guaranteed to be contiguous. This may 2174 * not return all contiguous folios if the batch gets filled up. 2175 * 2176 * Return: The number of folios found. 2177 * Also update @start to be positioned for traversal of the next folio. 2178 */ 2179 2180 unsigned filemap_get_folios_contig(struct address_space *mapping, 2181 pgoff_t *start, pgoff_t end, struct folio_batch *fbatch) 2182 { 2183 XA_STATE(xas, &mapping->i_pages, *start); 2184 unsigned long nr; 2185 struct folio *folio; 2186 2187 rcu_read_lock(); 2188 2189 for (folio = xas_load(&xas); folio && xas.xa_index <= end; 2190 folio = xas_next(&xas)) { 2191 if (xas_retry(&xas, folio)) 2192 continue; 2193 /* 2194 * If the entry has been swapped out, we can stop looking. 2195 * No current caller is looking for DAX entries. 2196 */ 2197 if (xa_is_value(folio)) 2198 goto update_start; 2199 2200 if (!folio_try_get_rcu(folio)) 2201 goto retry; 2202 2203 if (unlikely(folio != xas_reload(&xas))) 2204 goto put_folio; 2205 2206 if (!folio_batch_add(fbatch, folio)) { 2207 nr = folio_nr_pages(folio); 2208 2209 if (folio_test_hugetlb(folio)) 2210 nr = 1; 2211 *start = folio->index + nr; 2212 goto out; 2213 } 2214 continue; 2215 put_folio: 2216 folio_put(folio); 2217 2218 retry: 2219 xas_reset(&xas); 2220 } 2221 2222 update_start: 2223 nr = folio_batch_count(fbatch); 2224 2225 if (nr) { 2226 folio = fbatch->folios[nr - 1]; 2227 if (folio_test_hugetlb(folio)) 2228 *start = folio->index + 1; 2229 else 2230 *start = folio->index + folio_nr_pages(folio); 2231 } 2232 out: 2233 rcu_read_unlock(); 2234 return folio_batch_count(fbatch); 2235 } 2236 EXPORT_SYMBOL(filemap_get_folios_contig); 2237 2238 /** 2239 * filemap_get_folios_tag - Get a batch of folios matching @tag 2240 * @mapping: The address_space to search 2241 * @start: The starting page index 2242 * @end: The final page index (inclusive) 2243 * @tag: The tag index 2244 * @fbatch: The batch to fill 2245 * 2246 * Same as filemap_get_folios(), but only returning folios tagged with @tag. 2247 * 2248 * Return: The number of folios found. 2249 * Also update @start to index the next folio for traversal. 2250 */ 2251 unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, 2252 pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch) 2253 { 2254 XA_STATE(xas, &mapping->i_pages, *start); 2255 struct folio *folio; 2256 2257 rcu_read_lock(); 2258 while ((folio = find_get_entry(&xas, end, tag)) != NULL) { 2259 /* 2260 * Shadow entries should never be tagged, but this iteration 2261 * is lockless so there is a window for page reclaim to evict 2262 * a page we saw tagged. Skip over it. 2263 */ 2264 if (xa_is_value(folio)) 2265 continue; 2266 if (!folio_batch_add(fbatch, folio)) { 2267 unsigned long nr = folio_nr_pages(folio); 2268 2269 if (folio_test_hugetlb(folio)) 2270 nr = 1; 2271 *start = folio->index + nr; 2272 goto out; 2273 } 2274 } 2275 /* 2276 * We come here when there is no page beyond @end. We take care to not 2277 * overflow the index @start as it confuses some of the callers. This 2278 * breaks the iteration when there is a page at index -1 but that is 2279 * already broke anyway. 2280 */ 2281 if (end == (pgoff_t)-1) 2282 *start = (pgoff_t)-1; 2283 else 2284 *start = end + 1; 2285 out: 2286 rcu_read_unlock(); 2287 2288 return folio_batch_count(fbatch); 2289 } 2290 EXPORT_SYMBOL(filemap_get_folios_tag); 2291 2292 /* 2293 * CD/DVDs are error prone. When a medium error occurs, the driver may fail 2294 * a _large_ part of the i/o request. Imagine the worst scenario: 2295 * 2296 * ---R__________________________________________B__________ 2297 * ^ reading here ^ bad block(assume 4k) 2298 * 2299 * read(R) => miss => readahead(R...B) => media error => frustrating retries 2300 * => failing the whole request => read(R) => read(R+1) => 2301 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) => 2302 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) => 2303 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ...... 2304 * 2305 * It is going insane. Fix it by quickly scaling down the readahead size. 2306 */ 2307 static void shrink_readahead_size_eio(struct file_ra_state *ra) 2308 { 2309 ra->ra_pages /= 4; 2310 } 2311 2312 /* 2313 * filemap_get_read_batch - Get a batch of folios for read 2314 * 2315 * Get a batch of folios which represent a contiguous range of bytes in 2316 * the file. No exceptional entries will be returned. If @index is in 2317 * the middle of a folio, the entire folio will be returned. The last 2318 * folio in the batch may have the readahead flag set or the uptodate flag 2319 * clear so that the caller can take the appropriate action. 2320 */ 2321 static void filemap_get_read_batch(struct address_space *mapping, 2322 pgoff_t index, pgoff_t max, struct folio_batch *fbatch) 2323 { 2324 XA_STATE(xas, &mapping->i_pages, index); 2325 struct folio *folio; 2326 2327 rcu_read_lock(); 2328 for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { 2329 if (xas_retry(&xas, folio)) 2330 continue; 2331 if (xas.xa_index > max || xa_is_value(folio)) 2332 break; 2333 if (xa_is_sibling(folio)) 2334 break; 2335 if (!folio_try_get_rcu(folio)) 2336 goto retry; 2337 2338 if (unlikely(folio != xas_reload(&xas))) 2339 goto put_folio; 2340 2341 if (!folio_batch_add(fbatch, folio)) 2342 break; 2343 if (!folio_test_uptodate(folio)) 2344 break; 2345 if (folio_test_readahead(folio)) 2346 break; 2347 xas_advance(&xas, folio->index + folio_nr_pages(folio) - 1); 2348 continue; 2349 put_folio: 2350 folio_put(folio); 2351 retry: 2352 xas_reset(&xas); 2353 } 2354 rcu_read_unlock(); 2355 } 2356 2357 static int filemap_read_folio(struct file *file, filler_t filler, 2358 struct folio *folio) 2359 { 2360 bool workingset = folio_test_workingset(folio); 2361 unsigned long pflags; 2362 int error; 2363 2364 /* 2365 * A previous I/O error may have been due to temporary failures, 2366 * eg. multipath errors. PG_error will be set again if read_folio 2367 * fails. 2368 */ 2369 folio_clear_error(folio); 2370 2371 /* Start the actual read. The read will unlock the page. */ 2372 if (unlikely(workingset)) 2373 psi_memstall_enter(&pflags); 2374 error = filler(file, folio); 2375 if (unlikely(workingset)) 2376 psi_memstall_leave(&pflags); 2377 if (error) 2378 return error; 2379 2380 error = folio_wait_locked_killable(folio); 2381 if (error) 2382 return error; 2383 if (folio_test_uptodate(folio)) 2384 return 0; 2385 if (file) 2386 shrink_readahead_size_eio(&file->f_ra); 2387 return -EIO; 2388 } 2389 2390 static bool filemap_range_uptodate(struct address_space *mapping, 2391 loff_t pos, size_t count, struct folio *folio, 2392 bool need_uptodate) 2393 { 2394 if (folio_test_uptodate(folio)) 2395 return true; 2396 /* pipes can't handle partially uptodate pages */ 2397 if (need_uptodate) 2398 return false; 2399 if (!mapping->a_ops->is_partially_uptodate) 2400 return false; 2401 if (mapping->host->i_blkbits >= folio_shift(folio)) 2402 return false; 2403 2404 if (folio_pos(folio) > pos) { 2405 count -= folio_pos(folio) - pos; 2406 pos = 0; 2407 } else { 2408 pos -= folio_pos(folio); 2409 } 2410 2411 return mapping->a_ops->is_partially_uptodate(folio, pos, count); 2412 } 2413 2414 static int filemap_update_page(struct kiocb *iocb, 2415 struct address_space *mapping, size_t count, 2416 struct folio *folio, bool need_uptodate) 2417 { 2418 int error; 2419 2420 if (iocb->ki_flags & IOCB_NOWAIT) { 2421 if (!filemap_invalidate_trylock_shared(mapping)) 2422 return -EAGAIN; 2423 } else { 2424 filemap_invalidate_lock_shared(mapping); 2425 } 2426 2427 if (!folio_trylock(folio)) { 2428 error = -EAGAIN; 2429 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) 2430 goto unlock_mapping; 2431 if (!(iocb->ki_flags & IOCB_WAITQ)) { 2432 filemap_invalidate_unlock_shared(mapping); 2433 /* 2434 * This is where we usually end up waiting for a 2435 * previously submitted readahead to finish. 2436 */ 2437 folio_put_wait_locked(folio, TASK_KILLABLE); 2438 return AOP_TRUNCATED_PAGE; 2439 } 2440 error = __folio_lock_async(folio, iocb->ki_waitq); 2441 if (error) 2442 goto unlock_mapping; 2443 } 2444 2445 error = AOP_TRUNCATED_PAGE; 2446 if (!folio->mapping) 2447 goto unlock; 2448 2449 error = 0; 2450 if (filemap_range_uptodate(mapping, iocb->ki_pos, count, folio, 2451 need_uptodate)) 2452 goto unlock; 2453 2454 error = -EAGAIN; 2455 if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) 2456 goto unlock; 2457 2458 error = filemap_read_folio(iocb->ki_filp, mapping->a_ops->read_folio, 2459 folio); 2460 goto unlock_mapping; 2461 unlock: 2462 folio_unlock(folio); 2463 unlock_mapping: 2464 filemap_invalidate_unlock_shared(mapping); 2465 if (error == AOP_TRUNCATED_PAGE) 2466 folio_put(folio); 2467 return error; 2468 } 2469 2470 static int filemap_create_folio(struct file *file, 2471 struct address_space *mapping, pgoff_t index, 2472 struct folio_batch *fbatch) 2473 { 2474 struct folio *folio; 2475 int error; 2476 2477 folio = filemap_alloc_folio(mapping_gfp_mask(mapping), 0); 2478 if (!folio) 2479 return -ENOMEM; 2480 2481 /* 2482 * Protect against truncate / hole punch. Grabbing invalidate_lock 2483 * here assures we cannot instantiate and bring uptodate new 2484 * pagecache folios after evicting page cache during truncate 2485 * and before actually freeing blocks. Note that we could 2486 * release invalidate_lock after inserting the folio into 2487 * the page cache as the locked folio would then be enough to 2488 * synchronize with hole punching. But there are code paths 2489 * such as filemap_update_page() filling in partially uptodate 2490 * pages or ->readahead() that need to hold invalidate_lock 2491 * while mapping blocks for IO so let's hold the lock here as 2492 * well to keep locking rules simple. 2493 */ 2494 filemap_invalidate_lock_shared(mapping); 2495 error = filemap_add_folio(mapping, folio, index, 2496 mapping_gfp_constraint(mapping, GFP_KERNEL)); 2497 if (error == -EEXIST) 2498 error = AOP_TRUNCATED_PAGE; 2499 if (error) 2500 goto error; 2501 2502 error = filemap_read_folio(file, mapping->a_ops->read_folio, folio); 2503 if (error) 2504 goto error; 2505 2506 filemap_invalidate_unlock_shared(mapping); 2507 folio_batch_add(fbatch, folio); 2508 return 0; 2509 error: 2510 filemap_invalidate_unlock_shared(mapping); 2511 folio_put(folio); 2512 return error; 2513 } 2514 2515 static int filemap_readahead(struct kiocb *iocb, struct file *file, 2516 struct address_space *mapping, struct folio *folio, 2517 pgoff_t last_index) 2518 { 2519 DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, folio->index); 2520 2521 if (iocb->ki_flags & IOCB_NOIO) 2522 return -EAGAIN; 2523 page_cache_async_ra(&ractl, folio, last_index - folio->index); 2524 return 0; 2525 } 2526 2527 static int filemap_get_pages(struct kiocb *iocb, size_t count, 2528 struct folio_batch *fbatch, bool need_uptodate) 2529 { 2530 struct file *filp = iocb->ki_filp; 2531 struct address_space *mapping = filp->f_mapping; 2532 struct file_ra_state *ra = &filp->f_ra; 2533 pgoff_t index = iocb->ki_pos >> PAGE_SHIFT; 2534 pgoff_t last_index; 2535 struct folio *folio; 2536 int err = 0; 2537 2538 /* "last_index" is the index of the page beyond the end of the read */ 2539 last_index = DIV_ROUND_UP(iocb->ki_pos + count, PAGE_SIZE); 2540 retry: 2541 if (fatal_signal_pending(current)) 2542 return -EINTR; 2543 2544 filemap_get_read_batch(mapping, index, last_index - 1, fbatch); 2545 if (!folio_batch_count(fbatch)) { 2546 if (iocb->ki_flags & IOCB_NOIO) 2547 return -EAGAIN; 2548 page_cache_sync_readahead(mapping, ra, filp, index, 2549 last_index - index); 2550 filemap_get_read_batch(mapping, index, last_index - 1, fbatch); 2551 } 2552 if (!folio_batch_count(fbatch)) { 2553 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ)) 2554 return -EAGAIN; 2555 err = filemap_create_folio(filp, mapping, 2556 iocb->ki_pos >> PAGE_SHIFT, fbatch); 2557 if (err == AOP_TRUNCATED_PAGE) 2558 goto retry; 2559 return err; 2560 } 2561 2562 folio = fbatch->folios[folio_batch_count(fbatch) - 1]; 2563 if (folio_test_readahead(folio)) { 2564 err = filemap_readahead(iocb, filp, mapping, folio, last_index); 2565 if (err) 2566 goto err; 2567 } 2568 if (!folio_test_uptodate(folio)) { 2569 if ((iocb->ki_flags & IOCB_WAITQ) && 2570 folio_batch_count(fbatch) > 1) 2571 iocb->ki_flags |= IOCB_NOWAIT; 2572 err = filemap_update_page(iocb, mapping, count, folio, 2573 need_uptodate); 2574 if (err) 2575 goto err; 2576 } 2577 2578 return 0; 2579 err: 2580 if (err < 0) 2581 folio_put(folio); 2582 if (likely(--fbatch->nr)) 2583 return 0; 2584 if (err == AOP_TRUNCATED_PAGE) 2585 goto retry; 2586 return err; 2587 } 2588 2589 static inline bool pos_same_folio(loff_t pos1, loff_t pos2, struct folio *folio) 2590 { 2591 unsigned int shift = folio_shift(folio); 2592 2593 return (pos1 >> shift == pos2 >> shift); 2594 } 2595 2596 /** 2597 * filemap_read - Read data from the page cache. 2598 * @iocb: The iocb to read. 2599 * @iter: Destination for the data. 2600 * @already_read: Number of bytes already read by the caller. 2601 * 2602 * Copies data from the page cache. If the data is not currently present, 2603 * uses the readahead and read_folio address_space operations to fetch it. 2604 * 2605 * Return: Total number of bytes copied, including those already read by 2606 * the caller. If an error happens before any bytes are copied, returns 2607 * a negative error number. 2608 */ 2609 ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter, 2610 ssize_t already_read) 2611 { 2612 struct file *filp = iocb->ki_filp; 2613 struct file_ra_state *ra = &filp->f_ra; 2614 struct address_space *mapping = filp->f_mapping; 2615 struct inode *inode = mapping->host; 2616 struct folio_batch fbatch; 2617 int i, error = 0; 2618 bool writably_mapped; 2619 loff_t isize, end_offset; 2620 2621 if (unlikely(iocb->ki_pos >= inode->i_sb->s_maxbytes)) 2622 return 0; 2623 if (unlikely(!iov_iter_count(iter))) 2624 return 0; 2625 2626 iov_iter_truncate(iter, inode->i_sb->s_maxbytes); 2627 folio_batch_init(&fbatch); 2628 2629 do { 2630 cond_resched(); 2631 2632 /* 2633 * If we've already successfully copied some data, then we 2634 * can no longer safely return -EIOCBQUEUED. Hence mark 2635 * an async read NOWAIT at that point. 2636 */ 2637 if ((iocb->ki_flags & IOCB_WAITQ) && already_read) 2638 iocb->ki_flags |= IOCB_NOWAIT; 2639 2640 if (unlikely(iocb->ki_pos >= i_size_read(inode))) 2641 break; 2642 2643 error = filemap_get_pages(iocb, iter->count, &fbatch, false); 2644 if (error < 0) 2645 break; 2646 2647 /* 2648 * i_size must be checked after we know the pages are Uptodate. 2649 * 2650 * Checking i_size after the check allows us to calculate 2651 * the correct value for "nr", which means the zero-filled 2652 * part of the page is not copied back to userspace (unless 2653 * another truncate extends the file - this is desired though). 2654 */ 2655 isize = i_size_read(inode); 2656 if (unlikely(iocb->ki_pos >= isize)) 2657 goto put_folios; 2658 end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count); 2659 2660 /* 2661 * Once we start copying data, we don't want to be touching any 2662 * cachelines that might be contended: 2663 */ 2664 writably_mapped = mapping_writably_mapped(mapping); 2665 2666 /* 2667 * When a read accesses the same folio several times, only 2668 * mark it as accessed the first time. 2669 */ 2670 if (!pos_same_folio(iocb->ki_pos, ra->prev_pos - 1, 2671 fbatch.folios[0])) 2672 folio_mark_accessed(fbatch.folios[0]); 2673 2674 for (i = 0; i < folio_batch_count(&fbatch); i++) { 2675 struct folio *folio = fbatch.folios[i]; 2676 size_t fsize = folio_size(folio); 2677 size_t offset = iocb->ki_pos & (fsize - 1); 2678 size_t bytes = min_t(loff_t, end_offset - iocb->ki_pos, 2679 fsize - offset); 2680 size_t copied; 2681 2682 if (end_offset < folio_pos(folio)) 2683 break; 2684 if (i > 0) 2685 folio_mark_accessed(folio); 2686 /* 2687 * If users can be writing to this folio using arbitrary 2688 * virtual addresses, take care of potential aliasing 2689 * before reading the folio on the kernel side. 2690 */ 2691 if (writably_mapped) 2692 flush_dcache_folio(folio); 2693 2694 copied = copy_folio_to_iter(folio, offset, bytes, iter); 2695 2696 already_read += copied; 2697 iocb->ki_pos += copied; 2698 ra->prev_pos = iocb->ki_pos; 2699 2700 if (copied < bytes) { 2701 error = -EFAULT; 2702 break; 2703 } 2704 } 2705 put_folios: 2706 for (i = 0; i < folio_batch_count(&fbatch); i++) 2707 folio_put(fbatch.folios[i]); 2708 folio_batch_init(&fbatch); 2709 } while (iov_iter_count(iter) && iocb->ki_pos < isize && !error); 2710 2711 file_accessed(filp); 2712 2713 return already_read ? already_read : error; 2714 } 2715 EXPORT_SYMBOL_GPL(filemap_read); 2716 2717 int kiocb_write_and_wait(struct kiocb *iocb, size_t count) 2718 { 2719 struct address_space *mapping = iocb->ki_filp->f_mapping; 2720 loff_t pos = iocb->ki_pos; 2721 loff_t end = pos + count - 1; 2722 2723 if (iocb->ki_flags & IOCB_NOWAIT) { 2724 if (filemap_range_needs_writeback(mapping, pos, end)) 2725 return -EAGAIN; 2726 return 0; 2727 } 2728 2729 return filemap_write_and_wait_range(mapping, pos, end); 2730 } 2731 2732 int kiocb_invalidate_pages(struct kiocb *iocb, size_t count) 2733 { 2734 struct address_space *mapping = iocb->ki_filp->f_mapping; 2735 loff_t pos = iocb->ki_pos; 2736 loff_t end = pos + count - 1; 2737 int ret; 2738 2739 if (iocb->ki_flags & IOCB_NOWAIT) { 2740 /* we could block if there are any pages in the range */ 2741 if (filemap_range_has_page(mapping, pos, end)) 2742 return -EAGAIN; 2743 } else { 2744 ret = filemap_write_and_wait_range(mapping, pos, end); 2745 if (ret) 2746 return ret; 2747 } 2748 2749 /* 2750 * After a write we want buffered reads to be sure to go to disk to get 2751 * the new data. We invalidate clean cached page from the region we're 2752 * about to write. We do this *before* the write so that we can return 2753 * without clobbering -EIOCBQUEUED from ->direct_IO(). 2754 */ 2755 return invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, 2756 end >> PAGE_SHIFT); 2757 } 2758 2759 /** 2760 * generic_file_read_iter - generic filesystem read routine 2761 * @iocb: kernel I/O control block 2762 * @iter: destination for the data read 2763 * 2764 * This is the "read_iter()" routine for all filesystems 2765 * that can use the page cache directly. 2766 * 2767 * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall 2768 * be returned when no data can be read without waiting for I/O requests 2769 * to complete; it doesn't prevent readahead. 2770 * 2771 * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O 2772 * requests shall be made for the read or for readahead. When no data 2773 * can be read, -EAGAIN shall be returned. When readahead would be 2774 * triggered, a partial, possibly empty read shall be returned. 2775 * 2776 * Return: 2777 * * number of bytes copied, even for partial reads 2778 * * negative error code (or 0 if IOCB_NOIO) if nothing was read 2779 */ 2780 ssize_t 2781 generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) 2782 { 2783 size_t count = iov_iter_count(iter); 2784 ssize_t retval = 0; 2785 2786 if (!count) 2787 return 0; /* skip atime */ 2788 2789 if (iocb->ki_flags & IOCB_DIRECT) { 2790 struct file *file = iocb->ki_filp; 2791 struct address_space *mapping = file->f_mapping; 2792 struct inode *inode = mapping->host; 2793 2794 retval = kiocb_write_and_wait(iocb, count); 2795 if (retval < 0) 2796 return retval; 2797 file_accessed(file); 2798 2799 retval = mapping->a_ops->direct_IO(iocb, iter); 2800 if (retval >= 0) { 2801 iocb->ki_pos += retval; 2802 count -= retval; 2803 } 2804 if (retval != -EIOCBQUEUED) 2805 iov_iter_revert(iter, count - iov_iter_count(iter)); 2806 2807 /* 2808 * Btrfs can have a short DIO read if we encounter 2809 * compressed extents, so if there was an error, or if 2810 * we've already read everything we wanted to, or if 2811 * there was a short read because we hit EOF, go ahead 2812 * and return. Otherwise fallthrough to buffered io for 2813 * the rest of the read. Buffered reads will not work for 2814 * DAX files, so don't bother trying. 2815 */ 2816 if (retval < 0 || !count || IS_DAX(inode)) 2817 return retval; 2818 if (iocb->ki_pos >= i_size_read(inode)) 2819 return retval; 2820 } 2821 2822 return filemap_read(iocb, iter, retval); 2823 } 2824 EXPORT_SYMBOL(generic_file_read_iter); 2825 2826 /* 2827 * Splice subpages from a folio into a pipe. 2828 */ 2829 size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, 2830 struct folio *folio, loff_t fpos, size_t size) 2831 { 2832 struct page *page; 2833 size_t spliced = 0, offset = offset_in_folio(folio, fpos); 2834 2835 page = folio_page(folio, offset / PAGE_SIZE); 2836 size = min(size, folio_size(folio) - offset); 2837 offset %= PAGE_SIZE; 2838 2839 while (spliced < size && 2840 !pipe_full(pipe->head, pipe->tail, pipe->max_usage)) { 2841 struct pipe_buffer *buf = pipe_head_buf(pipe); 2842 size_t part = min_t(size_t, PAGE_SIZE - offset, size - spliced); 2843 2844 *buf = (struct pipe_buffer) { 2845 .ops = &page_cache_pipe_buf_ops, 2846 .page = page, 2847 .offset = offset, 2848 .len = part, 2849 }; 2850 folio_get(folio); 2851 pipe->head++; 2852 page++; 2853 spliced += part; 2854 offset = 0; 2855 } 2856 2857 return spliced; 2858 } 2859 2860 /** 2861 * filemap_splice_read - Splice data from a file's pagecache into a pipe 2862 * @in: The file to read from 2863 * @ppos: Pointer to the file position to read from 2864 * @pipe: The pipe to splice into 2865 * @len: The amount to splice 2866 * @flags: The SPLICE_F_* flags 2867 * 2868 * This function gets folios from a file's pagecache and splices them into the 2869 * pipe. Readahead will be called as necessary to fill more folios. This may 2870 * be used for blockdevs also. 2871 * 2872 * Return: On success, the number of bytes read will be returned and *@ppos 2873 * will be updated if appropriate; 0 will be returned if there is no more data 2874 * to be read; -EAGAIN will be returned if the pipe had no space, and some 2875 * other negative error code will be returned on error. A short read may occur 2876 * if the pipe has insufficient space, we reach the end of the data or we hit a 2877 * hole. 2878 */ 2879 ssize_t filemap_splice_read(struct file *in, loff_t *ppos, 2880 struct pipe_inode_info *pipe, 2881 size_t len, unsigned int flags) 2882 { 2883 struct folio_batch fbatch; 2884 struct kiocb iocb; 2885 size_t total_spliced = 0, used, npages; 2886 loff_t isize, end_offset; 2887 bool writably_mapped; 2888 int i, error = 0; 2889 2890 if (unlikely(*ppos >= in->f_mapping->host->i_sb->s_maxbytes)) 2891 return 0; 2892 2893 init_sync_kiocb(&iocb, in); 2894 iocb.ki_pos = *ppos; 2895 2896 /* Work out how much data we can actually add into the pipe */ 2897 used = pipe_occupancy(pipe->head, pipe->tail); 2898 npages = max_t(ssize_t, pipe->max_usage - used, 0); 2899 len = min_t(size_t, len, npages * PAGE_SIZE); 2900 2901 folio_batch_init(&fbatch); 2902 2903 do { 2904 cond_resched(); 2905 2906 if (*ppos >= i_size_read(in->f_mapping->host)) 2907 break; 2908 2909 iocb.ki_pos = *ppos; 2910 error = filemap_get_pages(&iocb, len, &fbatch, true); 2911 if (error < 0) 2912 break; 2913 2914 /* 2915 * i_size must be checked after we know the pages are Uptodate. 2916 * 2917 * Checking i_size after the check allows us to calculate 2918 * the correct value for "nr", which means the zero-filled 2919 * part of the page is not copied back to userspace (unless 2920 * another truncate extends the file - this is desired though). 2921 */ 2922 isize = i_size_read(in->f_mapping->host); 2923 if (unlikely(*ppos >= isize)) 2924 break; 2925 end_offset = min_t(loff_t, isize, *ppos + len); 2926 2927 /* 2928 * Once we start copying data, we don't want to be touching any 2929 * cachelines that might be contended: 2930 */ 2931 writably_mapped = mapping_writably_mapped(in->f_mapping); 2932 2933 for (i = 0; i < folio_batch_count(&fbatch); i++) { 2934 struct folio *folio = fbatch.folios[i]; 2935 size_t n; 2936 2937 if (folio_pos(folio) >= end_offset) 2938 goto out; 2939 folio_mark_accessed(folio); 2940 2941 /* 2942 * If users can be writing to this folio using arbitrary 2943 * virtual addresses, take care of potential aliasing 2944 * before reading the folio on the kernel side. 2945 */ 2946 if (writably_mapped) 2947 flush_dcache_folio(folio); 2948 2949 n = min_t(loff_t, len, isize - *ppos); 2950 n = splice_folio_into_pipe(pipe, folio, *ppos, n); 2951 if (!n) 2952 goto out; 2953 len -= n; 2954 total_spliced += n; 2955 *ppos += n; 2956 in->f_ra.prev_pos = *ppos; 2957 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) 2958 goto out; 2959 } 2960 2961 folio_batch_release(&fbatch); 2962 } while (len); 2963 2964 out: 2965 folio_batch_release(&fbatch); 2966 file_accessed(in); 2967 2968 return total_spliced ? total_spliced : error; 2969 } 2970 EXPORT_SYMBOL(filemap_splice_read); 2971 2972 static inline loff_t folio_seek_hole_data(struct xa_state *xas, 2973 struct address_space *mapping, struct folio *folio, 2974 loff_t start, loff_t end, bool seek_data) 2975 { 2976 const struct address_space_operations *ops = mapping->a_ops; 2977 size_t offset, bsz = i_blocksize(mapping->host); 2978 2979 if (xa_is_value(folio) || folio_test_uptodate(folio)) 2980 return seek_data ? start : end; 2981 if (!ops->is_partially_uptodate) 2982 return seek_data ? end : start; 2983 2984 xas_pause(xas); 2985 rcu_read_unlock(); 2986 folio_lock(folio); 2987 if (unlikely(folio->mapping != mapping)) 2988 goto unlock; 2989 2990 offset = offset_in_folio(folio, start) & ~(bsz - 1); 2991 2992 do { 2993 if (ops->is_partially_uptodate(folio, offset, bsz) == 2994 seek_data) 2995 break; 2996 start = (start + bsz) & ~(bsz - 1); 2997 offset += bsz; 2998 } while (offset < folio_size(folio)); 2999 unlock: 3000 folio_unlock(folio); 3001 rcu_read_lock(); 3002 return start; 3003 } 3004 3005 static inline size_t seek_folio_size(struct xa_state *xas, struct folio *folio) 3006 { 3007 if (xa_is_value(folio)) 3008 return PAGE_SIZE << xa_get_order(xas->xa, xas->xa_index); 3009 return folio_size(folio); 3010 } 3011 3012 /** 3013 * mapping_seek_hole_data - Seek for SEEK_DATA / SEEK_HOLE in the page cache. 3014 * @mapping: Address space to search. 3015 * @start: First byte to consider. 3016 * @end: Limit of search (exclusive). 3017 * @whence: Either SEEK_HOLE or SEEK_DATA. 3018 * 3019 * If the page cache knows which blocks contain holes and which blocks 3020 * contain data, your filesystem can use this function to implement 3021 * SEEK_HOLE and SEEK_DATA. This is useful for filesystems which are 3022 * entirely memory-based such as tmpfs, and filesystems which support 3023 * unwritten extents. 3024 * 3025 * Return: The requested offset on success, or -ENXIO if @whence specifies 3026 * SEEK_DATA and there is no data after @start. There is an implicit hole 3027 * after @end - 1, so SEEK_HOLE returns @end if all the bytes between @start 3028 * and @end contain data. 3029 */ 3030 loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, 3031 loff_t end, int whence) 3032 { 3033 XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT); 3034 pgoff_t max = (end - 1) >> PAGE_SHIFT; 3035 bool seek_data = (whence == SEEK_DATA); 3036 struct folio *folio; 3037 3038 if (end <= start) 3039 return -ENXIO; 3040 3041 rcu_read_lock(); 3042 while ((folio = find_get_entry(&xas, max, XA_PRESENT))) { 3043 loff_t pos = (u64)xas.xa_index << PAGE_SHIFT; 3044 size_t seek_size; 3045 3046 if (start < pos) { 3047 if (!seek_data) 3048 goto unlock; 3049 start = pos; 3050 } 3051 3052 seek_size = seek_folio_size(&xas, folio); 3053 pos = round_up((u64)pos + 1, seek_size); 3054 start = folio_seek_hole_data(&xas, mapping, folio, start, pos, 3055 seek_data); 3056 if (start < pos) 3057 goto unlock; 3058 if (start >= end) 3059 break; 3060 if (seek_size > PAGE_SIZE) 3061 xas_set(&xas, pos >> PAGE_SHIFT); 3062 if (!xa_is_value(folio)) 3063 folio_put(folio); 3064 } 3065 if (seek_data) 3066 start = -ENXIO; 3067 unlock: 3068 rcu_read_unlock(); 3069 if (folio && !xa_is_value(folio)) 3070 folio_put(folio); 3071 if (start > end) 3072 return end; 3073 return start; 3074 } 3075 3076 #ifdef CONFIG_MMU 3077 #define MMAP_LOTSAMISS (100) 3078 /* 3079 * lock_folio_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock 3080 * @vmf - the vm_fault for this fault. 3081 * @folio - the folio to lock. 3082 * @fpin - the pointer to the file we may pin (or is already pinned). 3083 * 3084 * This works similar to lock_folio_or_retry in that it can drop the 3085 * mmap_lock. It differs in that it actually returns the folio locked 3086 * if it returns 1 and 0 if it couldn't lock the folio. If we did have 3087 * to drop the mmap_lock then fpin will point to the pinned file and 3088 * needs to be fput()'ed at a later point. 3089 */ 3090 static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio, 3091 struct file **fpin) 3092 { 3093 if (folio_trylock(folio)) 3094 return 1; 3095 3096 /* 3097 * NOTE! This will make us return with VM_FAULT_RETRY, but with 3098 * the mmap_lock still held. That's how FAULT_FLAG_RETRY_NOWAIT 3099 * is supposed to work. We have way too many special cases.. 3100 */ 3101 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) 3102 return 0; 3103 3104 *fpin = maybe_unlock_mmap_for_io(vmf, *fpin); 3105 if (vmf->flags & FAULT_FLAG_KILLABLE) { 3106 if (__folio_lock_killable(folio)) { 3107 /* 3108 * We didn't have the right flags to drop the mmap_lock, 3109 * but all fault_handlers only check for fatal signals 3110 * if we return VM_FAULT_RETRY, so we need to drop the 3111 * mmap_lock here and return 0 if we don't have a fpin. 3112 */ 3113 if (*fpin == NULL) 3114 mmap_read_unlock(vmf->vma->vm_mm); 3115 return 0; 3116 } 3117 } else 3118 __folio_lock(folio); 3119 3120 return 1; 3121 } 3122 3123 /* 3124 * Synchronous readahead happens when we don't even find a page in the page 3125 * cache at all. We don't want to perform IO under the mmap sem, so if we have 3126 * to drop the mmap sem we return the file that was pinned in order for us to do 3127 * that. If we didn't pin a file then we return NULL. The file that is 3128 * returned needs to be fput()'ed when we're done with it. 3129 */ 3130 static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) 3131 { 3132 struct file *file = vmf->vma->vm_file; 3133 struct file_ra_state *ra = &file->f_ra; 3134 struct address_space *mapping = file->f_mapping; 3135 DEFINE_READAHEAD(ractl, file, ra, mapping, vmf->pgoff); 3136 struct file *fpin = NULL; 3137 unsigned long vm_flags = vmf->vma->vm_flags; 3138 unsigned int mmap_miss; 3139 3140 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3141 /* Use the readahead code, even if readahead is disabled */ 3142 if (vm_flags & VM_HUGEPAGE) { 3143 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3144 ractl._index &= ~((unsigned long)HPAGE_PMD_NR - 1); 3145 ra->size = HPAGE_PMD_NR; 3146 /* 3147 * Fetch two PMD folios, so we get the chance to actually 3148 * readahead, unless we've been told not to. 3149 */ 3150 if (!(vm_flags & VM_RAND_READ)) 3151 ra->size *= 2; 3152 ra->async_size = HPAGE_PMD_NR; 3153 page_cache_ra_order(&ractl, ra, HPAGE_PMD_ORDER); 3154 return fpin; 3155 } 3156 #endif 3157 3158 /* If we don't want any read-ahead, don't bother */ 3159 if (vm_flags & VM_RAND_READ) 3160 return fpin; 3161 if (!ra->ra_pages) 3162 return fpin; 3163 3164 if (vm_flags & VM_SEQ_READ) { 3165 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3166 page_cache_sync_ra(&ractl, ra->ra_pages); 3167 return fpin; 3168 } 3169 3170 /* Avoid banging the cache line if not needed */ 3171 mmap_miss = READ_ONCE(ra->mmap_miss); 3172 if (mmap_miss < MMAP_LOTSAMISS * 10) 3173 WRITE_ONCE(ra->mmap_miss, ++mmap_miss); 3174 3175 /* 3176 * Do we miss much more than hit in this file? If so, 3177 * stop bothering with read-ahead. It will only hurt. 3178 */ 3179 if (mmap_miss > MMAP_LOTSAMISS) 3180 return fpin; 3181 3182 /* 3183 * mmap read-around 3184 */ 3185 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3186 ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2); 3187 ra->size = ra->ra_pages; 3188 ra->async_size = ra->ra_pages / 4; 3189 ractl._index = ra->start; 3190 page_cache_ra_order(&ractl, ra, 0); 3191 return fpin; 3192 } 3193 3194 /* 3195 * Asynchronous readahead happens when we find the page and PG_readahead, 3196 * so we want to possibly extend the readahead further. We return the file that 3197 * was pinned if we have to drop the mmap_lock in order to do IO. 3198 */ 3199 static struct file *do_async_mmap_readahead(struct vm_fault *vmf, 3200 struct folio *folio) 3201 { 3202 struct file *file = vmf->vma->vm_file; 3203 struct file_ra_state *ra = &file->f_ra; 3204 DEFINE_READAHEAD(ractl, file, ra, file->f_mapping, vmf->pgoff); 3205 struct file *fpin = NULL; 3206 unsigned int mmap_miss; 3207 3208 /* If we don't want any read-ahead, don't bother */ 3209 if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages) 3210 return fpin; 3211 3212 mmap_miss = READ_ONCE(ra->mmap_miss); 3213 if (mmap_miss) 3214 WRITE_ONCE(ra->mmap_miss, --mmap_miss); 3215 3216 if (folio_test_readahead(folio)) { 3217 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3218 page_cache_async_ra(&ractl, folio, ra->ra_pages); 3219 } 3220 return fpin; 3221 } 3222 3223 /** 3224 * filemap_fault - read in file data for page fault handling 3225 * @vmf: struct vm_fault containing details of the fault 3226 * 3227 * filemap_fault() is invoked via the vma operations vector for a 3228 * mapped memory region to read in file data during a page fault. 3229 * 3230 * The goto's are kind of ugly, but this streamlines the normal case of having 3231 * it in the page cache, and handles the special cases reasonably without 3232 * having a lot of duplicated code. 3233 * 3234 * vma->vm_mm->mmap_lock must be held on entry. 3235 * 3236 * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock 3237 * may be dropped before doing I/O or by lock_folio_maybe_drop_mmap(). 3238 * 3239 * If our return value does not have VM_FAULT_RETRY set, the mmap_lock 3240 * has not been released. 3241 * 3242 * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set. 3243 * 3244 * Return: bitwise-OR of %VM_FAULT_ codes. 3245 */ 3246 vm_fault_t filemap_fault(struct vm_fault *vmf) 3247 { 3248 int error; 3249 struct file *file = vmf->vma->vm_file; 3250 struct file *fpin = NULL; 3251 struct address_space *mapping = file->f_mapping; 3252 struct inode *inode = mapping->host; 3253 pgoff_t max_idx, index = vmf->pgoff; 3254 struct folio *folio; 3255 vm_fault_t ret = 0; 3256 bool mapping_locked = false; 3257 3258 max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 3259 if (unlikely(index >= max_idx)) 3260 return VM_FAULT_SIGBUS; 3261 3262 /* 3263 * Do we have something in the page cache already? 3264 */ 3265 folio = filemap_get_folio(mapping, index); 3266 if (likely(!IS_ERR(folio))) { 3267 /* 3268 * We found the page, so try async readahead before waiting for 3269 * the lock. 3270 */ 3271 if (!(vmf->flags & FAULT_FLAG_TRIED)) 3272 fpin = do_async_mmap_readahead(vmf, folio); 3273 if (unlikely(!folio_test_uptodate(folio))) { 3274 filemap_invalidate_lock_shared(mapping); 3275 mapping_locked = true; 3276 } 3277 } else { 3278 /* No page in the page cache at all */ 3279 count_vm_event(PGMAJFAULT); 3280 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT); 3281 ret = VM_FAULT_MAJOR; 3282 fpin = do_sync_mmap_readahead(vmf); 3283 retry_find: 3284 /* 3285 * See comment in filemap_create_folio() why we need 3286 * invalidate_lock 3287 */ 3288 if (!mapping_locked) { 3289 filemap_invalidate_lock_shared(mapping); 3290 mapping_locked = true; 3291 } 3292 folio = __filemap_get_folio(mapping, index, 3293 FGP_CREAT|FGP_FOR_MMAP, 3294 vmf->gfp_mask); 3295 if (IS_ERR(folio)) { 3296 if (fpin) 3297 goto out_retry; 3298 filemap_invalidate_unlock_shared(mapping); 3299 return VM_FAULT_OOM; 3300 } 3301 } 3302 3303 if (!lock_folio_maybe_drop_mmap(vmf, folio, &fpin)) 3304 goto out_retry; 3305 3306 /* Did it get truncated? */ 3307 if (unlikely(folio->mapping != mapping)) { 3308 folio_unlock(folio); 3309 folio_put(folio); 3310 goto retry_find; 3311 } 3312 VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); 3313 3314 /* 3315 * We have a locked page in the page cache, now we need to check 3316 * that it's up-to-date. If not, it is going to be due to an error. 3317 */ 3318 if (unlikely(!folio_test_uptodate(folio))) { 3319 /* 3320 * The page was in cache and uptodate and now it is not. 3321 * Strange but possible since we didn't hold the page lock all 3322 * the time. Let's drop everything get the invalidate lock and 3323 * try again. 3324 */ 3325 if (!mapping_locked) { 3326 folio_unlock(folio); 3327 folio_put(folio); 3328 goto retry_find; 3329 } 3330 goto page_not_uptodate; 3331 } 3332 3333 /* 3334 * We've made it this far and we had to drop our mmap_lock, now is the 3335 * time to return to the upper layer and have it re-find the vma and 3336 * redo the fault. 3337 */ 3338 if (fpin) { 3339 folio_unlock(folio); 3340 goto out_retry; 3341 } 3342 if (mapping_locked) 3343 filemap_invalidate_unlock_shared(mapping); 3344 3345 /* 3346 * Found the page and have a reference on it. 3347 * We must recheck i_size under page lock. 3348 */ 3349 max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 3350 if (unlikely(index >= max_idx)) { 3351 folio_unlock(folio); 3352 folio_put(folio); 3353 return VM_FAULT_SIGBUS; 3354 } 3355 3356 vmf->page = folio_file_page(folio, index); 3357 return ret | VM_FAULT_LOCKED; 3358 3359 page_not_uptodate: 3360 /* 3361 * Umm, take care of errors if the page isn't up-to-date. 3362 * Try to re-read it _once_. We do this synchronously, 3363 * because there really aren't any performance issues here 3364 * and we need to check for errors. 3365 */ 3366 fpin = maybe_unlock_mmap_for_io(vmf, fpin); 3367 error = filemap_read_folio(file, mapping->a_ops->read_folio, folio); 3368 if (fpin) 3369 goto out_retry; 3370 folio_put(folio); 3371 3372 if (!error || error == AOP_TRUNCATED_PAGE) 3373 goto retry_find; 3374 filemap_invalidate_unlock_shared(mapping); 3375 3376 return VM_FAULT_SIGBUS; 3377 3378 out_retry: 3379 /* 3380 * We dropped the mmap_lock, we need to return to the fault handler to 3381 * re-find the vma and come back and find our hopefully still populated 3382 * page. 3383 */ 3384 if (!IS_ERR(folio)) 3385 folio_put(folio); 3386 if (mapping_locked) 3387 filemap_invalidate_unlock_shared(mapping); 3388 if (fpin) 3389 fput(fpin); 3390 return ret | VM_FAULT_RETRY; 3391 } 3392 EXPORT_SYMBOL(filemap_fault); 3393 3394 static bool filemap_map_pmd(struct vm_fault *vmf, struct folio *folio, 3395 pgoff_t start) 3396 { 3397 struct mm_struct *mm = vmf->vma->vm_mm; 3398 3399 /* Huge page is mapped? No need to proceed. */ 3400 if (pmd_trans_huge(*vmf->pmd)) { 3401 folio_unlock(folio); 3402 folio_put(folio); 3403 return true; 3404 } 3405 3406 if (pmd_none(*vmf->pmd) && folio_test_pmd_mappable(folio)) { 3407 struct page *page = folio_file_page(folio, start); 3408 vm_fault_t ret = do_set_pmd(vmf, page); 3409 if (!ret) { 3410 /* The page is mapped successfully, reference consumed. */ 3411 folio_unlock(folio); 3412 return true; 3413 } 3414 } 3415 3416 if (pmd_none(*vmf->pmd)) 3417 pmd_install(mm, vmf->pmd, &vmf->prealloc_pte); 3418 3419 return false; 3420 } 3421 3422 static struct folio *next_uptodate_page(struct folio *folio, 3423 struct address_space *mapping, 3424 struct xa_state *xas, pgoff_t end_pgoff) 3425 { 3426 unsigned long max_idx; 3427 3428 do { 3429 if (!folio) 3430 return NULL; 3431 if (xas_retry(xas, folio)) 3432 continue; 3433 if (xa_is_value(folio)) 3434 continue; 3435 if (folio_test_locked(folio)) 3436 continue; 3437 if (!folio_try_get_rcu(folio)) 3438 continue; 3439 /* Has the page moved or been split? */ 3440 if (unlikely(folio != xas_reload(xas))) 3441 goto skip; 3442 if (!folio_test_uptodate(folio) || folio_test_readahead(folio)) 3443 goto skip; 3444 if (!folio_trylock(folio)) 3445 goto skip; 3446 if (folio->mapping != mapping) 3447 goto unlock; 3448 if (!folio_test_uptodate(folio)) 3449 goto unlock; 3450 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 3451 if (xas->xa_index >= max_idx) 3452 goto unlock; 3453 return folio; 3454 unlock: 3455 folio_unlock(folio); 3456 skip: 3457 folio_put(folio); 3458 } while ((folio = xas_next_entry(xas, end_pgoff)) != NULL); 3459 3460 return NULL; 3461 } 3462 3463 static inline struct folio *first_map_page(struct address_space *mapping, 3464 struct xa_state *xas, 3465 pgoff_t end_pgoff) 3466 { 3467 return next_uptodate_page(xas_find(xas, end_pgoff), 3468 mapping, xas, end_pgoff); 3469 } 3470 3471 static inline struct folio *next_map_page(struct address_space *mapping, 3472 struct xa_state *xas, 3473 pgoff_t end_pgoff) 3474 { 3475 return next_uptodate_page(xas_next_entry(xas, end_pgoff), 3476 mapping, xas, end_pgoff); 3477 } 3478 3479 vm_fault_t filemap_map_pages(struct vm_fault *vmf, 3480 pgoff_t start_pgoff, pgoff_t end_pgoff) 3481 { 3482 struct vm_area_struct *vma = vmf->vma; 3483 struct file *file = vma->vm_file; 3484 struct address_space *mapping = file->f_mapping; 3485 pgoff_t last_pgoff = start_pgoff; 3486 unsigned long addr; 3487 XA_STATE(xas, &mapping->i_pages, start_pgoff); 3488 struct folio *folio; 3489 struct page *page; 3490 unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss); 3491 vm_fault_t ret = 0; 3492 3493 rcu_read_lock(); 3494 folio = first_map_page(mapping, &xas, end_pgoff); 3495 if (!folio) 3496 goto out; 3497 3498 if (filemap_map_pmd(vmf, folio, start_pgoff)) { 3499 ret = VM_FAULT_NOPAGE; 3500 goto out; 3501 } 3502 3503 addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); 3504 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); 3505 if (!vmf->pte) { 3506 folio_unlock(folio); 3507 folio_put(folio); 3508 goto out; 3509 } 3510 do { 3511 again: 3512 page = folio_file_page(folio, xas.xa_index); 3513 if (PageHWPoison(page)) 3514 goto unlock; 3515 3516 if (mmap_miss > 0) 3517 mmap_miss--; 3518 3519 addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT; 3520 vmf->pte += xas.xa_index - last_pgoff; 3521 last_pgoff = xas.xa_index; 3522 3523 /* 3524 * NOTE: If there're PTE markers, we'll leave them to be 3525 * handled in the specific fault path, and it'll prohibit the 3526 * fault-around logic. 3527 */ 3528 if (!pte_none(ptep_get(vmf->pte))) 3529 goto unlock; 3530 3531 /* We're about to handle the fault */ 3532 if (vmf->address == addr) 3533 ret = VM_FAULT_NOPAGE; 3534 3535 do_set_pte(vmf, page, addr); 3536 /* no need to invalidate: a not-present page won't be cached */ 3537 update_mmu_cache(vma, addr, vmf->pte); 3538 if (folio_more_pages(folio, xas.xa_index, end_pgoff)) { 3539 xas.xa_index++; 3540 folio_ref_inc(folio); 3541 goto again; 3542 } 3543 folio_unlock(folio); 3544 continue; 3545 unlock: 3546 if (folio_more_pages(folio, xas.xa_index, end_pgoff)) { 3547 xas.xa_index++; 3548 goto again; 3549 } 3550 folio_unlock(folio); 3551 folio_put(folio); 3552 } while ((folio = next_map_page(mapping, &xas, end_pgoff)) != NULL); 3553 pte_unmap_unlock(vmf->pte, vmf->ptl); 3554 out: 3555 rcu_read_unlock(); 3556 WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss); 3557 return ret; 3558 } 3559 EXPORT_SYMBOL(filemap_map_pages); 3560 3561 vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf) 3562 { 3563 struct address_space *mapping = vmf->vma->vm_file->f_mapping; 3564 struct folio *folio = page_folio(vmf->page); 3565 vm_fault_t ret = VM_FAULT_LOCKED; 3566 3567 sb_start_pagefault(mapping->host->i_sb); 3568 file_update_time(vmf->vma->vm_file); 3569 folio_lock(folio); 3570 if (folio->mapping != mapping) { 3571 folio_unlock(folio); 3572 ret = VM_FAULT_NOPAGE; 3573 goto out; 3574 } 3575 /* 3576 * We mark the folio dirty already here so that when freeze is in 3577 * progress, we are guaranteed that writeback during freezing will 3578 * see the dirty folio and writeprotect it again. 3579 */ 3580 folio_mark_dirty(folio); 3581 folio_wait_stable(folio); 3582 out: 3583 sb_end_pagefault(mapping->host->i_sb); 3584 return ret; 3585 } 3586 3587 const struct vm_operations_struct generic_file_vm_ops = { 3588 .fault = filemap_fault, 3589 .map_pages = filemap_map_pages, 3590 .page_mkwrite = filemap_page_mkwrite, 3591 }; 3592 3593 /* This is used for a general mmap of a disk file */ 3594 3595 int generic_file_mmap(struct file *file, struct vm_area_struct *vma) 3596 { 3597 struct address_space *mapping = file->f_mapping; 3598 3599 if (!mapping->a_ops->read_folio) 3600 return -ENOEXEC; 3601 file_accessed(file); 3602 vma->vm_ops = &generic_file_vm_ops; 3603 return 0; 3604 } 3605 3606 /* 3607 * This is for filesystems which do not implement ->writepage. 3608 */ 3609 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) 3610 { 3611 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) 3612 return -EINVAL; 3613 return generic_file_mmap(file, vma); 3614 } 3615 #else 3616 vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf) 3617 { 3618 return VM_FAULT_SIGBUS; 3619 } 3620 int generic_file_mmap(struct file *file, struct vm_area_struct *vma) 3621 { 3622 return -ENOSYS; 3623 } 3624 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) 3625 { 3626 return -ENOSYS; 3627 } 3628 #endif /* CONFIG_MMU */ 3629 3630 EXPORT_SYMBOL(filemap_page_mkwrite); 3631 EXPORT_SYMBOL(generic_file_mmap); 3632 EXPORT_SYMBOL(generic_file_readonly_mmap); 3633 3634 static struct folio *do_read_cache_folio(struct address_space *mapping, 3635 pgoff_t index, filler_t filler, struct file *file, gfp_t gfp) 3636 { 3637 struct folio *folio; 3638 int err; 3639 3640 if (!filler) 3641 filler = mapping->a_ops->read_folio; 3642 repeat: 3643 folio = filemap_get_folio(mapping, index); 3644 if (IS_ERR(folio)) { 3645 folio = filemap_alloc_folio(gfp, 0); 3646 if (!folio) 3647 return ERR_PTR(-ENOMEM); 3648 err = filemap_add_folio(mapping, folio, index, gfp); 3649 if (unlikely(err)) { 3650 folio_put(folio); 3651 if (err == -EEXIST) 3652 goto repeat; 3653 /* Presumably ENOMEM for xarray node */ 3654 return ERR_PTR(err); 3655 } 3656 3657 goto filler; 3658 } 3659 if (folio_test_uptodate(folio)) 3660 goto out; 3661 3662 if (!folio_trylock(folio)) { 3663 folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE); 3664 goto repeat; 3665 } 3666 3667 /* Folio was truncated from mapping */ 3668 if (!folio->mapping) { 3669 folio_unlock(folio); 3670 folio_put(folio); 3671 goto repeat; 3672 } 3673 3674 /* Someone else locked and filled the page in a very small window */ 3675 if (folio_test_uptodate(folio)) { 3676 folio_unlock(folio); 3677 goto out; 3678 } 3679 3680 filler: 3681 err = filemap_read_folio(file, filler, folio); 3682 if (err) { 3683 folio_put(folio); 3684 if (err == AOP_TRUNCATED_PAGE) 3685 goto repeat; 3686 return ERR_PTR(err); 3687 } 3688 3689 out: 3690 folio_mark_accessed(folio); 3691 return folio; 3692 } 3693 3694 /** 3695 * read_cache_folio - Read into page cache, fill it if needed. 3696 * @mapping: The address_space to read from. 3697 * @index: The index to read. 3698 * @filler: Function to perform the read, or NULL to use aops->read_folio(). 3699 * @file: Passed to filler function, may be NULL if not required. 3700 * 3701 * Read one page into the page cache. If it succeeds, the folio returned 3702 * will contain @index, but it may not be the first page of the folio. 3703 * 3704 * If the filler function returns an error, it will be returned to the 3705 * caller. 3706 * 3707 * Context: May sleep. Expects mapping->invalidate_lock to be held. 3708 * Return: An uptodate folio on success, ERR_PTR() on failure. 3709 */ 3710 struct folio *read_cache_folio(struct address_space *mapping, pgoff_t index, 3711 filler_t filler, struct file *file) 3712 { 3713 return do_read_cache_folio(mapping, index, filler, file, 3714 mapping_gfp_mask(mapping)); 3715 } 3716 EXPORT_SYMBOL(read_cache_folio); 3717 3718 /** 3719 * mapping_read_folio_gfp - Read into page cache, using specified allocation flags. 3720 * @mapping: The address_space for the folio. 3721 * @index: The index that the allocated folio will contain. 3722 * @gfp: The page allocator flags to use if allocating. 3723 * 3724 * This is the same as "read_cache_folio(mapping, index, NULL, NULL)", but with 3725 * any new memory allocations done using the specified allocation flags. 3726 * 3727 * The most likely error from this function is EIO, but ENOMEM is 3728 * possible and so is EINTR. If ->read_folio returns another error, 3729 * that will be returned to the caller. 3730 * 3731 * The function expects mapping->invalidate_lock to be already held. 3732 * 3733 * Return: Uptodate folio on success, ERR_PTR() on failure. 3734 */ 3735 struct folio *mapping_read_folio_gfp(struct address_space *mapping, 3736 pgoff_t index, gfp_t gfp) 3737 { 3738 return do_read_cache_folio(mapping, index, NULL, NULL, gfp); 3739 } 3740 EXPORT_SYMBOL(mapping_read_folio_gfp); 3741 3742 static struct page *do_read_cache_page(struct address_space *mapping, 3743 pgoff_t index, filler_t *filler, struct file *file, gfp_t gfp) 3744 { 3745 struct folio *folio; 3746 3747 folio = do_read_cache_folio(mapping, index, filler, file, gfp); 3748 if (IS_ERR(folio)) 3749 return &folio->page; 3750 return folio_file_page(folio, index); 3751 } 3752 3753 struct page *read_cache_page(struct address_space *mapping, 3754 pgoff_t index, filler_t *filler, struct file *file) 3755 { 3756 return do_read_cache_page(mapping, index, filler, file, 3757 mapping_gfp_mask(mapping)); 3758 } 3759 EXPORT_SYMBOL(read_cache_page); 3760 3761 /** 3762 * read_cache_page_gfp - read into page cache, using specified page allocation flags. 3763 * @mapping: the page's address_space 3764 * @index: the page index 3765 * @gfp: the page allocator flags to use if allocating 3766 * 3767 * This is the same as "read_mapping_page(mapping, index, NULL)", but with 3768 * any new page allocations done using the specified allocation flags. 3769 * 3770 * If the page does not get brought uptodate, return -EIO. 3771 * 3772 * The function expects mapping->invalidate_lock to be already held. 3773 * 3774 * Return: up to date page on success, ERR_PTR() on failure. 3775 */ 3776 struct page *read_cache_page_gfp(struct address_space *mapping, 3777 pgoff_t index, 3778 gfp_t gfp) 3779 { 3780 return do_read_cache_page(mapping, index, NULL, NULL, gfp); 3781 } 3782 EXPORT_SYMBOL(read_cache_page_gfp); 3783 3784 /* 3785 * Warn about a page cache invalidation failure during a direct I/O write. 3786 */ 3787 static void dio_warn_stale_pagecache(struct file *filp) 3788 { 3789 static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST); 3790 char pathname[128]; 3791 char *path; 3792 3793 errseq_set(&filp->f_mapping->wb_err, -EIO); 3794 if (__ratelimit(&_rs)) { 3795 path = file_path(filp, pathname, sizeof(pathname)); 3796 if (IS_ERR(path)) 3797 path = "(unknown)"; 3798 pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n"); 3799 pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid, 3800 current->comm); 3801 } 3802 } 3803 3804 void kiocb_invalidate_post_direct_write(struct kiocb *iocb, size_t count) 3805 { 3806 struct address_space *mapping = iocb->ki_filp->f_mapping; 3807 3808 if (mapping->nrpages && 3809 invalidate_inode_pages2_range(mapping, 3810 iocb->ki_pos >> PAGE_SHIFT, 3811 (iocb->ki_pos + count - 1) >> PAGE_SHIFT)) 3812 dio_warn_stale_pagecache(iocb->ki_filp); 3813 } 3814 3815 ssize_t 3816 generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from) 3817 { 3818 struct address_space *mapping = iocb->ki_filp->f_mapping; 3819 size_t write_len = iov_iter_count(from); 3820 ssize_t written; 3821 3822 /* 3823 * If a page can not be invalidated, return 0 to fall back 3824 * to buffered write. 3825 */ 3826 written = kiocb_invalidate_pages(iocb, write_len); 3827 if (written) { 3828 if (written == -EBUSY) 3829 return 0; 3830 return written; 3831 } 3832 3833 written = mapping->a_ops->direct_IO(iocb, from); 3834 3835 /* 3836 * Finally, try again to invalidate clean pages which might have been 3837 * cached by non-direct readahead, or faulted in by get_user_pages() 3838 * if the source of the write was an mmap'ed region of the file 3839 * we're writing. Either one is a pretty crazy thing to do, 3840 * so we don't support it 100%. If this invalidation 3841 * fails, tough, the write still worked... 3842 * 3843 * Most of the time we do not need this since dio_complete() will do 3844 * the invalidation for us. However there are some file systems that 3845 * do not end up with dio_complete() being called, so let's not break 3846 * them by removing it completely. 3847 * 3848 * Noticeable example is a blkdev_direct_IO(). 3849 * 3850 * Skip invalidation for async writes or if mapping has no pages. 3851 */ 3852 if (written > 0) { 3853 struct inode *inode = mapping->host; 3854 loff_t pos = iocb->ki_pos; 3855 3856 kiocb_invalidate_post_direct_write(iocb, written); 3857 pos += written; 3858 write_len -= written; 3859 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { 3860 i_size_write(inode, pos); 3861 mark_inode_dirty(inode); 3862 } 3863 iocb->ki_pos = pos; 3864 } 3865 if (written != -EIOCBQUEUED) 3866 iov_iter_revert(from, write_len - iov_iter_count(from)); 3867 return written; 3868 } 3869 EXPORT_SYMBOL(generic_file_direct_write); 3870 3871 ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i) 3872 { 3873 struct file *file = iocb->ki_filp; 3874 loff_t pos = iocb->ki_pos; 3875 struct address_space *mapping = file->f_mapping; 3876 const struct address_space_operations *a_ops = mapping->a_ops; 3877 long status = 0; 3878 ssize_t written = 0; 3879 3880 do { 3881 struct page *page; 3882 unsigned long offset; /* Offset into pagecache page */ 3883 unsigned long bytes; /* Bytes to write to page */ 3884 size_t copied; /* Bytes copied from user */ 3885 void *fsdata = NULL; 3886 3887 offset = (pos & (PAGE_SIZE - 1)); 3888 bytes = min_t(unsigned long, PAGE_SIZE - offset, 3889 iov_iter_count(i)); 3890 3891 again: 3892 /* 3893 * Bring in the user page that we will copy from _first_. 3894 * Otherwise there's a nasty deadlock on copying from the 3895 * same page as we're writing to, without it being marked 3896 * up-to-date. 3897 */ 3898 if (unlikely(fault_in_iov_iter_readable(i, bytes) == bytes)) { 3899 status = -EFAULT; 3900 break; 3901 } 3902 3903 if (fatal_signal_pending(current)) { 3904 status = -EINTR; 3905 break; 3906 } 3907 3908 status = a_ops->write_begin(file, mapping, pos, bytes, 3909 &page, &fsdata); 3910 if (unlikely(status < 0)) 3911 break; 3912 3913 if (mapping_writably_mapped(mapping)) 3914 flush_dcache_page(page); 3915 3916 copied = copy_page_from_iter_atomic(page, offset, bytes, i); 3917 flush_dcache_page(page); 3918 3919 status = a_ops->write_end(file, mapping, pos, bytes, copied, 3920 page, fsdata); 3921 if (unlikely(status != copied)) { 3922 iov_iter_revert(i, copied - max(status, 0L)); 3923 if (unlikely(status < 0)) 3924 break; 3925 } 3926 cond_resched(); 3927 3928 if (unlikely(status == 0)) { 3929 /* 3930 * A short copy made ->write_end() reject the 3931 * thing entirely. Might be memory poisoning 3932 * halfway through, might be a race with munmap, 3933 * might be severe memory pressure. 3934 */ 3935 if (copied) 3936 bytes = copied; 3937 goto again; 3938 } 3939 pos += status; 3940 written += status; 3941 3942 balance_dirty_pages_ratelimited(mapping); 3943 } while (iov_iter_count(i)); 3944 3945 if (!written) 3946 return status; 3947 iocb->ki_pos += written; 3948 return written; 3949 } 3950 EXPORT_SYMBOL(generic_perform_write); 3951 3952 /** 3953 * __generic_file_write_iter - write data to a file 3954 * @iocb: IO state structure (file, offset, etc.) 3955 * @from: iov_iter with data to write 3956 * 3957 * This function does all the work needed for actually writing data to a 3958 * file. It does all basic checks, removes SUID from the file, updates 3959 * modification times and calls proper subroutines depending on whether we 3960 * do direct IO or a standard buffered write. 3961 * 3962 * It expects i_rwsem to be grabbed unless we work on a block device or similar 3963 * object which does not need locking at all. 3964 * 3965 * This function does *not* take care of syncing data in case of O_SYNC write. 3966 * A caller has to handle it. This is mainly due to the fact that we want to 3967 * avoid syncing under i_rwsem. 3968 * 3969 * Return: 3970 * * number of bytes written, even for truncated writes 3971 * * negative error code if no data has been written at all 3972 */ 3973 ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 3974 { 3975 struct file *file = iocb->ki_filp; 3976 struct address_space *mapping = file->f_mapping; 3977 struct inode *inode = mapping->host; 3978 ssize_t ret; 3979 3980 ret = file_remove_privs(file); 3981 if (ret) 3982 return ret; 3983 3984 ret = file_update_time(file); 3985 if (ret) 3986 return ret; 3987 3988 if (iocb->ki_flags & IOCB_DIRECT) { 3989 ret = generic_file_direct_write(iocb, from); 3990 /* 3991 * If the write stopped short of completing, fall back to 3992 * buffered writes. Some filesystems do this for writes to 3993 * holes, for example. For DAX files, a buffered write will 3994 * not succeed (even if it did, DAX does not handle dirty 3995 * page-cache pages correctly). 3996 */ 3997 if (ret < 0 || !iov_iter_count(from) || IS_DAX(inode)) 3998 return ret; 3999 return direct_write_fallback(iocb, from, ret, 4000 generic_perform_write(iocb, from)); 4001 } 4002 4003 return generic_perform_write(iocb, from); 4004 } 4005 EXPORT_SYMBOL(__generic_file_write_iter); 4006 4007 /** 4008 * generic_file_write_iter - write data to a file 4009 * @iocb: IO state structure 4010 * @from: iov_iter with data to write 4011 * 4012 * This is a wrapper around __generic_file_write_iter() to be used by most 4013 * filesystems. It takes care of syncing the file in case of O_SYNC file 4014 * and acquires i_rwsem as needed. 4015 * Return: 4016 * * negative error code if no data has been written at all of 4017 * vfs_fsync_range() failed for a synchronous write 4018 * * number of bytes written, even for truncated writes 4019 */ 4020 ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 4021 { 4022 struct file *file = iocb->ki_filp; 4023 struct inode *inode = file->f_mapping->host; 4024 ssize_t ret; 4025 4026 inode_lock(inode); 4027 ret = generic_write_checks(iocb, from); 4028 if (ret > 0) 4029 ret = __generic_file_write_iter(iocb, from); 4030 inode_unlock(inode); 4031 4032 if (ret > 0) 4033 ret = generic_write_sync(iocb, ret); 4034 return ret; 4035 } 4036 EXPORT_SYMBOL(generic_file_write_iter); 4037 4038 /** 4039 * filemap_release_folio() - Release fs-specific metadata on a folio. 4040 * @folio: The folio which the kernel is trying to free. 4041 * @gfp: Memory allocation flags (and I/O mode). 4042 * 4043 * The address_space is trying to release any data attached to a folio 4044 * (presumably at folio->private). 4045 * 4046 * This will also be called if the private_2 flag is set on a page, 4047 * indicating that the folio has other metadata associated with it. 4048 * 4049 * The @gfp argument specifies whether I/O may be performed to release 4050 * this page (__GFP_IO), and whether the call may block 4051 * (__GFP_RECLAIM & __GFP_FS). 4052 * 4053 * Return: %true if the release was successful, otherwise %false. 4054 */ 4055 bool filemap_release_folio(struct folio *folio, gfp_t gfp) 4056 { 4057 struct address_space * const mapping = folio->mapping; 4058 4059 BUG_ON(!folio_test_locked(folio)); 4060 if (folio_test_writeback(folio)) 4061 return false; 4062 4063 if (mapping && mapping->a_ops->release_folio) 4064 return mapping->a_ops->release_folio(folio, gfp); 4065 return try_to_free_buffers(folio); 4066 } 4067 EXPORT_SYMBOL(filemap_release_folio); 4068 4069 #ifdef CONFIG_CACHESTAT_SYSCALL 4070 /** 4071 * filemap_cachestat() - compute the page cache statistics of a mapping 4072 * @mapping: The mapping to compute the statistics for. 4073 * @first_index: The starting page cache index. 4074 * @last_index: The final page index (inclusive). 4075 * @cs: the cachestat struct to write the result to. 4076 * 4077 * This will query the page cache statistics of a mapping in the 4078 * page range of [first_index, last_index] (inclusive). The statistics 4079 * queried include: number of dirty pages, number of pages marked for 4080 * writeback, and the number of (recently) evicted pages. 4081 */ 4082 static void filemap_cachestat(struct address_space *mapping, 4083 pgoff_t first_index, pgoff_t last_index, struct cachestat *cs) 4084 { 4085 XA_STATE(xas, &mapping->i_pages, first_index); 4086 struct folio *folio; 4087 4088 rcu_read_lock(); 4089 xas_for_each(&xas, folio, last_index) { 4090 unsigned long nr_pages; 4091 pgoff_t folio_first_index, folio_last_index; 4092 4093 if (xas_retry(&xas, folio)) 4094 continue; 4095 4096 if (xa_is_value(folio)) { 4097 /* page is evicted */ 4098 void *shadow = (void *)folio; 4099 bool workingset; /* not used */ 4100 int order = xa_get_order(xas.xa, xas.xa_index); 4101 4102 nr_pages = 1 << order; 4103 folio_first_index = round_down(xas.xa_index, 1 << order); 4104 folio_last_index = folio_first_index + nr_pages - 1; 4105 4106 /* Folios might straddle the range boundaries, only count covered pages */ 4107 if (folio_first_index < first_index) 4108 nr_pages -= first_index - folio_first_index; 4109 4110 if (folio_last_index > last_index) 4111 nr_pages -= folio_last_index - last_index; 4112 4113 cs->nr_evicted += nr_pages; 4114 4115 #ifdef CONFIG_SWAP /* implies CONFIG_MMU */ 4116 if (shmem_mapping(mapping)) { 4117 /* shmem file - in swap cache */ 4118 swp_entry_t swp = radix_to_swp_entry(folio); 4119 4120 shadow = get_shadow_from_swap_cache(swp); 4121 } 4122 #endif 4123 if (workingset_test_recent(shadow, true, &workingset)) 4124 cs->nr_recently_evicted += nr_pages; 4125 4126 goto resched; 4127 } 4128 4129 nr_pages = folio_nr_pages(folio); 4130 folio_first_index = folio_pgoff(folio); 4131 folio_last_index = folio_first_index + nr_pages - 1; 4132 4133 /* Folios might straddle the range boundaries, only count covered pages */ 4134 if (folio_first_index < first_index) 4135 nr_pages -= first_index - folio_first_index; 4136 4137 if (folio_last_index > last_index) 4138 nr_pages -= folio_last_index - last_index; 4139 4140 /* page is in cache */ 4141 cs->nr_cache += nr_pages; 4142 4143 if (folio_test_dirty(folio)) 4144 cs->nr_dirty += nr_pages; 4145 4146 if (folio_test_writeback(folio)) 4147 cs->nr_writeback += nr_pages; 4148 4149 resched: 4150 if (need_resched()) { 4151 xas_pause(&xas); 4152 cond_resched_rcu(); 4153 } 4154 } 4155 rcu_read_unlock(); 4156 } 4157 4158 /* 4159 * The cachestat(2) system call. 4160 * 4161 * cachestat() returns the page cache statistics of a file in the 4162 * bytes range specified by `off` and `len`: number of cached pages, 4163 * number of dirty pages, number of pages marked for writeback, 4164 * number of evicted pages, and number of recently evicted pages. 4165 * 4166 * An evicted page is a page that is previously in the page cache 4167 * but has been evicted since. A page is recently evicted if its last 4168 * eviction was recent enough that its reentry to the cache would 4169 * indicate that it is actively being used by the system, and that 4170 * there is memory pressure on the system. 4171 * 4172 * `off` and `len` must be non-negative integers. If `len` > 0, 4173 * the queried range is [`off`, `off` + `len`]. If `len` == 0, 4174 * we will query in the range from `off` to the end of the file. 4175 * 4176 * The `flags` argument is unused for now, but is included for future 4177 * extensibility. User should pass 0 (i.e no flag specified). 4178 * 4179 * Currently, hugetlbfs is not supported. 4180 * 4181 * Because the status of a page can change after cachestat() checks it 4182 * but before it returns to the application, the returned values may 4183 * contain stale information. 4184 * 4185 * return values: 4186 * zero - success 4187 * -EFAULT - cstat or cstat_range points to an illegal address 4188 * -EINVAL - invalid flags 4189 * -EBADF - invalid file descriptor 4190 * -EOPNOTSUPP - file descriptor is of a hugetlbfs file 4191 */ 4192 SYSCALL_DEFINE4(cachestat, unsigned int, fd, 4193 struct cachestat_range __user *, cstat_range, 4194 struct cachestat __user *, cstat, unsigned int, flags) 4195 { 4196 struct fd f = fdget(fd); 4197 struct address_space *mapping; 4198 struct cachestat_range csr; 4199 struct cachestat cs; 4200 pgoff_t first_index, last_index; 4201 4202 if (!f.file) 4203 return -EBADF; 4204 4205 if (copy_from_user(&csr, cstat_range, 4206 sizeof(struct cachestat_range))) { 4207 fdput(f); 4208 return -EFAULT; 4209 } 4210 4211 /* hugetlbfs is not supported */ 4212 if (is_file_hugepages(f.file)) { 4213 fdput(f); 4214 return -EOPNOTSUPP; 4215 } 4216 4217 if (flags != 0) { 4218 fdput(f); 4219 return -EINVAL; 4220 } 4221 4222 first_index = csr.off >> PAGE_SHIFT; 4223 last_index = 4224 csr.len == 0 ? ULONG_MAX : (csr.off + csr.len - 1) >> PAGE_SHIFT; 4225 memset(&cs, 0, sizeof(struct cachestat)); 4226 mapping = f.file->f_mapping; 4227 filemap_cachestat(mapping, first_index, last_index, &cs); 4228 fdput(f); 4229 4230 if (copy_to_user(cstat, &cs, sizeof(struct cachestat))) 4231 return -EFAULT; 4232 4233 return 0; 4234 } 4235 #endif /* CONFIG_CACHESTAT_SYSCALL */ 4236