1 /* 2 * fs/dax.c - Direct Access filesystem code 3 * Copyright (c) 2013-2014 Intel Corporation 4 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com> 5 * Author: Ross Zwisler <ross.zwisler@linux.intel.com> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms and conditions of the GNU General Public License, 9 * version 2, as published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 * more details. 15 */ 16 17 #include <linux/atomic.h> 18 #include <linux/blkdev.h> 19 #include <linux/buffer_head.h> 20 #include <linux/dax.h> 21 #include <linux/fs.h> 22 #include <linux/genhd.h> 23 #include <linux/highmem.h> 24 #include <linux/memcontrol.h> 25 #include <linux/mm.h> 26 #include <linux/mutex.h> 27 #include <linux/pagevec.h> 28 #include <linux/sched.h> 29 #include <linux/sched/signal.h> 30 #include <linux/uio.h> 31 #include <linux/vmstat.h> 32 #include <linux/pfn_t.h> 33 #include <linux/sizes.h> 34 #include <linux/mmu_notifier.h> 35 #include <linux/iomap.h> 36 #include "internal.h" 37 38 #define CREATE_TRACE_POINTS 39 #include <trace/events/fs_dax.h> 40 41 /* We choose 4096 entries - same as per-zone page wait tables */ 42 #define DAX_WAIT_TABLE_BITS 12 43 #define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS) 44 45 /* The 'colour' (ie low bits) within a PMD of a page offset. */ 46 #define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1) 47 #define PG_PMD_NR (PMD_SIZE >> PAGE_SHIFT) 48 49 static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES]; 50 51 static int __init init_dax_wait_table(void) 52 { 53 int i; 54 55 for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++) 56 init_waitqueue_head(wait_table + i); 57 return 0; 58 } 59 fs_initcall(init_dax_wait_table); 60 61 /* 62 * We use lowest available bit in exceptional entry for locking, one bit for 63 * the entry size (PMD) and two more to tell us if the entry is a zero page or 64 * an empty entry that is just used for locking. In total four special bits. 65 * 66 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE 67 * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem 68 * block allocation. 69 */ 70 #define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 4) 71 #define RADIX_DAX_ENTRY_LOCK (1 << RADIX_TREE_EXCEPTIONAL_SHIFT) 72 #define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1)) 73 #define RADIX_DAX_ZERO_PAGE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2)) 74 #define RADIX_DAX_EMPTY (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3)) 75 76 static unsigned long dax_radix_pfn(void *entry) 77 { 78 return (unsigned long)entry >> RADIX_DAX_SHIFT; 79 } 80 81 static void *dax_radix_locked_entry(unsigned long pfn, unsigned long flags) 82 { 83 return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags | 84 (pfn << RADIX_DAX_SHIFT) | RADIX_DAX_ENTRY_LOCK); 85 } 86 87 static unsigned int dax_radix_order(void *entry) 88 { 89 if ((unsigned long)entry & RADIX_DAX_PMD) 90 return PMD_SHIFT - PAGE_SHIFT; 91 return 0; 92 } 93 94 static int dax_is_pmd_entry(void *entry) 95 { 96 return (unsigned long)entry & RADIX_DAX_PMD; 97 } 98 99 static int dax_is_pte_entry(void *entry) 100 { 101 return !((unsigned long)entry & RADIX_DAX_PMD); 102 } 103 104 static int dax_is_zero_entry(void *entry) 105 { 106 return (unsigned long)entry & RADIX_DAX_ZERO_PAGE; 107 } 108 109 static int dax_is_empty_entry(void *entry) 110 { 111 return (unsigned long)entry & RADIX_DAX_EMPTY; 112 } 113 114 /* 115 * DAX radix tree locking 116 */ 117 struct exceptional_entry_key { 118 struct address_space *mapping; 119 pgoff_t entry_start; 120 }; 121 122 struct wait_exceptional_entry_queue { 123 wait_queue_entry_t wait; 124 struct exceptional_entry_key key; 125 }; 126 127 static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping, 128 pgoff_t index, void *entry, struct exceptional_entry_key *key) 129 { 130 unsigned long hash; 131 132 /* 133 * If 'entry' is a PMD, align the 'index' that we use for the wait 134 * queue to the start of that PMD. This ensures that all offsets in 135 * the range covered by the PMD map to the same bit lock. 136 */ 137 if (dax_is_pmd_entry(entry)) 138 index &= ~PG_PMD_COLOUR; 139 140 key->mapping = mapping; 141 key->entry_start = index; 142 143 hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS); 144 return wait_table + hash; 145 } 146 147 static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mode, 148 int sync, void *keyp) 149 { 150 struct exceptional_entry_key *key = keyp; 151 struct wait_exceptional_entry_queue *ewait = 152 container_of(wait, struct wait_exceptional_entry_queue, wait); 153 154 if (key->mapping != ewait->key.mapping || 155 key->entry_start != ewait->key.entry_start) 156 return 0; 157 return autoremove_wake_function(wait, mode, sync, NULL); 158 } 159 160 /* 161 * @entry may no longer be the entry at the index in the mapping. 162 * The important information it's conveying is whether the entry at 163 * this index used to be a PMD entry. 164 */ 165 static void dax_wake_mapping_entry_waiter(struct address_space *mapping, 166 pgoff_t index, void *entry, bool wake_all) 167 { 168 struct exceptional_entry_key key; 169 wait_queue_head_t *wq; 170 171 wq = dax_entry_waitqueue(mapping, index, entry, &key); 172 173 /* 174 * Checking for locked entry and prepare_to_wait_exclusive() happens 175 * under the i_pages lock, ditto for entry handling in our callers. 176 * So at this point all tasks that could have seen our entry locked 177 * must be in the waitqueue and the following check will see them. 178 */ 179 if (waitqueue_active(wq)) 180 __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key); 181 } 182 183 /* 184 * Check whether the given slot is locked. Must be called with the i_pages 185 * lock held. 186 */ 187 static inline int slot_locked(struct address_space *mapping, void **slot) 188 { 189 unsigned long entry = (unsigned long) 190 radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); 191 return entry & RADIX_DAX_ENTRY_LOCK; 192 } 193 194 /* 195 * Mark the given slot as locked. Must be called with the i_pages lock held. 196 */ 197 static inline void *lock_slot(struct address_space *mapping, void **slot) 198 { 199 unsigned long entry = (unsigned long) 200 radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); 201 202 entry |= RADIX_DAX_ENTRY_LOCK; 203 radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry); 204 return (void *)entry; 205 } 206 207 /* 208 * Mark the given slot as unlocked. Must be called with the i_pages lock held. 209 */ 210 static inline void *unlock_slot(struct address_space *mapping, void **slot) 211 { 212 unsigned long entry = (unsigned long) 213 radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); 214 215 entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK; 216 radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry); 217 return (void *)entry; 218 } 219 220 /* 221 * Lookup entry in radix tree, wait for it to become unlocked if it is 222 * exceptional entry and return it. The caller must call 223 * put_unlocked_mapping_entry() when he decided not to lock the entry or 224 * put_locked_mapping_entry() when he locked the entry and now wants to 225 * unlock it. 226 * 227 * Must be called with the i_pages lock held. 228 */ 229 static void *get_unlocked_mapping_entry(struct address_space *mapping, 230 pgoff_t index, void ***slotp) 231 { 232 void *entry, **slot; 233 struct wait_exceptional_entry_queue ewait; 234 wait_queue_head_t *wq; 235 236 init_wait(&ewait.wait); 237 ewait.wait.func = wake_exceptional_entry_func; 238 239 for (;;) { 240 entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, 241 &slot); 242 if (!entry || 243 WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) || 244 !slot_locked(mapping, slot)) { 245 if (slotp) 246 *slotp = slot; 247 return entry; 248 } 249 250 wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key); 251 prepare_to_wait_exclusive(wq, &ewait.wait, 252 TASK_UNINTERRUPTIBLE); 253 xa_unlock_irq(&mapping->i_pages); 254 schedule(); 255 finish_wait(wq, &ewait.wait); 256 xa_lock_irq(&mapping->i_pages); 257 } 258 } 259 260 static void dax_unlock_mapping_entry(struct address_space *mapping, 261 pgoff_t index) 262 { 263 void *entry, **slot; 264 265 xa_lock_irq(&mapping->i_pages); 266 entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, &slot); 267 if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) || 268 !slot_locked(mapping, slot))) { 269 xa_unlock_irq(&mapping->i_pages); 270 return; 271 } 272 unlock_slot(mapping, slot); 273 xa_unlock_irq(&mapping->i_pages); 274 dax_wake_mapping_entry_waiter(mapping, index, entry, false); 275 } 276 277 static void put_locked_mapping_entry(struct address_space *mapping, 278 pgoff_t index) 279 { 280 dax_unlock_mapping_entry(mapping, index); 281 } 282 283 /* 284 * Called when we are done with radix tree entry we looked up via 285 * get_unlocked_mapping_entry() and which we didn't lock in the end. 286 */ 287 static void put_unlocked_mapping_entry(struct address_space *mapping, 288 pgoff_t index, void *entry) 289 { 290 if (!entry) 291 return; 292 293 /* We have to wake up next waiter for the radix tree entry lock */ 294 dax_wake_mapping_entry_waiter(mapping, index, entry, false); 295 } 296 297 static unsigned long dax_entry_size(void *entry) 298 { 299 if (dax_is_zero_entry(entry)) 300 return 0; 301 else if (dax_is_empty_entry(entry)) 302 return 0; 303 else if (dax_is_pmd_entry(entry)) 304 return PMD_SIZE; 305 else 306 return PAGE_SIZE; 307 } 308 309 static unsigned long dax_radix_end_pfn(void *entry) 310 { 311 return dax_radix_pfn(entry) + dax_entry_size(entry) / PAGE_SIZE; 312 } 313 314 /* 315 * Iterate through all mapped pfns represented by an entry, i.e. skip 316 * 'empty' and 'zero' entries. 317 */ 318 #define for_each_mapped_pfn(entry, pfn) \ 319 for (pfn = dax_radix_pfn(entry); \ 320 pfn < dax_radix_end_pfn(entry); pfn++) 321 322 static void dax_associate_entry(void *entry, struct address_space *mapping) 323 { 324 unsigned long pfn; 325 326 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED)) 327 return; 328 329 for_each_mapped_pfn(entry, pfn) { 330 struct page *page = pfn_to_page(pfn); 331 332 WARN_ON_ONCE(page->mapping); 333 page->mapping = mapping; 334 } 335 } 336 337 static void dax_disassociate_entry(void *entry, struct address_space *mapping, 338 bool trunc) 339 { 340 unsigned long pfn; 341 342 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED)) 343 return; 344 345 for_each_mapped_pfn(entry, pfn) { 346 struct page *page = pfn_to_page(pfn); 347 348 WARN_ON_ONCE(trunc && page_ref_count(page) > 1); 349 WARN_ON_ONCE(page->mapping && page->mapping != mapping); 350 page->mapping = NULL; 351 } 352 } 353 354 static struct page *dax_busy_page(void *entry) 355 { 356 unsigned long pfn; 357 358 for_each_mapped_pfn(entry, pfn) { 359 struct page *page = pfn_to_page(pfn); 360 361 if (page_ref_count(page) > 1) 362 return page; 363 } 364 return NULL; 365 } 366 367 /* 368 * Find radix tree entry at given index. If it points to an exceptional entry, 369 * return it with the radix tree entry locked. If the radix tree doesn't 370 * contain given index, create an empty exceptional entry for the index and 371 * return with it locked. 372 * 373 * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will 374 * either return that locked entry or will return an error. This error will 375 * happen if there are any 4k entries within the 2MiB range that we are 376 * requesting. 377 * 378 * We always favor 4k entries over 2MiB entries. There isn't a flow where we 379 * evict 4k entries in order to 'upgrade' them to a 2MiB entry. A 2MiB 380 * insertion will fail if it finds any 4k entries already in the tree, and a 381 * 4k insertion will cause an existing 2MiB entry to be unmapped and 382 * downgraded to 4k entries. This happens for both 2MiB huge zero pages as 383 * well as 2MiB empty entries. 384 * 385 * The exception to this downgrade path is for 2MiB DAX PMD entries that have 386 * real storage backing them. We will leave these real 2MiB DAX entries in 387 * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry. 388 * 389 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For 390 * persistent memory the benefit is doubtful. We can add that later if we can 391 * show it helps. 392 */ 393 static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index, 394 unsigned long size_flag) 395 { 396 bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */ 397 void *entry, **slot; 398 399 restart: 400 xa_lock_irq(&mapping->i_pages); 401 entry = get_unlocked_mapping_entry(mapping, index, &slot); 402 403 if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) { 404 entry = ERR_PTR(-EIO); 405 goto out_unlock; 406 } 407 408 if (entry) { 409 if (size_flag & RADIX_DAX_PMD) { 410 if (dax_is_pte_entry(entry)) { 411 put_unlocked_mapping_entry(mapping, index, 412 entry); 413 entry = ERR_PTR(-EEXIST); 414 goto out_unlock; 415 } 416 } else { /* trying to grab a PTE entry */ 417 if (dax_is_pmd_entry(entry) && 418 (dax_is_zero_entry(entry) || 419 dax_is_empty_entry(entry))) { 420 pmd_downgrade = true; 421 } 422 } 423 } 424 425 /* No entry for given index? Make sure radix tree is big enough. */ 426 if (!entry || pmd_downgrade) { 427 int err; 428 429 if (pmd_downgrade) { 430 /* 431 * Make sure 'entry' remains valid while we drop 432 * the i_pages lock. 433 */ 434 entry = lock_slot(mapping, slot); 435 } 436 437 xa_unlock_irq(&mapping->i_pages); 438 /* 439 * Besides huge zero pages the only other thing that gets 440 * downgraded are empty entries which don't need to be 441 * unmapped. 442 */ 443 if (pmd_downgrade && dax_is_zero_entry(entry)) 444 unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR, 445 PG_PMD_NR, false); 446 447 err = radix_tree_preload( 448 mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM); 449 if (err) { 450 if (pmd_downgrade) 451 put_locked_mapping_entry(mapping, index); 452 return ERR_PTR(err); 453 } 454 xa_lock_irq(&mapping->i_pages); 455 456 if (!entry) { 457 /* 458 * We needed to drop the i_pages lock while calling 459 * radix_tree_preload() and we didn't have an entry to 460 * lock. See if another thread inserted an entry at 461 * our index during this time. 462 */ 463 entry = __radix_tree_lookup(&mapping->i_pages, index, 464 NULL, &slot); 465 if (entry) { 466 radix_tree_preload_end(); 467 xa_unlock_irq(&mapping->i_pages); 468 goto restart; 469 } 470 } 471 472 if (pmd_downgrade) { 473 dax_disassociate_entry(entry, mapping, false); 474 radix_tree_delete(&mapping->i_pages, index); 475 mapping->nrexceptional--; 476 dax_wake_mapping_entry_waiter(mapping, index, entry, 477 true); 478 } 479 480 entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY); 481 482 err = __radix_tree_insert(&mapping->i_pages, index, 483 dax_radix_order(entry), entry); 484 radix_tree_preload_end(); 485 if (err) { 486 xa_unlock_irq(&mapping->i_pages); 487 /* 488 * Our insertion of a DAX entry failed, most likely 489 * because we were inserting a PMD entry and it 490 * collided with a PTE sized entry at a different 491 * index in the PMD range. We haven't inserted 492 * anything into the radix tree and have no waiters to 493 * wake. 494 */ 495 return ERR_PTR(err); 496 } 497 /* Good, we have inserted empty locked entry into the tree. */ 498 mapping->nrexceptional++; 499 xa_unlock_irq(&mapping->i_pages); 500 return entry; 501 } 502 entry = lock_slot(mapping, slot); 503 out_unlock: 504 xa_unlock_irq(&mapping->i_pages); 505 return entry; 506 } 507 508 /** 509 * dax_layout_busy_page - find first pinned page in @mapping 510 * @mapping: address space to scan for a page with ref count > 1 511 * 512 * DAX requires ZONE_DEVICE mapped pages. These pages are never 513 * 'onlined' to the page allocator so they are considered idle when 514 * page->count == 1. A filesystem uses this interface to determine if 515 * any page in the mapping is busy, i.e. for DMA, or other 516 * get_user_pages() usages. 517 * 518 * It is expected that the filesystem is holding locks to block the 519 * establishment of new mappings in this address_space. I.e. it expects 520 * to be able to run unmap_mapping_range() and subsequently not race 521 * mapping_mapped() becoming true. 522 */ 523 struct page *dax_layout_busy_page(struct address_space *mapping) 524 { 525 pgoff_t indices[PAGEVEC_SIZE]; 526 struct page *page = NULL; 527 struct pagevec pvec; 528 pgoff_t index, end; 529 unsigned i; 530 531 /* 532 * In the 'limited' case get_user_pages() for dax is disabled. 533 */ 534 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED)) 535 return NULL; 536 537 if (!dax_mapping(mapping) || !mapping_mapped(mapping)) 538 return NULL; 539 540 pagevec_init(&pvec); 541 index = 0; 542 end = -1; 543 544 /* 545 * If we race get_user_pages_fast() here either we'll see the 546 * elevated page count in the pagevec_lookup and wait, or 547 * get_user_pages_fast() will see that the page it took a reference 548 * against is no longer mapped in the page tables and bail to the 549 * get_user_pages() slow path. The slow path is protected by 550 * pte_lock() and pmd_lock(). New references are not taken without 551 * holding those locks, and unmap_mapping_range() will not zero the 552 * pte or pmd without holding the respective lock, so we are 553 * guaranteed to either see new references or prevent new 554 * references from being established. 555 */ 556 unmap_mapping_range(mapping, 0, 0, 1); 557 558 while (index < end && pagevec_lookup_entries(&pvec, mapping, index, 559 min(end - index, (pgoff_t)PAGEVEC_SIZE), 560 indices)) { 561 for (i = 0; i < pagevec_count(&pvec); i++) { 562 struct page *pvec_ent = pvec.pages[i]; 563 void *entry; 564 565 index = indices[i]; 566 if (index >= end) 567 break; 568 569 if (WARN_ON_ONCE( 570 !radix_tree_exceptional_entry(pvec_ent))) 571 continue; 572 573 xa_lock_irq(&mapping->i_pages); 574 entry = get_unlocked_mapping_entry(mapping, index, NULL); 575 if (entry) 576 page = dax_busy_page(entry); 577 put_unlocked_mapping_entry(mapping, index, entry); 578 xa_unlock_irq(&mapping->i_pages); 579 if (page) 580 break; 581 } 582 583 /* 584 * We don't expect normal struct page entries to exist in our 585 * tree, but we keep these pagevec calls so that this code is 586 * consistent with the common pattern for handling pagevecs 587 * throughout the kernel. 588 */ 589 pagevec_remove_exceptionals(&pvec); 590 pagevec_release(&pvec); 591 index++; 592 593 if (page) 594 break; 595 } 596 return page; 597 } 598 EXPORT_SYMBOL_GPL(dax_layout_busy_page); 599 600 static int __dax_invalidate_mapping_entry(struct address_space *mapping, 601 pgoff_t index, bool trunc) 602 { 603 int ret = 0; 604 void *entry; 605 struct radix_tree_root *pages = &mapping->i_pages; 606 607 xa_lock_irq(pages); 608 entry = get_unlocked_mapping_entry(mapping, index, NULL); 609 if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry))) 610 goto out; 611 if (!trunc && 612 (radix_tree_tag_get(pages, index, PAGECACHE_TAG_DIRTY) || 613 radix_tree_tag_get(pages, index, PAGECACHE_TAG_TOWRITE))) 614 goto out; 615 dax_disassociate_entry(entry, mapping, trunc); 616 radix_tree_delete(pages, index); 617 mapping->nrexceptional--; 618 ret = 1; 619 out: 620 put_unlocked_mapping_entry(mapping, index, entry); 621 xa_unlock_irq(pages); 622 return ret; 623 } 624 /* 625 * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree 626 * entry to get unlocked before deleting it. 627 */ 628 int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index) 629 { 630 int ret = __dax_invalidate_mapping_entry(mapping, index, true); 631 632 /* 633 * This gets called from truncate / punch_hole path. As such, the caller 634 * must hold locks protecting against concurrent modifications of the 635 * radix tree (usually fs-private i_mmap_sem for writing). Since the 636 * caller has seen exceptional entry for this index, we better find it 637 * at that index as well... 638 */ 639 WARN_ON_ONCE(!ret); 640 return ret; 641 } 642 643 /* 644 * Invalidate exceptional DAX entry if it is clean. 645 */ 646 int dax_invalidate_mapping_entry_sync(struct address_space *mapping, 647 pgoff_t index) 648 { 649 return __dax_invalidate_mapping_entry(mapping, index, false); 650 } 651 652 static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev, 653 sector_t sector, size_t size, struct page *to, 654 unsigned long vaddr) 655 { 656 void *vto, *kaddr; 657 pgoff_t pgoff; 658 pfn_t pfn; 659 long rc; 660 int id; 661 662 rc = bdev_dax_pgoff(bdev, sector, size, &pgoff); 663 if (rc) 664 return rc; 665 666 id = dax_read_lock(); 667 rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn); 668 if (rc < 0) { 669 dax_read_unlock(id); 670 return rc; 671 } 672 vto = kmap_atomic(to); 673 copy_user_page(vto, (void __force *)kaddr, vaddr, to); 674 kunmap_atomic(vto); 675 dax_read_unlock(id); 676 return 0; 677 } 678 679 /* 680 * By this point grab_mapping_entry() has ensured that we have a locked entry 681 * of the appropriate size so we don't have to worry about downgrading PMDs to 682 * PTEs. If we happen to be trying to insert a PTE and there is a PMD 683 * already in the tree, we will skip the insertion and just dirty the PMD as 684 * appropriate. 685 */ 686 static void *dax_insert_mapping_entry(struct address_space *mapping, 687 struct vm_fault *vmf, 688 void *entry, pfn_t pfn_t, 689 unsigned long flags, bool dirty) 690 { 691 struct radix_tree_root *pages = &mapping->i_pages; 692 unsigned long pfn = pfn_t_to_pfn(pfn_t); 693 pgoff_t index = vmf->pgoff; 694 void *new_entry; 695 696 if (dirty) 697 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 698 699 if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_ZERO_PAGE)) { 700 /* we are replacing a zero page with block mapping */ 701 if (dax_is_pmd_entry(entry)) 702 unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR, 703 PG_PMD_NR, false); 704 else /* pte entry */ 705 unmap_mapping_pages(mapping, vmf->pgoff, 1, false); 706 } 707 708 xa_lock_irq(pages); 709 new_entry = dax_radix_locked_entry(pfn, flags); 710 if (dax_entry_size(entry) != dax_entry_size(new_entry)) { 711 dax_disassociate_entry(entry, mapping, false); 712 dax_associate_entry(new_entry, mapping); 713 } 714 715 if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) { 716 /* 717 * Only swap our new entry into the radix tree if the current 718 * entry is a zero page or an empty entry. If a normal PTE or 719 * PMD entry is already in the tree, we leave it alone. This 720 * means that if we are trying to insert a PTE and the 721 * existing entry is a PMD, we will just leave the PMD in the 722 * tree and dirty it if necessary. 723 */ 724 struct radix_tree_node *node; 725 void **slot; 726 void *ret; 727 728 ret = __radix_tree_lookup(pages, index, &node, &slot); 729 WARN_ON_ONCE(ret != entry); 730 __radix_tree_replace(pages, node, slot, 731 new_entry, NULL); 732 entry = new_entry; 733 } 734 735 if (dirty) 736 radix_tree_tag_set(pages, index, PAGECACHE_TAG_DIRTY); 737 738 xa_unlock_irq(pages); 739 return entry; 740 } 741 742 static inline unsigned long 743 pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma) 744 { 745 unsigned long address; 746 747 address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 748 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma); 749 return address; 750 } 751 752 /* Walk all mappings of a given index of a file and writeprotect them */ 753 static void dax_mapping_entry_mkclean(struct address_space *mapping, 754 pgoff_t index, unsigned long pfn) 755 { 756 struct vm_area_struct *vma; 757 pte_t pte, *ptep = NULL; 758 pmd_t *pmdp = NULL; 759 spinlock_t *ptl; 760 761 i_mmap_lock_read(mapping); 762 vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) { 763 unsigned long address, start, end; 764 765 cond_resched(); 766 767 if (!(vma->vm_flags & VM_SHARED)) 768 continue; 769 770 address = pgoff_address(index, vma); 771 772 /* 773 * Note because we provide start/end to follow_pte_pmd it will 774 * call mmu_notifier_invalidate_range_start() on our behalf 775 * before taking any lock. 776 */ 777 if (follow_pte_pmd(vma->vm_mm, address, &start, &end, &ptep, &pmdp, &ptl)) 778 continue; 779 780 /* 781 * No need to call mmu_notifier_invalidate_range() as we are 782 * downgrading page table protection not changing it to point 783 * to a new page. 784 * 785 * See Documentation/vm/mmu_notifier.rst 786 */ 787 if (pmdp) { 788 #ifdef CONFIG_FS_DAX_PMD 789 pmd_t pmd; 790 791 if (pfn != pmd_pfn(*pmdp)) 792 goto unlock_pmd; 793 if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp)) 794 goto unlock_pmd; 795 796 flush_cache_page(vma, address, pfn); 797 pmd = pmdp_huge_clear_flush(vma, address, pmdp); 798 pmd = pmd_wrprotect(pmd); 799 pmd = pmd_mkclean(pmd); 800 set_pmd_at(vma->vm_mm, address, pmdp, pmd); 801 unlock_pmd: 802 #endif 803 spin_unlock(ptl); 804 } else { 805 if (pfn != pte_pfn(*ptep)) 806 goto unlock_pte; 807 if (!pte_dirty(*ptep) && !pte_write(*ptep)) 808 goto unlock_pte; 809 810 flush_cache_page(vma, address, pfn); 811 pte = ptep_clear_flush(vma, address, ptep); 812 pte = pte_wrprotect(pte); 813 pte = pte_mkclean(pte); 814 set_pte_at(vma->vm_mm, address, ptep, pte); 815 unlock_pte: 816 pte_unmap_unlock(ptep, ptl); 817 } 818 819 mmu_notifier_invalidate_range_end(vma->vm_mm, start, end); 820 } 821 i_mmap_unlock_read(mapping); 822 } 823 824 static int dax_writeback_one(struct dax_device *dax_dev, 825 struct address_space *mapping, pgoff_t index, void *entry) 826 { 827 struct radix_tree_root *pages = &mapping->i_pages; 828 void *entry2, **slot; 829 unsigned long pfn; 830 long ret = 0; 831 size_t size; 832 833 /* 834 * A page got tagged dirty in DAX mapping? Something is seriously 835 * wrong. 836 */ 837 if (WARN_ON(!radix_tree_exceptional_entry(entry))) 838 return -EIO; 839 840 xa_lock_irq(pages); 841 entry2 = get_unlocked_mapping_entry(mapping, index, &slot); 842 /* Entry got punched out / reallocated? */ 843 if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2))) 844 goto put_unlocked; 845 /* 846 * Entry got reallocated elsewhere? No need to writeback. We have to 847 * compare pfns as we must not bail out due to difference in lockbit 848 * or entry type. 849 */ 850 if (dax_radix_pfn(entry2) != dax_radix_pfn(entry)) 851 goto put_unlocked; 852 if (WARN_ON_ONCE(dax_is_empty_entry(entry) || 853 dax_is_zero_entry(entry))) { 854 ret = -EIO; 855 goto put_unlocked; 856 } 857 858 /* Another fsync thread may have already written back this entry */ 859 if (!radix_tree_tag_get(pages, index, PAGECACHE_TAG_TOWRITE)) 860 goto put_unlocked; 861 /* Lock the entry to serialize with page faults */ 862 entry = lock_slot(mapping, slot); 863 /* 864 * We can clear the tag now but we have to be careful so that concurrent 865 * dax_writeback_one() calls for the same index cannot finish before we 866 * actually flush the caches. This is achieved as the calls will look 867 * at the entry only under the i_pages lock and once they do that 868 * they will see the entry locked and wait for it to unlock. 869 */ 870 radix_tree_tag_clear(pages, index, PAGECACHE_TAG_TOWRITE); 871 xa_unlock_irq(pages); 872 873 /* 874 * Even if dax_writeback_mapping_range() was given a wbc->range_start 875 * in the middle of a PMD, the 'index' we are given will be aligned to 876 * the start index of the PMD, as will the pfn we pull from 'entry'. 877 * This allows us to flush for PMD_SIZE and not have to worry about 878 * partial PMD writebacks. 879 */ 880 pfn = dax_radix_pfn(entry); 881 size = PAGE_SIZE << dax_radix_order(entry); 882 883 dax_mapping_entry_mkclean(mapping, index, pfn); 884 dax_flush(dax_dev, page_address(pfn_to_page(pfn)), size); 885 /* 886 * After we have flushed the cache, we can clear the dirty tag. There 887 * cannot be new dirty data in the pfn after the flush has completed as 888 * the pfn mappings are writeprotected and fault waits for mapping 889 * entry lock. 890 */ 891 xa_lock_irq(pages); 892 radix_tree_tag_clear(pages, index, PAGECACHE_TAG_DIRTY); 893 xa_unlock_irq(pages); 894 trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT); 895 put_locked_mapping_entry(mapping, index); 896 return ret; 897 898 put_unlocked: 899 put_unlocked_mapping_entry(mapping, index, entry2); 900 xa_unlock_irq(pages); 901 return ret; 902 } 903 904 /* 905 * Flush the mapping to the persistent domain within the byte range of [start, 906 * end]. This is required by data integrity operations to ensure file data is 907 * on persistent storage prior to completion of the operation. 908 */ 909 int dax_writeback_mapping_range(struct address_space *mapping, 910 struct block_device *bdev, struct writeback_control *wbc) 911 { 912 struct inode *inode = mapping->host; 913 pgoff_t start_index, end_index; 914 pgoff_t indices[PAGEVEC_SIZE]; 915 struct dax_device *dax_dev; 916 struct pagevec pvec; 917 bool done = false; 918 int i, ret = 0; 919 920 if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT)) 921 return -EIO; 922 923 if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL) 924 return 0; 925 926 dax_dev = dax_get_by_host(bdev->bd_disk->disk_name); 927 if (!dax_dev) 928 return -EIO; 929 930 start_index = wbc->range_start >> PAGE_SHIFT; 931 end_index = wbc->range_end >> PAGE_SHIFT; 932 933 trace_dax_writeback_range(inode, start_index, end_index); 934 935 tag_pages_for_writeback(mapping, start_index, end_index); 936 937 pagevec_init(&pvec); 938 while (!done) { 939 pvec.nr = find_get_entries_tag(mapping, start_index, 940 PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE, 941 pvec.pages, indices); 942 943 if (pvec.nr == 0) 944 break; 945 946 for (i = 0; i < pvec.nr; i++) { 947 if (indices[i] > end_index) { 948 done = true; 949 break; 950 } 951 952 ret = dax_writeback_one(dax_dev, mapping, indices[i], 953 pvec.pages[i]); 954 if (ret < 0) { 955 mapping_set_error(mapping, ret); 956 goto out; 957 } 958 } 959 start_index = indices[pvec.nr - 1] + 1; 960 } 961 out: 962 put_dax(dax_dev); 963 trace_dax_writeback_range_done(inode, start_index, end_index); 964 return (ret < 0 ? ret : 0); 965 } 966 EXPORT_SYMBOL_GPL(dax_writeback_mapping_range); 967 968 static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos) 969 { 970 return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9; 971 } 972 973 static int dax_iomap_pfn(struct iomap *iomap, loff_t pos, size_t size, 974 pfn_t *pfnp) 975 { 976 const sector_t sector = dax_iomap_sector(iomap, pos); 977 pgoff_t pgoff; 978 void *kaddr; 979 int id, rc; 980 long length; 981 982 rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff); 983 if (rc) 984 return rc; 985 id = dax_read_lock(); 986 length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size), 987 &kaddr, pfnp); 988 if (length < 0) { 989 rc = length; 990 goto out; 991 } 992 rc = -EINVAL; 993 if (PFN_PHYS(length) < size) 994 goto out; 995 if (pfn_t_to_pfn(*pfnp) & (PHYS_PFN(size)-1)) 996 goto out; 997 /* For larger pages we need devmap */ 998 if (length > 1 && !pfn_t_devmap(*pfnp)) 999 goto out; 1000 rc = 0; 1001 out: 1002 dax_read_unlock(id); 1003 return rc; 1004 } 1005 1006 /* 1007 * The user has performed a load from a hole in the file. Allocating a new 1008 * page in the file would cause excessive storage usage for workloads with 1009 * sparse files. Instead we insert a read-only mapping of the 4k zero page. 1010 * If this page is ever written to we will re-fault and change the mapping to 1011 * point to real DAX storage instead. 1012 */ 1013 static vm_fault_t dax_load_hole(struct address_space *mapping, void *entry, 1014 struct vm_fault *vmf) 1015 { 1016 struct inode *inode = mapping->host; 1017 unsigned long vaddr = vmf->address; 1018 vm_fault_t ret = VM_FAULT_NOPAGE; 1019 struct page *zero_page; 1020 pfn_t pfn; 1021 1022 zero_page = ZERO_PAGE(0); 1023 if (unlikely(!zero_page)) { 1024 ret = VM_FAULT_OOM; 1025 goto out; 1026 } 1027 1028 pfn = page_to_pfn_t(zero_page); 1029 dax_insert_mapping_entry(mapping, vmf, entry, pfn, RADIX_DAX_ZERO_PAGE, 1030 false); 1031 ret = vmf_insert_mixed(vmf->vma, vaddr, pfn); 1032 out: 1033 trace_dax_load_hole(inode, vmf, ret); 1034 return ret; 1035 } 1036 1037 static bool dax_range_is_aligned(struct block_device *bdev, 1038 unsigned int offset, unsigned int length) 1039 { 1040 unsigned short sector_size = bdev_logical_block_size(bdev); 1041 1042 if (!IS_ALIGNED(offset, sector_size)) 1043 return false; 1044 if (!IS_ALIGNED(length, sector_size)) 1045 return false; 1046 1047 return true; 1048 } 1049 1050 int __dax_zero_page_range(struct block_device *bdev, 1051 struct dax_device *dax_dev, sector_t sector, 1052 unsigned int offset, unsigned int size) 1053 { 1054 if (dax_range_is_aligned(bdev, offset, size)) { 1055 sector_t start_sector = sector + (offset >> 9); 1056 1057 return blkdev_issue_zeroout(bdev, start_sector, 1058 size >> 9, GFP_NOFS, 0); 1059 } else { 1060 pgoff_t pgoff; 1061 long rc, id; 1062 void *kaddr; 1063 pfn_t pfn; 1064 1065 rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff); 1066 if (rc) 1067 return rc; 1068 1069 id = dax_read_lock(); 1070 rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr, 1071 &pfn); 1072 if (rc < 0) { 1073 dax_read_unlock(id); 1074 return rc; 1075 } 1076 memset(kaddr + offset, 0, size); 1077 dax_flush(dax_dev, kaddr + offset, size); 1078 dax_read_unlock(id); 1079 } 1080 return 0; 1081 } 1082 EXPORT_SYMBOL_GPL(__dax_zero_page_range); 1083 1084 static loff_t 1085 dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data, 1086 struct iomap *iomap) 1087 { 1088 struct block_device *bdev = iomap->bdev; 1089 struct dax_device *dax_dev = iomap->dax_dev; 1090 struct iov_iter *iter = data; 1091 loff_t end = pos + length, done = 0; 1092 ssize_t ret = 0; 1093 size_t xfer; 1094 int id; 1095 1096 if (iov_iter_rw(iter) == READ) { 1097 end = min(end, i_size_read(inode)); 1098 if (pos >= end) 1099 return 0; 1100 1101 if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN) 1102 return iov_iter_zero(min(length, end - pos), iter); 1103 } 1104 1105 if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED)) 1106 return -EIO; 1107 1108 /* 1109 * Write can allocate block for an area which has a hole page mapped 1110 * into page tables. We have to tear down these mappings so that data 1111 * written by write(2) is visible in mmap. 1112 */ 1113 if (iomap->flags & IOMAP_F_NEW) { 1114 invalidate_inode_pages2_range(inode->i_mapping, 1115 pos >> PAGE_SHIFT, 1116 (end - 1) >> PAGE_SHIFT); 1117 } 1118 1119 id = dax_read_lock(); 1120 while (pos < end) { 1121 unsigned offset = pos & (PAGE_SIZE - 1); 1122 const size_t size = ALIGN(length + offset, PAGE_SIZE); 1123 const sector_t sector = dax_iomap_sector(iomap, pos); 1124 ssize_t map_len; 1125 pgoff_t pgoff; 1126 void *kaddr; 1127 pfn_t pfn; 1128 1129 if (fatal_signal_pending(current)) { 1130 ret = -EINTR; 1131 break; 1132 } 1133 1134 ret = bdev_dax_pgoff(bdev, sector, size, &pgoff); 1135 if (ret) 1136 break; 1137 1138 map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), 1139 &kaddr, &pfn); 1140 if (map_len < 0) { 1141 ret = map_len; 1142 break; 1143 } 1144 1145 map_len = PFN_PHYS(map_len); 1146 kaddr += offset; 1147 map_len -= offset; 1148 if (map_len > end - pos) 1149 map_len = end - pos; 1150 1151 /* 1152 * The userspace address for the memory copy has already been 1153 * validated via access_ok() in either vfs_read() or 1154 * vfs_write(), depending on which operation we are doing. 1155 */ 1156 if (iov_iter_rw(iter) == WRITE) 1157 xfer = dax_copy_from_iter(dax_dev, pgoff, kaddr, 1158 map_len, iter); 1159 else 1160 xfer = dax_copy_to_iter(dax_dev, pgoff, kaddr, 1161 map_len, iter); 1162 1163 pos += xfer; 1164 length -= xfer; 1165 done += xfer; 1166 1167 if (xfer == 0) 1168 ret = -EFAULT; 1169 if (xfer < map_len) 1170 break; 1171 } 1172 dax_read_unlock(id); 1173 1174 return done ? done : ret; 1175 } 1176 1177 /** 1178 * dax_iomap_rw - Perform I/O to a DAX file 1179 * @iocb: The control block for this I/O 1180 * @iter: The addresses to do I/O from or to 1181 * @ops: iomap ops passed from the file system 1182 * 1183 * This function performs read and write operations to directly mapped 1184 * persistent memory. The callers needs to take care of read/write exclusion 1185 * and evicting any page cache pages in the region under I/O. 1186 */ 1187 ssize_t 1188 dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, 1189 const struct iomap_ops *ops) 1190 { 1191 struct address_space *mapping = iocb->ki_filp->f_mapping; 1192 struct inode *inode = mapping->host; 1193 loff_t pos = iocb->ki_pos, ret = 0, done = 0; 1194 unsigned flags = 0; 1195 1196 if (iov_iter_rw(iter) == WRITE) { 1197 lockdep_assert_held_exclusive(&inode->i_rwsem); 1198 flags |= IOMAP_WRITE; 1199 } else { 1200 lockdep_assert_held(&inode->i_rwsem); 1201 } 1202 1203 while (iov_iter_count(iter)) { 1204 ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops, 1205 iter, dax_iomap_actor); 1206 if (ret <= 0) 1207 break; 1208 pos += ret; 1209 done += ret; 1210 } 1211 1212 iocb->ki_pos += done; 1213 return done ? done : ret; 1214 } 1215 EXPORT_SYMBOL_GPL(dax_iomap_rw); 1216 1217 static vm_fault_t dax_fault_return(int error) 1218 { 1219 if (error == 0) 1220 return VM_FAULT_NOPAGE; 1221 if (error == -ENOMEM) 1222 return VM_FAULT_OOM; 1223 return VM_FAULT_SIGBUS; 1224 } 1225 1226 /* 1227 * MAP_SYNC on a dax mapping guarantees dirty metadata is 1228 * flushed on write-faults (non-cow), but not read-faults. 1229 */ 1230 static bool dax_fault_is_synchronous(unsigned long flags, 1231 struct vm_area_struct *vma, struct iomap *iomap) 1232 { 1233 return (flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC) 1234 && (iomap->flags & IOMAP_F_DIRTY); 1235 } 1236 1237 static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp, 1238 int *iomap_errp, const struct iomap_ops *ops) 1239 { 1240 struct vm_area_struct *vma = vmf->vma; 1241 struct address_space *mapping = vma->vm_file->f_mapping; 1242 struct inode *inode = mapping->host; 1243 unsigned long vaddr = vmf->address; 1244 loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT; 1245 struct iomap iomap = { 0 }; 1246 unsigned flags = IOMAP_FAULT; 1247 int error, major = 0; 1248 bool write = vmf->flags & FAULT_FLAG_WRITE; 1249 bool sync; 1250 vm_fault_t ret = 0; 1251 void *entry; 1252 pfn_t pfn; 1253 1254 trace_dax_pte_fault(inode, vmf, ret); 1255 /* 1256 * Check whether offset isn't beyond end of file now. Caller is supposed 1257 * to hold locks serializing us with truncate / punch hole so this is 1258 * a reliable test. 1259 */ 1260 if (pos >= i_size_read(inode)) { 1261 ret = VM_FAULT_SIGBUS; 1262 goto out; 1263 } 1264 1265 if (write && !vmf->cow_page) 1266 flags |= IOMAP_WRITE; 1267 1268 entry = grab_mapping_entry(mapping, vmf->pgoff, 0); 1269 if (IS_ERR(entry)) { 1270 ret = dax_fault_return(PTR_ERR(entry)); 1271 goto out; 1272 } 1273 1274 /* 1275 * It is possible, particularly with mixed reads & writes to private 1276 * mappings, that we have raced with a PMD fault that overlaps with 1277 * the PTE we need to set up. If so just return and the fault will be 1278 * retried. 1279 */ 1280 if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) { 1281 ret = VM_FAULT_NOPAGE; 1282 goto unlock_entry; 1283 } 1284 1285 /* 1286 * Note that we don't bother to use iomap_apply here: DAX required 1287 * the file system block size to be equal the page size, which means 1288 * that we never have to deal with more than a single extent here. 1289 */ 1290 error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap); 1291 if (iomap_errp) 1292 *iomap_errp = error; 1293 if (error) { 1294 ret = dax_fault_return(error); 1295 goto unlock_entry; 1296 } 1297 if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) { 1298 error = -EIO; /* fs corruption? */ 1299 goto error_finish_iomap; 1300 } 1301 1302 if (vmf->cow_page) { 1303 sector_t sector = dax_iomap_sector(&iomap, pos); 1304 1305 switch (iomap.type) { 1306 case IOMAP_HOLE: 1307 case IOMAP_UNWRITTEN: 1308 clear_user_highpage(vmf->cow_page, vaddr); 1309 break; 1310 case IOMAP_MAPPED: 1311 error = copy_user_dax(iomap.bdev, iomap.dax_dev, 1312 sector, PAGE_SIZE, vmf->cow_page, vaddr); 1313 break; 1314 default: 1315 WARN_ON_ONCE(1); 1316 error = -EIO; 1317 break; 1318 } 1319 1320 if (error) 1321 goto error_finish_iomap; 1322 1323 __SetPageUptodate(vmf->cow_page); 1324 ret = finish_fault(vmf); 1325 if (!ret) 1326 ret = VM_FAULT_DONE_COW; 1327 goto finish_iomap; 1328 } 1329 1330 sync = dax_fault_is_synchronous(flags, vma, &iomap); 1331 1332 switch (iomap.type) { 1333 case IOMAP_MAPPED: 1334 if (iomap.flags & IOMAP_F_NEW) { 1335 count_vm_event(PGMAJFAULT); 1336 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT); 1337 major = VM_FAULT_MAJOR; 1338 } 1339 error = dax_iomap_pfn(&iomap, pos, PAGE_SIZE, &pfn); 1340 if (error < 0) 1341 goto error_finish_iomap; 1342 1343 entry = dax_insert_mapping_entry(mapping, vmf, entry, pfn, 1344 0, write && !sync); 1345 1346 /* 1347 * If we are doing synchronous page fault and inode needs fsync, 1348 * we can insert PTE into page tables only after that happens. 1349 * Skip insertion for now and return the pfn so that caller can 1350 * insert it after fsync is done. 1351 */ 1352 if (sync) { 1353 if (WARN_ON_ONCE(!pfnp)) { 1354 error = -EIO; 1355 goto error_finish_iomap; 1356 } 1357 *pfnp = pfn; 1358 ret = VM_FAULT_NEEDDSYNC | major; 1359 goto finish_iomap; 1360 } 1361 trace_dax_insert_mapping(inode, vmf, entry); 1362 if (write) 1363 ret = vmf_insert_mixed_mkwrite(vma, vaddr, pfn); 1364 else 1365 ret = vmf_insert_mixed(vma, vaddr, pfn); 1366 1367 goto finish_iomap; 1368 case IOMAP_UNWRITTEN: 1369 case IOMAP_HOLE: 1370 if (!write) { 1371 ret = dax_load_hole(mapping, entry, vmf); 1372 goto finish_iomap; 1373 } 1374 /*FALLTHRU*/ 1375 default: 1376 WARN_ON_ONCE(1); 1377 error = -EIO; 1378 break; 1379 } 1380 1381 error_finish_iomap: 1382 ret = dax_fault_return(error); 1383 finish_iomap: 1384 if (ops->iomap_end) { 1385 int copied = PAGE_SIZE; 1386 1387 if (ret & VM_FAULT_ERROR) 1388 copied = 0; 1389 /* 1390 * The fault is done by now and there's no way back (other 1391 * thread may be already happily using PTE we have installed). 1392 * Just ignore error from ->iomap_end since we cannot do much 1393 * with it. 1394 */ 1395 ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap); 1396 } 1397 unlock_entry: 1398 put_locked_mapping_entry(mapping, vmf->pgoff); 1399 out: 1400 trace_dax_pte_fault_done(inode, vmf, ret); 1401 return ret | major; 1402 } 1403 1404 #ifdef CONFIG_FS_DAX_PMD 1405 static vm_fault_t dax_pmd_load_hole(struct vm_fault *vmf, struct iomap *iomap, 1406 void *entry) 1407 { 1408 struct address_space *mapping = vmf->vma->vm_file->f_mapping; 1409 unsigned long pmd_addr = vmf->address & PMD_MASK; 1410 struct inode *inode = mapping->host; 1411 struct page *zero_page; 1412 void *ret = NULL; 1413 spinlock_t *ptl; 1414 pmd_t pmd_entry; 1415 pfn_t pfn; 1416 1417 zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm); 1418 1419 if (unlikely(!zero_page)) 1420 goto fallback; 1421 1422 pfn = page_to_pfn_t(zero_page); 1423 ret = dax_insert_mapping_entry(mapping, vmf, entry, pfn, 1424 RADIX_DAX_PMD | RADIX_DAX_ZERO_PAGE, false); 1425 1426 ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); 1427 if (!pmd_none(*(vmf->pmd))) { 1428 spin_unlock(ptl); 1429 goto fallback; 1430 } 1431 1432 pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot); 1433 pmd_entry = pmd_mkhuge(pmd_entry); 1434 set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry); 1435 spin_unlock(ptl); 1436 trace_dax_pmd_load_hole(inode, vmf, zero_page, ret); 1437 return VM_FAULT_NOPAGE; 1438 1439 fallback: 1440 trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, ret); 1441 return VM_FAULT_FALLBACK; 1442 } 1443 1444 static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, 1445 const struct iomap_ops *ops) 1446 { 1447 struct vm_area_struct *vma = vmf->vma; 1448 struct address_space *mapping = vma->vm_file->f_mapping; 1449 unsigned long pmd_addr = vmf->address & PMD_MASK; 1450 bool write = vmf->flags & FAULT_FLAG_WRITE; 1451 bool sync; 1452 unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT; 1453 struct inode *inode = mapping->host; 1454 vm_fault_t result = VM_FAULT_FALLBACK; 1455 struct iomap iomap = { 0 }; 1456 pgoff_t max_pgoff, pgoff; 1457 void *entry; 1458 loff_t pos; 1459 int error; 1460 pfn_t pfn; 1461 1462 /* 1463 * Check whether offset isn't beyond end of file now. Caller is 1464 * supposed to hold locks serializing us with truncate / punch hole so 1465 * this is a reliable test. 1466 */ 1467 pgoff = linear_page_index(vma, pmd_addr); 1468 max_pgoff = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 1469 1470 trace_dax_pmd_fault(inode, vmf, max_pgoff, 0); 1471 1472 /* 1473 * Make sure that the faulting address's PMD offset (color) matches 1474 * the PMD offset from the start of the file. This is necessary so 1475 * that a PMD range in the page table overlaps exactly with a PMD 1476 * range in the radix tree. 1477 */ 1478 if ((vmf->pgoff & PG_PMD_COLOUR) != 1479 ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR)) 1480 goto fallback; 1481 1482 /* Fall back to PTEs if we're going to COW */ 1483 if (write && !(vma->vm_flags & VM_SHARED)) 1484 goto fallback; 1485 1486 /* If the PMD would extend outside the VMA */ 1487 if (pmd_addr < vma->vm_start) 1488 goto fallback; 1489 if ((pmd_addr + PMD_SIZE) > vma->vm_end) 1490 goto fallback; 1491 1492 if (pgoff >= max_pgoff) { 1493 result = VM_FAULT_SIGBUS; 1494 goto out; 1495 } 1496 1497 /* If the PMD would extend beyond the file size */ 1498 if ((pgoff | PG_PMD_COLOUR) >= max_pgoff) 1499 goto fallback; 1500 1501 /* 1502 * grab_mapping_entry() will make sure we get a 2MiB empty entry, a 1503 * 2MiB zero page entry or a DAX PMD. If it can't (because a 4k page 1504 * is already in the tree, for instance), it will return -EEXIST and 1505 * we just fall back to 4k entries. 1506 */ 1507 entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD); 1508 if (IS_ERR(entry)) 1509 goto fallback; 1510 1511 /* 1512 * It is possible, particularly with mixed reads & writes to private 1513 * mappings, that we have raced with a PTE fault that overlaps with 1514 * the PMD we need to set up. If so just return and the fault will be 1515 * retried. 1516 */ 1517 if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) && 1518 !pmd_devmap(*vmf->pmd)) { 1519 result = 0; 1520 goto unlock_entry; 1521 } 1522 1523 /* 1524 * Note that we don't use iomap_apply here. We aren't doing I/O, only 1525 * setting up a mapping, so really we're using iomap_begin() as a way 1526 * to look up our filesystem block. 1527 */ 1528 pos = (loff_t)pgoff << PAGE_SHIFT; 1529 error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap); 1530 if (error) 1531 goto unlock_entry; 1532 1533 if (iomap.offset + iomap.length < pos + PMD_SIZE) 1534 goto finish_iomap; 1535 1536 sync = dax_fault_is_synchronous(iomap_flags, vma, &iomap); 1537 1538 switch (iomap.type) { 1539 case IOMAP_MAPPED: 1540 error = dax_iomap_pfn(&iomap, pos, PMD_SIZE, &pfn); 1541 if (error < 0) 1542 goto finish_iomap; 1543 1544 entry = dax_insert_mapping_entry(mapping, vmf, entry, pfn, 1545 RADIX_DAX_PMD, write && !sync); 1546 1547 /* 1548 * If we are doing synchronous page fault and inode needs fsync, 1549 * we can insert PMD into page tables only after that happens. 1550 * Skip insertion for now and return the pfn so that caller can 1551 * insert it after fsync is done. 1552 */ 1553 if (sync) { 1554 if (WARN_ON_ONCE(!pfnp)) 1555 goto finish_iomap; 1556 *pfnp = pfn; 1557 result = VM_FAULT_NEEDDSYNC; 1558 goto finish_iomap; 1559 } 1560 1561 trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry); 1562 result = vmf_insert_pfn_pmd(vma, vmf->address, vmf->pmd, pfn, 1563 write); 1564 break; 1565 case IOMAP_UNWRITTEN: 1566 case IOMAP_HOLE: 1567 if (WARN_ON_ONCE(write)) 1568 break; 1569 result = dax_pmd_load_hole(vmf, &iomap, entry); 1570 break; 1571 default: 1572 WARN_ON_ONCE(1); 1573 break; 1574 } 1575 1576 finish_iomap: 1577 if (ops->iomap_end) { 1578 int copied = PMD_SIZE; 1579 1580 if (result == VM_FAULT_FALLBACK) 1581 copied = 0; 1582 /* 1583 * The fault is done by now and there's no way back (other 1584 * thread may be already happily using PMD we have installed). 1585 * Just ignore error from ->iomap_end since we cannot do much 1586 * with it. 1587 */ 1588 ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags, 1589 &iomap); 1590 } 1591 unlock_entry: 1592 put_locked_mapping_entry(mapping, pgoff); 1593 fallback: 1594 if (result == VM_FAULT_FALLBACK) { 1595 split_huge_pmd(vma, vmf->pmd, vmf->address); 1596 count_vm_event(THP_FAULT_FALLBACK); 1597 } 1598 out: 1599 trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result); 1600 return result; 1601 } 1602 #else 1603 static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, 1604 const struct iomap_ops *ops) 1605 { 1606 return VM_FAULT_FALLBACK; 1607 } 1608 #endif /* CONFIG_FS_DAX_PMD */ 1609 1610 /** 1611 * dax_iomap_fault - handle a page fault on a DAX file 1612 * @vmf: The description of the fault 1613 * @pe_size: Size of the page to fault in 1614 * @pfnp: PFN to insert for synchronous faults if fsync is required 1615 * @iomap_errp: Storage for detailed error code in case of error 1616 * @ops: Iomap ops passed from the file system 1617 * 1618 * When a page fault occurs, filesystems may call this helper in 1619 * their fault handler for DAX files. dax_iomap_fault() assumes the caller 1620 * has done all the necessary locking for page fault to proceed 1621 * successfully. 1622 */ 1623 vm_fault_t dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size, 1624 pfn_t *pfnp, int *iomap_errp, const struct iomap_ops *ops) 1625 { 1626 switch (pe_size) { 1627 case PE_SIZE_PTE: 1628 return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, ops); 1629 case PE_SIZE_PMD: 1630 return dax_iomap_pmd_fault(vmf, pfnp, ops); 1631 default: 1632 return VM_FAULT_FALLBACK; 1633 } 1634 } 1635 EXPORT_SYMBOL_GPL(dax_iomap_fault); 1636 1637 /** 1638 * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables 1639 * @vmf: The description of the fault 1640 * @pe_size: Size of entry to be inserted 1641 * @pfn: PFN to insert 1642 * 1643 * This function inserts writeable PTE or PMD entry into page tables for mmaped 1644 * DAX file. It takes care of marking corresponding radix tree entry as dirty 1645 * as well. 1646 */ 1647 static vm_fault_t dax_insert_pfn_mkwrite(struct vm_fault *vmf, 1648 enum page_entry_size pe_size, 1649 pfn_t pfn) 1650 { 1651 struct address_space *mapping = vmf->vma->vm_file->f_mapping; 1652 void *entry, **slot; 1653 pgoff_t index = vmf->pgoff; 1654 vm_fault_t ret; 1655 1656 xa_lock_irq(&mapping->i_pages); 1657 entry = get_unlocked_mapping_entry(mapping, index, &slot); 1658 /* Did we race with someone splitting entry or so? */ 1659 if (!entry || 1660 (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) || 1661 (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) { 1662 put_unlocked_mapping_entry(mapping, index, entry); 1663 xa_unlock_irq(&mapping->i_pages); 1664 trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf, 1665 VM_FAULT_NOPAGE); 1666 return VM_FAULT_NOPAGE; 1667 } 1668 radix_tree_tag_set(&mapping->i_pages, index, PAGECACHE_TAG_DIRTY); 1669 entry = lock_slot(mapping, slot); 1670 xa_unlock_irq(&mapping->i_pages); 1671 switch (pe_size) { 1672 case PE_SIZE_PTE: 1673 ret = vmf_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn); 1674 break; 1675 #ifdef CONFIG_FS_DAX_PMD 1676 case PE_SIZE_PMD: 1677 ret = vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, 1678 pfn, true); 1679 break; 1680 #endif 1681 default: 1682 ret = VM_FAULT_FALLBACK; 1683 } 1684 put_locked_mapping_entry(mapping, index); 1685 trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret); 1686 return ret; 1687 } 1688 1689 /** 1690 * dax_finish_sync_fault - finish synchronous page fault 1691 * @vmf: The description of the fault 1692 * @pe_size: Size of entry to be inserted 1693 * @pfn: PFN to insert 1694 * 1695 * This function ensures that the file range touched by the page fault is 1696 * stored persistently on the media and handles inserting of appropriate page 1697 * table entry. 1698 */ 1699 vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, 1700 enum page_entry_size pe_size, pfn_t pfn) 1701 { 1702 int err; 1703 loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT; 1704 size_t len = 0; 1705 1706 if (pe_size == PE_SIZE_PTE) 1707 len = PAGE_SIZE; 1708 else if (pe_size == PE_SIZE_PMD) 1709 len = PMD_SIZE; 1710 else 1711 WARN_ON_ONCE(1); 1712 err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1); 1713 if (err) 1714 return VM_FAULT_SIGBUS; 1715 return dax_insert_pfn_mkwrite(vmf, pe_size, pfn); 1716 } 1717 EXPORT_SYMBOL_GPL(dax_finish_sync_fault); 1718