1 /* 2 * fs/dax.c - Direct Access filesystem code 3 * Copyright (c) 2013-2014 Intel Corporation 4 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com> 5 * Author: Ross Zwisler <ross.zwisler@linux.intel.com> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms and conditions of the GNU General Public License, 9 * version 2, as published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 * more details. 15 */ 16 17 #include <linux/atomic.h> 18 #include <linux/blkdev.h> 19 #include <linux/buffer_head.h> 20 #include <linux/dax.h> 21 #include <linux/fs.h> 22 #include <linux/genhd.h> 23 #include <linux/highmem.h> 24 #include <linux/memcontrol.h> 25 #include <linux/mm.h> 26 #include <linux/mutex.h> 27 #include <linux/pagevec.h> 28 #include <linux/sched.h> 29 #include <linux/sched/signal.h> 30 #include <linux/uio.h> 31 #include <linux/vmstat.h> 32 #include <linux/pfn_t.h> 33 #include <linux/sizes.h> 34 #include <linux/mmu_notifier.h> 35 #include <linux/iomap.h> 36 #include "internal.h" 37 38 #define CREATE_TRACE_POINTS 39 #include <trace/events/fs_dax.h> 40 41 /* We choose 4096 entries - same as per-zone page wait tables */ 42 #define DAX_WAIT_TABLE_BITS 12 43 #define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS) 44 45 /* The 'colour' (ie low bits) within a PMD of a page offset. */ 46 #define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1) 47 48 static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES]; 49 50 static int __init init_dax_wait_table(void) 51 { 52 int i; 53 54 for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++) 55 init_waitqueue_head(wait_table + i); 56 return 0; 57 } 58 fs_initcall(init_dax_wait_table); 59 60 /* 61 * We use lowest available bit in exceptional entry for locking, one bit for 62 * the entry size (PMD) and two more to tell us if the entry is a zero page or 63 * an empty entry that is just used for locking. In total four special bits. 64 * 65 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE 66 * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem 67 * block allocation. 68 */ 69 #define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 4) 70 #define RADIX_DAX_ENTRY_LOCK (1 << RADIX_TREE_EXCEPTIONAL_SHIFT) 71 #define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1)) 72 #define RADIX_DAX_ZERO_PAGE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2)) 73 #define RADIX_DAX_EMPTY (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3)) 74 75 static unsigned long dax_radix_sector(void *entry) 76 { 77 return (unsigned long)entry >> RADIX_DAX_SHIFT; 78 } 79 80 static void *dax_radix_locked_entry(sector_t sector, unsigned long flags) 81 { 82 return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags | 83 ((unsigned long)sector << RADIX_DAX_SHIFT) | 84 RADIX_DAX_ENTRY_LOCK); 85 } 86 87 static unsigned int dax_radix_order(void *entry) 88 { 89 if ((unsigned long)entry & RADIX_DAX_PMD) 90 return PMD_SHIFT - PAGE_SHIFT; 91 return 0; 92 } 93 94 static int dax_is_pmd_entry(void *entry) 95 { 96 return (unsigned long)entry & RADIX_DAX_PMD; 97 } 98 99 static int dax_is_pte_entry(void *entry) 100 { 101 return !((unsigned long)entry & RADIX_DAX_PMD); 102 } 103 104 static int dax_is_zero_entry(void *entry) 105 { 106 return (unsigned long)entry & RADIX_DAX_ZERO_PAGE; 107 } 108 109 static int dax_is_empty_entry(void *entry) 110 { 111 return (unsigned long)entry & RADIX_DAX_EMPTY; 112 } 113 114 /* 115 * DAX radix tree locking 116 */ 117 struct exceptional_entry_key { 118 struct address_space *mapping; 119 pgoff_t entry_start; 120 }; 121 122 struct wait_exceptional_entry_queue { 123 wait_queue_entry_t wait; 124 struct exceptional_entry_key key; 125 }; 126 127 static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping, 128 pgoff_t index, void *entry, struct exceptional_entry_key *key) 129 { 130 unsigned long hash; 131 132 /* 133 * If 'entry' is a PMD, align the 'index' that we use for the wait 134 * queue to the start of that PMD. This ensures that all offsets in 135 * the range covered by the PMD map to the same bit lock. 136 */ 137 if (dax_is_pmd_entry(entry)) 138 index &= ~PG_PMD_COLOUR; 139 140 key->mapping = mapping; 141 key->entry_start = index; 142 143 hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS); 144 return wait_table + hash; 145 } 146 147 static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mode, 148 int sync, void *keyp) 149 { 150 struct exceptional_entry_key *key = keyp; 151 struct wait_exceptional_entry_queue *ewait = 152 container_of(wait, struct wait_exceptional_entry_queue, wait); 153 154 if (key->mapping != ewait->key.mapping || 155 key->entry_start != ewait->key.entry_start) 156 return 0; 157 return autoremove_wake_function(wait, mode, sync, NULL); 158 } 159 160 /* 161 * We do not necessarily hold the mapping->tree_lock when we call this 162 * function so it is possible that 'entry' is no longer a valid item in the 163 * radix tree. This is okay because all we really need to do is to find the 164 * correct waitqueue where tasks might be waiting for that old 'entry' and 165 * wake them. 166 */ 167 static void dax_wake_mapping_entry_waiter(struct address_space *mapping, 168 pgoff_t index, void *entry, bool wake_all) 169 { 170 struct exceptional_entry_key key; 171 wait_queue_head_t *wq; 172 173 wq = dax_entry_waitqueue(mapping, index, entry, &key); 174 175 /* 176 * Checking for locked entry and prepare_to_wait_exclusive() happens 177 * under mapping->tree_lock, ditto for entry handling in our callers. 178 * So at this point all tasks that could have seen our entry locked 179 * must be in the waitqueue and the following check will see them. 180 */ 181 if (waitqueue_active(wq)) 182 __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key); 183 } 184 185 /* 186 * Check whether the given slot is locked. The function must be called with 187 * mapping->tree_lock held 188 */ 189 static inline int slot_locked(struct address_space *mapping, void **slot) 190 { 191 unsigned long entry = (unsigned long) 192 radix_tree_deref_slot_protected(slot, &mapping->tree_lock); 193 return entry & RADIX_DAX_ENTRY_LOCK; 194 } 195 196 /* 197 * Mark the given slot is locked. The function must be called with 198 * mapping->tree_lock held 199 */ 200 static inline void *lock_slot(struct address_space *mapping, void **slot) 201 { 202 unsigned long entry = (unsigned long) 203 radix_tree_deref_slot_protected(slot, &mapping->tree_lock); 204 205 entry |= RADIX_DAX_ENTRY_LOCK; 206 radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry); 207 return (void *)entry; 208 } 209 210 /* 211 * Mark the given slot is unlocked. The function must be called with 212 * mapping->tree_lock held 213 */ 214 static inline void *unlock_slot(struct address_space *mapping, void **slot) 215 { 216 unsigned long entry = (unsigned long) 217 radix_tree_deref_slot_protected(slot, &mapping->tree_lock); 218 219 entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK; 220 radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry); 221 return (void *)entry; 222 } 223 224 /* 225 * Lookup entry in radix tree, wait for it to become unlocked if it is 226 * exceptional entry and return it. The caller must call 227 * put_unlocked_mapping_entry() when he decided not to lock the entry or 228 * put_locked_mapping_entry() when he locked the entry and now wants to 229 * unlock it. 230 * 231 * The function must be called with mapping->tree_lock held. 232 */ 233 static void *get_unlocked_mapping_entry(struct address_space *mapping, 234 pgoff_t index, void ***slotp) 235 { 236 void *entry, **slot; 237 struct wait_exceptional_entry_queue ewait; 238 wait_queue_head_t *wq; 239 240 init_wait(&ewait.wait); 241 ewait.wait.func = wake_exceptional_entry_func; 242 243 for (;;) { 244 entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, 245 &slot); 246 if (!entry || 247 WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) || 248 !slot_locked(mapping, slot)) { 249 if (slotp) 250 *slotp = slot; 251 return entry; 252 } 253 254 wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key); 255 prepare_to_wait_exclusive(wq, &ewait.wait, 256 TASK_UNINTERRUPTIBLE); 257 spin_unlock_irq(&mapping->tree_lock); 258 schedule(); 259 finish_wait(wq, &ewait.wait); 260 spin_lock_irq(&mapping->tree_lock); 261 } 262 } 263 264 static void dax_unlock_mapping_entry(struct address_space *mapping, 265 pgoff_t index) 266 { 267 void *entry, **slot; 268 269 spin_lock_irq(&mapping->tree_lock); 270 entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot); 271 if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) || 272 !slot_locked(mapping, slot))) { 273 spin_unlock_irq(&mapping->tree_lock); 274 return; 275 } 276 unlock_slot(mapping, slot); 277 spin_unlock_irq(&mapping->tree_lock); 278 dax_wake_mapping_entry_waiter(mapping, index, entry, false); 279 } 280 281 static void put_locked_mapping_entry(struct address_space *mapping, 282 pgoff_t index) 283 { 284 dax_unlock_mapping_entry(mapping, index); 285 } 286 287 /* 288 * Called when we are done with radix tree entry we looked up via 289 * get_unlocked_mapping_entry() and which we didn't lock in the end. 290 */ 291 static void put_unlocked_mapping_entry(struct address_space *mapping, 292 pgoff_t index, void *entry) 293 { 294 if (!entry) 295 return; 296 297 /* We have to wake up next waiter for the radix tree entry lock */ 298 dax_wake_mapping_entry_waiter(mapping, index, entry, false); 299 } 300 301 /* 302 * Find radix tree entry at given index. If it points to an exceptional entry, 303 * return it with the radix tree entry locked. If the radix tree doesn't 304 * contain given index, create an empty exceptional entry for the index and 305 * return with it locked. 306 * 307 * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will 308 * either return that locked entry or will return an error. This error will 309 * happen if there are any 4k entries within the 2MiB range that we are 310 * requesting. 311 * 312 * We always favor 4k entries over 2MiB entries. There isn't a flow where we 313 * evict 4k entries in order to 'upgrade' them to a 2MiB entry. A 2MiB 314 * insertion will fail if it finds any 4k entries already in the tree, and a 315 * 4k insertion will cause an existing 2MiB entry to be unmapped and 316 * downgraded to 4k entries. This happens for both 2MiB huge zero pages as 317 * well as 2MiB empty entries. 318 * 319 * The exception to this downgrade path is for 2MiB DAX PMD entries that have 320 * real storage backing them. We will leave these real 2MiB DAX entries in 321 * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry. 322 * 323 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For 324 * persistent memory the benefit is doubtful. We can add that later if we can 325 * show it helps. 326 */ 327 static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index, 328 unsigned long size_flag) 329 { 330 bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */ 331 void *entry, **slot; 332 333 restart: 334 spin_lock_irq(&mapping->tree_lock); 335 entry = get_unlocked_mapping_entry(mapping, index, &slot); 336 337 if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) { 338 entry = ERR_PTR(-EIO); 339 goto out_unlock; 340 } 341 342 if (entry) { 343 if (size_flag & RADIX_DAX_PMD) { 344 if (dax_is_pte_entry(entry)) { 345 put_unlocked_mapping_entry(mapping, index, 346 entry); 347 entry = ERR_PTR(-EEXIST); 348 goto out_unlock; 349 } 350 } else { /* trying to grab a PTE entry */ 351 if (dax_is_pmd_entry(entry) && 352 (dax_is_zero_entry(entry) || 353 dax_is_empty_entry(entry))) { 354 pmd_downgrade = true; 355 } 356 } 357 } 358 359 /* No entry for given index? Make sure radix tree is big enough. */ 360 if (!entry || pmd_downgrade) { 361 int err; 362 363 if (pmd_downgrade) { 364 /* 365 * Make sure 'entry' remains valid while we drop 366 * mapping->tree_lock. 367 */ 368 entry = lock_slot(mapping, slot); 369 } 370 371 spin_unlock_irq(&mapping->tree_lock); 372 /* 373 * Besides huge zero pages the only other thing that gets 374 * downgraded are empty entries which don't need to be 375 * unmapped. 376 */ 377 if (pmd_downgrade && dax_is_zero_entry(entry)) 378 unmap_mapping_range(mapping, 379 (index << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0); 380 381 err = radix_tree_preload( 382 mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM); 383 if (err) { 384 if (pmd_downgrade) 385 put_locked_mapping_entry(mapping, index); 386 return ERR_PTR(err); 387 } 388 spin_lock_irq(&mapping->tree_lock); 389 390 if (!entry) { 391 /* 392 * We needed to drop the page_tree lock while calling 393 * radix_tree_preload() and we didn't have an entry to 394 * lock. See if another thread inserted an entry at 395 * our index during this time. 396 */ 397 entry = __radix_tree_lookup(&mapping->page_tree, index, 398 NULL, &slot); 399 if (entry) { 400 radix_tree_preload_end(); 401 spin_unlock_irq(&mapping->tree_lock); 402 goto restart; 403 } 404 } 405 406 if (pmd_downgrade) { 407 radix_tree_delete(&mapping->page_tree, index); 408 mapping->nrexceptional--; 409 dax_wake_mapping_entry_waiter(mapping, index, entry, 410 true); 411 } 412 413 entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY); 414 415 err = __radix_tree_insert(&mapping->page_tree, index, 416 dax_radix_order(entry), entry); 417 radix_tree_preload_end(); 418 if (err) { 419 spin_unlock_irq(&mapping->tree_lock); 420 /* 421 * Our insertion of a DAX entry failed, most likely 422 * because we were inserting a PMD entry and it 423 * collided with a PTE sized entry at a different 424 * index in the PMD range. We haven't inserted 425 * anything into the radix tree and have no waiters to 426 * wake. 427 */ 428 return ERR_PTR(err); 429 } 430 /* Good, we have inserted empty locked entry into the tree. */ 431 mapping->nrexceptional++; 432 spin_unlock_irq(&mapping->tree_lock); 433 return entry; 434 } 435 entry = lock_slot(mapping, slot); 436 out_unlock: 437 spin_unlock_irq(&mapping->tree_lock); 438 return entry; 439 } 440 441 static int __dax_invalidate_mapping_entry(struct address_space *mapping, 442 pgoff_t index, bool trunc) 443 { 444 int ret = 0; 445 void *entry; 446 struct radix_tree_root *page_tree = &mapping->page_tree; 447 448 spin_lock_irq(&mapping->tree_lock); 449 entry = get_unlocked_mapping_entry(mapping, index, NULL); 450 if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry))) 451 goto out; 452 if (!trunc && 453 (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) || 454 radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))) 455 goto out; 456 radix_tree_delete(page_tree, index); 457 mapping->nrexceptional--; 458 ret = 1; 459 out: 460 put_unlocked_mapping_entry(mapping, index, entry); 461 spin_unlock_irq(&mapping->tree_lock); 462 return ret; 463 } 464 /* 465 * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree 466 * entry to get unlocked before deleting it. 467 */ 468 int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index) 469 { 470 int ret = __dax_invalidate_mapping_entry(mapping, index, true); 471 472 /* 473 * This gets called from truncate / punch_hole path. As such, the caller 474 * must hold locks protecting against concurrent modifications of the 475 * radix tree (usually fs-private i_mmap_sem for writing). Since the 476 * caller has seen exceptional entry for this index, we better find it 477 * at that index as well... 478 */ 479 WARN_ON_ONCE(!ret); 480 return ret; 481 } 482 483 /* 484 * Invalidate exceptional DAX entry if it is clean. 485 */ 486 int dax_invalidate_mapping_entry_sync(struct address_space *mapping, 487 pgoff_t index) 488 { 489 return __dax_invalidate_mapping_entry(mapping, index, false); 490 } 491 492 static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev, 493 sector_t sector, size_t size, struct page *to, 494 unsigned long vaddr) 495 { 496 void *vto, *kaddr; 497 pgoff_t pgoff; 498 pfn_t pfn; 499 long rc; 500 int id; 501 502 rc = bdev_dax_pgoff(bdev, sector, size, &pgoff); 503 if (rc) 504 return rc; 505 506 id = dax_read_lock(); 507 rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn); 508 if (rc < 0) { 509 dax_read_unlock(id); 510 return rc; 511 } 512 vto = kmap_atomic(to); 513 copy_user_page(vto, (void __force *)kaddr, vaddr, to); 514 kunmap_atomic(vto); 515 dax_read_unlock(id); 516 return 0; 517 } 518 519 /* 520 * By this point grab_mapping_entry() has ensured that we have a locked entry 521 * of the appropriate size so we don't have to worry about downgrading PMDs to 522 * PTEs. If we happen to be trying to insert a PTE and there is a PMD 523 * already in the tree, we will skip the insertion and just dirty the PMD as 524 * appropriate. 525 */ 526 static void *dax_insert_mapping_entry(struct address_space *mapping, 527 struct vm_fault *vmf, 528 void *entry, sector_t sector, 529 unsigned long flags, bool dirty) 530 { 531 struct radix_tree_root *page_tree = &mapping->page_tree; 532 void *new_entry; 533 pgoff_t index = vmf->pgoff; 534 535 if (dirty) 536 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 537 538 if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_ZERO_PAGE)) { 539 /* we are replacing a zero page with block mapping */ 540 if (dax_is_pmd_entry(entry)) 541 unmap_mapping_range(mapping, 542 (vmf->pgoff << PAGE_SHIFT) & PMD_MASK, 543 PMD_SIZE, 0); 544 else /* pte entry */ 545 unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT, 546 PAGE_SIZE, 0); 547 } 548 549 spin_lock_irq(&mapping->tree_lock); 550 new_entry = dax_radix_locked_entry(sector, flags); 551 552 if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) { 553 /* 554 * Only swap our new entry into the radix tree if the current 555 * entry is a zero page or an empty entry. If a normal PTE or 556 * PMD entry is already in the tree, we leave it alone. This 557 * means that if we are trying to insert a PTE and the 558 * existing entry is a PMD, we will just leave the PMD in the 559 * tree and dirty it if necessary. 560 */ 561 struct radix_tree_node *node; 562 void **slot; 563 void *ret; 564 565 ret = __radix_tree_lookup(page_tree, index, &node, &slot); 566 WARN_ON_ONCE(ret != entry); 567 __radix_tree_replace(page_tree, node, slot, 568 new_entry, NULL); 569 entry = new_entry; 570 } 571 572 if (dirty) 573 radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY); 574 575 spin_unlock_irq(&mapping->tree_lock); 576 return entry; 577 } 578 579 static inline unsigned long 580 pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma) 581 { 582 unsigned long address; 583 584 address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); 585 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma); 586 return address; 587 } 588 589 /* Walk all mappings of a given index of a file and writeprotect them */ 590 static void dax_mapping_entry_mkclean(struct address_space *mapping, 591 pgoff_t index, unsigned long pfn) 592 { 593 struct vm_area_struct *vma; 594 pte_t pte, *ptep = NULL; 595 pmd_t *pmdp = NULL; 596 spinlock_t *ptl; 597 598 i_mmap_lock_read(mapping); 599 vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) { 600 unsigned long address, start, end; 601 602 cond_resched(); 603 604 if (!(vma->vm_flags & VM_SHARED)) 605 continue; 606 607 address = pgoff_address(index, vma); 608 609 /* 610 * Note because we provide start/end to follow_pte_pmd it will 611 * call mmu_notifier_invalidate_range_start() on our behalf 612 * before taking any lock. 613 */ 614 if (follow_pte_pmd(vma->vm_mm, address, &start, &end, &ptep, &pmdp, &ptl)) 615 continue; 616 617 /* 618 * No need to call mmu_notifier_invalidate_range() as we are 619 * downgrading page table protection not changing it to point 620 * to a new page. 621 * 622 * See Documentation/vm/mmu_notifier.txt 623 */ 624 if (pmdp) { 625 #ifdef CONFIG_FS_DAX_PMD 626 pmd_t pmd; 627 628 if (pfn != pmd_pfn(*pmdp)) 629 goto unlock_pmd; 630 if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp)) 631 goto unlock_pmd; 632 633 flush_cache_page(vma, address, pfn); 634 pmd = pmdp_huge_clear_flush(vma, address, pmdp); 635 pmd = pmd_wrprotect(pmd); 636 pmd = pmd_mkclean(pmd); 637 set_pmd_at(vma->vm_mm, address, pmdp, pmd); 638 unlock_pmd: 639 spin_unlock(ptl); 640 #endif 641 } else { 642 if (pfn != pte_pfn(*ptep)) 643 goto unlock_pte; 644 if (!pte_dirty(*ptep) && !pte_write(*ptep)) 645 goto unlock_pte; 646 647 flush_cache_page(vma, address, pfn); 648 pte = ptep_clear_flush(vma, address, ptep); 649 pte = pte_wrprotect(pte); 650 pte = pte_mkclean(pte); 651 set_pte_at(vma->vm_mm, address, ptep, pte); 652 unlock_pte: 653 pte_unmap_unlock(ptep, ptl); 654 } 655 656 mmu_notifier_invalidate_range_end(vma->vm_mm, start, end); 657 } 658 i_mmap_unlock_read(mapping); 659 } 660 661 static int dax_writeback_one(struct block_device *bdev, 662 struct dax_device *dax_dev, struct address_space *mapping, 663 pgoff_t index, void *entry) 664 { 665 struct radix_tree_root *page_tree = &mapping->page_tree; 666 void *entry2, **slot, *kaddr; 667 long ret = 0, id; 668 sector_t sector; 669 pgoff_t pgoff; 670 size_t size; 671 pfn_t pfn; 672 673 /* 674 * A page got tagged dirty in DAX mapping? Something is seriously 675 * wrong. 676 */ 677 if (WARN_ON(!radix_tree_exceptional_entry(entry))) 678 return -EIO; 679 680 spin_lock_irq(&mapping->tree_lock); 681 entry2 = get_unlocked_mapping_entry(mapping, index, &slot); 682 /* Entry got punched out / reallocated? */ 683 if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2))) 684 goto put_unlocked; 685 /* 686 * Entry got reallocated elsewhere? No need to writeback. We have to 687 * compare sectors as we must not bail out due to difference in lockbit 688 * or entry type. 689 */ 690 if (dax_radix_sector(entry2) != dax_radix_sector(entry)) 691 goto put_unlocked; 692 if (WARN_ON_ONCE(dax_is_empty_entry(entry) || 693 dax_is_zero_entry(entry))) { 694 ret = -EIO; 695 goto put_unlocked; 696 } 697 698 /* Another fsync thread may have already written back this entry */ 699 if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)) 700 goto put_unlocked; 701 /* Lock the entry to serialize with page faults */ 702 entry = lock_slot(mapping, slot); 703 /* 704 * We can clear the tag now but we have to be careful so that concurrent 705 * dax_writeback_one() calls for the same index cannot finish before we 706 * actually flush the caches. This is achieved as the calls will look 707 * at the entry only under tree_lock and once they do that they will 708 * see the entry locked and wait for it to unlock. 709 */ 710 radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE); 711 spin_unlock_irq(&mapping->tree_lock); 712 713 /* 714 * Even if dax_writeback_mapping_range() was given a wbc->range_start 715 * in the middle of a PMD, the 'index' we are given will be aligned to 716 * the start index of the PMD, as will the sector we pull from 717 * 'entry'. This allows us to flush for PMD_SIZE and not have to 718 * worry about partial PMD writebacks. 719 */ 720 sector = dax_radix_sector(entry); 721 size = PAGE_SIZE << dax_radix_order(entry); 722 723 id = dax_read_lock(); 724 ret = bdev_dax_pgoff(bdev, sector, size, &pgoff); 725 if (ret) 726 goto dax_unlock; 727 728 /* 729 * dax_direct_access() may sleep, so cannot hold tree_lock over 730 * its invocation. 731 */ 732 ret = dax_direct_access(dax_dev, pgoff, size / PAGE_SIZE, &kaddr, &pfn); 733 if (ret < 0) 734 goto dax_unlock; 735 736 if (WARN_ON_ONCE(ret < size / PAGE_SIZE)) { 737 ret = -EIO; 738 goto dax_unlock; 739 } 740 741 dax_mapping_entry_mkclean(mapping, index, pfn_t_to_pfn(pfn)); 742 dax_flush(dax_dev, kaddr, size); 743 /* 744 * After we have flushed the cache, we can clear the dirty tag. There 745 * cannot be new dirty data in the pfn after the flush has completed as 746 * the pfn mappings are writeprotected and fault waits for mapping 747 * entry lock. 748 */ 749 spin_lock_irq(&mapping->tree_lock); 750 radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY); 751 spin_unlock_irq(&mapping->tree_lock); 752 trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT); 753 dax_unlock: 754 dax_read_unlock(id); 755 put_locked_mapping_entry(mapping, index); 756 return ret; 757 758 put_unlocked: 759 put_unlocked_mapping_entry(mapping, index, entry2); 760 spin_unlock_irq(&mapping->tree_lock); 761 return ret; 762 } 763 764 /* 765 * Flush the mapping to the persistent domain within the byte range of [start, 766 * end]. This is required by data integrity operations to ensure file data is 767 * on persistent storage prior to completion of the operation. 768 */ 769 int dax_writeback_mapping_range(struct address_space *mapping, 770 struct block_device *bdev, struct writeback_control *wbc) 771 { 772 struct inode *inode = mapping->host; 773 pgoff_t start_index, end_index; 774 pgoff_t indices[PAGEVEC_SIZE]; 775 struct dax_device *dax_dev; 776 struct pagevec pvec; 777 bool done = false; 778 int i, ret = 0; 779 780 if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT)) 781 return -EIO; 782 783 if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL) 784 return 0; 785 786 dax_dev = dax_get_by_host(bdev->bd_disk->disk_name); 787 if (!dax_dev) 788 return -EIO; 789 790 start_index = wbc->range_start >> PAGE_SHIFT; 791 end_index = wbc->range_end >> PAGE_SHIFT; 792 793 trace_dax_writeback_range(inode, start_index, end_index); 794 795 tag_pages_for_writeback(mapping, start_index, end_index); 796 797 pagevec_init(&pvec); 798 while (!done) { 799 pvec.nr = find_get_entries_tag(mapping, start_index, 800 PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE, 801 pvec.pages, indices); 802 803 if (pvec.nr == 0) 804 break; 805 806 for (i = 0; i < pvec.nr; i++) { 807 if (indices[i] > end_index) { 808 done = true; 809 break; 810 } 811 812 ret = dax_writeback_one(bdev, dax_dev, mapping, 813 indices[i], pvec.pages[i]); 814 if (ret < 0) { 815 mapping_set_error(mapping, ret); 816 goto out; 817 } 818 } 819 start_index = indices[pvec.nr - 1] + 1; 820 } 821 out: 822 put_dax(dax_dev); 823 trace_dax_writeback_range_done(inode, start_index, end_index); 824 return (ret < 0 ? ret : 0); 825 } 826 EXPORT_SYMBOL_GPL(dax_writeback_mapping_range); 827 828 static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos) 829 { 830 return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9; 831 } 832 833 static int dax_iomap_pfn(struct iomap *iomap, loff_t pos, size_t size, 834 pfn_t *pfnp) 835 { 836 const sector_t sector = dax_iomap_sector(iomap, pos); 837 pgoff_t pgoff; 838 void *kaddr; 839 int id, rc; 840 long length; 841 842 rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff); 843 if (rc) 844 return rc; 845 id = dax_read_lock(); 846 length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size), 847 &kaddr, pfnp); 848 if (length < 0) { 849 rc = length; 850 goto out; 851 } 852 rc = -EINVAL; 853 if (PFN_PHYS(length) < size) 854 goto out; 855 if (pfn_t_to_pfn(*pfnp) & (PHYS_PFN(size)-1)) 856 goto out; 857 /* For larger pages we need devmap */ 858 if (length > 1 && !pfn_t_devmap(*pfnp)) 859 goto out; 860 rc = 0; 861 out: 862 dax_read_unlock(id); 863 return rc; 864 } 865 866 /* 867 * The user has performed a load from a hole in the file. Allocating a new 868 * page in the file would cause excessive storage usage for workloads with 869 * sparse files. Instead we insert a read-only mapping of the 4k zero page. 870 * If this page is ever written to we will re-fault and change the mapping to 871 * point to real DAX storage instead. 872 */ 873 static int dax_load_hole(struct address_space *mapping, void *entry, 874 struct vm_fault *vmf) 875 { 876 struct inode *inode = mapping->host; 877 unsigned long vaddr = vmf->address; 878 int ret = VM_FAULT_NOPAGE; 879 struct page *zero_page; 880 void *entry2; 881 882 zero_page = ZERO_PAGE(0); 883 if (unlikely(!zero_page)) { 884 ret = VM_FAULT_OOM; 885 goto out; 886 } 887 888 entry2 = dax_insert_mapping_entry(mapping, vmf, entry, 0, 889 RADIX_DAX_ZERO_PAGE, false); 890 if (IS_ERR(entry2)) { 891 ret = VM_FAULT_SIGBUS; 892 goto out; 893 } 894 895 vm_insert_mixed(vmf->vma, vaddr, page_to_pfn_t(zero_page)); 896 out: 897 trace_dax_load_hole(inode, vmf, ret); 898 return ret; 899 } 900 901 static bool dax_range_is_aligned(struct block_device *bdev, 902 unsigned int offset, unsigned int length) 903 { 904 unsigned short sector_size = bdev_logical_block_size(bdev); 905 906 if (!IS_ALIGNED(offset, sector_size)) 907 return false; 908 if (!IS_ALIGNED(length, sector_size)) 909 return false; 910 911 return true; 912 } 913 914 int __dax_zero_page_range(struct block_device *bdev, 915 struct dax_device *dax_dev, sector_t sector, 916 unsigned int offset, unsigned int size) 917 { 918 if (dax_range_is_aligned(bdev, offset, size)) { 919 sector_t start_sector = sector + (offset >> 9); 920 921 return blkdev_issue_zeroout(bdev, start_sector, 922 size >> 9, GFP_NOFS, 0); 923 } else { 924 pgoff_t pgoff; 925 long rc, id; 926 void *kaddr; 927 pfn_t pfn; 928 929 rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff); 930 if (rc) 931 return rc; 932 933 id = dax_read_lock(); 934 rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr, 935 &pfn); 936 if (rc < 0) { 937 dax_read_unlock(id); 938 return rc; 939 } 940 memset(kaddr + offset, 0, size); 941 dax_flush(dax_dev, kaddr + offset, size); 942 dax_read_unlock(id); 943 } 944 return 0; 945 } 946 EXPORT_SYMBOL_GPL(__dax_zero_page_range); 947 948 static loff_t 949 dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data, 950 struct iomap *iomap) 951 { 952 struct block_device *bdev = iomap->bdev; 953 struct dax_device *dax_dev = iomap->dax_dev; 954 struct iov_iter *iter = data; 955 loff_t end = pos + length, done = 0; 956 ssize_t ret = 0; 957 int id; 958 959 if (iov_iter_rw(iter) == READ) { 960 end = min(end, i_size_read(inode)); 961 if (pos >= end) 962 return 0; 963 964 if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN) 965 return iov_iter_zero(min(length, end - pos), iter); 966 } 967 968 if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED)) 969 return -EIO; 970 971 /* 972 * Write can allocate block for an area which has a hole page mapped 973 * into page tables. We have to tear down these mappings so that data 974 * written by write(2) is visible in mmap. 975 */ 976 if (iomap->flags & IOMAP_F_NEW) { 977 invalidate_inode_pages2_range(inode->i_mapping, 978 pos >> PAGE_SHIFT, 979 (end - 1) >> PAGE_SHIFT); 980 } 981 982 id = dax_read_lock(); 983 while (pos < end) { 984 unsigned offset = pos & (PAGE_SIZE - 1); 985 const size_t size = ALIGN(length + offset, PAGE_SIZE); 986 const sector_t sector = dax_iomap_sector(iomap, pos); 987 ssize_t map_len; 988 pgoff_t pgoff; 989 void *kaddr; 990 pfn_t pfn; 991 992 if (fatal_signal_pending(current)) { 993 ret = -EINTR; 994 break; 995 } 996 997 ret = bdev_dax_pgoff(bdev, sector, size, &pgoff); 998 if (ret) 999 break; 1000 1001 map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), 1002 &kaddr, &pfn); 1003 if (map_len < 0) { 1004 ret = map_len; 1005 break; 1006 } 1007 1008 map_len = PFN_PHYS(map_len); 1009 kaddr += offset; 1010 map_len -= offset; 1011 if (map_len > end - pos) 1012 map_len = end - pos; 1013 1014 /* 1015 * The userspace address for the memory copy has already been 1016 * validated via access_ok() in either vfs_read() or 1017 * vfs_write(), depending on which operation we are doing. 1018 */ 1019 if (iov_iter_rw(iter) == WRITE) 1020 map_len = dax_copy_from_iter(dax_dev, pgoff, kaddr, 1021 map_len, iter); 1022 else 1023 map_len = copy_to_iter(kaddr, map_len, iter); 1024 if (map_len <= 0) { 1025 ret = map_len ? map_len : -EFAULT; 1026 break; 1027 } 1028 1029 pos += map_len; 1030 length -= map_len; 1031 done += map_len; 1032 } 1033 dax_read_unlock(id); 1034 1035 return done ? done : ret; 1036 } 1037 1038 /** 1039 * dax_iomap_rw - Perform I/O to a DAX file 1040 * @iocb: The control block for this I/O 1041 * @iter: The addresses to do I/O from or to 1042 * @ops: iomap ops passed from the file system 1043 * 1044 * This function performs read and write operations to directly mapped 1045 * persistent memory. The callers needs to take care of read/write exclusion 1046 * and evicting any page cache pages in the region under I/O. 1047 */ 1048 ssize_t 1049 dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, 1050 const struct iomap_ops *ops) 1051 { 1052 struct address_space *mapping = iocb->ki_filp->f_mapping; 1053 struct inode *inode = mapping->host; 1054 loff_t pos = iocb->ki_pos, ret = 0, done = 0; 1055 unsigned flags = 0; 1056 1057 if (iov_iter_rw(iter) == WRITE) { 1058 lockdep_assert_held_exclusive(&inode->i_rwsem); 1059 flags |= IOMAP_WRITE; 1060 } else { 1061 lockdep_assert_held(&inode->i_rwsem); 1062 } 1063 1064 while (iov_iter_count(iter)) { 1065 ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops, 1066 iter, dax_iomap_actor); 1067 if (ret <= 0) 1068 break; 1069 pos += ret; 1070 done += ret; 1071 } 1072 1073 iocb->ki_pos += done; 1074 return done ? done : ret; 1075 } 1076 EXPORT_SYMBOL_GPL(dax_iomap_rw); 1077 1078 static int dax_fault_return(int error) 1079 { 1080 if (error == 0) 1081 return VM_FAULT_NOPAGE; 1082 if (error == -ENOMEM) 1083 return VM_FAULT_OOM; 1084 return VM_FAULT_SIGBUS; 1085 } 1086 1087 /* 1088 * MAP_SYNC on a dax mapping guarantees dirty metadata is 1089 * flushed on write-faults (non-cow), but not read-faults. 1090 */ 1091 static bool dax_fault_is_synchronous(unsigned long flags, 1092 struct vm_area_struct *vma, struct iomap *iomap) 1093 { 1094 return (flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC) 1095 && (iomap->flags & IOMAP_F_DIRTY); 1096 } 1097 1098 static int dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp, 1099 const struct iomap_ops *ops) 1100 { 1101 struct vm_area_struct *vma = vmf->vma; 1102 struct address_space *mapping = vma->vm_file->f_mapping; 1103 struct inode *inode = mapping->host; 1104 unsigned long vaddr = vmf->address; 1105 loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT; 1106 struct iomap iomap = { 0 }; 1107 unsigned flags = IOMAP_FAULT; 1108 int error, major = 0; 1109 bool write = vmf->flags & FAULT_FLAG_WRITE; 1110 bool sync; 1111 int vmf_ret = 0; 1112 void *entry; 1113 pfn_t pfn; 1114 1115 trace_dax_pte_fault(inode, vmf, vmf_ret); 1116 /* 1117 * Check whether offset isn't beyond end of file now. Caller is supposed 1118 * to hold locks serializing us with truncate / punch hole so this is 1119 * a reliable test. 1120 */ 1121 if (pos >= i_size_read(inode)) { 1122 vmf_ret = VM_FAULT_SIGBUS; 1123 goto out; 1124 } 1125 1126 if (write && !vmf->cow_page) 1127 flags |= IOMAP_WRITE; 1128 1129 entry = grab_mapping_entry(mapping, vmf->pgoff, 0); 1130 if (IS_ERR(entry)) { 1131 vmf_ret = dax_fault_return(PTR_ERR(entry)); 1132 goto out; 1133 } 1134 1135 /* 1136 * It is possible, particularly with mixed reads & writes to private 1137 * mappings, that we have raced with a PMD fault that overlaps with 1138 * the PTE we need to set up. If so just return and the fault will be 1139 * retried. 1140 */ 1141 if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) { 1142 vmf_ret = VM_FAULT_NOPAGE; 1143 goto unlock_entry; 1144 } 1145 1146 /* 1147 * Note that we don't bother to use iomap_apply here: DAX required 1148 * the file system block size to be equal the page size, which means 1149 * that we never have to deal with more than a single extent here. 1150 */ 1151 error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap); 1152 if (error) { 1153 vmf_ret = dax_fault_return(error); 1154 goto unlock_entry; 1155 } 1156 if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) { 1157 error = -EIO; /* fs corruption? */ 1158 goto error_finish_iomap; 1159 } 1160 1161 if (vmf->cow_page) { 1162 sector_t sector = dax_iomap_sector(&iomap, pos); 1163 1164 switch (iomap.type) { 1165 case IOMAP_HOLE: 1166 case IOMAP_UNWRITTEN: 1167 clear_user_highpage(vmf->cow_page, vaddr); 1168 break; 1169 case IOMAP_MAPPED: 1170 error = copy_user_dax(iomap.bdev, iomap.dax_dev, 1171 sector, PAGE_SIZE, vmf->cow_page, vaddr); 1172 break; 1173 default: 1174 WARN_ON_ONCE(1); 1175 error = -EIO; 1176 break; 1177 } 1178 1179 if (error) 1180 goto error_finish_iomap; 1181 1182 __SetPageUptodate(vmf->cow_page); 1183 vmf_ret = finish_fault(vmf); 1184 if (!vmf_ret) 1185 vmf_ret = VM_FAULT_DONE_COW; 1186 goto finish_iomap; 1187 } 1188 1189 sync = dax_fault_is_synchronous(flags, vma, &iomap); 1190 1191 switch (iomap.type) { 1192 case IOMAP_MAPPED: 1193 if (iomap.flags & IOMAP_F_NEW) { 1194 count_vm_event(PGMAJFAULT); 1195 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT); 1196 major = VM_FAULT_MAJOR; 1197 } 1198 error = dax_iomap_pfn(&iomap, pos, PAGE_SIZE, &pfn); 1199 if (error < 0) 1200 goto error_finish_iomap; 1201 1202 entry = dax_insert_mapping_entry(mapping, vmf, entry, 1203 dax_iomap_sector(&iomap, pos), 1204 0, write && !sync); 1205 if (IS_ERR(entry)) { 1206 error = PTR_ERR(entry); 1207 goto error_finish_iomap; 1208 } 1209 1210 /* 1211 * If we are doing synchronous page fault and inode needs fsync, 1212 * we can insert PTE into page tables only after that happens. 1213 * Skip insertion for now and return the pfn so that caller can 1214 * insert it after fsync is done. 1215 */ 1216 if (sync) { 1217 if (WARN_ON_ONCE(!pfnp)) { 1218 error = -EIO; 1219 goto error_finish_iomap; 1220 } 1221 *pfnp = pfn; 1222 vmf_ret = VM_FAULT_NEEDDSYNC | major; 1223 goto finish_iomap; 1224 } 1225 trace_dax_insert_mapping(inode, vmf, entry); 1226 if (write) 1227 error = vm_insert_mixed_mkwrite(vma, vaddr, pfn); 1228 else 1229 error = vm_insert_mixed(vma, vaddr, pfn); 1230 1231 /* -EBUSY is fine, somebody else faulted on the same PTE */ 1232 if (error == -EBUSY) 1233 error = 0; 1234 break; 1235 case IOMAP_UNWRITTEN: 1236 case IOMAP_HOLE: 1237 if (!write) { 1238 vmf_ret = dax_load_hole(mapping, entry, vmf); 1239 goto finish_iomap; 1240 } 1241 /*FALLTHRU*/ 1242 default: 1243 WARN_ON_ONCE(1); 1244 error = -EIO; 1245 break; 1246 } 1247 1248 error_finish_iomap: 1249 vmf_ret = dax_fault_return(error) | major; 1250 finish_iomap: 1251 if (ops->iomap_end) { 1252 int copied = PAGE_SIZE; 1253 1254 if (vmf_ret & VM_FAULT_ERROR) 1255 copied = 0; 1256 /* 1257 * The fault is done by now and there's no way back (other 1258 * thread may be already happily using PTE we have installed). 1259 * Just ignore error from ->iomap_end since we cannot do much 1260 * with it. 1261 */ 1262 ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap); 1263 } 1264 unlock_entry: 1265 put_locked_mapping_entry(mapping, vmf->pgoff); 1266 out: 1267 trace_dax_pte_fault_done(inode, vmf, vmf_ret); 1268 return vmf_ret; 1269 } 1270 1271 #ifdef CONFIG_FS_DAX_PMD 1272 /* 1273 * The 'colour' (ie low bits) within a PMD of a page offset. This comes up 1274 * more often than one might expect in the below functions. 1275 */ 1276 #define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1) 1277 1278 static int dax_pmd_load_hole(struct vm_fault *vmf, struct iomap *iomap, 1279 void *entry) 1280 { 1281 struct address_space *mapping = vmf->vma->vm_file->f_mapping; 1282 unsigned long pmd_addr = vmf->address & PMD_MASK; 1283 struct inode *inode = mapping->host; 1284 struct page *zero_page; 1285 void *ret = NULL; 1286 spinlock_t *ptl; 1287 pmd_t pmd_entry; 1288 1289 zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm); 1290 1291 if (unlikely(!zero_page)) 1292 goto fallback; 1293 1294 ret = dax_insert_mapping_entry(mapping, vmf, entry, 0, 1295 RADIX_DAX_PMD | RADIX_DAX_ZERO_PAGE, false); 1296 if (IS_ERR(ret)) 1297 goto fallback; 1298 1299 ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); 1300 if (!pmd_none(*(vmf->pmd))) { 1301 spin_unlock(ptl); 1302 goto fallback; 1303 } 1304 1305 pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot); 1306 pmd_entry = pmd_mkhuge(pmd_entry); 1307 set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry); 1308 spin_unlock(ptl); 1309 trace_dax_pmd_load_hole(inode, vmf, zero_page, ret); 1310 return VM_FAULT_NOPAGE; 1311 1312 fallback: 1313 trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, ret); 1314 return VM_FAULT_FALLBACK; 1315 } 1316 1317 static int dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, 1318 const struct iomap_ops *ops) 1319 { 1320 struct vm_area_struct *vma = vmf->vma; 1321 struct address_space *mapping = vma->vm_file->f_mapping; 1322 unsigned long pmd_addr = vmf->address & PMD_MASK; 1323 bool write = vmf->flags & FAULT_FLAG_WRITE; 1324 bool sync; 1325 unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT; 1326 struct inode *inode = mapping->host; 1327 int result = VM_FAULT_FALLBACK; 1328 struct iomap iomap = { 0 }; 1329 pgoff_t max_pgoff, pgoff; 1330 void *entry; 1331 loff_t pos; 1332 int error; 1333 pfn_t pfn; 1334 1335 /* 1336 * Check whether offset isn't beyond end of file now. Caller is 1337 * supposed to hold locks serializing us with truncate / punch hole so 1338 * this is a reliable test. 1339 */ 1340 pgoff = linear_page_index(vma, pmd_addr); 1341 max_pgoff = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 1342 1343 trace_dax_pmd_fault(inode, vmf, max_pgoff, 0); 1344 1345 /* 1346 * Make sure that the faulting address's PMD offset (color) matches 1347 * the PMD offset from the start of the file. This is necessary so 1348 * that a PMD range in the page table overlaps exactly with a PMD 1349 * range in the radix tree. 1350 */ 1351 if ((vmf->pgoff & PG_PMD_COLOUR) != 1352 ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR)) 1353 goto fallback; 1354 1355 /* Fall back to PTEs if we're going to COW */ 1356 if (write && !(vma->vm_flags & VM_SHARED)) 1357 goto fallback; 1358 1359 /* If the PMD would extend outside the VMA */ 1360 if (pmd_addr < vma->vm_start) 1361 goto fallback; 1362 if ((pmd_addr + PMD_SIZE) > vma->vm_end) 1363 goto fallback; 1364 1365 if (pgoff >= max_pgoff) { 1366 result = VM_FAULT_SIGBUS; 1367 goto out; 1368 } 1369 1370 /* If the PMD would extend beyond the file size */ 1371 if ((pgoff | PG_PMD_COLOUR) >= max_pgoff) 1372 goto fallback; 1373 1374 /* 1375 * grab_mapping_entry() will make sure we get a 2MiB empty entry, a 1376 * 2MiB zero page entry or a DAX PMD. If it can't (because a 4k page 1377 * is already in the tree, for instance), it will return -EEXIST and 1378 * we just fall back to 4k entries. 1379 */ 1380 entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD); 1381 if (IS_ERR(entry)) 1382 goto fallback; 1383 1384 /* 1385 * It is possible, particularly with mixed reads & writes to private 1386 * mappings, that we have raced with a PTE fault that overlaps with 1387 * the PMD we need to set up. If so just return and the fault will be 1388 * retried. 1389 */ 1390 if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) && 1391 !pmd_devmap(*vmf->pmd)) { 1392 result = 0; 1393 goto unlock_entry; 1394 } 1395 1396 /* 1397 * Note that we don't use iomap_apply here. We aren't doing I/O, only 1398 * setting up a mapping, so really we're using iomap_begin() as a way 1399 * to look up our filesystem block. 1400 */ 1401 pos = (loff_t)pgoff << PAGE_SHIFT; 1402 error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap); 1403 if (error) 1404 goto unlock_entry; 1405 1406 if (iomap.offset + iomap.length < pos + PMD_SIZE) 1407 goto finish_iomap; 1408 1409 sync = dax_fault_is_synchronous(iomap_flags, vma, &iomap); 1410 1411 switch (iomap.type) { 1412 case IOMAP_MAPPED: 1413 error = dax_iomap_pfn(&iomap, pos, PMD_SIZE, &pfn); 1414 if (error < 0) 1415 goto finish_iomap; 1416 1417 entry = dax_insert_mapping_entry(mapping, vmf, entry, 1418 dax_iomap_sector(&iomap, pos), 1419 RADIX_DAX_PMD, write && !sync); 1420 if (IS_ERR(entry)) 1421 goto finish_iomap; 1422 1423 /* 1424 * If we are doing synchronous page fault and inode needs fsync, 1425 * we can insert PMD into page tables only after that happens. 1426 * Skip insertion for now and return the pfn so that caller can 1427 * insert it after fsync is done. 1428 */ 1429 if (sync) { 1430 if (WARN_ON_ONCE(!pfnp)) 1431 goto finish_iomap; 1432 *pfnp = pfn; 1433 result = VM_FAULT_NEEDDSYNC; 1434 goto finish_iomap; 1435 } 1436 1437 trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry); 1438 result = vmf_insert_pfn_pmd(vma, vmf->address, vmf->pmd, pfn, 1439 write); 1440 break; 1441 case IOMAP_UNWRITTEN: 1442 case IOMAP_HOLE: 1443 if (WARN_ON_ONCE(write)) 1444 break; 1445 result = dax_pmd_load_hole(vmf, &iomap, entry); 1446 break; 1447 default: 1448 WARN_ON_ONCE(1); 1449 break; 1450 } 1451 1452 finish_iomap: 1453 if (ops->iomap_end) { 1454 int copied = PMD_SIZE; 1455 1456 if (result == VM_FAULT_FALLBACK) 1457 copied = 0; 1458 /* 1459 * The fault is done by now and there's no way back (other 1460 * thread may be already happily using PMD we have installed). 1461 * Just ignore error from ->iomap_end since we cannot do much 1462 * with it. 1463 */ 1464 ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags, 1465 &iomap); 1466 } 1467 unlock_entry: 1468 put_locked_mapping_entry(mapping, pgoff); 1469 fallback: 1470 if (result == VM_FAULT_FALLBACK) { 1471 split_huge_pmd(vma, vmf->pmd, vmf->address); 1472 count_vm_event(THP_FAULT_FALLBACK); 1473 } 1474 out: 1475 trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result); 1476 return result; 1477 } 1478 #else 1479 static int dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, 1480 const struct iomap_ops *ops) 1481 { 1482 return VM_FAULT_FALLBACK; 1483 } 1484 #endif /* CONFIG_FS_DAX_PMD */ 1485 1486 /** 1487 * dax_iomap_fault - handle a page fault on a DAX file 1488 * @vmf: The description of the fault 1489 * @pe_size: Size of the page to fault in 1490 * @pfnp: PFN to insert for synchronous faults if fsync is required 1491 * @ops: Iomap ops passed from the file system 1492 * 1493 * When a page fault occurs, filesystems may call this helper in 1494 * their fault handler for DAX files. dax_iomap_fault() assumes the caller 1495 * has done all the necessary locking for page fault to proceed 1496 * successfully. 1497 */ 1498 int dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size, 1499 pfn_t *pfnp, const struct iomap_ops *ops) 1500 { 1501 switch (pe_size) { 1502 case PE_SIZE_PTE: 1503 return dax_iomap_pte_fault(vmf, pfnp, ops); 1504 case PE_SIZE_PMD: 1505 return dax_iomap_pmd_fault(vmf, pfnp, ops); 1506 default: 1507 return VM_FAULT_FALLBACK; 1508 } 1509 } 1510 EXPORT_SYMBOL_GPL(dax_iomap_fault); 1511 1512 /** 1513 * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables 1514 * @vmf: The description of the fault 1515 * @pe_size: Size of entry to be inserted 1516 * @pfn: PFN to insert 1517 * 1518 * This function inserts writeable PTE or PMD entry into page tables for mmaped 1519 * DAX file. It takes care of marking corresponding radix tree entry as dirty 1520 * as well. 1521 */ 1522 static int dax_insert_pfn_mkwrite(struct vm_fault *vmf, 1523 enum page_entry_size pe_size, 1524 pfn_t pfn) 1525 { 1526 struct address_space *mapping = vmf->vma->vm_file->f_mapping; 1527 void *entry, **slot; 1528 pgoff_t index = vmf->pgoff; 1529 int vmf_ret, error; 1530 1531 spin_lock_irq(&mapping->tree_lock); 1532 entry = get_unlocked_mapping_entry(mapping, index, &slot); 1533 /* Did we race with someone splitting entry or so? */ 1534 if (!entry || 1535 (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) || 1536 (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) { 1537 put_unlocked_mapping_entry(mapping, index, entry); 1538 spin_unlock_irq(&mapping->tree_lock); 1539 trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf, 1540 VM_FAULT_NOPAGE); 1541 return VM_FAULT_NOPAGE; 1542 } 1543 radix_tree_tag_set(&mapping->page_tree, index, PAGECACHE_TAG_DIRTY); 1544 entry = lock_slot(mapping, slot); 1545 spin_unlock_irq(&mapping->tree_lock); 1546 switch (pe_size) { 1547 case PE_SIZE_PTE: 1548 error = vm_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn); 1549 vmf_ret = dax_fault_return(error); 1550 break; 1551 #ifdef CONFIG_FS_DAX_PMD 1552 case PE_SIZE_PMD: 1553 vmf_ret = vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, 1554 pfn, true); 1555 break; 1556 #endif 1557 default: 1558 vmf_ret = VM_FAULT_FALLBACK; 1559 } 1560 put_locked_mapping_entry(mapping, index); 1561 trace_dax_insert_pfn_mkwrite(mapping->host, vmf, vmf_ret); 1562 return vmf_ret; 1563 } 1564 1565 /** 1566 * dax_finish_sync_fault - finish synchronous page fault 1567 * @vmf: The description of the fault 1568 * @pe_size: Size of entry to be inserted 1569 * @pfn: PFN to insert 1570 * 1571 * This function ensures that the file range touched by the page fault is 1572 * stored persistently on the media and handles inserting of appropriate page 1573 * table entry. 1574 */ 1575 int dax_finish_sync_fault(struct vm_fault *vmf, enum page_entry_size pe_size, 1576 pfn_t pfn) 1577 { 1578 int err; 1579 loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT; 1580 size_t len = 0; 1581 1582 if (pe_size == PE_SIZE_PTE) 1583 len = PAGE_SIZE; 1584 else if (pe_size == PE_SIZE_PMD) 1585 len = PMD_SIZE; 1586 else 1587 WARN_ON_ONCE(1); 1588 err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1); 1589 if (err) 1590 return VM_FAULT_SIGBUS; 1591 return dax_insert_pfn_mkwrite(vmf, pe_size, pfn); 1592 } 1593 EXPORT_SYMBOL_GPL(dax_finish_sync_fault); 1594