1 /* 2 * linux/mm/memory.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 */ 6 7 /* 8 * demand-loading started 01.12.91 - seems it is high on the list of 9 * things wanted, and it should be easy to implement. - Linus 10 */ 11 12 /* 13 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared 14 * pages started 02.12.91, seems to work. - Linus. 15 * 16 * Tested sharing by executing about 30 /bin/sh: under the old kernel it 17 * would have taken more than the 6M I have free, but it worked well as 18 * far as I could see. 19 * 20 * Also corrected some "invalidate()"s - I wasn't doing enough of them. 21 */ 22 23 /* 24 * Real VM (paging to/from disk) started 18.12.91. Much more work and 25 * thought has to go into this. Oh, well.. 26 * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why. 27 * Found it. Everything seems to work now. 28 * 20.12.91 - Ok, making the swap-device changeable like the root. 29 */ 30 31 /* 32 * 05.04.94 - Multi-page memory management added for v1.1. 33 * Idea by Alex Bligh (alex@cconcepts.co.uk) 34 * 35 * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG 36 * (Gerhard.Wichert@pdb.siemens.de) 37 * 38 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) 39 */ 40 41 #include <linux/kernel_stat.h> 42 #include <linux/mm.h> 43 #include <linux/hugetlb.h> 44 #include <linux/mman.h> 45 #include <linux/swap.h> 46 #include <linux/highmem.h> 47 #include <linux/pagemap.h> 48 #include <linux/ksm.h> 49 #include <linux/rmap.h> 50 #include <linux/export.h> 51 #include <linux/delayacct.h> 52 #include <linux/init.h> 53 #include <linux/pfn_t.h> 54 #include <linux/writeback.h> 55 #include <linux/memcontrol.h> 56 #include <linux/mmu_notifier.h> 57 #include <linux/kallsyms.h> 58 #include <linux/swapops.h> 59 #include <linux/elf.h> 60 #include <linux/gfp.h> 61 #include <linux/migrate.h> 62 #include <linux/string.h> 63 #include <linux/dma-debug.h> 64 #include <linux/debugfs.h> 65 #include <linux/userfaultfd_k.h> 66 #include <linux/dax.h> 67 68 #include <asm/io.h> 69 #include <asm/mmu_context.h> 70 #include <asm/pgalloc.h> 71 #include <linux/uaccess.h> 72 #include <asm/tlb.h> 73 #include <asm/tlbflush.h> 74 #include <asm/pgtable.h> 75 76 #include "internal.h" 77 78 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS 79 #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid. 80 #endif 81 82 #ifndef CONFIG_NEED_MULTIPLE_NODES 83 /* use the per-pgdat data instead for discontigmem - mbligh */ 84 unsigned long max_mapnr; 85 EXPORT_SYMBOL(max_mapnr); 86 87 struct page *mem_map; 88 EXPORT_SYMBOL(mem_map); 89 #endif 90 91 /* 92 * A number of key systems in x86 including ioremap() rely on the assumption 93 * that high_memory defines the upper bound on direct map memory, then end 94 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and 95 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL 96 * and ZONE_HIGHMEM. 97 */ 98 void *high_memory; 99 EXPORT_SYMBOL(high_memory); 100 101 /* 102 * Randomize the address space (stacks, mmaps, brk, etc.). 103 * 104 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization, 105 * as ancient (libc5 based) binaries can segfault. ) 106 */ 107 int randomize_va_space __read_mostly = 108 #ifdef CONFIG_COMPAT_BRK 109 1; 110 #else 111 2; 112 #endif 113 114 static int __init disable_randmaps(char *s) 115 { 116 randomize_va_space = 0; 117 return 1; 118 } 119 __setup("norandmaps", disable_randmaps); 120 121 unsigned long zero_pfn __read_mostly; 122 EXPORT_SYMBOL(zero_pfn); 123 124 unsigned long highest_memmap_pfn __read_mostly; 125 126 /* 127 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init() 128 */ 129 static int __init init_zero_pfn(void) 130 { 131 zero_pfn = page_to_pfn(ZERO_PAGE(0)); 132 return 0; 133 } 134 core_initcall(init_zero_pfn); 135 136 137 #if defined(SPLIT_RSS_COUNTING) 138 139 void sync_mm_rss(struct mm_struct *mm) 140 { 141 int i; 142 143 for (i = 0; i < NR_MM_COUNTERS; i++) { 144 if (current->rss_stat.count[i]) { 145 add_mm_counter(mm, i, current->rss_stat.count[i]); 146 current->rss_stat.count[i] = 0; 147 } 148 } 149 current->rss_stat.events = 0; 150 } 151 152 static void add_mm_counter_fast(struct mm_struct *mm, int member, int val) 153 { 154 struct task_struct *task = current; 155 156 if (likely(task->mm == mm)) 157 task->rss_stat.count[member] += val; 158 else 159 add_mm_counter(mm, member, val); 160 } 161 #define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1) 162 #define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1) 163 164 /* sync counter once per 64 page faults */ 165 #define TASK_RSS_EVENTS_THRESH (64) 166 static void check_sync_rss_stat(struct task_struct *task) 167 { 168 if (unlikely(task != current)) 169 return; 170 if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH)) 171 sync_mm_rss(task->mm); 172 } 173 #else /* SPLIT_RSS_COUNTING */ 174 175 #define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member) 176 #define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member) 177 178 static void check_sync_rss_stat(struct task_struct *task) 179 { 180 } 181 182 #endif /* SPLIT_RSS_COUNTING */ 183 184 #ifdef HAVE_GENERIC_MMU_GATHER 185 186 static bool tlb_next_batch(struct mmu_gather *tlb) 187 { 188 struct mmu_gather_batch *batch; 189 190 batch = tlb->active; 191 if (batch->next) { 192 tlb->active = batch->next; 193 return true; 194 } 195 196 if (tlb->batch_count == MAX_GATHER_BATCH_COUNT) 197 return false; 198 199 batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0); 200 if (!batch) 201 return false; 202 203 tlb->batch_count++; 204 batch->next = NULL; 205 batch->nr = 0; 206 batch->max = MAX_GATHER_BATCH; 207 208 tlb->active->next = batch; 209 tlb->active = batch; 210 211 return true; 212 } 213 214 /* tlb_gather_mmu 215 * Called to initialize an (on-stack) mmu_gather structure for page-table 216 * tear-down from @mm. The @fullmm argument is used when @mm is without 217 * users and we're going to destroy the full address space (exit/execve). 218 */ 219 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end) 220 { 221 tlb->mm = mm; 222 223 /* Is it from 0 to ~0? */ 224 tlb->fullmm = !(start | (end+1)); 225 tlb->need_flush_all = 0; 226 tlb->local.next = NULL; 227 tlb->local.nr = 0; 228 tlb->local.max = ARRAY_SIZE(tlb->__pages); 229 tlb->active = &tlb->local; 230 tlb->batch_count = 0; 231 232 #ifdef CONFIG_HAVE_RCU_TABLE_FREE 233 tlb->batch = NULL; 234 #endif 235 tlb->page_size = 0; 236 237 __tlb_reset_range(tlb); 238 } 239 240 static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb) 241 { 242 if (!tlb->end) 243 return; 244 245 tlb_flush(tlb); 246 mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end); 247 #ifdef CONFIG_HAVE_RCU_TABLE_FREE 248 tlb_table_flush(tlb); 249 #endif 250 __tlb_reset_range(tlb); 251 } 252 253 static void tlb_flush_mmu_free(struct mmu_gather *tlb) 254 { 255 struct mmu_gather_batch *batch; 256 257 for (batch = &tlb->local; batch && batch->nr; batch = batch->next) { 258 free_pages_and_swap_cache(batch->pages, batch->nr); 259 batch->nr = 0; 260 } 261 tlb->active = &tlb->local; 262 } 263 264 void tlb_flush_mmu(struct mmu_gather *tlb) 265 { 266 tlb_flush_mmu_tlbonly(tlb); 267 tlb_flush_mmu_free(tlb); 268 } 269 270 /* tlb_finish_mmu 271 * Called at the end of the shootdown operation to free up any resources 272 * that were required. 273 */ 274 void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) 275 { 276 struct mmu_gather_batch *batch, *next; 277 278 tlb_flush_mmu(tlb); 279 280 /* keep the page table cache within bounds */ 281 check_pgt_cache(); 282 283 for (batch = tlb->local.next; batch; batch = next) { 284 next = batch->next; 285 free_pages((unsigned long)batch, 0); 286 } 287 tlb->local.next = NULL; 288 } 289 290 /* __tlb_remove_page 291 * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while 292 * handling the additional races in SMP caused by other CPUs caching valid 293 * mappings in their TLBs. Returns the number of free page slots left. 294 * When out of page slots we must call tlb_flush_mmu(). 295 *returns true if the caller should flush. 296 */ 297 bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size) 298 { 299 struct mmu_gather_batch *batch; 300 301 VM_BUG_ON(!tlb->end); 302 VM_WARN_ON(tlb->page_size != page_size); 303 304 batch = tlb->active; 305 /* 306 * Add the page and check if we are full. If so 307 * force a flush. 308 */ 309 batch->pages[batch->nr++] = page; 310 if (batch->nr == batch->max) { 311 if (!tlb_next_batch(tlb)) 312 return true; 313 batch = tlb->active; 314 } 315 VM_BUG_ON_PAGE(batch->nr > batch->max, page); 316 317 return false; 318 } 319 320 #endif /* HAVE_GENERIC_MMU_GATHER */ 321 322 #ifdef CONFIG_HAVE_RCU_TABLE_FREE 323 324 /* 325 * See the comment near struct mmu_table_batch. 326 */ 327 328 static void tlb_remove_table_smp_sync(void *arg) 329 { 330 /* Simply deliver the interrupt */ 331 } 332 333 static void tlb_remove_table_one(void *table) 334 { 335 /* 336 * This isn't an RCU grace period and hence the page-tables cannot be 337 * assumed to be actually RCU-freed. 338 * 339 * It is however sufficient for software page-table walkers that rely on 340 * IRQ disabling. See the comment near struct mmu_table_batch. 341 */ 342 smp_call_function(tlb_remove_table_smp_sync, NULL, 1); 343 __tlb_remove_table(table); 344 } 345 346 static void tlb_remove_table_rcu(struct rcu_head *head) 347 { 348 struct mmu_table_batch *batch; 349 int i; 350 351 batch = container_of(head, struct mmu_table_batch, rcu); 352 353 for (i = 0; i < batch->nr; i++) 354 __tlb_remove_table(batch->tables[i]); 355 356 free_page((unsigned long)batch); 357 } 358 359 void tlb_table_flush(struct mmu_gather *tlb) 360 { 361 struct mmu_table_batch **batch = &tlb->batch; 362 363 if (*batch) { 364 call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu); 365 *batch = NULL; 366 } 367 } 368 369 void tlb_remove_table(struct mmu_gather *tlb, void *table) 370 { 371 struct mmu_table_batch **batch = &tlb->batch; 372 373 /* 374 * When there's less then two users of this mm there cannot be a 375 * concurrent page-table walk. 376 */ 377 if (atomic_read(&tlb->mm->mm_users) < 2) { 378 __tlb_remove_table(table); 379 return; 380 } 381 382 if (*batch == NULL) { 383 *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN); 384 if (*batch == NULL) { 385 tlb_remove_table_one(table); 386 return; 387 } 388 (*batch)->nr = 0; 389 } 390 (*batch)->tables[(*batch)->nr++] = table; 391 if ((*batch)->nr == MAX_TABLE_BATCH) 392 tlb_table_flush(tlb); 393 } 394 395 #endif /* CONFIG_HAVE_RCU_TABLE_FREE */ 396 397 /* 398 * Note: this doesn't free the actual pages themselves. That 399 * has been handled earlier when unmapping all the memory regions. 400 */ 401 static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, 402 unsigned long addr) 403 { 404 pgtable_t token = pmd_pgtable(*pmd); 405 pmd_clear(pmd); 406 pte_free_tlb(tlb, token, addr); 407 atomic_long_dec(&tlb->mm->nr_ptes); 408 } 409 410 static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, 411 unsigned long addr, unsigned long end, 412 unsigned long floor, unsigned long ceiling) 413 { 414 pmd_t *pmd; 415 unsigned long next; 416 unsigned long start; 417 418 start = addr; 419 pmd = pmd_offset(pud, addr); 420 do { 421 next = pmd_addr_end(addr, end); 422 if (pmd_none_or_clear_bad(pmd)) 423 continue; 424 free_pte_range(tlb, pmd, addr); 425 } while (pmd++, addr = next, addr != end); 426 427 start &= PUD_MASK; 428 if (start < floor) 429 return; 430 if (ceiling) { 431 ceiling &= PUD_MASK; 432 if (!ceiling) 433 return; 434 } 435 if (end - 1 > ceiling - 1) 436 return; 437 438 pmd = pmd_offset(pud, start); 439 pud_clear(pud); 440 pmd_free_tlb(tlb, pmd, start); 441 mm_dec_nr_pmds(tlb->mm); 442 } 443 444 static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, 445 unsigned long addr, unsigned long end, 446 unsigned long floor, unsigned long ceiling) 447 { 448 pud_t *pud; 449 unsigned long next; 450 unsigned long start; 451 452 start = addr; 453 pud = pud_offset(pgd, addr); 454 do { 455 next = pud_addr_end(addr, end); 456 if (pud_none_or_clear_bad(pud)) 457 continue; 458 free_pmd_range(tlb, pud, addr, next, floor, ceiling); 459 } while (pud++, addr = next, addr != end); 460 461 start &= PGDIR_MASK; 462 if (start < floor) 463 return; 464 if (ceiling) { 465 ceiling &= PGDIR_MASK; 466 if (!ceiling) 467 return; 468 } 469 if (end - 1 > ceiling - 1) 470 return; 471 472 pud = pud_offset(pgd, start); 473 pgd_clear(pgd); 474 pud_free_tlb(tlb, pud, start); 475 } 476 477 /* 478 * This function frees user-level page tables of a process. 479 */ 480 void free_pgd_range(struct mmu_gather *tlb, 481 unsigned long addr, unsigned long end, 482 unsigned long floor, unsigned long ceiling) 483 { 484 pgd_t *pgd; 485 unsigned long next; 486 487 /* 488 * The next few lines have given us lots of grief... 489 * 490 * Why are we testing PMD* at this top level? Because often 491 * there will be no work to do at all, and we'd prefer not to 492 * go all the way down to the bottom just to discover that. 493 * 494 * Why all these "- 1"s? Because 0 represents both the bottom 495 * of the address space and the top of it (using -1 for the 496 * top wouldn't help much: the masks would do the wrong thing). 497 * The rule is that addr 0 and floor 0 refer to the bottom of 498 * the address space, but end 0 and ceiling 0 refer to the top 499 * Comparisons need to use "end - 1" and "ceiling - 1" (though 500 * that end 0 case should be mythical). 501 * 502 * Wherever addr is brought up or ceiling brought down, we must 503 * be careful to reject "the opposite 0" before it confuses the 504 * subsequent tests. But what about where end is brought down 505 * by PMD_SIZE below? no, end can't go down to 0 there. 506 * 507 * Whereas we round start (addr) and ceiling down, by different 508 * masks at different levels, in order to test whether a table 509 * now has no other vmas using it, so can be freed, we don't 510 * bother to round floor or end up - the tests don't need that. 511 */ 512 513 addr &= PMD_MASK; 514 if (addr < floor) { 515 addr += PMD_SIZE; 516 if (!addr) 517 return; 518 } 519 if (ceiling) { 520 ceiling &= PMD_MASK; 521 if (!ceiling) 522 return; 523 } 524 if (end - 1 > ceiling - 1) 525 end -= PMD_SIZE; 526 if (addr > end - 1) 527 return; 528 /* 529 * We add page table cache pages with PAGE_SIZE, 530 * (see pte_free_tlb()), flush the tlb if we need 531 */ 532 tlb_remove_check_page_size_change(tlb, PAGE_SIZE); 533 pgd = pgd_offset(tlb->mm, addr); 534 do { 535 next = pgd_addr_end(addr, end); 536 if (pgd_none_or_clear_bad(pgd)) 537 continue; 538 free_pud_range(tlb, pgd, addr, next, floor, ceiling); 539 } while (pgd++, addr = next, addr != end); 540 } 541 542 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, 543 unsigned long floor, unsigned long ceiling) 544 { 545 while (vma) { 546 struct vm_area_struct *next = vma->vm_next; 547 unsigned long addr = vma->vm_start; 548 549 /* 550 * Hide vma from rmap and truncate_pagecache before freeing 551 * pgtables 552 */ 553 unlink_anon_vmas(vma); 554 unlink_file_vma(vma); 555 556 if (is_vm_hugetlb_page(vma)) { 557 hugetlb_free_pgd_range(tlb, addr, vma->vm_end, 558 floor, next ? next->vm_start : ceiling); 559 } else { 560 /* 561 * Optimization: gather nearby vmas into one call down 562 */ 563 while (next && next->vm_start <= vma->vm_end + PMD_SIZE 564 && !is_vm_hugetlb_page(next)) { 565 vma = next; 566 next = vma->vm_next; 567 unlink_anon_vmas(vma); 568 unlink_file_vma(vma); 569 } 570 free_pgd_range(tlb, addr, vma->vm_end, 571 floor, next ? next->vm_start : ceiling); 572 } 573 vma = next; 574 } 575 } 576 577 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address) 578 { 579 spinlock_t *ptl; 580 pgtable_t new = pte_alloc_one(mm, address); 581 if (!new) 582 return -ENOMEM; 583 584 /* 585 * Ensure all pte setup (eg. pte page lock and page clearing) are 586 * visible before the pte is made visible to other CPUs by being 587 * put into page tables. 588 * 589 * The other side of the story is the pointer chasing in the page 590 * table walking code (when walking the page table without locking; 591 * ie. most of the time). Fortunately, these data accesses consist 592 * of a chain of data-dependent loads, meaning most CPUs (alpha 593 * being the notable exception) will already guarantee loads are 594 * seen in-order. See the alpha page table accessors for the 595 * smp_read_barrier_depends() barriers in page table walking code. 596 */ 597 smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */ 598 599 ptl = pmd_lock(mm, pmd); 600 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ 601 atomic_long_inc(&mm->nr_ptes); 602 pmd_populate(mm, pmd, new); 603 new = NULL; 604 } 605 spin_unlock(ptl); 606 if (new) 607 pte_free(mm, new); 608 return 0; 609 } 610 611 int __pte_alloc_kernel(pmd_t *pmd, unsigned long address) 612 { 613 pte_t *new = pte_alloc_one_kernel(&init_mm, address); 614 if (!new) 615 return -ENOMEM; 616 617 smp_wmb(); /* See comment in __pte_alloc */ 618 619 spin_lock(&init_mm.page_table_lock); 620 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ 621 pmd_populate_kernel(&init_mm, pmd, new); 622 new = NULL; 623 } 624 spin_unlock(&init_mm.page_table_lock); 625 if (new) 626 pte_free_kernel(&init_mm, new); 627 return 0; 628 } 629 630 static inline void init_rss_vec(int *rss) 631 { 632 memset(rss, 0, sizeof(int) * NR_MM_COUNTERS); 633 } 634 635 static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss) 636 { 637 int i; 638 639 if (current->mm == mm) 640 sync_mm_rss(mm); 641 for (i = 0; i < NR_MM_COUNTERS; i++) 642 if (rss[i]) 643 add_mm_counter(mm, i, rss[i]); 644 } 645 646 /* 647 * This function is called to print an error when a bad pte 648 * is found. For example, we might have a PFN-mapped pte in 649 * a region that doesn't allow it. 650 * 651 * The calling function must still handle the error. 652 */ 653 static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, 654 pte_t pte, struct page *page) 655 { 656 pgd_t *pgd = pgd_offset(vma->vm_mm, addr); 657 pud_t *pud = pud_offset(pgd, addr); 658 pmd_t *pmd = pmd_offset(pud, addr); 659 struct address_space *mapping; 660 pgoff_t index; 661 static unsigned long resume; 662 static unsigned long nr_shown; 663 static unsigned long nr_unshown; 664 665 /* 666 * Allow a burst of 60 reports, then keep quiet for that minute; 667 * or allow a steady drip of one report per second. 668 */ 669 if (nr_shown == 60) { 670 if (time_before(jiffies, resume)) { 671 nr_unshown++; 672 return; 673 } 674 if (nr_unshown) { 675 pr_alert("BUG: Bad page map: %lu messages suppressed\n", 676 nr_unshown); 677 nr_unshown = 0; 678 } 679 nr_shown = 0; 680 } 681 if (nr_shown++ == 0) 682 resume = jiffies + 60 * HZ; 683 684 mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; 685 index = linear_page_index(vma, addr); 686 687 pr_alert("BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n", 688 current->comm, 689 (long long)pte_val(pte), (long long)pmd_val(*pmd)); 690 if (page) 691 dump_page(page, "bad pte"); 692 pr_alert("addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n", 693 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index); 694 /* 695 * Choose text because data symbols depend on CONFIG_KALLSYMS_ALL=y 696 */ 697 pr_alert("file:%pD fault:%pf mmap:%pf readpage:%pf\n", 698 vma->vm_file, 699 vma->vm_ops ? vma->vm_ops->fault : NULL, 700 vma->vm_file ? vma->vm_file->f_op->mmap : NULL, 701 mapping ? mapping->a_ops->readpage : NULL); 702 dump_stack(); 703 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 704 } 705 706 /* 707 * vm_normal_page -- This function gets the "struct page" associated with a pte. 708 * 709 * "Special" mappings do not wish to be associated with a "struct page" (either 710 * it doesn't exist, or it exists but they don't want to touch it). In this 711 * case, NULL is returned here. "Normal" mappings do have a struct page. 712 * 713 * There are 2 broad cases. Firstly, an architecture may define a pte_special() 714 * pte bit, in which case this function is trivial. Secondly, an architecture 715 * may not have a spare pte bit, which requires a more complicated scheme, 716 * described below. 717 * 718 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a 719 * special mapping (even if there are underlying and valid "struct pages"). 720 * COWed pages of a VM_PFNMAP are always normal. 721 * 722 * The way we recognize COWed pages within VM_PFNMAP mappings is through the 723 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit 724 * set, and the vm_pgoff will point to the first PFN mapped: thus every special 725 * mapping will always honor the rule 726 * 727 * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT) 728 * 729 * And for normal mappings this is false. 730 * 731 * This restricts such mappings to be a linear translation from virtual address 732 * to pfn. To get around this restriction, we allow arbitrary mappings so long 733 * as the vma is not a COW mapping; in that case, we know that all ptes are 734 * special (because none can have been COWed). 735 * 736 * 737 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP. 738 * 739 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct 740 * page" backing, however the difference is that _all_ pages with a struct 741 * page (that is, those where pfn_valid is true) are refcounted and considered 742 * normal pages by the VM. The disadvantage is that pages are refcounted 743 * (which can be slower and simply not an option for some PFNMAP users). The 744 * advantage is that we don't have to follow the strict linearity rule of 745 * PFNMAP mappings in order to support COWable mappings. 746 * 747 */ 748 #ifdef __HAVE_ARCH_PTE_SPECIAL 749 # define HAVE_PTE_SPECIAL 1 750 #else 751 # define HAVE_PTE_SPECIAL 0 752 #endif 753 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, 754 pte_t pte) 755 { 756 unsigned long pfn = pte_pfn(pte); 757 758 if (HAVE_PTE_SPECIAL) { 759 if (likely(!pte_special(pte))) 760 goto check_pfn; 761 if (vma->vm_ops && vma->vm_ops->find_special_page) 762 return vma->vm_ops->find_special_page(vma, addr); 763 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) 764 return NULL; 765 if (!is_zero_pfn(pfn)) 766 print_bad_pte(vma, addr, pte, NULL); 767 return NULL; 768 } 769 770 /* !HAVE_PTE_SPECIAL case follows: */ 771 772 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) { 773 if (vma->vm_flags & VM_MIXEDMAP) { 774 if (!pfn_valid(pfn)) 775 return NULL; 776 goto out; 777 } else { 778 unsigned long off; 779 off = (addr - vma->vm_start) >> PAGE_SHIFT; 780 if (pfn == vma->vm_pgoff + off) 781 return NULL; 782 if (!is_cow_mapping(vma->vm_flags)) 783 return NULL; 784 } 785 } 786 787 if (is_zero_pfn(pfn)) 788 return NULL; 789 check_pfn: 790 if (unlikely(pfn > highest_memmap_pfn)) { 791 print_bad_pte(vma, addr, pte, NULL); 792 return NULL; 793 } 794 795 /* 796 * NOTE! We still have PageReserved() pages in the page tables. 797 * eg. VDSO mappings can cause them to exist. 798 */ 799 out: 800 return pfn_to_page(pfn); 801 } 802 803 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 804 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, 805 pmd_t pmd) 806 { 807 unsigned long pfn = pmd_pfn(pmd); 808 809 /* 810 * There is no pmd_special() but there may be special pmds, e.g. 811 * in a direct-access (dax) mapping, so let's just replicate the 812 * !HAVE_PTE_SPECIAL case from vm_normal_page() here. 813 */ 814 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) { 815 if (vma->vm_flags & VM_MIXEDMAP) { 816 if (!pfn_valid(pfn)) 817 return NULL; 818 goto out; 819 } else { 820 unsigned long off; 821 off = (addr - vma->vm_start) >> PAGE_SHIFT; 822 if (pfn == vma->vm_pgoff + off) 823 return NULL; 824 if (!is_cow_mapping(vma->vm_flags)) 825 return NULL; 826 } 827 } 828 829 if (is_zero_pfn(pfn)) 830 return NULL; 831 if (unlikely(pfn > highest_memmap_pfn)) 832 return NULL; 833 834 /* 835 * NOTE! We still have PageReserved() pages in the page tables. 836 * eg. VDSO mappings can cause them to exist. 837 */ 838 out: 839 return pfn_to_page(pfn); 840 } 841 #endif 842 843 /* 844 * copy one vm_area from one task to the other. Assumes the page tables 845 * already present in the new task to be cleared in the whole range 846 * covered by this vma. 847 */ 848 849 static inline unsigned long 850 copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, 851 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma, 852 unsigned long addr, int *rss) 853 { 854 unsigned long vm_flags = vma->vm_flags; 855 pte_t pte = *src_pte; 856 struct page *page; 857 858 /* pte contains position in swap or file, so copy. */ 859 if (unlikely(!pte_present(pte))) { 860 swp_entry_t entry = pte_to_swp_entry(pte); 861 862 if (likely(!non_swap_entry(entry))) { 863 if (swap_duplicate(entry) < 0) 864 return entry.val; 865 866 /* make sure dst_mm is on swapoff's mmlist. */ 867 if (unlikely(list_empty(&dst_mm->mmlist))) { 868 spin_lock(&mmlist_lock); 869 if (list_empty(&dst_mm->mmlist)) 870 list_add(&dst_mm->mmlist, 871 &src_mm->mmlist); 872 spin_unlock(&mmlist_lock); 873 } 874 rss[MM_SWAPENTS]++; 875 } else if (is_migration_entry(entry)) { 876 page = migration_entry_to_page(entry); 877 878 rss[mm_counter(page)]++; 879 880 if (is_write_migration_entry(entry) && 881 is_cow_mapping(vm_flags)) { 882 /* 883 * COW mappings require pages in both 884 * parent and child to be set to read. 885 */ 886 make_migration_entry_read(&entry); 887 pte = swp_entry_to_pte(entry); 888 if (pte_swp_soft_dirty(*src_pte)) 889 pte = pte_swp_mksoft_dirty(pte); 890 set_pte_at(src_mm, addr, src_pte, pte); 891 } 892 } 893 goto out_set_pte; 894 } 895 896 /* 897 * If it's a COW mapping, write protect it both 898 * in the parent and the child 899 */ 900 if (is_cow_mapping(vm_flags)) { 901 ptep_set_wrprotect(src_mm, addr, src_pte); 902 pte = pte_wrprotect(pte); 903 } 904 905 /* 906 * If it's a shared mapping, mark it clean in 907 * the child 908 */ 909 if (vm_flags & VM_SHARED) 910 pte = pte_mkclean(pte); 911 pte = pte_mkold(pte); 912 913 page = vm_normal_page(vma, addr, pte); 914 if (page) { 915 get_page(page); 916 page_dup_rmap(page, false); 917 rss[mm_counter(page)]++; 918 } 919 920 out_set_pte: 921 set_pte_at(dst_mm, addr, dst_pte, pte); 922 return 0; 923 } 924 925 static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 926 pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma, 927 unsigned long addr, unsigned long end) 928 { 929 pte_t *orig_src_pte, *orig_dst_pte; 930 pte_t *src_pte, *dst_pte; 931 spinlock_t *src_ptl, *dst_ptl; 932 int progress = 0; 933 int rss[NR_MM_COUNTERS]; 934 swp_entry_t entry = (swp_entry_t){0}; 935 936 again: 937 init_rss_vec(rss); 938 939 dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl); 940 if (!dst_pte) 941 return -ENOMEM; 942 src_pte = pte_offset_map(src_pmd, addr); 943 src_ptl = pte_lockptr(src_mm, src_pmd); 944 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 945 orig_src_pte = src_pte; 946 orig_dst_pte = dst_pte; 947 arch_enter_lazy_mmu_mode(); 948 949 do { 950 /* 951 * We are holding two locks at this point - either of them 952 * could generate latencies in another task on another CPU. 953 */ 954 if (progress >= 32) { 955 progress = 0; 956 if (need_resched() || 957 spin_needbreak(src_ptl) || spin_needbreak(dst_ptl)) 958 break; 959 } 960 if (pte_none(*src_pte)) { 961 progress++; 962 continue; 963 } 964 entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, 965 vma, addr, rss); 966 if (entry.val) 967 break; 968 progress += 8; 969 } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); 970 971 arch_leave_lazy_mmu_mode(); 972 spin_unlock(src_ptl); 973 pte_unmap(orig_src_pte); 974 add_mm_rss_vec(dst_mm, rss); 975 pte_unmap_unlock(orig_dst_pte, dst_ptl); 976 cond_resched(); 977 978 if (entry.val) { 979 if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) 980 return -ENOMEM; 981 progress = 0; 982 } 983 if (addr != end) 984 goto again; 985 return 0; 986 } 987 988 static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 989 pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma, 990 unsigned long addr, unsigned long end) 991 { 992 pmd_t *src_pmd, *dst_pmd; 993 unsigned long next; 994 995 dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); 996 if (!dst_pmd) 997 return -ENOMEM; 998 src_pmd = pmd_offset(src_pud, addr); 999 do { 1000 next = pmd_addr_end(addr, end); 1001 if (pmd_trans_huge(*src_pmd) || pmd_devmap(*src_pmd)) { 1002 int err; 1003 VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, vma); 1004 err = copy_huge_pmd(dst_mm, src_mm, 1005 dst_pmd, src_pmd, addr, vma); 1006 if (err == -ENOMEM) 1007 return -ENOMEM; 1008 if (!err) 1009 continue; 1010 /* fall through */ 1011 } 1012 if (pmd_none_or_clear_bad(src_pmd)) 1013 continue; 1014 if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd, 1015 vma, addr, next)) 1016 return -ENOMEM; 1017 } while (dst_pmd++, src_pmd++, addr = next, addr != end); 1018 return 0; 1019 } 1020 1021 static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1022 pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma, 1023 unsigned long addr, unsigned long end) 1024 { 1025 pud_t *src_pud, *dst_pud; 1026 unsigned long next; 1027 1028 dst_pud = pud_alloc(dst_mm, dst_pgd, addr); 1029 if (!dst_pud) 1030 return -ENOMEM; 1031 src_pud = pud_offset(src_pgd, addr); 1032 do { 1033 next = pud_addr_end(addr, end); 1034 if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) { 1035 int err; 1036 1037 VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, vma); 1038 err = copy_huge_pud(dst_mm, src_mm, 1039 dst_pud, src_pud, addr, vma); 1040 if (err == -ENOMEM) 1041 return -ENOMEM; 1042 if (!err) 1043 continue; 1044 /* fall through */ 1045 } 1046 if (pud_none_or_clear_bad(src_pud)) 1047 continue; 1048 if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud, 1049 vma, addr, next)) 1050 return -ENOMEM; 1051 } while (dst_pud++, src_pud++, addr = next, addr != end); 1052 return 0; 1053 } 1054 1055 int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1056 struct vm_area_struct *vma) 1057 { 1058 pgd_t *src_pgd, *dst_pgd; 1059 unsigned long next; 1060 unsigned long addr = vma->vm_start; 1061 unsigned long end = vma->vm_end; 1062 unsigned long mmun_start; /* For mmu_notifiers */ 1063 unsigned long mmun_end; /* For mmu_notifiers */ 1064 bool is_cow; 1065 int ret; 1066 1067 /* 1068 * Don't copy ptes where a page fault will fill them correctly. 1069 * Fork becomes much lighter when there are big shared or private 1070 * readonly mappings. The tradeoff is that copy_page_range is more 1071 * efficient than faulting. 1072 */ 1073 if (!(vma->vm_flags & (VM_HUGETLB | VM_PFNMAP | VM_MIXEDMAP)) && 1074 !vma->anon_vma) 1075 return 0; 1076 1077 if (is_vm_hugetlb_page(vma)) 1078 return copy_hugetlb_page_range(dst_mm, src_mm, vma); 1079 1080 if (unlikely(vma->vm_flags & VM_PFNMAP)) { 1081 /* 1082 * We do not free on error cases below as remove_vma 1083 * gets called on error from higher level routine 1084 */ 1085 ret = track_pfn_copy(vma); 1086 if (ret) 1087 return ret; 1088 } 1089 1090 /* 1091 * We need to invalidate the secondary MMU mappings only when 1092 * there could be a permission downgrade on the ptes of the 1093 * parent mm. And a permission downgrade will only happen if 1094 * is_cow_mapping() returns true. 1095 */ 1096 is_cow = is_cow_mapping(vma->vm_flags); 1097 mmun_start = addr; 1098 mmun_end = end; 1099 if (is_cow) 1100 mmu_notifier_invalidate_range_start(src_mm, mmun_start, 1101 mmun_end); 1102 1103 ret = 0; 1104 dst_pgd = pgd_offset(dst_mm, addr); 1105 src_pgd = pgd_offset(src_mm, addr); 1106 do { 1107 next = pgd_addr_end(addr, end); 1108 if (pgd_none_or_clear_bad(src_pgd)) 1109 continue; 1110 if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd, 1111 vma, addr, next))) { 1112 ret = -ENOMEM; 1113 break; 1114 } 1115 } while (dst_pgd++, src_pgd++, addr = next, addr != end); 1116 1117 if (is_cow) 1118 mmu_notifier_invalidate_range_end(src_mm, mmun_start, mmun_end); 1119 return ret; 1120 } 1121 1122 static unsigned long zap_pte_range(struct mmu_gather *tlb, 1123 struct vm_area_struct *vma, pmd_t *pmd, 1124 unsigned long addr, unsigned long end, 1125 struct zap_details *details) 1126 { 1127 struct mm_struct *mm = tlb->mm; 1128 int force_flush = 0; 1129 int rss[NR_MM_COUNTERS]; 1130 spinlock_t *ptl; 1131 pte_t *start_pte; 1132 pte_t *pte; 1133 swp_entry_t entry; 1134 1135 tlb_remove_check_page_size_change(tlb, PAGE_SIZE); 1136 again: 1137 init_rss_vec(rss); 1138 start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 1139 pte = start_pte; 1140 arch_enter_lazy_mmu_mode(); 1141 do { 1142 pte_t ptent = *pte; 1143 if (pte_none(ptent)) 1144 continue; 1145 1146 if (pte_present(ptent)) { 1147 struct page *page; 1148 1149 page = vm_normal_page(vma, addr, ptent); 1150 if (unlikely(details) && page) { 1151 /* 1152 * unmap_shared_mapping_pages() wants to 1153 * invalidate cache without truncating: 1154 * unmap shared but keep private pages. 1155 */ 1156 if (details->check_mapping && 1157 details->check_mapping != page_rmapping(page)) 1158 continue; 1159 } 1160 ptent = ptep_get_and_clear_full(mm, addr, pte, 1161 tlb->fullmm); 1162 tlb_remove_tlb_entry(tlb, pte, addr); 1163 if (unlikely(!page)) 1164 continue; 1165 1166 if (!PageAnon(page)) { 1167 if (pte_dirty(ptent)) { 1168 force_flush = 1; 1169 set_page_dirty(page); 1170 } 1171 if (pte_young(ptent) && 1172 likely(!(vma->vm_flags & VM_SEQ_READ))) 1173 mark_page_accessed(page); 1174 } 1175 rss[mm_counter(page)]--; 1176 page_remove_rmap(page, false); 1177 if (unlikely(page_mapcount(page) < 0)) 1178 print_bad_pte(vma, addr, ptent, page); 1179 if (unlikely(__tlb_remove_page(tlb, page))) { 1180 force_flush = 1; 1181 addr += PAGE_SIZE; 1182 break; 1183 } 1184 continue; 1185 } 1186 /* If details->check_mapping, we leave swap entries. */ 1187 if (unlikely(details)) 1188 continue; 1189 1190 entry = pte_to_swp_entry(ptent); 1191 if (!non_swap_entry(entry)) 1192 rss[MM_SWAPENTS]--; 1193 else if (is_migration_entry(entry)) { 1194 struct page *page; 1195 1196 page = migration_entry_to_page(entry); 1197 rss[mm_counter(page)]--; 1198 } 1199 if (unlikely(!free_swap_and_cache(entry))) 1200 print_bad_pte(vma, addr, ptent, NULL); 1201 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); 1202 } while (pte++, addr += PAGE_SIZE, addr != end); 1203 1204 add_mm_rss_vec(mm, rss); 1205 arch_leave_lazy_mmu_mode(); 1206 1207 /* Do the actual TLB flush before dropping ptl */ 1208 if (force_flush) 1209 tlb_flush_mmu_tlbonly(tlb); 1210 pte_unmap_unlock(start_pte, ptl); 1211 1212 /* 1213 * If we forced a TLB flush (either due to running out of 1214 * batch buffers or because we needed to flush dirty TLB 1215 * entries before releasing the ptl), free the batched 1216 * memory too. Restart if we didn't do everything. 1217 */ 1218 if (force_flush) { 1219 force_flush = 0; 1220 tlb_flush_mmu_free(tlb); 1221 if (addr != end) 1222 goto again; 1223 } 1224 1225 return addr; 1226 } 1227 1228 static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, 1229 struct vm_area_struct *vma, pud_t *pud, 1230 unsigned long addr, unsigned long end, 1231 struct zap_details *details) 1232 { 1233 pmd_t *pmd; 1234 unsigned long next; 1235 1236 pmd = pmd_offset(pud, addr); 1237 do { 1238 next = pmd_addr_end(addr, end); 1239 if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) { 1240 if (next - addr != HPAGE_PMD_SIZE) { 1241 VM_BUG_ON_VMA(vma_is_anonymous(vma) && 1242 !rwsem_is_locked(&tlb->mm->mmap_sem), vma); 1243 __split_huge_pmd(vma, pmd, addr, false, NULL); 1244 } else if (zap_huge_pmd(tlb, vma, pmd, addr)) 1245 goto next; 1246 /* fall through */ 1247 } 1248 /* 1249 * Here there can be other concurrent MADV_DONTNEED or 1250 * trans huge page faults running, and if the pmd is 1251 * none or trans huge it can change under us. This is 1252 * because MADV_DONTNEED holds the mmap_sem in read 1253 * mode. 1254 */ 1255 if (pmd_none_or_trans_huge_or_clear_bad(pmd)) 1256 goto next; 1257 next = zap_pte_range(tlb, vma, pmd, addr, next, details); 1258 next: 1259 cond_resched(); 1260 } while (pmd++, addr = next, addr != end); 1261 1262 return addr; 1263 } 1264 1265 static inline unsigned long zap_pud_range(struct mmu_gather *tlb, 1266 struct vm_area_struct *vma, pgd_t *pgd, 1267 unsigned long addr, unsigned long end, 1268 struct zap_details *details) 1269 { 1270 pud_t *pud; 1271 unsigned long next; 1272 1273 pud = pud_offset(pgd, addr); 1274 do { 1275 next = pud_addr_end(addr, end); 1276 if (pud_trans_huge(*pud) || pud_devmap(*pud)) { 1277 if (next - addr != HPAGE_PUD_SIZE) { 1278 VM_BUG_ON_VMA(!rwsem_is_locked(&tlb->mm->mmap_sem), vma); 1279 split_huge_pud(vma, pud, addr); 1280 } else if (zap_huge_pud(tlb, vma, pud, addr)) 1281 goto next; 1282 /* fall through */ 1283 } 1284 if (pud_none_or_clear_bad(pud)) 1285 continue; 1286 next = zap_pmd_range(tlb, vma, pud, addr, next, details); 1287 next: 1288 cond_resched(); 1289 } while (pud++, addr = next, addr != end); 1290 1291 return addr; 1292 } 1293 1294 void unmap_page_range(struct mmu_gather *tlb, 1295 struct vm_area_struct *vma, 1296 unsigned long addr, unsigned long end, 1297 struct zap_details *details) 1298 { 1299 pgd_t *pgd; 1300 unsigned long next; 1301 1302 BUG_ON(addr >= end); 1303 tlb_start_vma(tlb, vma); 1304 pgd = pgd_offset(vma->vm_mm, addr); 1305 do { 1306 next = pgd_addr_end(addr, end); 1307 if (pgd_none_or_clear_bad(pgd)) 1308 continue; 1309 next = zap_pud_range(tlb, vma, pgd, addr, next, details); 1310 } while (pgd++, addr = next, addr != end); 1311 tlb_end_vma(tlb, vma); 1312 } 1313 1314 1315 static void unmap_single_vma(struct mmu_gather *tlb, 1316 struct vm_area_struct *vma, unsigned long start_addr, 1317 unsigned long end_addr, 1318 struct zap_details *details) 1319 { 1320 unsigned long start = max(vma->vm_start, start_addr); 1321 unsigned long end; 1322 1323 if (start >= vma->vm_end) 1324 return; 1325 end = min(vma->vm_end, end_addr); 1326 if (end <= vma->vm_start) 1327 return; 1328 1329 if (vma->vm_file) 1330 uprobe_munmap(vma, start, end); 1331 1332 if (unlikely(vma->vm_flags & VM_PFNMAP)) 1333 untrack_pfn(vma, 0, 0); 1334 1335 if (start != end) { 1336 if (unlikely(is_vm_hugetlb_page(vma))) { 1337 /* 1338 * It is undesirable to test vma->vm_file as it 1339 * should be non-null for valid hugetlb area. 1340 * However, vm_file will be NULL in the error 1341 * cleanup path of mmap_region. When 1342 * hugetlbfs ->mmap method fails, 1343 * mmap_region() nullifies vma->vm_file 1344 * before calling this function to clean up. 1345 * Since no pte has actually been setup, it is 1346 * safe to do nothing in this case. 1347 */ 1348 if (vma->vm_file) { 1349 i_mmap_lock_write(vma->vm_file->f_mapping); 1350 __unmap_hugepage_range_final(tlb, vma, start, end, NULL); 1351 i_mmap_unlock_write(vma->vm_file->f_mapping); 1352 } 1353 } else 1354 unmap_page_range(tlb, vma, start, end, details); 1355 } 1356 } 1357 1358 /** 1359 * unmap_vmas - unmap a range of memory covered by a list of vma's 1360 * @tlb: address of the caller's struct mmu_gather 1361 * @vma: the starting vma 1362 * @start_addr: virtual address at which to start unmapping 1363 * @end_addr: virtual address at which to end unmapping 1364 * 1365 * Unmap all pages in the vma list. 1366 * 1367 * Only addresses between `start' and `end' will be unmapped. 1368 * 1369 * The VMA list must be sorted in ascending virtual address order. 1370 * 1371 * unmap_vmas() assumes that the caller will flush the whole unmapped address 1372 * range after unmap_vmas() returns. So the only responsibility here is to 1373 * ensure that any thus-far unmapped pages are flushed before unmap_vmas() 1374 * drops the lock and schedules. 1375 */ 1376 void unmap_vmas(struct mmu_gather *tlb, 1377 struct vm_area_struct *vma, unsigned long start_addr, 1378 unsigned long end_addr) 1379 { 1380 struct mm_struct *mm = vma->vm_mm; 1381 1382 mmu_notifier_invalidate_range_start(mm, start_addr, end_addr); 1383 for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) 1384 unmap_single_vma(tlb, vma, start_addr, end_addr, NULL); 1385 mmu_notifier_invalidate_range_end(mm, start_addr, end_addr); 1386 } 1387 1388 /** 1389 * zap_page_range - remove user pages in a given range 1390 * @vma: vm_area_struct holding the applicable pages 1391 * @start: starting address of pages to zap 1392 * @size: number of bytes to zap 1393 * 1394 * Caller must protect the VMA list 1395 */ 1396 void zap_page_range(struct vm_area_struct *vma, unsigned long start, 1397 unsigned long size) 1398 { 1399 struct mm_struct *mm = vma->vm_mm; 1400 struct mmu_gather tlb; 1401 unsigned long end = start + size; 1402 1403 lru_add_drain(); 1404 tlb_gather_mmu(&tlb, mm, start, end); 1405 update_hiwater_rss(mm); 1406 mmu_notifier_invalidate_range_start(mm, start, end); 1407 for ( ; vma && vma->vm_start < end; vma = vma->vm_next) 1408 unmap_single_vma(&tlb, vma, start, end, NULL); 1409 mmu_notifier_invalidate_range_end(mm, start, end); 1410 tlb_finish_mmu(&tlb, start, end); 1411 } 1412 1413 /** 1414 * zap_page_range_single - remove user pages in a given range 1415 * @vma: vm_area_struct holding the applicable pages 1416 * @address: starting address of pages to zap 1417 * @size: number of bytes to zap 1418 * @details: details of shared cache invalidation 1419 * 1420 * The range must fit into one VMA. 1421 */ 1422 static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, 1423 unsigned long size, struct zap_details *details) 1424 { 1425 struct mm_struct *mm = vma->vm_mm; 1426 struct mmu_gather tlb; 1427 unsigned long end = address + size; 1428 1429 lru_add_drain(); 1430 tlb_gather_mmu(&tlb, mm, address, end); 1431 update_hiwater_rss(mm); 1432 mmu_notifier_invalidate_range_start(mm, address, end); 1433 unmap_single_vma(&tlb, vma, address, end, details); 1434 mmu_notifier_invalidate_range_end(mm, address, end); 1435 tlb_finish_mmu(&tlb, address, end); 1436 } 1437 1438 /** 1439 * zap_vma_ptes - remove ptes mapping the vma 1440 * @vma: vm_area_struct holding ptes to be zapped 1441 * @address: starting address of pages to zap 1442 * @size: number of bytes to zap 1443 * 1444 * This function only unmaps ptes assigned to VM_PFNMAP vmas. 1445 * 1446 * The entire address range must be fully contained within the vma. 1447 * 1448 * Returns 0 if successful. 1449 */ 1450 int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 1451 unsigned long size) 1452 { 1453 if (address < vma->vm_start || address + size > vma->vm_end || 1454 !(vma->vm_flags & VM_PFNMAP)) 1455 return -1; 1456 zap_page_range_single(vma, address, size, NULL); 1457 return 0; 1458 } 1459 EXPORT_SYMBOL_GPL(zap_vma_ptes); 1460 1461 pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr, 1462 spinlock_t **ptl) 1463 { 1464 pgd_t *pgd = pgd_offset(mm, addr); 1465 pud_t *pud = pud_alloc(mm, pgd, addr); 1466 if (pud) { 1467 pmd_t *pmd = pmd_alloc(mm, pud, addr); 1468 if (pmd) { 1469 VM_BUG_ON(pmd_trans_huge(*pmd)); 1470 return pte_alloc_map_lock(mm, pmd, addr, ptl); 1471 } 1472 } 1473 return NULL; 1474 } 1475 1476 /* 1477 * This is the old fallback for page remapping. 1478 * 1479 * For historical reasons, it only allows reserved pages. Only 1480 * old drivers should use this, and they needed to mark their 1481 * pages reserved for the old functions anyway. 1482 */ 1483 static int insert_page(struct vm_area_struct *vma, unsigned long addr, 1484 struct page *page, pgprot_t prot) 1485 { 1486 struct mm_struct *mm = vma->vm_mm; 1487 int retval; 1488 pte_t *pte; 1489 spinlock_t *ptl; 1490 1491 retval = -EINVAL; 1492 if (PageAnon(page)) 1493 goto out; 1494 retval = -ENOMEM; 1495 flush_dcache_page(page); 1496 pte = get_locked_pte(mm, addr, &ptl); 1497 if (!pte) 1498 goto out; 1499 retval = -EBUSY; 1500 if (!pte_none(*pte)) 1501 goto out_unlock; 1502 1503 /* Ok, finally just insert the thing.. */ 1504 get_page(page); 1505 inc_mm_counter_fast(mm, mm_counter_file(page)); 1506 page_add_file_rmap(page, false); 1507 set_pte_at(mm, addr, pte, mk_pte(page, prot)); 1508 1509 retval = 0; 1510 pte_unmap_unlock(pte, ptl); 1511 return retval; 1512 out_unlock: 1513 pte_unmap_unlock(pte, ptl); 1514 out: 1515 return retval; 1516 } 1517 1518 /** 1519 * vm_insert_page - insert single page into user vma 1520 * @vma: user vma to map to 1521 * @addr: target user address of this page 1522 * @page: source kernel page 1523 * 1524 * This allows drivers to insert individual pages they've allocated 1525 * into a user vma. 1526 * 1527 * The page has to be a nice clean _individual_ kernel allocation. 1528 * If you allocate a compound page, you need to have marked it as 1529 * such (__GFP_COMP), or manually just split the page up yourself 1530 * (see split_page()). 1531 * 1532 * NOTE! Traditionally this was done with "remap_pfn_range()" which 1533 * took an arbitrary page protection parameter. This doesn't allow 1534 * that. Your vma protection will have to be set up correctly, which 1535 * means that if you want a shared writable mapping, you'd better 1536 * ask for a shared writable mapping! 1537 * 1538 * The page does not need to be reserved. 1539 * 1540 * Usually this function is called from f_op->mmap() handler 1541 * under mm->mmap_sem write-lock, so it can change vma->vm_flags. 1542 * Caller must set VM_MIXEDMAP on vma if it wants to call this 1543 * function from other places, for example from page-fault handler. 1544 */ 1545 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, 1546 struct page *page) 1547 { 1548 if (addr < vma->vm_start || addr >= vma->vm_end) 1549 return -EFAULT; 1550 if (!page_count(page)) 1551 return -EINVAL; 1552 if (!(vma->vm_flags & VM_MIXEDMAP)) { 1553 BUG_ON(down_read_trylock(&vma->vm_mm->mmap_sem)); 1554 BUG_ON(vma->vm_flags & VM_PFNMAP); 1555 vma->vm_flags |= VM_MIXEDMAP; 1556 } 1557 return insert_page(vma, addr, page, vma->vm_page_prot); 1558 } 1559 EXPORT_SYMBOL(vm_insert_page); 1560 1561 static int insert_pfn(struct vm_area_struct *vma, unsigned long addr, 1562 pfn_t pfn, pgprot_t prot) 1563 { 1564 struct mm_struct *mm = vma->vm_mm; 1565 int retval; 1566 pte_t *pte, entry; 1567 spinlock_t *ptl; 1568 1569 retval = -ENOMEM; 1570 pte = get_locked_pte(mm, addr, &ptl); 1571 if (!pte) 1572 goto out; 1573 retval = -EBUSY; 1574 if (!pte_none(*pte)) 1575 goto out_unlock; 1576 1577 /* Ok, finally just insert the thing.. */ 1578 if (pfn_t_devmap(pfn)) 1579 entry = pte_mkdevmap(pfn_t_pte(pfn, prot)); 1580 else 1581 entry = pte_mkspecial(pfn_t_pte(pfn, prot)); 1582 set_pte_at(mm, addr, pte, entry); 1583 update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */ 1584 1585 retval = 0; 1586 out_unlock: 1587 pte_unmap_unlock(pte, ptl); 1588 out: 1589 return retval; 1590 } 1591 1592 /** 1593 * vm_insert_pfn - insert single pfn into user vma 1594 * @vma: user vma to map to 1595 * @addr: target user address of this page 1596 * @pfn: source kernel pfn 1597 * 1598 * Similar to vm_insert_page, this allows drivers to insert individual pages 1599 * they've allocated into a user vma. Same comments apply. 1600 * 1601 * This function should only be called from a vm_ops->fault handler, and 1602 * in that case the handler should return NULL. 1603 * 1604 * vma cannot be a COW mapping. 1605 * 1606 * As this is called only for pages that do not currently exist, we 1607 * do not need to flush old virtual caches or the TLB. 1608 */ 1609 int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 1610 unsigned long pfn) 1611 { 1612 return vm_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot); 1613 } 1614 EXPORT_SYMBOL(vm_insert_pfn); 1615 1616 /** 1617 * vm_insert_pfn_prot - insert single pfn into user vma with specified pgprot 1618 * @vma: user vma to map to 1619 * @addr: target user address of this page 1620 * @pfn: source kernel pfn 1621 * @pgprot: pgprot flags for the inserted page 1622 * 1623 * This is exactly like vm_insert_pfn, except that it allows drivers to 1624 * to override pgprot on a per-page basis. 1625 * 1626 * This only makes sense for IO mappings, and it makes no sense for 1627 * cow mappings. In general, using multiple vmas is preferable; 1628 * vm_insert_pfn_prot should only be used if using multiple VMAs is 1629 * impractical. 1630 */ 1631 int vm_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr, 1632 unsigned long pfn, pgprot_t pgprot) 1633 { 1634 int ret; 1635 /* 1636 * Technically, architectures with pte_special can avoid all these 1637 * restrictions (same for remap_pfn_range). However we would like 1638 * consistency in testing and feature parity among all, so we should 1639 * try to keep these invariants in place for everybody. 1640 */ 1641 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1642 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1643 (VM_PFNMAP|VM_MIXEDMAP)); 1644 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1645 BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); 1646 1647 if (addr < vma->vm_start || addr >= vma->vm_end) 1648 return -EFAULT; 1649 1650 track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV)); 1651 1652 ret = insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot); 1653 1654 return ret; 1655 } 1656 EXPORT_SYMBOL(vm_insert_pfn_prot); 1657 1658 int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 1659 pfn_t pfn) 1660 { 1661 pgprot_t pgprot = vma->vm_page_prot; 1662 1663 BUG_ON(!(vma->vm_flags & VM_MIXEDMAP)); 1664 1665 if (addr < vma->vm_start || addr >= vma->vm_end) 1666 return -EFAULT; 1667 1668 track_pfn_insert(vma, &pgprot, pfn); 1669 1670 /* 1671 * If we don't have pte special, then we have to use the pfn_valid() 1672 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must* 1673 * refcount the page if pfn_valid is true (hence insert_page rather 1674 * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP 1675 * without pte special, it would there be refcounted as a normal page. 1676 */ 1677 if (!HAVE_PTE_SPECIAL && !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) { 1678 struct page *page; 1679 1680 /* 1681 * At this point we are committed to insert_page() 1682 * regardless of whether the caller specified flags that 1683 * result in pfn_t_has_page() == false. 1684 */ 1685 page = pfn_to_page(pfn_t_to_pfn(pfn)); 1686 return insert_page(vma, addr, page, pgprot); 1687 } 1688 return insert_pfn(vma, addr, pfn, pgprot); 1689 } 1690 EXPORT_SYMBOL(vm_insert_mixed); 1691 1692 /* 1693 * maps a range of physical memory into the requested pages. the old 1694 * mappings are removed. any references to nonexistent pages results 1695 * in null mappings (currently treated as "copy-on-access") 1696 */ 1697 static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, 1698 unsigned long addr, unsigned long end, 1699 unsigned long pfn, pgprot_t prot) 1700 { 1701 pte_t *pte; 1702 spinlock_t *ptl; 1703 1704 pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); 1705 if (!pte) 1706 return -ENOMEM; 1707 arch_enter_lazy_mmu_mode(); 1708 do { 1709 BUG_ON(!pte_none(*pte)); 1710 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); 1711 pfn++; 1712 } while (pte++, addr += PAGE_SIZE, addr != end); 1713 arch_leave_lazy_mmu_mode(); 1714 pte_unmap_unlock(pte - 1, ptl); 1715 return 0; 1716 } 1717 1718 static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, 1719 unsigned long addr, unsigned long end, 1720 unsigned long pfn, pgprot_t prot) 1721 { 1722 pmd_t *pmd; 1723 unsigned long next; 1724 1725 pfn -= addr >> PAGE_SHIFT; 1726 pmd = pmd_alloc(mm, pud, addr); 1727 if (!pmd) 1728 return -ENOMEM; 1729 VM_BUG_ON(pmd_trans_huge(*pmd)); 1730 do { 1731 next = pmd_addr_end(addr, end); 1732 if (remap_pte_range(mm, pmd, addr, next, 1733 pfn + (addr >> PAGE_SHIFT), prot)) 1734 return -ENOMEM; 1735 } while (pmd++, addr = next, addr != end); 1736 return 0; 1737 } 1738 1739 static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd, 1740 unsigned long addr, unsigned long end, 1741 unsigned long pfn, pgprot_t prot) 1742 { 1743 pud_t *pud; 1744 unsigned long next; 1745 1746 pfn -= addr >> PAGE_SHIFT; 1747 pud = pud_alloc(mm, pgd, addr); 1748 if (!pud) 1749 return -ENOMEM; 1750 do { 1751 next = pud_addr_end(addr, end); 1752 if (remap_pmd_range(mm, pud, addr, next, 1753 pfn + (addr >> PAGE_SHIFT), prot)) 1754 return -ENOMEM; 1755 } while (pud++, addr = next, addr != end); 1756 return 0; 1757 } 1758 1759 /** 1760 * remap_pfn_range - remap kernel memory to userspace 1761 * @vma: user vma to map to 1762 * @addr: target user address to start at 1763 * @pfn: physical address of kernel memory 1764 * @size: size of map area 1765 * @prot: page protection flags for this mapping 1766 * 1767 * Note: this is only safe if the mm semaphore is held when called. 1768 */ 1769 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 1770 unsigned long pfn, unsigned long size, pgprot_t prot) 1771 { 1772 pgd_t *pgd; 1773 unsigned long next; 1774 unsigned long end = addr + PAGE_ALIGN(size); 1775 struct mm_struct *mm = vma->vm_mm; 1776 unsigned long remap_pfn = pfn; 1777 int err; 1778 1779 /* 1780 * Physically remapped pages are special. Tell the 1781 * rest of the world about it: 1782 * VM_IO tells people not to look at these pages 1783 * (accesses can have side effects). 1784 * VM_PFNMAP tells the core MM that the base pages are just 1785 * raw PFN mappings, and do not have a "struct page" associated 1786 * with them. 1787 * VM_DONTEXPAND 1788 * Disable vma merging and expanding with mremap(). 1789 * VM_DONTDUMP 1790 * Omit vma from core dump, even when VM_IO turned off. 1791 * 1792 * There's a horrible special case to handle copy-on-write 1793 * behaviour that some programs depend on. We mark the "original" 1794 * un-COW'ed pages by matching them up with "vma->vm_pgoff". 1795 * See vm_normal_page() for details. 1796 */ 1797 if (is_cow_mapping(vma->vm_flags)) { 1798 if (addr != vma->vm_start || end != vma->vm_end) 1799 return -EINVAL; 1800 vma->vm_pgoff = pfn; 1801 } 1802 1803 err = track_pfn_remap(vma, &prot, remap_pfn, addr, PAGE_ALIGN(size)); 1804 if (err) 1805 return -EINVAL; 1806 1807 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 1808 1809 BUG_ON(addr >= end); 1810 pfn -= addr >> PAGE_SHIFT; 1811 pgd = pgd_offset(mm, addr); 1812 flush_cache_range(vma, addr, end); 1813 do { 1814 next = pgd_addr_end(addr, end); 1815 err = remap_pud_range(mm, pgd, addr, next, 1816 pfn + (addr >> PAGE_SHIFT), prot); 1817 if (err) 1818 break; 1819 } while (pgd++, addr = next, addr != end); 1820 1821 if (err) 1822 untrack_pfn(vma, remap_pfn, PAGE_ALIGN(size)); 1823 1824 return err; 1825 } 1826 EXPORT_SYMBOL(remap_pfn_range); 1827 1828 /** 1829 * vm_iomap_memory - remap memory to userspace 1830 * @vma: user vma to map to 1831 * @start: start of area 1832 * @len: size of area 1833 * 1834 * This is a simplified io_remap_pfn_range() for common driver use. The 1835 * driver just needs to give us the physical memory range to be mapped, 1836 * we'll figure out the rest from the vma information. 1837 * 1838 * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get 1839 * whatever write-combining details or similar. 1840 */ 1841 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len) 1842 { 1843 unsigned long vm_len, pfn, pages; 1844 1845 /* Check that the physical memory area passed in looks valid */ 1846 if (start + len < start) 1847 return -EINVAL; 1848 /* 1849 * You *really* shouldn't map things that aren't page-aligned, 1850 * but we've historically allowed it because IO memory might 1851 * just have smaller alignment. 1852 */ 1853 len += start & ~PAGE_MASK; 1854 pfn = start >> PAGE_SHIFT; 1855 pages = (len + ~PAGE_MASK) >> PAGE_SHIFT; 1856 if (pfn + pages < pfn) 1857 return -EINVAL; 1858 1859 /* We start the mapping 'vm_pgoff' pages into the area */ 1860 if (vma->vm_pgoff > pages) 1861 return -EINVAL; 1862 pfn += vma->vm_pgoff; 1863 pages -= vma->vm_pgoff; 1864 1865 /* Can we fit all of the mapping? */ 1866 vm_len = vma->vm_end - vma->vm_start; 1867 if (vm_len >> PAGE_SHIFT > pages) 1868 return -EINVAL; 1869 1870 /* Ok, let it rip */ 1871 return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot); 1872 } 1873 EXPORT_SYMBOL(vm_iomap_memory); 1874 1875 static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, 1876 unsigned long addr, unsigned long end, 1877 pte_fn_t fn, void *data) 1878 { 1879 pte_t *pte; 1880 int err; 1881 pgtable_t token; 1882 spinlock_t *uninitialized_var(ptl); 1883 1884 pte = (mm == &init_mm) ? 1885 pte_alloc_kernel(pmd, addr) : 1886 pte_alloc_map_lock(mm, pmd, addr, &ptl); 1887 if (!pte) 1888 return -ENOMEM; 1889 1890 BUG_ON(pmd_huge(*pmd)); 1891 1892 arch_enter_lazy_mmu_mode(); 1893 1894 token = pmd_pgtable(*pmd); 1895 1896 do { 1897 err = fn(pte++, token, addr, data); 1898 if (err) 1899 break; 1900 } while (addr += PAGE_SIZE, addr != end); 1901 1902 arch_leave_lazy_mmu_mode(); 1903 1904 if (mm != &init_mm) 1905 pte_unmap_unlock(pte-1, ptl); 1906 return err; 1907 } 1908 1909 static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, 1910 unsigned long addr, unsigned long end, 1911 pte_fn_t fn, void *data) 1912 { 1913 pmd_t *pmd; 1914 unsigned long next; 1915 int err; 1916 1917 BUG_ON(pud_huge(*pud)); 1918 1919 pmd = pmd_alloc(mm, pud, addr); 1920 if (!pmd) 1921 return -ENOMEM; 1922 do { 1923 next = pmd_addr_end(addr, end); 1924 err = apply_to_pte_range(mm, pmd, addr, next, fn, data); 1925 if (err) 1926 break; 1927 } while (pmd++, addr = next, addr != end); 1928 return err; 1929 } 1930 1931 static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd, 1932 unsigned long addr, unsigned long end, 1933 pte_fn_t fn, void *data) 1934 { 1935 pud_t *pud; 1936 unsigned long next; 1937 int err; 1938 1939 pud = pud_alloc(mm, pgd, addr); 1940 if (!pud) 1941 return -ENOMEM; 1942 do { 1943 next = pud_addr_end(addr, end); 1944 err = apply_to_pmd_range(mm, pud, addr, next, fn, data); 1945 if (err) 1946 break; 1947 } while (pud++, addr = next, addr != end); 1948 return err; 1949 } 1950 1951 /* 1952 * Scan a region of virtual memory, filling in page tables as necessary 1953 * and calling a provided function on each leaf page table. 1954 */ 1955 int apply_to_page_range(struct mm_struct *mm, unsigned long addr, 1956 unsigned long size, pte_fn_t fn, void *data) 1957 { 1958 pgd_t *pgd; 1959 unsigned long next; 1960 unsigned long end = addr + size; 1961 int err; 1962 1963 if (WARN_ON(addr >= end)) 1964 return -EINVAL; 1965 1966 pgd = pgd_offset(mm, addr); 1967 do { 1968 next = pgd_addr_end(addr, end); 1969 err = apply_to_pud_range(mm, pgd, addr, next, fn, data); 1970 if (err) 1971 break; 1972 } while (pgd++, addr = next, addr != end); 1973 1974 return err; 1975 } 1976 EXPORT_SYMBOL_GPL(apply_to_page_range); 1977 1978 /* 1979 * handle_pte_fault chooses page fault handler according to an entry which was 1980 * read non-atomically. Before making any commitment, on those architectures 1981 * or configurations (e.g. i386 with PAE) which might give a mix of unmatched 1982 * parts, do_swap_page must check under lock before unmapping the pte and 1983 * proceeding (but do_wp_page is only called after already making such a check; 1984 * and do_anonymous_page can safely check later on). 1985 */ 1986 static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd, 1987 pte_t *page_table, pte_t orig_pte) 1988 { 1989 int same = 1; 1990 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT) 1991 if (sizeof(pte_t) > sizeof(unsigned long)) { 1992 spinlock_t *ptl = pte_lockptr(mm, pmd); 1993 spin_lock(ptl); 1994 same = pte_same(*page_table, orig_pte); 1995 spin_unlock(ptl); 1996 } 1997 #endif 1998 pte_unmap(page_table); 1999 return same; 2000 } 2001 2002 static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma) 2003 { 2004 debug_dma_assert_idle(src); 2005 2006 /* 2007 * If the source page was a PFN mapping, we don't have 2008 * a "struct page" for it. We do a best-effort copy by 2009 * just copying from the original user address. If that 2010 * fails, we just zero-fill it. Live with it. 2011 */ 2012 if (unlikely(!src)) { 2013 void *kaddr = kmap_atomic(dst); 2014 void __user *uaddr = (void __user *)(va & PAGE_MASK); 2015 2016 /* 2017 * This really shouldn't fail, because the page is there 2018 * in the page tables. But it might just be unreadable, 2019 * in which case we just give up and fill the result with 2020 * zeroes. 2021 */ 2022 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) 2023 clear_page(kaddr); 2024 kunmap_atomic(kaddr); 2025 flush_dcache_page(dst); 2026 } else 2027 copy_user_highpage(dst, src, va, vma); 2028 } 2029 2030 static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma) 2031 { 2032 struct file *vm_file = vma->vm_file; 2033 2034 if (vm_file) 2035 return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO; 2036 2037 /* 2038 * Special mappings (e.g. VDSO) do not have any file so fake 2039 * a default GFP_KERNEL for them. 2040 */ 2041 return GFP_KERNEL; 2042 } 2043 2044 /* 2045 * Notify the address space that the page is about to become writable so that 2046 * it can prohibit this or wait for the page to get into an appropriate state. 2047 * 2048 * We do this without the lock held, so that it can sleep if it needs to. 2049 */ 2050 static int do_page_mkwrite(struct vm_fault *vmf) 2051 { 2052 int ret; 2053 struct page *page = vmf->page; 2054 unsigned int old_flags = vmf->flags; 2055 2056 vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE; 2057 2058 ret = vmf->vma->vm_ops->page_mkwrite(vmf); 2059 /* Restore original flags so that caller is not surprised */ 2060 vmf->flags = old_flags; 2061 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) 2062 return ret; 2063 if (unlikely(!(ret & VM_FAULT_LOCKED))) { 2064 lock_page(page); 2065 if (!page->mapping) { 2066 unlock_page(page); 2067 return 0; /* retry */ 2068 } 2069 ret |= VM_FAULT_LOCKED; 2070 } else 2071 VM_BUG_ON_PAGE(!PageLocked(page), page); 2072 return ret; 2073 } 2074 2075 /* 2076 * Handle dirtying of a page in shared file mapping on a write fault. 2077 * 2078 * The function expects the page to be locked and unlocks it. 2079 */ 2080 static void fault_dirty_shared_page(struct vm_area_struct *vma, 2081 struct page *page) 2082 { 2083 struct address_space *mapping; 2084 bool dirtied; 2085 bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite; 2086 2087 dirtied = set_page_dirty(page); 2088 VM_BUG_ON_PAGE(PageAnon(page), page); 2089 /* 2090 * Take a local copy of the address_space - page.mapping may be zeroed 2091 * by truncate after unlock_page(). The address_space itself remains 2092 * pinned by vma->vm_file's reference. We rely on unlock_page()'s 2093 * release semantics to prevent the compiler from undoing this copying. 2094 */ 2095 mapping = page_rmapping(page); 2096 unlock_page(page); 2097 2098 if ((dirtied || page_mkwrite) && mapping) { 2099 /* 2100 * Some device drivers do not set page.mapping 2101 * but still dirty their pages 2102 */ 2103 balance_dirty_pages_ratelimited(mapping); 2104 } 2105 2106 if (!page_mkwrite) 2107 file_update_time(vma->vm_file); 2108 } 2109 2110 /* 2111 * Handle write page faults for pages that can be reused in the current vma 2112 * 2113 * This can happen either due to the mapping being with the VM_SHARED flag, 2114 * or due to us being the last reference standing to the page. In either 2115 * case, all we need to do here is to mark the page as writable and update 2116 * any related book-keeping. 2117 */ 2118 static inline void wp_page_reuse(struct vm_fault *vmf) 2119 __releases(vmf->ptl) 2120 { 2121 struct vm_area_struct *vma = vmf->vma; 2122 struct page *page = vmf->page; 2123 pte_t entry; 2124 /* 2125 * Clear the pages cpupid information as the existing 2126 * information potentially belongs to a now completely 2127 * unrelated process. 2128 */ 2129 if (page) 2130 page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1); 2131 2132 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); 2133 entry = pte_mkyoung(vmf->orig_pte); 2134 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2135 if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1)) 2136 update_mmu_cache(vma, vmf->address, vmf->pte); 2137 pte_unmap_unlock(vmf->pte, vmf->ptl); 2138 } 2139 2140 /* 2141 * Handle the case of a page which we actually need to copy to a new page. 2142 * 2143 * Called with mmap_sem locked and the old page referenced, but 2144 * without the ptl held. 2145 * 2146 * High level logic flow: 2147 * 2148 * - Allocate a page, copy the content of the old page to the new one. 2149 * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc. 2150 * - Take the PTL. If the pte changed, bail out and release the allocated page 2151 * - If the pte is still the way we remember it, update the page table and all 2152 * relevant references. This includes dropping the reference the page-table 2153 * held to the old page, as well as updating the rmap. 2154 * - In any case, unlock the PTL and drop the reference we took to the old page. 2155 */ 2156 static int wp_page_copy(struct vm_fault *vmf) 2157 { 2158 struct vm_area_struct *vma = vmf->vma; 2159 struct mm_struct *mm = vma->vm_mm; 2160 struct page *old_page = vmf->page; 2161 struct page *new_page = NULL; 2162 pte_t entry; 2163 int page_copied = 0; 2164 const unsigned long mmun_start = vmf->address & PAGE_MASK; 2165 const unsigned long mmun_end = mmun_start + PAGE_SIZE; 2166 struct mem_cgroup *memcg; 2167 2168 if (unlikely(anon_vma_prepare(vma))) 2169 goto oom; 2170 2171 if (is_zero_pfn(pte_pfn(vmf->orig_pte))) { 2172 new_page = alloc_zeroed_user_highpage_movable(vma, 2173 vmf->address); 2174 if (!new_page) 2175 goto oom; 2176 } else { 2177 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, 2178 vmf->address); 2179 if (!new_page) 2180 goto oom; 2181 cow_user_page(new_page, old_page, vmf->address, vma); 2182 } 2183 2184 if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg, false)) 2185 goto oom_free_new; 2186 2187 __SetPageUptodate(new_page); 2188 2189 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 2190 2191 /* 2192 * Re-check the pte - we dropped the lock 2193 */ 2194 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); 2195 if (likely(pte_same(*vmf->pte, vmf->orig_pte))) { 2196 if (old_page) { 2197 if (!PageAnon(old_page)) { 2198 dec_mm_counter_fast(mm, 2199 mm_counter_file(old_page)); 2200 inc_mm_counter_fast(mm, MM_ANONPAGES); 2201 } 2202 } else { 2203 inc_mm_counter_fast(mm, MM_ANONPAGES); 2204 } 2205 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); 2206 entry = mk_pte(new_page, vma->vm_page_prot); 2207 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2208 /* 2209 * Clear the pte entry and flush it first, before updating the 2210 * pte with the new entry. This will avoid a race condition 2211 * seen in the presence of one thread doing SMC and another 2212 * thread doing COW. 2213 */ 2214 ptep_clear_flush_notify(vma, vmf->address, vmf->pte); 2215 page_add_new_anon_rmap(new_page, vma, vmf->address, false); 2216 mem_cgroup_commit_charge(new_page, memcg, false, false); 2217 lru_cache_add_active_or_unevictable(new_page, vma); 2218 /* 2219 * We call the notify macro here because, when using secondary 2220 * mmu page tables (such as kvm shadow page tables), we want the 2221 * new page to be mapped directly into the secondary page table. 2222 */ 2223 set_pte_at_notify(mm, vmf->address, vmf->pte, entry); 2224 update_mmu_cache(vma, vmf->address, vmf->pte); 2225 if (old_page) { 2226 /* 2227 * Only after switching the pte to the new page may 2228 * we remove the mapcount here. Otherwise another 2229 * process may come and find the rmap count decremented 2230 * before the pte is switched to the new page, and 2231 * "reuse" the old page writing into it while our pte 2232 * here still points into it and can be read by other 2233 * threads. 2234 * 2235 * The critical issue is to order this 2236 * page_remove_rmap with the ptp_clear_flush above. 2237 * Those stores are ordered by (if nothing else,) 2238 * the barrier present in the atomic_add_negative 2239 * in page_remove_rmap. 2240 * 2241 * Then the TLB flush in ptep_clear_flush ensures that 2242 * no process can access the old page before the 2243 * decremented mapcount is visible. And the old page 2244 * cannot be reused until after the decremented 2245 * mapcount is visible. So transitively, TLBs to 2246 * old page will be flushed before it can be reused. 2247 */ 2248 page_remove_rmap(old_page, false); 2249 } 2250 2251 /* Free the old page.. */ 2252 new_page = old_page; 2253 page_copied = 1; 2254 } else { 2255 mem_cgroup_cancel_charge(new_page, memcg, false); 2256 } 2257 2258 if (new_page) 2259 put_page(new_page); 2260 2261 pte_unmap_unlock(vmf->pte, vmf->ptl); 2262 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2263 if (old_page) { 2264 /* 2265 * Don't let another task, with possibly unlocked vma, 2266 * keep the mlocked page. 2267 */ 2268 if (page_copied && (vma->vm_flags & VM_LOCKED)) { 2269 lock_page(old_page); /* LRU manipulation */ 2270 if (PageMlocked(old_page)) 2271 munlock_vma_page(old_page); 2272 unlock_page(old_page); 2273 } 2274 put_page(old_page); 2275 } 2276 return page_copied ? VM_FAULT_WRITE : 0; 2277 oom_free_new: 2278 put_page(new_page); 2279 oom: 2280 if (old_page) 2281 put_page(old_page); 2282 return VM_FAULT_OOM; 2283 } 2284 2285 /** 2286 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE 2287 * writeable once the page is prepared 2288 * 2289 * @vmf: structure describing the fault 2290 * 2291 * This function handles all that is needed to finish a write page fault in a 2292 * shared mapping due to PTE being read-only once the mapped page is prepared. 2293 * It handles locking of PTE and modifying it. The function returns 2294 * VM_FAULT_WRITE on success, 0 when PTE got changed before we acquired PTE 2295 * lock. 2296 * 2297 * The function expects the page to be locked or other protection against 2298 * concurrent faults / writeback (such as DAX radix tree locks). 2299 */ 2300 int finish_mkwrite_fault(struct vm_fault *vmf) 2301 { 2302 WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED)); 2303 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address, 2304 &vmf->ptl); 2305 /* 2306 * We might have raced with another page fault while we released the 2307 * pte_offset_map_lock. 2308 */ 2309 if (!pte_same(*vmf->pte, vmf->orig_pte)) { 2310 pte_unmap_unlock(vmf->pte, vmf->ptl); 2311 return VM_FAULT_NOPAGE; 2312 } 2313 wp_page_reuse(vmf); 2314 return 0; 2315 } 2316 2317 /* 2318 * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED 2319 * mapping 2320 */ 2321 static int wp_pfn_shared(struct vm_fault *vmf) 2322 { 2323 struct vm_area_struct *vma = vmf->vma; 2324 2325 if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) { 2326 int ret; 2327 2328 pte_unmap_unlock(vmf->pte, vmf->ptl); 2329 vmf->flags |= FAULT_FLAG_MKWRITE; 2330 ret = vma->vm_ops->pfn_mkwrite(vmf); 2331 if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)) 2332 return ret; 2333 return finish_mkwrite_fault(vmf); 2334 } 2335 wp_page_reuse(vmf); 2336 return VM_FAULT_WRITE; 2337 } 2338 2339 static int wp_page_shared(struct vm_fault *vmf) 2340 __releases(vmf->ptl) 2341 { 2342 struct vm_area_struct *vma = vmf->vma; 2343 2344 get_page(vmf->page); 2345 2346 if (vma->vm_ops && vma->vm_ops->page_mkwrite) { 2347 int tmp; 2348 2349 pte_unmap_unlock(vmf->pte, vmf->ptl); 2350 tmp = do_page_mkwrite(vmf); 2351 if (unlikely(!tmp || (tmp & 2352 (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) { 2353 put_page(vmf->page); 2354 return tmp; 2355 } 2356 tmp = finish_mkwrite_fault(vmf); 2357 if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) { 2358 unlock_page(vmf->page); 2359 put_page(vmf->page); 2360 return tmp; 2361 } 2362 } else { 2363 wp_page_reuse(vmf); 2364 lock_page(vmf->page); 2365 } 2366 fault_dirty_shared_page(vma, vmf->page); 2367 put_page(vmf->page); 2368 2369 return VM_FAULT_WRITE; 2370 } 2371 2372 /* 2373 * This routine handles present pages, when users try to write 2374 * to a shared page. It is done by copying the page to a new address 2375 * and decrementing the shared-page counter for the old page. 2376 * 2377 * Note that this routine assumes that the protection checks have been 2378 * done by the caller (the low-level page fault routine in most cases). 2379 * Thus we can safely just mark it writable once we've done any necessary 2380 * COW. 2381 * 2382 * We also mark the page dirty at this point even though the page will 2383 * change only once the write actually happens. This avoids a few races, 2384 * and potentially makes it more efficient. 2385 * 2386 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2387 * but allow concurrent faults), with pte both mapped and locked. 2388 * We return with mmap_sem still held, but pte unmapped and unlocked. 2389 */ 2390 static int do_wp_page(struct vm_fault *vmf) 2391 __releases(vmf->ptl) 2392 { 2393 struct vm_area_struct *vma = vmf->vma; 2394 2395 vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte); 2396 if (!vmf->page) { 2397 /* 2398 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a 2399 * VM_PFNMAP VMA. 2400 * 2401 * We should not cow pages in a shared writeable mapping. 2402 * Just mark the pages writable and/or call ops->pfn_mkwrite. 2403 */ 2404 if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 2405 (VM_WRITE|VM_SHARED)) 2406 return wp_pfn_shared(vmf); 2407 2408 pte_unmap_unlock(vmf->pte, vmf->ptl); 2409 return wp_page_copy(vmf); 2410 } 2411 2412 /* 2413 * Take out anonymous pages first, anonymous shared vmas are 2414 * not dirty accountable. 2415 */ 2416 if (PageAnon(vmf->page) && !PageKsm(vmf->page)) { 2417 int total_mapcount; 2418 if (!trylock_page(vmf->page)) { 2419 get_page(vmf->page); 2420 pte_unmap_unlock(vmf->pte, vmf->ptl); 2421 lock_page(vmf->page); 2422 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, 2423 vmf->address, &vmf->ptl); 2424 if (!pte_same(*vmf->pte, vmf->orig_pte)) { 2425 unlock_page(vmf->page); 2426 pte_unmap_unlock(vmf->pte, vmf->ptl); 2427 put_page(vmf->page); 2428 return 0; 2429 } 2430 put_page(vmf->page); 2431 } 2432 if (reuse_swap_page(vmf->page, &total_mapcount)) { 2433 if (total_mapcount == 1) { 2434 /* 2435 * The page is all ours. Move it to 2436 * our anon_vma so the rmap code will 2437 * not search our parent or siblings. 2438 * Protected against the rmap code by 2439 * the page lock. 2440 */ 2441 page_move_anon_rmap(vmf->page, vma); 2442 } 2443 unlock_page(vmf->page); 2444 wp_page_reuse(vmf); 2445 return VM_FAULT_WRITE; 2446 } 2447 unlock_page(vmf->page); 2448 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 2449 (VM_WRITE|VM_SHARED))) { 2450 return wp_page_shared(vmf); 2451 } 2452 2453 /* 2454 * Ok, we need to copy. Oh, well.. 2455 */ 2456 get_page(vmf->page); 2457 2458 pte_unmap_unlock(vmf->pte, vmf->ptl); 2459 return wp_page_copy(vmf); 2460 } 2461 2462 static void unmap_mapping_range_vma(struct vm_area_struct *vma, 2463 unsigned long start_addr, unsigned long end_addr, 2464 struct zap_details *details) 2465 { 2466 zap_page_range_single(vma, start_addr, end_addr - start_addr, details); 2467 } 2468 2469 static inline void unmap_mapping_range_tree(struct rb_root *root, 2470 struct zap_details *details) 2471 { 2472 struct vm_area_struct *vma; 2473 pgoff_t vba, vea, zba, zea; 2474 2475 vma_interval_tree_foreach(vma, root, 2476 details->first_index, details->last_index) { 2477 2478 vba = vma->vm_pgoff; 2479 vea = vba + vma_pages(vma) - 1; 2480 zba = details->first_index; 2481 if (zba < vba) 2482 zba = vba; 2483 zea = details->last_index; 2484 if (zea > vea) 2485 zea = vea; 2486 2487 unmap_mapping_range_vma(vma, 2488 ((zba - vba) << PAGE_SHIFT) + vma->vm_start, 2489 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start, 2490 details); 2491 } 2492 } 2493 2494 /** 2495 * unmap_mapping_range - unmap the portion of all mmaps in the specified 2496 * address_space corresponding to the specified page range in the underlying 2497 * file. 2498 * 2499 * @mapping: the address space containing mmaps to be unmapped. 2500 * @holebegin: byte in first page to unmap, relative to the start of 2501 * the underlying file. This will be rounded down to a PAGE_SIZE 2502 * boundary. Note that this is different from truncate_pagecache(), which 2503 * must keep the partial page. In contrast, we must get rid of 2504 * partial pages. 2505 * @holelen: size of prospective hole in bytes. This will be rounded 2506 * up to a PAGE_SIZE boundary. A holelen of zero truncates to the 2507 * end of the file. 2508 * @even_cows: 1 when truncating a file, unmap even private COWed pages; 2509 * but 0 when invalidating pagecache, don't throw away private data. 2510 */ 2511 void unmap_mapping_range(struct address_space *mapping, 2512 loff_t const holebegin, loff_t const holelen, int even_cows) 2513 { 2514 struct zap_details details = { }; 2515 pgoff_t hba = holebegin >> PAGE_SHIFT; 2516 pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2517 2518 /* Check for overflow. */ 2519 if (sizeof(holelen) > sizeof(hlen)) { 2520 long long holeend = 2521 (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2522 if (holeend & ~(long long)ULONG_MAX) 2523 hlen = ULONG_MAX - hba + 1; 2524 } 2525 2526 details.check_mapping = even_cows ? NULL : mapping; 2527 details.first_index = hba; 2528 details.last_index = hba + hlen - 1; 2529 if (details.last_index < details.first_index) 2530 details.last_index = ULONG_MAX; 2531 2532 i_mmap_lock_write(mapping); 2533 if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap))) 2534 unmap_mapping_range_tree(&mapping->i_mmap, &details); 2535 i_mmap_unlock_write(mapping); 2536 } 2537 EXPORT_SYMBOL(unmap_mapping_range); 2538 2539 /* 2540 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2541 * but allow concurrent faults), and pte mapped but not yet locked. 2542 * We return with pte unmapped and unlocked. 2543 * 2544 * We return with the mmap_sem locked or unlocked in the same cases 2545 * as does filemap_fault(). 2546 */ 2547 int do_swap_page(struct vm_fault *vmf) 2548 { 2549 struct vm_area_struct *vma = vmf->vma; 2550 struct page *page, *swapcache; 2551 struct mem_cgroup *memcg; 2552 swp_entry_t entry; 2553 pte_t pte; 2554 int locked; 2555 int exclusive = 0; 2556 int ret = 0; 2557 2558 if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, vmf->orig_pte)) 2559 goto out; 2560 2561 entry = pte_to_swp_entry(vmf->orig_pte); 2562 if (unlikely(non_swap_entry(entry))) { 2563 if (is_migration_entry(entry)) { 2564 migration_entry_wait(vma->vm_mm, vmf->pmd, 2565 vmf->address); 2566 } else if (is_hwpoison_entry(entry)) { 2567 ret = VM_FAULT_HWPOISON; 2568 } else { 2569 print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL); 2570 ret = VM_FAULT_SIGBUS; 2571 } 2572 goto out; 2573 } 2574 delayacct_set_flag(DELAYACCT_PF_SWAPIN); 2575 page = lookup_swap_cache(entry); 2576 if (!page) { 2577 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, vma, 2578 vmf->address); 2579 if (!page) { 2580 /* 2581 * Back out if somebody else faulted in this pte 2582 * while we released the pte lock. 2583 */ 2584 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, 2585 vmf->address, &vmf->ptl); 2586 if (likely(pte_same(*vmf->pte, vmf->orig_pte))) 2587 ret = VM_FAULT_OOM; 2588 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2589 goto unlock; 2590 } 2591 2592 /* Had to read the page from swap area: Major fault */ 2593 ret = VM_FAULT_MAJOR; 2594 count_vm_event(PGMAJFAULT); 2595 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); 2596 } else if (PageHWPoison(page)) { 2597 /* 2598 * hwpoisoned dirty swapcache pages are kept for killing 2599 * owner processes (which may be unknown at hwpoison time) 2600 */ 2601 ret = VM_FAULT_HWPOISON; 2602 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2603 swapcache = page; 2604 goto out_release; 2605 } 2606 2607 swapcache = page; 2608 locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags); 2609 2610 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2611 if (!locked) { 2612 ret |= VM_FAULT_RETRY; 2613 goto out_release; 2614 } 2615 2616 /* 2617 * Make sure try_to_free_swap or reuse_swap_page or swapoff did not 2618 * release the swapcache from under us. The page pin, and pte_same 2619 * test below, are not enough to exclude that. Even if it is still 2620 * swapcache, we need to check that the page's swap has not changed. 2621 */ 2622 if (unlikely(!PageSwapCache(page) || page_private(page) != entry.val)) 2623 goto out_page; 2624 2625 page = ksm_might_need_to_copy(page, vma, vmf->address); 2626 if (unlikely(!page)) { 2627 ret = VM_FAULT_OOM; 2628 page = swapcache; 2629 goto out_page; 2630 } 2631 2632 if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, 2633 &memcg, false)) { 2634 ret = VM_FAULT_OOM; 2635 goto out_page; 2636 } 2637 2638 /* 2639 * Back out if somebody else already faulted in this pte. 2640 */ 2641 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, 2642 &vmf->ptl); 2643 if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) 2644 goto out_nomap; 2645 2646 if (unlikely(!PageUptodate(page))) { 2647 ret = VM_FAULT_SIGBUS; 2648 goto out_nomap; 2649 } 2650 2651 /* 2652 * The page isn't present yet, go ahead with the fault. 2653 * 2654 * Be careful about the sequence of operations here. 2655 * To get its accounting right, reuse_swap_page() must be called 2656 * while the page is counted on swap but not yet in mapcount i.e. 2657 * before page_add_anon_rmap() and swap_free(); try_to_free_swap() 2658 * must be called after the swap_free(), or it will never succeed. 2659 */ 2660 2661 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); 2662 dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS); 2663 pte = mk_pte(page, vma->vm_page_prot); 2664 if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) { 2665 pte = maybe_mkwrite(pte_mkdirty(pte), vma); 2666 vmf->flags &= ~FAULT_FLAG_WRITE; 2667 ret |= VM_FAULT_WRITE; 2668 exclusive = RMAP_EXCLUSIVE; 2669 } 2670 flush_icache_page(vma, page); 2671 if (pte_swp_soft_dirty(vmf->orig_pte)) 2672 pte = pte_mksoft_dirty(pte); 2673 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); 2674 vmf->orig_pte = pte; 2675 if (page == swapcache) { 2676 do_page_add_anon_rmap(page, vma, vmf->address, exclusive); 2677 mem_cgroup_commit_charge(page, memcg, true, false); 2678 activate_page(page); 2679 } else { /* ksm created a completely new copy */ 2680 page_add_new_anon_rmap(page, vma, vmf->address, false); 2681 mem_cgroup_commit_charge(page, memcg, false, false); 2682 lru_cache_add_active_or_unevictable(page, vma); 2683 } 2684 2685 swap_free(entry); 2686 if (mem_cgroup_swap_full(page) || 2687 (vma->vm_flags & VM_LOCKED) || PageMlocked(page)) 2688 try_to_free_swap(page); 2689 unlock_page(page); 2690 if (page != swapcache) { 2691 /* 2692 * Hold the lock to avoid the swap entry to be reused 2693 * until we take the PT lock for the pte_same() check 2694 * (to avoid false positives from pte_same). For 2695 * further safety release the lock after the swap_free 2696 * so that the swap count won't change under a 2697 * parallel locked swapcache. 2698 */ 2699 unlock_page(swapcache); 2700 put_page(swapcache); 2701 } 2702 2703 if (vmf->flags & FAULT_FLAG_WRITE) { 2704 ret |= do_wp_page(vmf); 2705 if (ret & VM_FAULT_ERROR) 2706 ret &= VM_FAULT_ERROR; 2707 goto out; 2708 } 2709 2710 /* No need to invalidate - it was non-present before */ 2711 update_mmu_cache(vma, vmf->address, vmf->pte); 2712 unlock: 2713 pte_unmap_unlock(vmf->pte, vmf->ptl); 2714 out: 2715 return ret; 2716 out_nomap: 2717 mem_cgroup_cancel_charge(page, memcg, false); 2718 pte_unmap_unlock(vmf->pte, vmf->ptl); 2719 out_page: 2720 unlock_page(page); 2721 out_release: 2722 put_page(page); 2723 if (page != swapcache) { 2724 unlock_page(swapcache); 2725 put_page(swapcache); 2726 } 2727 return ret; 2728 } 2729 2730 /* 2731 * This is like a special single-page "expand_{down|up}wards()", 2732 * except we must first make sure that 'address{-|+}PAGE_SIZE' 2733 * doesn't hit another vma. 2734 */ 2735 static inline int check_stack_guard_page(struct vm_area_struct *vma, unsigned long address) 2736 { 2737 address &= PAGE_MASK; 2738 if ((vma->vm_flags & VM_GROWSDOWN) && address == vma->vm_start) { 2739 struct vm_area_struct *prev = vma->vm_prev; 2740 2741 /* 2742 * Is there a mapping abutting this one below? 2743 * 2744 * That's only ok if it's the same stack mapping 2745 * that has gotten split.. 2746 */ 2747 if (prev && prev->vm_end == address) 2748 return prev->vm_flags & VM_GROWSDOWN ? 0 : -ENOMEM; 2749 2750 return expand_downwards(vma, address - PAGE_SIZE); 2751 } 2752 if ((vma->vm_flags & VM_GROWSUP) && address + PAGE_SIZE == vma->vm_end) { 2753 struct vm_area_struct *next = vma->vm_next; 2754 2755 /* As VM_GROWSDOWN but s/below/above/ */ 2756 if (next && next->vm_start == address + PAGE_SIZE) 2757 return next->vm_flags & VM_GROWSUP ? 0 : -ENOMEM; 2758 2759 return expand_upwards(vma, address + PAGE_SIZE); 2760 } 2761 return 0; 2762 } 2763 2764 /* 2765 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2766 * but allow concurrent faults), and pte mapped but not yet locked. 2767 * We return with mmap_sem still held, but pte unmapped and unlocked. 2768 */ 2769 static int do_anonymous_page(struct vm_fault *vmf) 2770 { 2771 struct vm_area_struct *vma = vmf->vma; 2772 struct mem_cgroup *memcg; 2773 struct page *page; 2774 pte_t entry; 2775 2776 /* File mapping without ->vm_ops ? */ 2777 if (vma->vm_flags & VM_SHARED) 2778 return VM_FAULT_SIGBUS; 2779 2780 /* Check if we need to add a guard page to the stack */ 2781 if (check_stack_guard_page(vma, vmf->address) < 0) 2782 return VM_FAULT_SIGSEGV; 2783 2784 /* 2785 * Use pte_alloc() instead of pte_alloc_map(). We can't run 2786 * pte_offset_map() on pmds where a huge pmd might be created 2787 * from a different thread. 2788 * 2789 * pte_alloc_map() is safe to use under down_write(mmap_sem) or when 2790 * parallel threads are excluded by other means. 2791 * 2792 * Here we only have down_read(mmap_sem). 2793 */ 2794 if (pte_alloc(vma->vm_mm, vmf->pmd, vmf->address)) 2795 return VM_FAULT_OOM; 2796 2797 /* See the comment in pte_alloc_one_map() */ 2798 if (unlikely(pmd_trans_unstable(vmf->pmd))) 2799 return 0; 2800 2801 /* Use the zero-page for reads */ 2802 if (!(vmf->flags & FAULT_FLAG_WRITE) && 2803 !mm_forbids_zeropage(vma->vm_mm)) { 2804 entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address), 2805 vma->vm_page_prot)); 2806 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, 2807 vmf->address, &vmf->ptl); 2808 if (!pte_none(*vmf->pte)) 2809 goto unlock; 2810 /* Deliver the page fault to userland, check inside PT lock */ 2811 if (userfaultfd_missing(vma)) { 2812 pte_unmap_unlock(vmf->pte, vmf->ptl); 2813 return handle_userfault(vmf, VM_UFFD_MISSING); 2814 } 2815 goto setpte; 2816 } 2817 2818 /* Allocate our own private page. */ 2819 if (unlikely(anon_vma_prepare(vma))) 2820 goto oom; 2821 page = alloc_zeroed_user_highpage_movable(vma, vmf->address); 2822 if (!page) 2823 goto oom; 2824 2825 if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg, false)) 2826 goto oom_free_page; 2827 2828 /* 2829 * The memory barrier inside __SetPageUptodate makes sure that 2830 * preceeding stores to the page contents become visible before 2831 * the set_pte_at() write. 2832 */ 2833 __SetPageUptodate(page); 2834 2835 entry = mk_pte(page, vma->vm_page_prot); 2836 if (vma->vm_flags & VM_WRITE) 2837 entry = pte_mkwrite(pte_mkdirty(entry)); 2838 2839 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, 2840 &vmf->ptl); 2841 if (!pte_none(*vmf->pte)) 2842 goto release; 2843 2844 /* Deliver the page fault to userland, check inside PT lock */ 2845 if (userfaultfd_missing(vma)) { 2846 pte_unmap_unlock(vmf->pte, vmf->ptl); 2847 mem_cgroup_cancel_charge(page, memcg, false); 2848 put_page(page); 2849 return handle_userfault(vmf, VM_UFFD_MISSING); 2850 } 2851 2852 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); 2853 page_add_new_anon_rmap(page, vma, vmf->address, false); 2854 mem_cgroup_commit_charge(page, memcg, false, false); 2855 lru_cache_add_active_or_unevictable(page, vma); 2856 setpte: 2857 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry); 2858 2859 /* No need to invalidate - it was non-present before */ 2860 update_mmu_cache(vma, vmf->address, vmf->pte); 2861 unlock: 2862 pte_unmap_unlock(vmf->pte, vmf->ptl); 2863 return 0; 2864 release: 2865 mem_cgroup_cancel_charge(page, memcg, false); 2866 put_page(page); 2867 goto unlock; 2868 oom_free_page: 2869 put_page(page); 2870 oom: 2871 return VM_FAULT_OOM; 2872 } 2873 2874 /* 2875 * The mmap_sem must have been held on entry, and may have been 2876 * released depending on flags and vma->vm_ops->fault() return value. 2877 * See filemap_fault() and __lock_page_retry(). 2878 */ 2879 static int __do_fault(struct vm_fault *vmf) 2880 { 2881 struct vm_area_struct *vma = vmf->vma; 2882 int ret; 2883 2884 ret = vma->vm_ops->fault(vmf); 2885 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY | 2886 VM_FAULT_DONE_COW))) 2887 return ret; 2888 2889 if (unlikely(PageHWPoison(vmf->page))) { 2890 if (ret & VM_FAULT_LOCKED) 2891 unlock_page(vmf->page); 2892 put_page(vmf->page); 2893 vmf->page = NULL; 2894 return VM_FAULT_HWPOISON; 2895 } 2896 2897 if (unlikely(!(ret & VM_FAULT_LOCKED))) 2898 lock_page(vmf->page); 2899 else 2900 VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page); 2901 2902 return ret; 2903 } 2904 2905 static int pte_alloc_one_map(struct vm_fault *vmf) 2906 { 2907 struct vm_area_struct *vma = vmf->vma; 2908 2909 if (!pmd_none(*vmf->pmd)) 2910 goto map_pte; 2911 if (vmf->prealloc_pte) { 2912 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2913 if (unlikely(!pmd_none(*vmf->pmd))) { 2914 spin_unlock(vmf->ptl); 2915 goto map_pte; 2916 } 2917 2918 atomic_long_inc(&vma->vm_mm->nr_ptes); 2919 pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte); 2920 spin_unlock(vmf->ptl); 2921 vmf->prealloc_pte = NULL; 2922 } else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd, vmf->address))) { 2923 return VM_FAULT_OOM; 2924 } 2925 map_pte: 2926 /* 2927 * If a huge pmd materialized under us just retry later. Use 2928 * pmd_trans_unstable() instead of pmd_trans_huge() to ensure the pmd 2929 * didn't become pmd_trans_huge under us and then back to pmd_none, as 2930 * a result of MADV_DONTNEED running immediately after a huge pmd fault 2931 * in a different thread of this mm, in turn leading to a misleading 2932 * pmd_trans_huge() retval. All we have to ensure is that it is a 2933 * regular pmd that we can walk with pte_offset_map() and we can do that 2934 * through an atomic read in C, which is what pmd_trans_unstable() 2935 * provides. 2936 */ 2937 if (pmd_trans_unstable(vmf->pmd) || pmd_devmap(*vmf->pmd)) 2938 return VM_FAULT_NOPAGE; 2939 2940 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, 2941 &vmf->ptl); 2942 return 0; 2943 } 2944 2945 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 2946 2947 #define HPAGE_CACHE_INDEX_MASK (HPAGE_PMD_NR - 1) 2948 static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, 2949 unsigned long haddr) 2950 { 2951 if (((vma->vm_start >> PAGE_SHIFT) & HPAGE_CACHE_INDEX_MASK) != 2952 (vma->vm_pgoff & HPAGE_CACHE_INDEX_MASK)) 2953 return false; 2954 if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end) 2955 return false; 2956 return true; 2957 } 2958 2959 static void deposit_prealloc_pte(struct vm_fault *vmf) 2960 { 2961 struct vm_area_struct *vma = vmf->vma; 2962 2963 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte); 2964 /* 2965 * We are going to consume the prealloc table, 2966 * count that as nr_ptes. 2967 */ 2968 atomic_long_inc(&vma->vm_mm->nr_ptes); 2969 vmf->prealloc_pte = NULL; 2970 } 2971 2972 static int do_set_pmd(struct vm_fault *vmf, struct page *page) 2973 { 2974 struct vm_area_struct *vma = vmf->vma; 2975 bool write = vmf->flags & FAULT_FLAG_WRITE; 2976 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 2977 pmd_t entry; 2978 int i, ret; 2979 2980 if (!transhuge_vma_suitable(vma, haddr)) 2981 return VM_FAULT_FALLBACK; 2982 2983 ret = VM_FAULT_FALLBACK; 2984 page = compound_head(page); 2985 2986 /* 2987 * Archs like ppc64 need additonal space to store information 2988 * related to pte entry. Use the preallocated table for that. 2989 */ 2990 if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) { 2991 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm, vmf->address); 2992 if (!vmf->prealloc_pte) 2993 return VM_FAULT_OOM; 2994 smp_wmb(); /* See comment in __pte_alloc() */ 2995 } 2996 2997 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2998 if (unlikely(!pmd_none(*vmf->pmd))) 2999 goto out; 3000 3001 for (i = 0; i < HPAGE_PMD_NR; i++) 3002 flush_icache_page(vma, page + i); 3003 3004 entry = mk_huge_pmd(page, vma->vm_page_prot); 3005 if (write) 3006 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 3007 3008 add_mm_counter(vma->vm_mm, MM_FILEPAGES, HPAGE_PMD_NR); 3009 page_add_file_rmap(page, true); 3010 /* 3011 * deposit and withdraw with pmd lock held 3012 */ 3013 if (arch_needs_pgtable_deposit()) 3014 deposit_prealloc_pte(vmf); 3015 3016 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); 3017 3018 update_mmu_cache_pmd(vma, haddr, vmf->pmd); 3019 3020 /* fault is handled */ 3021 ret = 0; 3022 count_vm_event(THP_FILE_MAPPED); 3023 out: 3024 spin_unlock(vmf->ptl); 3025 return ret; 3026 } 3027 #else 3028 static int do_set_pmd(struct vm_fault *vmf, struct page *page) 3029 { 3030 BUILD_BUG(); 3031 return 0; 3032 } 3033 #endif 3034 3035 /** 3036 * alloc_set_pte - setup new PTE entry for given page and add reverse page 3037 * mapping. If needed, the fucntion allocates page table or use pre-allocated. 3038 * 3039 * @vmf: fault environment 3040 * @memcg: memcg to charge page (only for private mappings) 3041 * @page: page to map 3042 * 3043 * Caller must take care of unlocking vmf->ptl, if vmf->pte is non-NULL on 3044 * return. 3045 * 3046 * Target users are page handler itself and implementations of 3047 * vm_ops->map_pages. 3048 */ 3049 int alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg, 3050 struct page *page) 3051 { 3052 struct vm_area_struct *vma = vmf->vma; 3053 bool write = vmf->flags & FAULT_FLAG_WRITE; 3054 pte_t entry; 3055 int ret; 3056 3057 if (pmd_none(*vmf->pmd) && PageTransCompound(page) && 3058 IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) { 3059 /* THP on COW? */ 3060 VM_BUG_ON_PAGE(memcg, page); 3061 3062 ret = do_set_pmd(vmf, page); 3063 if (ret != VM_FAULT_FALLBACK) 3064 return ret; 3065 } 3066 3067 if (!vmf->pte) { 3068 ret = pte_alloc_one_map(vmf); 3069 if (ret) 3070 return ret; 3071 } 3072 3073 /* Re-check under ptl */ 3074 if (unlikely(!pte_none(*vmf->pte))) 3075 return VM_FAULT_NOPAGE; 3076 3077 flush_icache_page(vma, page); 3078 entry = mk_pte(page, vma->vm_page_prot); 3079 if (write) 3080 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 3081 /* copy-on-write page */ 3082 if (write && !(vma->vm_flags & VM_SHARED)) { 3083 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); 3084 page_add_new_anon_rmap(page, vma, vmf->address, false); 3085 mem_cgroup_commit_charge(page, memcg, false, false); 3086 lru_cache_add_active_or_unevictable(page, vma); 3087 } else { 3088 inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page)); 3089 page_add_file_rmap(page, false); 3090 } 3091 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry); 3092 3093 /* no need to invalidate: a not-present page won't be cached */ 3094 update_mmu_cache(vma, vmf->address, vmf->pte); 3095 3096 return 0; 3097 } 3098 3099 3100 /** 3101 * finish_fault - finish page fault once we have prepared the page to fault 3102 * 3103 * @vmf: structure describing the fault 3104 * 3105 * This function handles all that is needed to finish a page fault once the 3106 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for 3107 * given page, adds reverse page mapping, handles memcg charges and LRU 3108 * addition. The function returns 0 on success, VM_FAULT_ code in case of 3109 * error. 3110 * 3111 * The function expects the page to be locked and on success it consumes a 3112 * reference of a page being mapped (for the PTE which maps it). 3113 */ 3114 int finish_fault(struct vm_fault *vmf) 3115 { 3116 struct page *page; 3117 int ret; 3118 3119 /* Did we COW the page? */ 3120 if ((vmf->flags & FAULT_FLAG_WRITE) && 3121 !(vmf->vma->vm_flags & VM_SHARED)) 3122 page = vmf->cow_page; 3123 else 3124 page = vmf->page; 3125 ret = alloc_set_pte(vmf, vmf->memcg, page); 3126 if (vmf->pte) 3127 pte_unmap_unlock(vmf->pte, vmf->ptl); 3128 return ret; 3129 } 3130 3131 static unsigned long fault_around_bytes __read_mostly = 3132 rounddown_pow_of_two(65536); 3133 3134 #ifdef CONFIG_DEBUG_FS 3135 static int fault_around_bytes_get(void *data, u64 *val) 3136 { 3137 *val = fault_around_bytes; 3138 return 0; 3139 } 3140 3141 /* 3142 * fault_around_pages() and fault_around_mask() expects fault_around_bytes 3143 * rounded down to nearest page order. It's what do_fault_around() expects to 3144 * see. 3145 */ 3146 static int fault_around_bytes_set(void *data, u64 val) 3147 { 3148 if (val / PAGE_SIZE > PTRS_PER_PTE) 3149 return -EINVAL; 3150 if (val > PAGE_SIZE) 3151 fault_around_bytes = rounddown_pow_of_two(val); 3152 else 3153 fault_around_bytes = PAGE_SIZE; /* rounddown_pow_of_two(0) is undefined */ 3154 return 0; 3155 } 3156 DEFINE_SIMPLE_ATTRIBUTE(fault_around_bytes_fops, 3157 fault_around_bytes_get, fault_around_bytes_set, "%llu\n"); 3158 3159 static int __init fault_around_debugfs(void) 3160 { 3161 void *ret; 3162 3163 ret = debugfs_create_file("fault_around_bytes", 0644, NULL, NULL, 3164 &fault_around_bytes_fops); 3165 if (!ret) 3166 pr_warn("Failed to create fault_around_bytes in debugfs"); 3167 return 0; 3168 } 3169 late_initcall(fault_around_debugfs); 3170 #endif 3171 3172 /* 3173 * do_fault_around() tries to map few pages around the fault address. The hope 3174 * is that the pages will be needed soon and this will lower the number of 3175 * faults to handle. 3176 * 3177 * It uses vm_ops->map_pages() to map the pages, which skips the page if it's 3178 * not ready to be mapped: not up-to-date, locked, etc. 3179 * 3180 * This function is called with the page table lock taken. In the split ptlock 3181 * case the page table lock only protects only those entries which belong to 3182 * the page table corresponding to the fault address. 3183 * 3184 * This function doesn't cross the VMA boundaries, in order to call map_pages() 3185 * only once. 3186 * 3187 * fault_around_pages() defines how many pages we'll try to map. 3188 * do_fault_around() expects it to return a power of two less than or equal to 3189 * PTRS_PER_PTE. 3190 * 3191 * The virtual address of the area that we map is naturally aligned to the 3192 * fault_around_pages() value (and therefore to page order). This way it's 3193 * easier to guarantee that we don't cross page table boundaries. 3194 */ 3195 static int do_fault_around(struct vm_fault *vmf) 3196 { 3197 unsigned long address = vmf->address, nr_pages, mask; 3198 pgoff_t start_pgoff = vmf->pgoff; 3199 pgoff_t end_pgoff; 3200 int off, ret = 0; 3201 3202 nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT; 3203 mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK; 3204 3205 vmf->address = max(address & mask, vmf->vma->vm_start); 3206 off = ((address - vmf->address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); 3207 start_pgoff -= off; 3208 3209 /* 3210 * end_pgoff is either end of page table or end of vma 3211 * or fault_around_pages() from start_pgoff, depending what is nearest. 3212 */ 3213 end_pgoff = start_pgoff - 3214 ((vmf->address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) + 3215 PTRS_PER_PTE - 1; 3216 end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1, 3217 start_pgoff + nr_pages - 1); 3218 3219 if (pmd_none(*vmf->pmd)) { 3220 vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm, 3221 vmf->address); 3222 if (!vmf->prealloc_pte) 3223 goto out; 3224 smp_wmb(); /* See comment in __pte_alloc() */ 3225 } 3226 3227 vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff); 3228 3229 /* Huge page is mapped? Page fault is solved */ 3230 if (pmd_trans_huge(*vmf->pmd)) { 3231 ret = VM_FAULT_NOPAGE; 3232 goto out; 3233 } 3234 3235 /* ->map_pages() haven't done anything useful. Cold page cache? */ 3236 if (!vmf->pte) 3237 goto out; 3238 3239 /* check if the page fault is solved */ 3240 vmf->pte -= (vmf->address >> PAGE_SHIFT) - (address >> PAGE_SHIFT); 3241 if (!pte_none(*vmf->pte)) 3242 ret = VM_FAULT_NOPAGE; 3243 pte_unmap_unlock(vmf->pte, vmf->ptl); 3244 out: 3245 vmf->address = address; 3246 vmf->pte = NULL; 3247 return ret; 3248 } 3249 3250 static int do_read_fault(struct vm_fault *vmf) 3251 { 3252 struct vm_area_struct *vma = vmf->vma; 3253 int ret = 0; 3254 3255 /* 3256 * Let's call ->map_pages() first and use ->fault() as fallback 3257 * if page by the offset is not ready to be mapped (cold cache or 3258 * something). 3259 */ 3260 if (vma->vm_ops->map_pages && fault_around_bytes >> PAGE_SHIFT > 1) { 3261 ret = do_fault_around(vmf); 3262 if (ret) 3263 return ret; 3264 } 3265 3266 ret = __do_fault(vmf); 3267 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) 3268 return ret; 3269 3270 ret |= finish_fault(vmf); 3271 unlock_page(vmf->page); 3272 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) 3273 put_page(vmf->page); 3274 return ret; 3275 } 3276 3277 static int do_cow_fault(struct vm_fault *vmf) 3278 { 3279 struct vm_area_struct *vma = vmf->vma; 3280 int ret; 3281 3282 if (unlikely(anon_vma_prepare(vma))) 3283 return VM_FAULT_OOM; 3284 3285 vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address); 3286 if (!vmf->cow_page) 3287 return VM_FAULT_OOM; 3288 3289 if (mem_cgroup_try_charge(vmf->cow_page, vma->vm_mm, GFP_KERNEL, 3290 &vmf->memcg, false)) { 3291 put_page(vmf->cow_page); 3292 return VM_FAULT_OOM; 3293 } 3294 3295 ret = __do_fault(vmf); 3296 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) 3297 goto uncharge_out; 3298 if (ret & VM_FAULT_DONE_COW) 3299 return ret; 3300 3301 copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma); 3302 __SetPageUptodate(vmf->cow_page); 3303 3304 ret |= finish_fault(vmf); 3305 unlock_page(vmf->page); 3306 put_page(vmf->page); 3307 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) 3308 goto uncharge_out; 3309 return ret; 3310 uncharge_out: 3311 mem_cgroup_cancel_charge(vmf->cow_page, vmf->memcg, false); 3312 put_page(vmf->cow_page); 3313 return ret; 3314 } 3315 3316 static int do_shared_fault(struct vm_fault *vmf) 3317 { 3318 struct vm_area_struct *vma = vmf->vma; 3319 int ret, tmp; 3320 3321 ret = __do_fault(vmf); 3322 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) 3323 return ret; 3324 3325 /* 3326 * Check if the backing address space wants to know that the page is 3327 * about to become writable 3328 */ 3329 if (vma->vm_ops->page_mkwrite) { 3330 unlock_page(vmf->page); 3331 tmp = do_page_mkwrite(vmf); 3332 if (unlikely(!tmp || 3333 (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) { 3334 put_page(vmf->page); 3335 return tmp; 3336 } 3337 } 3338 3339 ret |= finish_fault(vmf); 3340 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | 3341 VM_FAULT_RETRY))) { 3342 unlock_page(vmf->page); 3343 put_page(vmf->page); 3344 return ret; 3345 } 3346 3347 fault_dirty_shared_page(vma, vmf->page); 3348 return ret; 3349 } 3350 3351 /* 3352 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3353 * but allow concurrent faults). 3354 * The mmap_sem may have been released depending on flags and our 3355 * return value. See filemap_fault() and __lock_page_or_retry(). 3356 */ 3357 static int do_fault(struct vm_fault *vmf) 3358 { 3359 struct vm_area_struct *vma = vmf->vma; 3360 int ret; 3361 3362 /* The VMA was not fully populated on mmap() or missing VM_DONTEXPAND */ 3363 if (!vma->vm_ops->fault) 3364 ret = VM_FAULT_SIGBUS; 3365 else if (!(vmf->flags & FAULT_FLAG_WRITE)) 3366 ret = do_read_fault(vmf); 3367 else if (!(vma->vm_flags & VM_SHARED)) 3368 ret = do_cow_fault(vmf); 3369 else 3370 ret = do_shared_fault(vmf); 3371 3372 /* preallocated pagetable is unused: free it */ 3373 if (vmf->prealloc_pte) { 3374 pte_free(vma->vm_mm, vmf->prealloc_pte); 3375 vmf->prealloc_pte = NULL; 3376 } 3377 return ret; 3378 } 3379 3380 static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma, 3381 unsigned long addr, int page_nid, 3382 int *flags) 3383 { 3384 get_page(page); 3385 3386 count_vm_numa_event(NUMA_HINT_FAULTS); 3387 if (page_nid == numa_node_id()) { 3388 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); 3389 *flags |= TNF_FAULT_LOCAL; 3390 } 3391 3392 return mpol_misplaced(page, vma, addr); 3393 } 3394 3395 static int do_numa_page(struct vm_fault *vmf) 3396 { 3397 struct vm_area_struct *vma = vmf->vma; 3398 struct page *page = NULL; 3399 int page_nid = -1; 3400 int last_cpupid; 3401 int target_nid; 3402 bool migrated = false; 3403 pte_t pte; 3404 bool was_writable = pte_savedwrite(vmf->orig_pte); 3405 int flags = 0; 3406 3407 /* 3408 * The "pte" at this point cannot be used safely without 3409 * validation through pte_unmap_same(). It's of NUMA type but 3410 * the pfn may be screwed if the read is non atomic. 3411 */ 3412 vmf->ptl = pte_lockptr(vma->vm_mm, vmf->pmd); 3413 spin_lock(vmf->ptl); 3414 if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) { 3415 pte_unmap_unlock(vmf->pte, vmf->ptl); 3416 goto out; 3417 } 3418 3419 /* 3420 * Make it present again, Depending on how arch implementes non 3421 * accessible ptes, some can allow access by kernel mode. 3422 */ 3423 pte = ptep_modify_prot_start(vma->vm_mm, vmf->address, vmf->pte); 3424 pte = pte_modify(pte, vma->vm_page_prot); 3425 pte = pte_mkyoung(pte); 3426 if (was_writable) 3427 pte = pte_mkwrite(pte); 3428 ptep_modify_prot_commit(vma->vm_mm, vmf->address, vmf->pte, pte); 3429 update_mmu_cache(vma, vmf->address, vmf->pte); 3430 3431 page = vm_normal_page(vma, vmf->address, pte); 3432 if (!page) { 3433 pte_unmap_unlock(vmf->pte, vmf->ptl); 3434 return 0; 3435 } 3436 3437 /* TODO: handle PTE-mapped THP */ 3438 if (PageCompound(page)) { 3439 pte_unmap_unlock(vmf->pte, vmf->ptl); 3440 return 0; 3441 } 3442 3443 /* 3444 * Avoid grouping on RO pages in general. RO pages shouldn't hurt as 3445 * much anyway since they can be in shared cache state. This misses 3446 * the case where a mapping is writable but the process never writes 3447 * to it but pte_write gets cleared during protection updates and 3448 * pte_dirty has unpredictable behaviour between PTE scan updates, 3449 * background writeback, dirty balancing and application behaviour. 3450 */ 3451 if (!pte_write(pte)) 3452 flags |= TNF_NO_GROUP; 3453 3454 /* 3455 * Flag if the page is shared between multiple address spaces. This 3456 * is later used when determining whether to group tasks together 3457 */ 3458 if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED)) 3459 flags |= TNF_SHARED; 3460 3461 last_cpupid = page_cpupid_last(page); 3462 page_nid = page_to_nid(page); 3463 target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid, 3464 &flags); 3465 pte_unmap_unlock(vmf->pte, vmf->ptl); 3466 if (target_nid == -1) { 3467 put_page(page); 3468 goto out; 3469 } 3470 3471 /* Migrate to the requested node */ 3472 migrated = migrate_misplaced_page(page, vma, target_nid); 3473 if (migrated) { 3474 page_nid = target_nid; 3475 flags |= TNF_MIGRATED; 3476 } else 3477 flags |= TNF_MIGRATE_FAIL; 3478 3479 out: 3480 if (page_nid != -1) 3481 task_numa_fault(last_cpupid, page_nid, 1, flags); 3482 return 0; 3483 } 3484 3485 static int create_huge_pmd(struct vm_fault *vmf) 3486 { 3487 if (vma_is_anonymous(vmf->vma)) 3488 return do_huge_pmd_anonymous_page(vmf); 3489 if (vmf->vma->vm_ops->huge_fault) 3490 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD); 3491 return VM_FAULT_FALLBACK; 3492 } 3493 3494 static int wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd) 3495 { 3496 if (vma_is_anonymous(vmf->vma)) 3497 return do_huge_pmd_wp_page(vmf, orig_pmd); 3498 if (vmf->vma->vm_ops->huge_fault) 3499 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD); 3500 3501 /* COW handled on pte level: split pmd */ 3502 VM_BUG_ON_VMA(vmf->vma->vm_flags & VM_SHARED, vmf->vma); 3503 __split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL); 3504 3505 return VM_FAULT_FALLBACK; 3506 } 3507 3508 static inline bool vma_is_accessible(struct vm_area_struct *vma) 3509 { 3510 return vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE); 3511 } 3512 3513 static int create_huge_pud(struct vm_fault *vmf) 3514 { 3515 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3516 /* No support for anonymous transparent PUD pages yet */ 3517 if (vma_is_anonymous(vmf->vma)) 3518 return VM_FAULT_FALLBACK; 3519 if (vmf->vma->vm_ops->huge_fault) 3520 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD); 3521 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 3522 return VM_FAULT_FALLBACK; 3523 } 3524 3525 static int wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud) 3526 { 3527 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3528 /* No support for anonymous transparent PUD pages yet */ 3529 if (vma_is_anonymous(vmf->vma)) 3530 return VM_FAULT_FALLBACK; 3531 if (vmf->vma->vm_ops->huge_fault) 3532 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD); 3533 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 3534 return VM_FAULT_FALLBACK; 3535 } 3536 3537 /* 3538 * These routines also need to handle stuff like marking pages dirty 3539 * and/or accessed for architectures that don't do it in hardware (most 3540 * RISC architectures). The early dirtying is also good on the i386. 3541 * 3542 * There is also a hook called "update_mmu_cache()" that architectures 3543 * with external mmu caches can use to update those (ie the Sparc or 3544 * PowerPC hashed page tables that act as extended TLBs). 3545 * 3546 * We enter with non-exclusive mmap_sem (to exclude vma changes, but allow 3547 * concurrent faults). 3548 * 3549 * The mmap_sem may have been released depending on flags and our return value. 3550 * See filemap_fault() and __lock_page_or_retry(). 3551 */ 3552 static int handle_pte_fault(struct vm_fault *vmf) 3553 { 3554 pte_t entry; 3555 3556 if (unlikely(pmd_none(*vmf->pmd))) { 3557 /* 3558 * Leave __pte_alloc() until later: because vm_ops->fault may 3559 * want to allocate huge page, and if we expose page table 3560 * for an instant, it will be difficult to retract from 3561 * concurrent faults and from rmap lookups. 3562 */ 3563 vmf->pte = NULL; 3564 } else { 3565 /* See comment in pte_alloc_one_map() */ 3566 if (pmd_trans_unstable(vmf->pmd) || pmd_devmap(*vmf->pmd)) 3567 return 0; 3568 /* 3569 * A regular pmd is established and it can't morph into a huge 3570 * pmd from under us anymore at this point because we hold the 3571 * mmap_sem read mode and khugepaged takes it in write mode. 3572 * So now it's safe to run pte_offset_map(). 3573 */ 3574 vmf->pte = pte_offset_map(vmf->pmd, vmf->address); 3575 vmf->orig_pte = *vmf->pte; 3576 3577 /* 3578 * some architectures can have larger ptes than wordsize, 3579 * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and 3580 * CONFIG_32BIT=y, so READ_ONCE or ACCESS_ONCE cannot guarantee 3581 * atomic accesses. The code below just needs a consistent 3582 * view for the ifs and we later double check anyway with the 3583 * ptl lock held. So here a barrier will do. 3584 */ 3585 barrier(); 3586 if (pte_none(vmf->orig_pte)) { 3587 pte_unmap(vmf->pte); 3588 vmf->pte = NULL; 3589 } 3590 } 3591 3592 if (!vmf->pte) { 3593 if (vma_is_anonymous(vmf->vma)) 3594 return do_anonymous_page(vmf); 3595 else 3596 return do_fault(vmf); 3597 } 3598 3599 if (!pte_present(vmf->orig_pte)) 3600 return do_swap_page(vmf); 3601 3602 if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) 3603 return do_numa_page(vmf); 3604 3605 vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd); 3606 spin_lock(vmf->ptl); 3607 entry = vmf->orig_pte; 3608 if (unlikely(!pte_same(*vmf->pte, entry))) 3609 goto unlock; 3610 if (vmf->flags & FAULT_FLAG_WRITE) { 3611 if (!pte_write(entry)) 3612 return do_wp_page(vmf); 3613 entry = pte_mkdirty(entry); 3614 } 3615 entry = pte_mkyoung(entry); 3616 if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry, 3617 vmf->flags & FAULT_FLAG_WRITE)) { 3618 update_mmu_cache(vmf->vma, vmf->address, vmf->pte); 3619 } else { 3620 /* 3621 * This is needed only for protection faults but the arch code 3622 * is not yet telling us if this is a protection fault or not. 3623 * This still avoids useless tlb flushes for .text page faults 3624 * with threads. 3625 */ 3626 if (vmf->flags & FAULT_FLAG_WRITE) 3627 flush_tlb_fix_spurious_fault(vmf->vma, vmf->address); 3628 } 3629 unlock: 3630 pte_unmap_unlock(vmf->pte, vmf->ptl); 3631 return 0; 3632 } 3633 3634 /* 3635 * By the time we get here, we already hold the mm semaphore 3636 * 3637 * The mmap_sem may have been released depending on flags and our 3638 * return value. See filemap_fault() and __lock_page_or_retry(). 3639 */ 3640 static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address, 3641 unsigned int flags) 3642 { 3643 struct vm_fault vmf = { 3644 .vma = vma, 3645 .address = address & PAGE_MASK, 3646 .flags = flags, 3647 .pgoff = linear_page_index(vma, address), 3648 .gfp_mask = __get_fault_gfp_mask(vma), 3649 }; 3650 struct mm_struct *mm = vma->vm_mm; 3651 pgd_t *pgd; 3652 int ret; 3653 3654 pgd = pgd_offset(mm, address); 3655 3656 vmf.pud = pud_alloc(mm, pgd, address); 3657 if (!vmf.pud) 3658 return VM_FAULT_OOM; 3659 if (pud_none(*vmf.pud) && transparent_hugepage_enabled(vma)) { 3660 ret = create_huge_pud(&vmf); 3661 if (!(ret & VM_FAULT_FALLBACK)) 3662 return ret; 3663 } else { 3664 pud_t orig_pud = *vmf.pud; 3665 3666 barrier(); 3667 if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) { 3668 unsigned int dirty = flags & FAULT_FLAG_WRITE; 3669 3670 /* NUMA case for anonymous PUDs would go here */ 3671 3672 if (dirty && !pud_write(orig_pud)) { 3673 ret = wp_huge_pud(&vmf, orig_pud); 3674 if (!(ret & VM_FAULT_FALLBACK)) 3675 return ret; 3676 } else { 3677 huge_pud_set_accessed(&vmf, orig_pud); 3678 return 0; 3679 } 3680 } 3681 } 3682 3683 vmf.pmd = pmd_alloc(mm, vmf.pud, address); 3684 if (!vmf.pmd) 3685 return VM_FAULT_OOM; 3686 if (pmd_none(*vmf.pmd) && transparent_hugepage_enabled(vma)) { 3687 ret = create_huge_pmd(&vmf); 3688 if (!(ret & VM_FAULT_FALLBACK)) 3689 return ret; 3690 } else { 3691 pmd_t orig_pmd = *vmf.pmd; 3692 3693 barrier(); 3694 if (pmd_trans_huge(orig_pmd) || pmd_devmap(orig_pmd)) { 3695 if (pmd_protnone(orig_pmd) && vma_is_accessible(vma)) 3696 return do_huge_pmd_numa_page(&vmf, orig_pmd); 3697 3698 if ((vmf.flags & FAULT_FLAG_WRITE) && 3699 !pmd_write(orig_pmd)) { 3700 ret = wp_huge_pmd(&vmf, orig_pmd); 3701 if (!(ret & VM_FAULT_FALLBACK)) 3702 return ret; 3703 } else { 3704 huge_pmd_set_accessed(&vmf, orig_pmd); 3705 return 0; 3706 } 3707 } 3708 } 3709 3710 return handle_pte_fault(&vmf); 3711 } 3712 3713 /* 3714 * By the time we get here, we already hold the mm semaphore 3715 * 3716 * The mmap_sem may have been released depending on flags and our 3717 * return value. See filemap_fault() and __lock_page_or_retry(). 3718 */ 3719 int handle_mm_fault(struct vm_area_struct *vma, unsigned long address, 3720 unsigned int flags) 3721 { 3722 int ret; 3723 3724 __set_current_state(TASK_RUNNING); 3725 3726 count_vm_event(PGFAULT); 3727 mem_cgroup_count_vm_event(vma->vm_mm, PGFAULT); 3728 3729 /* do counter updates before entering really critical section. */ 3730 check_sync_rss_stat(current); 3731 3732 /* 3733 * Enable the memcg OOM handling for faults triggered in user 3734 * space. Kernel faults are handled more gracefully. 3735 */ 3736 if (flags & FAULT_FLAG_USER) 3737 mem_cgroup_oom_enable(); 3738 3739 if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE, 3740 flags & FAULT_FLAG_INSTRUCTION, 3741 flags & FAULT_FLAG_REMOTE)) 3742 return VM_FAULT_SIGSEGV; 3743 3744 if (unlikely(is_vm_hugetlb_page(vma))) 3745 ret = hugetlb_fault(vma->vm_mm, vma, address, flags); 3746 else 3747 ret = __handle_mm_fault(vma, address, flags); 3748 3749 if (flags & FAULT_FLAG_USER) { 3750 mem_cgroup_oom_disable(); 3751 /* 3752 * The task may have entered a memcg OOM situation but 3753 * if the allocation error was handled gracefully (no 3754 * VM_FAULT_OOM), there is no need to kill anything. 3755 * Just clean up the OOM state peacefully. 3756 */ 3757 if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM)) 3758 mem_cgroup_oom_synchronize(false); 3759 } 3760 3761 /* 3762 * This mm has been already reaped by the oom reaper and so the 3763 * refault cannot be trusted in general. Anonymous refaults would 3764 * lose data and give a zero page instead e.g. This is especially 3765 * problem for use_mm() because regular tasks will just die and 3766 * the corrupted data will not be visible anywhere while kthread 3767 * will outlive the oom victim and potentially propagate the date 3768 * further. 3769 */ 3770 if (unlikely((current->flags & PF_KTHREAD) && !(ret & VM_FAULT_ERROR) 3771 && test_bit(MMF_UNSTABLE, &vma->vm_mm->flags))) 3772 ret = VM_FAULT_SIGBUS; 3773 3774 return ret; 3775 } 3776 EXPORT_SYMBOL_GPL(handle_mm_fault); 3777 3778 #ifndef __PAGETABLE_PUD_FOLDED 3779 /* 3780 * Allocate page upper directory. 3781 * We've already handled the fast-path in-line. 3782 */ 3783 int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) 3784 { 3785 pud_t *new = pud_alloc_one(mm, address); 3786 if (!new) 3787 return -ENOMEM; 3788 3789 smp_wmb(); /* See comment in __pte_alloc */ 3790 3791 spin_lock(&mm->page_table_lock); 3792 if (pgd_present(*pgd)) /* Another has populated it */ 3793 pud_free(mm, new); 3794 else 3795 pgd_populate(mm, pgd, new); 3796 spin_unlock(&mm->page_table_lock); 3797 return 0; 3798 } 3799 #endif /* __PAGETABLE_PUD_FOLDED */ 3800 3801 #ifndef __PAGETABLE_PMD_FOLDED 3802 /* 3803 * Allocate page middle directory. 3804 * We've already handled the fast-path in-line. 3805 */ 3806 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 3807 { 3808 spinlock_t *ptl; 3809 pmd_t *new = pmd_alloc_one(mm, address); 3810 if (!new) 3811 return -ENOMEM; 3812 3813 smp_wmb(); /* See comment in __pte_alloc */ 3814 3815 ptl = pud_lock(mm, pud); 3816 #ifndef __ARCH_HAS_4LEVEL_HACK 3817 if (!pud_present(*pud)) { 3818 mm_inc_nr_pmds(mm); 3819 pud_populate(mm, pud, new); 3820 } else /* Another has populated it */ 3821 pmd_free(mm, new); 3822 #else 3823 if (!pgd_present(*pud)) { 3824 mm_inc_nr_pmds(mm); 3825 pgd_populate(mm, pud, new); 3826 } else /* Another has populated it */ 3827 pmd_free(mm, new); 3828 #endif /* __ARCH_HAS_4LEVEL_HACK */ 3829 spin_unlock(ptl); 3830 return 0; 3831 } 3832 #endif /* __PAGETABLE_PMD_FOLDED */ 3833 3834 static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address, 3835 pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp) 3836 { 3837 pgd_t *pgd; 3838 pud_t *pud; 3839 pmd_t *pmd; 3840 pte_t *ptep; 3841 3842 pgd = pgd_offset(mm, address); 3843 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 3844 goto out; 3845 3846 pud = pud_offset(pgd, address); 3847 if (pud_none(*pud) || unlikely(pud_bad(*pud))) 3848 goto out; 3849 3850 pmd = pmd_offset(pud, address); 3851 VM_BUG_ON(pmd_trans_huge(*pmd)); 3852 3853 if (pmd_huge(*pmd)) { 3854 if (!pmdpp) 3855 goto out; 3856 3857 *ptlp = pmd_lock(mm, pmd); 3858 if (pmd_huge(*pmd)) { 3859 *pmdpp = pmd; 3860 return 0; 3861 } 3862 spin_unlock(*ptlp); 3863 } 3864 3865 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) 3866 goto out; 3867 3868 ptep = pte_offset_map_lock(mm, pmd, address, ptlp); 3869 if (!ptep) 3870 goto out; 3871 if (!pte_present(*ptep)) 3872 goto unlock; 3873 *ptepp = ptep; 3874 return 0; 3875 unlock: 3876 pte_unmap_unlock(ptep, *ptlp); 3877 out: 3878 return -EINVAL; 3879 } 3880 3881 static inline int follow_pte(struct mm_struct *mm, unsigned long address, 3882 pte_t **ptepp, spinlock_t **ptlp) 3883 { 3884 int res; 3885 3886 /* (void) is needed to make gcc happy */ 3887 (void) __cond_lock(*ptlp, 3888 !(res = __follow_pte_pmd(mm, address, ptepp, NULL, 3889 ptlp))); 3890 return res; 3891 } 3892 3893 int follow_pte_pmd(struct mm_struct *mm, unsigned long address, 3894 pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp) 3895 { 3896 int res; 3897 3898 /* (void) is needed to make gcc happy */ 3899 (void) __cond_lock(*ptlp, 3900 !(res = __follow_pte_pmd(mm, address, ptepp, pmdpp, 3901 ptlp))); 3902 return res; 3903 } 3904 EXPORT_SYMBOL(follow_pte_pmd); 3905 3906 /** 3907 * follow_pfn - look up PFN at a user virtual address 3908 * @vma: memory mapping 3909 * @address: user virtual address 3910 * @pfn: location to store found PFN 3911 * 3912 * Only IO mappings and raw PFN mappings are allowed. 3913 * 3914 * Returns zero and the pfn at @pfn on success, -ve otherwise. 3915 */ 3916 int follow_pfn(struct vm_area_struct *vma, unsigned long address, 3917 unsigned long *pfn) 3918 { 3919 int ret = -EINVAL; 3920 spinlock_t *ptl; 3921 pte_t *ptep; 3922 3923 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 3924 return ret; 3925 3926 ret = follow_pte(vma->vm_mm, address, &ptep, &ptl); 3927 if (ret) 3928 return ret; 3929 *pfn = pte_pfn(*ptep); 3930 pte_unmap_unlock(ptep, ptl); 3931 return 0; 3932 } 3933 EXPORT_SYMBOL(follow_pfn); 3934 3935 #ifdef CONFIG_HAVE_IOREMAP_PROT 3936 int follow_phys(struct vm_area_struct *vma, 3937 unsigned long address, unsigned int flags, 3938 unsigned long *prot, resource_size_t *phys) 3939 { 3940 int ret = -EINVAL; 3941 pte_t *ptep, pte; 3942 spinlock_t *ptl; 3943 3944 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 3945 goto out; 3946 3947 if (follow_pte(vma->vm_mm, address, &ptep, &ptl)) 3948 goto out; 3949 pte = *ptep; 3950 3951 if ((flags & FOLL_WRITE) && !pte_write(pte)) 3952 goto unlock; 3953 3954 *prot = pgprot_val(pte_pgprot(pte)); 3955 *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT; 3956 3957 ret = 0; 3958 unlock: 3959 pte_unmap_unlock(ptep, ptl); 3960 out: 3961 return ret; 3962 } 3963 3964 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, 3965 void *buf, int len, int write) 3966 { 3967 resource_size_t phys_addr; 3968 unsigned long prot = 0; 3969 void __iomem *maddr; 3970 int offset = addr & (PAGE_SIZE-1); 3971 3972 if (follow_phys(vma, addr, write, &prot, &phys_addr)) 3973 return -EINVAL; 3974 3975 maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot); 3976 if (write) 3977 memcpy_toio(maddr + offset, buf, len); 3978 else 3979 memcpy_fromio(buf, maddr + offset, len); 3980 iounmap(maddr); 3981 3982 return len; 3983 } 3984 EXPORT_SYMBOL_GPL(generic_access_phys); 3985 #endif 3986 3987 /* 3988 * Access another process' address space as given in mm. If non-NULL, use the 3989 * given task for page fault accounting. 3990 */ 3991 int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, 3992 unsigned long addr, void *buf, int len, unsigned int gup_flags) 3993 { 3994 struct vm_area_struct *vma; 3995 void *old_buf = buf; 3996 int write = gup_flags & FOLL_WRITE; 3997 3998 down_read(&mm->mmap_sem); 3999 /* ignore errors, just check how much was successfully transferred */ 4000 while (len) { 4001 int bytes, ret, offset; 4002 void *maddr; 4003 struct page *page = NULL; 4004 4005 ret = get_user_pages_remote(tsk, mm, addr, 1, 4006 gup_flags, &page, &vma, NULL); 4007 if (ret <= 0) { 4008 #ifndef CONFIG_HAVE_IOREMAP_PROT 4009 break; 4010 #else 4011 /* 4012 * Check if this is a VM_IO | VM_PFNMAP VMA, which 4013 * we can access using slightly different code. 4014 */ 4015 vma = find_vma(mm, addr); 4016 if (!vma || vma->vm_start > addr) 4017 break; 4018 if (vma->vm_ops && vma->vm_ops->access) 4019 ret = vma->vm_ops->access(vma, addr, buf, 4020 len, write); 4021 if (ret <= 0) 4022 break; 4023 bytes = ret; 4024 #endif 4025 } else { 4026 bytes = len; 4027 offset = addr & (PAGE_SIZE-1); 4028 if (bytes > PAGE_SIZE-offset) 4029 bytes = PAGE_SIZE-offset; 4030 4031 maddr = kmap(page); 4032 if (write) { 4033 copy_to_user_page(vma, page, addr, 4034 maddr + offset, buf, bytes); 4035 set_page_dirty_lock(page); 4036 } else { 4037 copy_from_user_page(vma, page, addr, 4038 buf, maddr + offset, bytes); 4039 } 4040 kunmap(page); 4041 put_page(page); 4042 } 4043 len -= bytes; 4044 buf += bytes; 4045 addr += bytes; 4046 } 4047 up_read(&mm->mmap_sem); 4048 4049 return buf - old_buf; 4050 } 4051 4052 /** 4053 * access_remote_vm - access another process' address space 4054 * @mm: the mm_struct of the target address space 4055 * @addr: start address to access 4056 * @buf: source or destination buffer 4057 * @len: number of bytes to transfer 4058 * @gup_flags: flags modifying lookup behaviour 4059 * 4060 * The caller must hold a reference on @mm. 4061 */ 4062 int access_remote_vm(struct mm_struct *mm, unsigned long addr, 4063 void *buf, int len, unsigned int gup_flags) 4064 { 4065 return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags); 4066 } 4067 4068 /* 4069 * Access another process' address space. 4070 * Source/target buffer must be kernel space, 4071 * Do not walk the page table directly, use get_user_pages 4072 */ 4073 int access_process_vm(struct task_struct *tsk, unsigned long addr, 4074 void *buf, int len, unsigned int gup_flags) 4075 { 4076 struct mm_struct *mm; 4077 int ret; 4078 4079 mm = get_task_mm(tsk); 4080 if (!mm) 4081 return 0; 4082 4083 ret = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags); 4084 4085 mmput(mm); 4086 4087 return ret; 4088 } 4089 EXPORT_SYMBOL_GPL(access_process_vm); 4090 4091 /* 4092 * Print the name of a VMA. 4093 */ 4094 void print_vma_addr(char *prefix, unsigned long ip) 4095 { 4096 struct mm_struct *mm = current->mm; 4097 struct vm_area_struct *vma; 4098 4099 /* 4100 * Do not print if we are in atomic 4101 * contexts (in exception stacks, etc.): 4102 */ 4103 if (preempt_count()) 4104 return; 4105 4106 down_read(&mm->mmap_sem); 4107 vma = find_vma(mm, ip); 4108 if (vma && vma->vm_file) { 4109 struct file *f = vma->vm_file; 4110 char *buf = (char *)__get_free_page(GFP_KERNEL); 4111 if (buf) { 4112 char *p; 4113 4114 p = file_path(f, buf, PAGE_SIZE); 4115 if (IS_ERR(p)) 4116 p = "?"; 4117 printk("%s%s[%lx+%lx]", prefix, kbasename(p), 4118 vma->vm_start, 4119 vma->vm_end - vma->vm_start); 4120 free_page((unsigned long)buf); 4121 } 4122 } 4123 up_read(&mm->mmap_sem); 4124 } 4125 4126 #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP) 4127 void __might_fault(const char *file, int line) 4128 { 4129 /* 4130 * Some code (nfs/sunrpc) uses socket ops on kernel memory while 4131 * holding the mmap_sem, this is safe because kernel memory doesn't 4132 * get paged out, therefore we'll never actually fault, and the 4133 * below annotations will generate false positives. 4134 */ 4135 if (segment_eq(get_fs(), KERNEL_DS)) 4136 return; 4137 if (pagefault_disabled()) 4138 return; 4139 __might_sleep(file, line, 0); 4140 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) 4141 if (current->mm) 4142 might_lock_read(¤t->mm->mmap_sem); 4143 #endif 4144 } 4145 EXPORT_SYMBOL(__might_fault); 4146 #endif 4147 4148 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) 4149 static void clear_gigantic_page(struct page *page, 4150 unsigned long addr, 4151 unsigned int pages_per_huge_page) 4152 { 4153 int i; 4154 struct page *p = page; 4155 4156 might_sleep(); 4157 for (i = 0; i < pages_per_huge_page; 4158 i++, p = mem_map_next(p, page, i)) { 4159 cond_resched(); 4160 clear_user_highpage(p, addr + i * PAGE_SIZE); 4161 } 4162 } 4163 void clear_huge_page(struct page *page, 4164 unsigned long addr, unsigned int pages_per_huge_page) 4165 { 4166 int i; 4167 4168 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { 4169 clear_gigantic_page(page, addr, pages_per_huge_page); 4170 return; 4171 } 4172 4173 might_sleep(); 4174 for (i = 0; i < pages_per_huge_page; i++) { 4175 cond_resched(); 4176 clear_user_highpage(page + i, addr + i * PAGE_SIZE); 4177 } 4178 } 4179 4180 static void copy_user_gigantic_page(struct page *dst, struct page *src, 4181 unsigned long addr, 4182 struct vm_area_struct *vma, 4183 unsigned int pages_per_huge_page) 4184 { 4185 int i; 4186 struct page *dst_base = dst; 4187 struct page *src_base = src; 4188 4189 for (i = 0; i < pages_per_huge_page; ) { 4190 cond_resched(); 4191 copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma); 4192 4193 i++; 4194 dst = mem_map_next(dst, dst_base, i); 4195 src = mem_map_next(src, src_base, i); 4196 } 4197 } 4198 4199 void copy_user_huge_page(struct page *dst, struct page *src, 4200 unsigned long addr, struct vm_area_struct *vma, 4201 unsigned int pages_per_huge_page) 4202 { 4203 int i; 4204 4205 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { 4206 copy_user_gigantic_page(dst, src, addr, vma, 4207 pages_per_huge_page); 4208 return; 4209 } 4210 4211 might_sleep(); 4212 for (i = 0; i < pages_per_huge_page; i++) { 4213 cond_resched(); 4214 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma); 4215 } 4216 } 4217 4218 long copy_huge_page_from_user(struct page *dst_page, 4219 const void __user *usr_src, 4220 unsigned int pages_per_huge_page, 4221 bool allow_pagefault) 4222 { 4223 void *src = (void *)usr_src; 4224 void *page_kaddr; 4225 unsigned long i, rc = 0; 4226 unsigned long ret_val = pages_per_huge_page * PAGE_SIZE; 4227 4228 for (i = 0; i < pages_per_huge_page; i++) { 4229 if (allow_pagefault) 4230 page_kaddr = kmap(dst_page + i); 4231 else 4232 page_kaddr = kmap_atomic(dst_page + i); 4233 rc = copy_from_user(page_kaddr, 4234 (const void __user *)(src + i * PAGE_SIZE), 4235 PAGE_SIZE); 4236 if (allow_pagefault) 4237 kunmap(dst_page + i); 4238 else 4239 kunmap_atomic(page_kaddr); 4240 4241 ret_val -= (PAGE_SIZE - rc); 4242 if (rc) 4243 break; 4244 4245 cond_resched(); 4246 } 4247 return ret_val; 4248 } 4249 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ 4250 4251 #if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS 4252 4253 static struct kmem_cache *page_ptl_cachep; 4254 4255 void __init ptlock_cache_init(void) 4256 { 4257 page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0, 4258 SLAB_PANIC, NULL); 4259 } 4260 4261 bool ptlock_alloc(struct page *page) 4262 { 4263 spinlock_t *ptl; 4264 4265 ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL); 4266 if (!ptl) 4267 return false; 4268 page->ptl = ptl; 4269 return true; 4270 } 4271 4272 void ptlock_free(struct page *page) 4273 { 4274 kmem_cache_free(page_ptl_cachep, page->ptl); 4275 } 4276 #endif 4277