1 /* 2 * linux/mm/memory.c 3 * 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 5 */ 6 7 /* 8 * demand-loading started 01.12.91 - seems it is high on the list of 9 * things wanted, and it should be easy to implement. - Linus 10 */ 11 12 /* 13 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared 14 * pages started 02.12.91, seems to work. - Linus. 15 * 16 * Tested sharing by executing about 30 /bin/sh: under the old kernel it 17 * would have taken more than the 6M I have free, but it worked well as 18 * far as I could see. 19 * 20 * Also corrected some "invalidate()"s - I wasn't doing enough of them. 21 */ 22 23 /* 24 * Real VM (paging to/from disk) started 18.12.91. Much more work and 25 * thought has to go into this. Oh, well.. 26 * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why. 27 * Found it. Everything seems to work now. 28 * 20.12.91 - Ok, making the swap-device changeable like the root. 29 */ 30 31 /* 32 * 05.04.94 - Multi-page memory management added for v1.1. 33 * Idea by Alex Bligh (alex@cconcepts.co.uk) 34 * 35 * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG 36 * (Gerhard.Wichert@pdb.siemens.de) 37 * 38 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) 39 */ 40 41 #include <linux/kernel_stat.h> 42 #include <linux/mm.h> 43 #include <linux/hugetlb.h> 44 #include <linux/mman.h> 45 #include <linux/swap.h> 46 #include <linux/highmem.h> 47 #include <linux/pagemap.h> 48 #include <linux/ksm.h> 49 #include <linux/rmap.h> 50 #include <linux/export.h> 51 #include <linux/delayacct.h> 52 #include <linux/init.h> 53 #include <linux/writeback.h> 54 #include <linux/memcontrol.h> 55 #include <linux/mmu_notifier.h> 56 #include <linux/kallsyms.h> 57 #include <linux/swapops.h> 58 #include <linux/elf.h> 59 #include <linux/gfp.h> 60 #include <linux/migrate.h> 61 #include <linux/string.h> 62 63 #include <asm/io.h> 64 #include <asm/pgalloc.h> 65 #include <asm/uaccess.h> 66 #include <asm/tlb.h> 67 #include <asm/tlbflush.h> 68 #include <asm/pgtable.h> 69 70 #include "internal.h" 71 72 #ifndef CONFIG_NEED_MULTIPLE_NODES 73 /* use the per-pgdat data instead for discontigmem - mbligh */ 74 unsigned long max_mapnr; 75 struct page *mem_map; 76 77 EXPORT_SYMBOL(max_mapnr); 78 EXPORT_SYMBOL(mem_map); 79 #endif 80 81 unsigned long num_physpages; 82 /* 83 * A number of key systems in x86 including ioremap() rely on the assumption 84 * that high_memory defines the upper bound on direct map memory, then end 85 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and 86 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL 87 * and ZONE_HIGHMEM. 88 */ 89 void * high_memory; 90 91 EXPORT_SYMBOL(num_physpages); 92 EXPORT_SYMBOL(high_memory); 93 94 /* 95 * Randomize the address space (stacks, mmaps, brk, etc.). 96 * 97 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization, 98 * as ancient (libc5 based) binaries can segfault. ) 99 */ 100 int randomize_va_space __read_mostly = 101 #ifdef CONFIG_COMPAT_BRK 102 1; 103 #else 104 2; 105 #endif 106 107 static int __init disable_randmaps(char *s) 108 { 109 randomize_va_space = 0; 110 return 1; 111 } 112 __setup("norandmaps", disable_randmaps); 113 114 unsigned long zero_pfn __read_mostly; 115 unsigned long highest_memmap_pfn __read_mostly; 116 117 /* 118 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init() 119 */ 120 static int __init init_zero_pfn(void) 121 { 122 zero_pfn = page_to_pfn(ZERO_PAGE(0)); 123 return 0; 124 } 125 core_initcall(init_zero_pfn); 126 127 128 #if defined(SPLIT_RSS_COUNTING) 129 130 void sync_mm_rss(struct mm_struct *mm) 131 { 132 int i; 133 134 for (i = 0; i < NR_MM_COUNTERS; i++) { 135 if (current->rss_stat.count[i]) { 136 add_mm_counter(mm, i, current->rss_stat.count[i]); 137 current->rss_stat.count[i] = 0; 138 } 139 } 140 current->rss_stat.events = 0; 141 } 142 143 static void add_mm_counter_fast(struct mm_struct *mm, int member, int val) 144 { 145 struct task_struct *task = current; 146 147 if (likely(task->mm == mm)) 148 task->rss_stat.count[member] += val; 149 else 150 add_mm_counter(mm, member, val); 151 } 152 #define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1) 153 #define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1) 154 155 /* sync counter once per 64 page faults */ 156 #define TASK_RSS_EVENTS_THRESH (64) 157 static void check_sync_rss_stat(struct task_struct *task) 158 { 159 if (unlikely(task != current)) 160 return; 161 if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH)) 162 sync_mm_rss(task->mm); 163 } 164 #else /* SPLIT_RSS_COUNTING */ 165 166 #define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member) 167 #define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member) 168 169 static void check_sync_rss_stat(struct task_struct *task) 170 { 171 } 172 173 #endif /* SPLIT_RSS_COUNTING */ 174 175 #ifdef HAVE_GENERIC_MMU_GATHER 176 177 static int tlb_next_batch(struct mmu_gather *tlb) 178 { 179 struct mmu_gather_batch *batch; 180 181 batch = tlb->active; 182 if (batch->next) { 183 tlb->active = batch->next; 184 return 1; 185 } 186 187 batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0); 188 if (!batch) 189 return 0; 190 191 batch->next = NULL; 192 batch->nr = 0; 193 batch->max = MAX_GATHER_BATCH; 194 195 tlb->active->next = batch; 196 tlb->active = batch; 197 198 return 1; 199 } 200 201 /* tlb_gather_mmu 202 * Called to initialize an (on-stack) mmu_gather structure for page-table 203 * tear-down from @mm. The @fullmm argument is used when @mm is without 204 * users and we're going to destroy the full address space (exit/execve). 205 */ 206 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm) 207 { 208 tlb->mm = mm; 209 210 tlb->fullmm = fullmm; 211 tlb->start = -1UL; 212 tlb->end = 0; 213 tlb->need_flush = 0; 214 tlb->fast_mode = (num_possible_cpus() == 1); 215 tlb->local.next = NULL; 216 tlb->local.nr = 0; 217 tlb->local.max = ARRAY_SIZE(tlb->__pages); 218 tlb->active = &tlb->local; 219 220 #ifdef CONFIG_HAVE_RCU_TABLE_FREE 221 tlb->batch = NULL; 222 #endif 223 } 224 225 void tlb_flush_mmu(struct mmu_gather *tlb) 226 { 227 struct mmu_gather_batch *batch; 228 229 if (!tlb->need_flush) 230 return; 231 tlb->need_flush = 0; 232 tlb_flush(tlb); 233 #ifdef CONFIG_HAVE_RCU_TABLE_FREE 234 tlb_table_flush(tlb); 235 #endif 236 237 if (tlb_fast_mode(tlb)) 238 return; 239 240 for (batch = &tlb->local; batch; batch = batch->next) { 241 free_pages_and_swap_cache(batch->pages, batch->nr); 242 batch->nr = 0; 243 } 244 tlb->active = &tlb->local; 245 } 246 247 /* tlb_finish_mmu 248 * Called at the end of the shootdown operation to free up any resources 249 * that were required. 250 */ 251 void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) 252 { 253 struct mmu_gather_batch *batch, *next; 254 255 tlb->start = start; 256 tlb->end = end; 257 tlb_flush_mmu(tlb); 258 259 /* keep the page table cache within bounds */ 260 check_pgt_cache(); 261 262 for (batch = tlb->local.next; batch; batch = next) { 263 next = batch->next; 264 free_pages((unsigned long)batch, 0); 265 } 266 tlb->local.next = NULL; 267 } 268 269 /* __tlb_remove_page 270 * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while 271 * handling the additional races in SMP caused by other CPUs caching valid 272 * mappings in their TLBs. Returns the number of free page slots left. 273 * When out of page slots we must call tlb_flush_mmu(). 274 */ 275 int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) 276 { 277 struct mmu_gather_batch *batch; 278 279 VM_BUG_ON(!tlb->need_flush); 280 281 if (tlb_fast_mode(tlb)) { 282 free_page_and_swap_cache(page); 283 return 1; /* avoid calling tlb_flush_mmu() */ 284 } 285 286 batch = tlb->active; 287 batch->pages[batch->nr++] = page; 288 if (batch->nr == batch->max) { 289 if (!tlb_next_batch(tlb)) 290 return 0; 291 batch = tlb->active; 292 } 293 VM_BUG_ON(batch->nr > batch->max); 294 295 return batch->max - batch->nr; 296 } 297 298 #endif /* HAVE_GENERIC_MMU_GATHER */ 299 300 #ifdef CONFIG_HAVE_RCU_TABLE_FREE 301 302 /* 303 * See the comment near struct mmu_table_batch. 304 */ 305 306 static void tlb_remove_table_smp_sync(void *arg) 307 { 308 /* Simply deliver the interrupt */ 309 } 310 311 static void tlb_remove_table_one(void *table) 312 { 313 /* 314 * This isn't an RCU grace period and hence the page-tables cannot be 315 * assumed to be actually RCU-freed. 316 * 317 * It is however sufficient for software page-table walkers that rely on 318 * IRQ disabling. See the comment near struct mmu_table_batch. 319 */ 320 smp_call_function(tlb_remove_table_smp_sync, NULL, 1); 321 __tlb_remove_table(table); 322 } 323 324 static void tlb_remove_table_rcu(struct rcu_head *head) 325 { 326 struct mmu_table_batch *batch; 327 int i; 328 329 batch = container_of(head, struct mmu_table_batch, rcu); 330 331 for (i = 0; i < batch->nr; i++) 332 __tlb_remove_table(batch->tables[i]); 333 334 free_page((unsigned long)batch); 335 } 336 337 void tlb_table_flush(struct mmu_gather *tlb) 338 { 339 struct mmu_table_batch **batch = &tlb->batch; 340 341 if (*batch) { 342 call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu); 343 *batch = NULL; 344 } 345 } 346 347 void tlb_remove_table(struct mmu_gather *tlb, void *table) 348 { 349 struct mmu_table_batch **batch = &tlb->batch; 350 351 tlb->need_flush = 1; 352 353 /* 354 * When there's less then two users of this mm there cannot be a 355 * concurrent page-table walk. 356 */ 357 if (atomic_read(&tlb->mm->mm_users) < 2) { 358 __tlb_remove_table(table); 359 return; 360 } 361 362 if (*batch == NULL) { 363 *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN); 364 if (*batch == NULL) { 365 tlb_remove_table_one(table); 366 return; 367 } 368 (*batch)->nr = 0; 369 } 370 (*batch)->tables[(*batch)->nr++] = table; 371 if ((*batch)->nr == MAX_TABLE_BATCH) 372 tlb_table_flush(tlb); 373 } 374 375 #endif /* CONFIG_HAVE_RCU_TABLE_FREE */ 376 377 /* 378 * If a p?d_bad entry is found while walking page tables, report 379 * the error, before resetting entry to p?d_none. Usually (but 380 * very seldom) called out from the p?d_none_or_clear_bad macros. 381 */ 382 383 void pgd_clear_bad(pgd_t *pgd) 384 { 385 pgd_ERROR(*pgd); 386 pgd_clear(pgd); 387 } 388 389 void pud_clear_bad(pud_t *pud) 390 { 391 pud_ERROR(*pud); 392 pud_clear(pud); 393 } 394 395 void pmd_clear_bad(pmd_t *pmd) 396 { 397 pmd_ERROR(*pmd); 398 pmd_clear(pmd); 399 } 400 401 /* 402 * Note: this doesn't free the actual pages themselves. That 403 * has been handled earlier when unmapping all the memory regions. 404 */ 405 static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, 406 unsigned long addr) 407 { 408 pgtable_t token = pmd_pgtable(*pmd); 409 pmd_clear(pmd); 410 pte_free_tlb(tlb, token, addr); 411 tlb->mm->nr_ptes--; 412 } 413 414 static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, 415 unsigned long addr, unsigned long end, 416 unsigned long floor, unsigned long ceiling) 417 { 418 pmd_t *pmd; 419 unsigned long next; 420 unsigned long start; 421 422 start = addr; 423 pmd = pmd_offset(pud, addr); 424 do { 425 next = pmd_addr_end(addr, end); 426 if (pmd_none_or_clear_bad(pmd)) 427 continue; 428 free_pte_range(tlb, pmd, addr); 429 } while (pmd++, addr = next, addr != end); 430 431 start &= PUD_MASK; 432 if (start < floor) 433 return; 434 if (ceiling) { 435 ceiling &= PUD_MASK; 436 if (!ceiling) 437 return; 438 } 439 if (end - 1 > ceiling - 1) 440 return; 441 442 pmd = pmd_offset(pud, start); 443 pud_clear(pud); 444 pmd_free_tlb(tlb, pmd, start); 445 } 446 447 static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, 448 unsigned long addr, unsigned long end, 449 unsigned long floor, unsigned long ceiling) 450 { 451 pud_t *pud; 452 unsigned long next; 453 unsigned long start; 454 455 start = addr; 456 pud = pud_offset(pgd, addr); 457 do { 458 next = pud_addr_end(addr, end); 459 if (pud_none_or_clear_bad(pud)) 460 continue; 461 free_pmd_range(tlb, pud, addr, next, floor, ceiling); 462 } while (pud++, addr = next, addr != end); 463 464 start &= PGDIR_MASK; 465 if (start < floor) 466 return; 467 if (ceiling) { 468 ceiling &= PGDIR_MASK; 469 if (!ceiling) 470 return; 471 } 472 if (end - 1 > ceiling - 1) 473 return; 474 475 pud = pud_offset(pgd, start); 476 pgd_clear(pgd); 477 pud_free_tlb(tlb, pud, start); 478 } 479 480 /* 481 * This function frees user-level page tables of a process. 482 * 483 * Must be called with pagetable lock held. 484 */ 485 void free_pgd_range(struct mmu_gather *tlb, 486 unsigned long addr, unsigned long end, 487 unsigned long floor, unsigned long ceiling) 488 { 489 pgd_t *pgd; 490 unsigned long next; 491 492 /* 493 * The next few lines have given us lots of grief... 494 * 495 * Why are we testing PMD* at this top level? Because often 496 * there will be no work to do at all, and we'd prefer not to 497 * go all the way down to the bottom just to discover that. 498 * 499 * Why all these "- 1"s? Because 0 represents both the bottom 500 * of the address space and the top of it (using -1 for the 501 * top wouldn't help much: the masks would do the wrong thing). 502 * The rule is that addr 0 and floor 0 refer to the bottom of 503 * the address space, but end 0 and ceiling 0 refer to the top 504 * Comparisons need to use "end - 1" and "ceiling - 1" (though 505 * that end 0 case should be mythical). 506 * 507 * Wherever addr is brought up or ceiling brought down, we must 508 * be careful to reject "the opposite 0" before it confuses the 509 * subsequent tests. But what about where end is brought down 510 * by PMD_SIZE below? no, end can't go down to 0 there. 511 * 512 * Whereas we round start (addr) and ceiling down, by different 513 * masks at different levels, in order to test whether a table 514 * now has no other vmas using it, so can be freed, we don't 515 * bother to round floor or end up - the tests don't need that. 516 */ 517 518 addr &= PMD_MASK; 519 if (addr < floor) { 520 addr += PMD_SIZE; 521 if (!addr) 522 return; 523 } 524 if (ceiling) { 525 ceiling &= PMD_MASK; 526 if (!ceiling) 527 return; 528 } 529 if (end - 1 > ceiling - 1) 530 end -= PMD_SIZE; 531 if (addr > end - 1) 532 return; 533 534 pgd = pgd_offset(tlb->mm, addr); 535 do { 536 next = pgd_addr_end(addr, end); 537 if (pgd_none_or_clear_bad(pgd)) 538 continue; 539 free_pud_range(tlb, pgd, addr, next, floor, ceiling); 540 } while (pgd++, addr = next, addr != end); 541 } 542 543 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, 544 unsigned long floor, unsigned long ceiling) 545 { 546 while (vma) { 547 struct vm_area_struct *next = vma->vm_next; 548 unsigned long addr = vma->vm_start; 549 550 /* 551 * Hide vma from rmap and truncate_pagecache before freeing 552 * pgtables 553 */ 554 unlink_anon_vmas(vma); 555 unlink_file_vma(vma); 556 557 if (is_vm_hugetlb_page(vma)) { 558 hugetlb_free_pgd_range(tlb, addr, vma->vm_end, 559 floor, next? next->vm_start: ceiling); 560 } else { 561 /* 562 * Optimization: gather nearby vmas into one call down 563 */ 564 while (next && next->vm_start <= vma->vm_end + PMD_SIZE 565 && !is_vm_hugetlb_page(next)) { 566 vma = next; 567 next = vma->vm_next; 568 unlink_anon_vmas(vma); 569 unlink_file_vma(vma); 570 } 571 free_pgd_range(tlb, addr, vma->vm_end, 572 floor, next? next->vm_start: ceiling); 573 } 574 vma = next; 575 } 576 } 577 578 int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, 579 pmd_t *pmd, unsigned long address) 580 { 581 pgtable_t new = pte_alloc_one(mm, address); 582 int wait_split_huge_page; 583 if (!new) 584 return -ENOMEM; 585 586 /* 587 * Ensure all pte setup (eg. pte page lock and page clearing) are 588 * visible before the pte is made visible to other CPUs by being 589 * put into page tables. 590 * 591 * The other side of the story is the pointer chasing in the page 592 * table walking code (when walking the page table without locking; 593 * ie. most of the time). Fortunately, these data accesses consist 594 * of a chain of data-dependent loads, meaning most CPUs (alpha 595 * being the notable exception) will already guarantee loads are 596 * seen in-order. See the alpha page table accessors for the 597 * smp_read_barrier_depends() barriers in page table walking code. 598 */ 599 smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */ 600 601 spin_lock(&mm->page_table_lock); 602 wait_split_huge_page = 0; 603 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ 604 mm->nr_ptes++; 605 pmd_populate(mm, pmd, new); 606 new = NULL; 607 } else if (unlikely(pmd_trans_splitting(*pmd))) 608 wait_split_huge_page = 1; 609 spin_unlock(&mm->page_table_lock); 610 if (new) 611 pte_free(mm, new); 612 if (wait_split_huge_page) 613 wait_split_huge_page(vma->anon_vma, pmd); 614 return 0; 615 } 616 617 int __pte_alloc_kernel(pmd_t *pmd, unsigned long address) 618 { 619 pte_t *new = pte_alloc_one_kernel(&init_mm, address); 620 if (!new) 621 return -ENOMEM; 622 623 smp_wmb(); /* See comment in __pte_alloc */ 624 625 spin_lock(&init_mm.page_table_lock); 626 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ 627 pmd_populate_kernel(&init_mm, pmd, new); 628 new = NULL; 629 } else 630 VM_BUG_ON(pmd_trans_splitting(*pmd)); 631 spin_unlock(&init_mm.page_table_lock); 632 if (new) 633 pte_free_kernel(&init_mm, new); 634 return 0; 635 } 636 637 static inline void init_rss_vec(int *rss) 638 { 639 memset(rss, 0, sizeof(int) * NR_MM_COUNTERS); 640 } 641 642 static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss) 643 { 644 int i; 645 646 if (current->mm == mm) 647 sync_mm_rss(mm); 648 for (i = 0; i < NR_MM_COUNTERS; i++) 649 if (rss[i]) 650 add_mm_counter(mm, i, rss[i]); 651 } 652 653 /* 654 * This function is called to print an error when a bad pte 655 * is found. For example, we might have a PFN-mapped pte in 656 * a region that doesn't allow it. 657 * 658 * The calling function must still handle the error. 659 */ 660 static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, 661 pte_t pte, struct page *page) 662 { 663 pgd_t *pgd = pgd_offset(vma->vm_mm, addr); 664 pud_t *pud = pud_offset(pgd, addr); 665 pmd_t *pmd = pmd_offset(pud, addr); 666 struct address_space *mapping; 667 pgoff_t index; 668 static unsigned long resume; 669 static unsigned long nr_shown; 670 static unsigned long nr_unshown; 671 672 /* 673 * Allow a burst of 60 reports, then keep quiet for that minute; 674 * or allow a steady drip of one report per second. 675 */ 676 if (nr_shown == 60) { 677 if (time_before(jiffies, resume)) { 678 nr_unshown++; 679 return; 680 } 681 if (nr_unshown) { 682 printk(KERN_ALERT 683 "BUG: Bad page map: %lu messages suppressed\n", 684 nr_unshown); 685 nr_unshown = 0; 686 } 687 nr_shown = 0; 688 } 689 if (nr_shown++ == 0) 690 resume = jiffies + 60 * HZ; 691 692 mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL; 693 index = linear_page_index(vma, addr); 694 695 printk(KERN_ALERT 696 "BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n", 697 current->comm, 698 (long long)pte_val(pte), (long long)pmd_val(*pmd)); 699 if (page) 700 dump_page(page); 701 printk(KERN_ALERT 702 "addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n", 703 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index); 704 /* 705 * Choose text because data symbols depend on CONFIG_KALLSYMS_ALL=y 706 */ 707 if (vma->vm_ops) 708 print_symbol(KERN_ALERT "vma->vm_ops->fault: %s\n", 709 (unsigned long)vma->vm_ops->fault); 710 if (vma->vm_file && vma->vm_file->f_op) 711 print_symbol(KERN_ALERT "vma->vm_file->f_op->mmap: %s\n", 712 (unsigned long)vma->vm_file->f_op->mmap); 713 dump_stack(); 714 add_taint(TAINT_BAD_PAGE); 715 } 716 717 static inline bool is_cow_mapping(vm_flags_t flags) 718 { 719 return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; 720 } 721 722 /* 723 * vm_normal_page -- This function gets the "struct page" associated with a pte. 724 * 725 * "Special" mappings do not wish to be associated with a "struct page" (either 726 * it doesn't exist, or it exists but they don't want to touch it). In this 727 * case, NULL is returned here. "Normal" mappings do have a struct page. 728 * 729 * There are 2 broad cases. Firstly, an architecture may define a pte_special() 730 * pte bit, in which case this function is trivial. Secondly, an architecture 731 * may not have a spare pte bit, which requires a more complicated scheme, 732 * described below. 733 * 734 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a 735 * special mapping (even if there are underlying and valid "struct pages"). 736 * COWed pages of a VM_PFNMAP are always normal. 737 * 738 * The way we recognize COWed pages within VM_PFNMAP mappings is through the 739 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit 740 * set, and the vm_pgoff will point to the first PFN mapped: thus every special 741 * mapping will always honor the rule 742 * 743 * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT) 744 * 745 * And for normal mappings this is false. 746 * 747 * This restricts such mappings to be a linear translation from virtual address 748 * to pfn. To get around this restriction, we allow arbitrary mappings so long 749 * as the vma is not a COW mapping; in that case, we know that all ptes are 750 * special (because none can have been COWed). 751 * 752 * 753 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP. 754 * 755 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct 756 * page" backing, however the difference is that _all_ pages with a struct 757 * page (that is, those where pfn_valid is true) are refcounted and considered 758 * normal pages by the VM. The disadvantage is that pages are refcounted 759 * (which can be slower and simply not an option for some PFNMAP users). The 760 * advantage is that we don't have to follow the strict linearity rule of 761 * PFNMAP mappings in order to support COWable mappings. 762 * 763 */ 764 #ifdef __HAVE_ARCH_PTE_SPECIAL 765 # define HAVE_PTE_SPECIAL 1 766 #else 767 # define HAVE_PTE_SPECIAL 0 768 #endif 769 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, 770 pte_t pte) 771 { 772 unsigned long pfn = pte_pfn(pte); 773 774 if (HAVE_PTE_SPECIAL) { 775 if (likely(!pte_special(pte))) 776 goto check_pfn; 777 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) 778 return NULL; 779 if (!is_zero_pfn(pfn)) 780 print_bad_pte(vma, addr, pte, NULL); 781 return NULL; 782 } 783 784 /* !HAVE_PTE_SPECIAL case follows: */ 785 786 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) { 787 if (vma->vm_flags & VM_MIXEDMAP) { 788 if (!pfn_valid(pfn)) 789 return NULL; 790 goto out; 791 } else { 792 unsigned long off; 793 off = (addr - vma->vm_start) >> PAGE_SHIFT; 794 if (pfn == vma->vm_pgoff + off) 795 return NULL; 796 if (!is_cow_mapping(vma->vm_flags)) 797 return NULL; 798 } 799 } 800 801 if (is_zero_pfn(pfn)) 802 return NULL; 803 check_pfn: 804 if (unlikely(pfn > highest_memmap_pfn)) { 805 print_bad_pte(vma, addr, pte, NULL); 806 return NULL; 807 } 808 809 /* 810 * NOTE! We still have PageReserved() pages in the page tables. 811 * eg. VDSO mappings can cause them to exist. 812 */ 813 out: 814 return pfn_to_page(pfn); 815 } 816 817 /* 818 * copy one vm_area from one task to the other. Assumes the page tables 819 * already present in the new task to be cleared in the whole range 820 * covered by this vma. 821 */ 822 823 static inline unsigned long 824 copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, 825 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma, 826 unsigned long addr, int *rss) 827 { 828 unsigned long vm_flags = vma->vm_flags; 829 pte_t pte = *src_pte; 830 struct page *page; 831 832 /* pte contains position in swap or file, so copy. */ 833 if (unlikely(!pte_present(pte))) { 834 if (!pte_file(pte)) { 835 swp_entry_t entry = pte_to_swp_entry(pte); 836 837 if (swap_duplicate(entry) < 0) 838 return entry.val; 839 840 /* make sure dst_mm is on swapoff's mmlist. */ 841 if (unlikely(list_empty(&dst_mm->mmlist))) { 842 spin_lock(&mmlist_lock); 843 if (list_empty(&dst_mm->mmlist)) 844 list_add(&dst_mm->mmlist, 845 &src_mm->mmlist); 846 spin_unlock(&mmlist_lock); 847 } 848 if (likely(!non_swap_entry(entry))) 849 rss[MM_SWAPENTS]++; 850 else if (is_migration_entry(entry)) { 851 page = migration_entry_to_page(entry); 852 853 if (PageAnon(page)) 854 rss[MM_ANONPAGES]++; 855 else 856 rss[MM_FILEPAGES]++; 857 858 if (is_write_migration_entry(entry) && 859 is_cow_mapping(vm_flags)) { 860 /* 861 * COW mappings require pages in both 862 * parent and child to be set to read. 863 */ 864 make_migration_entry_read(&entry); 865 pte = swp_entry_to_pte(entry); 866 set_pte_at(src_mm, addr, src_pte, pte); 867 } 868 } 869 } 870 goto out_set_pte; 871 } 872 873 /* 874 * If it's a COW mapping, write protect it both 875 * in the parent and the child 876 */ 877 if (is_cow_mapping(vm_flags)) { 878 ptep_set_wrprotect(src_mm, addr, src_pte); 879 pte = pte_wrprotect(pte); 880 } 881 882 /* 883 * If it's a shared mapping, mark it clean in 884 * the child 885 */ 886 if (vm_flags & VM_SHARED) 887 pte = pte_mkclean(pte); 888 pte = pte_mkold(pte); 889 890 page = vm_normal_page(vma, addr, pte); 891 if (page) { 892 get_page(page); 893 page_dup_rmap(page); 894 if (PageAnon(page)) 895 rss[MM_ANONPAGES]++; 896 else 897 rss[MM_FILEPAGES]++; 898 } 899 900 out_set_pte: 901 set_pte_at(dst_mm, addr, dst_pte, pte); 902 return 0; 903 } 904 905 int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 906 pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma, 907 unsigned long addr, unsigned long end) 908 { 909 pte_t *orig_src_pte, *orig_dst_pte; 910 pte_t *src_pte, *dst_pte; 911 spinlock_t *src_ptl, *dst_ptl; 912 int progress = 0; 913 int rss[NR_MM_COUNTERS]; 914 swp_entry_t entry = (swp_entry_t){0}; 915 916 again: 917 init_rss_vec(rss); 918 919 dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl); 920 if (!dst_pte) 921 return -ENOMEM; 922 src_pte = pte_offset_map(src_pmd, addr); 923 src_ptl = pte_lockptr(src_mm, src_pmd); 924 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 925 orig_src_pte = src_pte; 926 orig_dst_pte = dst_pte; 927 arch_enter_lazy_mmu_mode(); 928 929 do { 930 /* 931 * We are holding two locks at this point - either of them 932 * could generate latencies in another task on another CPU. 933 */ 934 if (progress >= 32) { 935 progress = 0; 936 if (need_resched() || 937 spin_needbreak(src_ptl) || spin_needbreak(dst_ptl)) 938 break; 939 } 940 if (pte_none(*src_pte)) { 941 progress++; 942 continue; 943 } 944 entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, 945 vma, addr, rss); 946 if (entry.val) 947 break; 948 progress += 8; 949 } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); 950 951 arch_leave_lazy_mmu_mode(); 952 spin_unlock(src_ptl); 953 pte_unmap(orig_src_pte); 954 add_mm_rss_vec(dst_mm, rss); 955 pte_unmap_unlock(orig_dst_pte, dst_ptl); 956 cond_resched(); 957 958 if (entry.val) { 959 if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) 960 return -ENOMEM; 961 progress = 0; 962 } 963 if (addr != end) 964 goto again; 965 return 0; 966 } 967 968 static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 969 pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma, 970 unsigned long addr, unsigned long end) 971 { 972 pmd_t *src_pmd, *dst_pmd; 973 unsigned long next; 974 975 dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); 976 if (!dst_pmd) 977 return -ENOMEM; 978 src_pmd = pmd_offset(src_pud, addr); 979 do { 980 next = pmd_addr_end(addr, end); 981 if (pmd_trans_huge(*src_pmd)) { 982 int err; 983 VM_BUG_ON(next-addr != HPAGE_PMD_SIZE); 984 err = copy_huge_pmd(dst_mm, src_mm, 985 dst_pmd, src_pmd, addr, vma); 986 if (err == -ENOMEM) 987 return -ENOMEM; 988 if (!err) 989 continue; 990 /* fall through */ 991 } 992 if (pmd_none_or_clear_bad(src_pmd)) 993 continue; 994 if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd, 995 vma, addr, next)) 996 return -ENOMEM; 997 } while (dst_pmd++, src_pmd++, addr = next, addr != end); 998 return 0; 999 } 1000 1001 static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1002 pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma, 1003 unsigned long addr, unsigned long end) 1004 { 1005 pud_t *src_pud, *dst_pud; 1006 unsigned long next; 1007 1008 dst_pud = pud_alloc(dst_mm, dst_pgd, addr); 1009 if (!dst_pud) 1010 return -ENOMEM; 1011 src_pud = pud_offset(src_pgd, addr); 1012 do { 1013 next = pud_addr_end(addr, end); 1014 if (pud_none_or_clear_bad(src_pud)) 1015 continue; 1016 if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud, 1017 vma, addr, next)) 1018 return -ENOMEM; 1019 } while (dst_pud++, src_pud++, addr = next, addr != end); 1020 return 0; 1021 } 1022 1023 int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1024 struct vm_area_struct *vma) 1025 { 1026 pgd_t *src_pgd, *dst_pgd; 1027 unsigned long next; 1028 unsigned long addr = vma->vm_start; 1029 unsigned long end = vma->vm_end; 1030 unsigned long mmun_start; /* For mmu_notifiers */ 1031 unsigned long mmun_end; /* For mmu_notifiers */ 1032 bool is_cow; 1033 int ret; 1034 1035 /* 1036 * Don't copy ptes where a page fault will fill them correctly. 1037 * Fork becomes much lighter when there are big shared or private 1038 * readonly mappings. The tradeoff is that copy_page_range is more 1039 * efficient than faulting. 1040 */ 1041 if (!(vma->vm_flags & (VM_HUGETLB | VM_NONLINEAR | 1042 VM_PFNMAP | VM_MIXEDMAP))) { 1043 if (!vma->anon_vma) 1044 return 0; 1045 } 1046 1047 if (is_vm_hugetlb_page(vma)) 1048 return copy_hugetlb_page_range(dst_mm, src_mm, vma); 1049 1050 if (unlikely(vma->vm_flags & VM_PFNMAP)) { 1051 /* 1052 * We do not free on error cases below as remove_vma 1053 * gets called on error from higher level routine 1054 */ 1055 ret = track_pfn_copy(vma); 1056 if (ret) 1057 return ret; 1058 } 1059 1060 /* 1061 * We need to invalidate the secondary MMU mappings only when 1062 * there could be a permission downgrade on the ptes of the 1063 * parent mm. And a permission downgrade will only happen if 1064 * is_cow_mapping() returns true. 1065 */ 1066 is_cow = is_cow_mapping(vma->vm_flags); 1067 mmun_start = addr; 1068 mmun_end = end; 1069 if (is_cow) 1070 mmu_notifier_invalidate_range_start(src_mm, mmun_start, 1071 mmun_end); 1072 1073 ret = 0; 1074 dst_pgd = pgd_offset(dst_mm, addr); 1075 src_pgd = pgd_offset(src_mm, addr); 1076 do { 1077 next = pgd_addr_end(addr, end); 1078 if (pgd_none_or_clear_bad(src_pgd)) 1079 continue; 1080 if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd, 1081 vma, addr, next))) { 1082 ret = -ENOMEM; 1083 break; 1084 } 1085 } while (dst_pgd++, src_pgd++, addr = next, addr != end); 1086 1087 if (is_cow) 1088 mmu_notifier_invalidate_range_end(src_mm, mmun_start, mmun_end); 1089 return ret; 1090 } 1091 1092 static unsigned long zap_pte_range(struct mmu_gather *tlb, 1093 struct vm_area_struct *vma, pmd_t *pmd, 1094 unsigned long addr, unsigned long end, 1095 struct zap_details *details) 1096 { 1097 struct mm_struct *mm = tlb->mm; 1098 int force_flush = 0; 1099 int rss[NR_MM_COUNTERS]; 1100 spinlock_t *ptl; 1101 pte_t *start_pte; 1102 pte_t *pte; 1103 1104 again: 1105 init_rss_vec(rss); 1106 start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl); 1107 pte = start_pte; 1108 arch_enter_lazy_mmu_mode(); 1109 do { 1110 pte_t ptent = *pte; 1111 if (pte_none(ptent)) { 1112 continue; 1113 } 1114 1115 if (pte_present(ptent)) { 1116 struct page *page; 1117 1118 page = vm_normal_page(vma, addr, ptent); 1119 if (unlikely(details) && page) { 1120 /* 1121 * unmap_shared_mapping_pages() wants to 1122 * invalidate cache without truncating: 1123 * unmap shared but keep private pages. 1124 */ 1125 if (details->check_mapping && 1126 details->check_mapping != page->mapping) 1127 continue; 1128 /* 1129 * Each page->index must be checked when 1130 * invalidating or truncating nonlinear. 1131 */ 1132 if (details->nonlinear_vma && 1133 (page->index < details->first_index || 1134 page->index > details->last_index)) 1135 continue; 1136 } 1137 ptent = ptep_get_and_clear_full(mm, addr, pte, 1138 tlb->fullmm); 1139 tlb_remove_tlb_entry(tlb, pte, addr); 1140 if (unlikely(!page)) 1141 continue; 1142 if (unlikely(details) && details->nonlinear_vma 1143 && linear_page_index(details->nonlinear_vma, 1144 addr) != page->index) 1145 set_pte_at(mm, addr, pte, 1146 pgoff_to_pte(page->index)); 1147 if (PageAnon(page)) 1148 rss[MM_ANONPAGES]--; 1149 else { 1150 if (pte_dirty(ptent)) 1151 set_page_dirty(page); 1152 if (pte_young(ptent) && 1153 likely(!VM_SequentialReadHint(vma))) 1154 mark_page_accessed(page); 1155 rss[MM_FILEPAGES]--; 1156 } 1157 page_remove_rmap(page); 1158 if (unlikely(page_mapcount(page) < 0)) 1159 print_bad_pte(vma, addr, ptent, page); 1160 force_flush = !__tlb_remove_page(tlb, page); 1161 if (force_flush) 1162 break; 1163 continue; 1164 } 1165 /* 1166 * If details->check_mapping, we leave swap entries; 1167 * if details->nonlinear_vma, we leave file entries. 1168 */ 1169 if (unlikely(details)) 1170 continue; 1171 if (pte_file(ptent)) { 1172 if (unlikely(!(vma->vm_flags & VM_NONLINEAR))) 1173 print_bad_pte(vma, addr, ptent, NULL); 1174 } else { 1175 swp_entry_t entry = pte_to_swp_entry(ptent); 1176 1177 if (!non_swap_entry(entry)) 1178 rss[MM_SWAPENTS]--; 1179 else if (is_migration_entry(entry)) { 1180 struct page *page; 1181 1182 page = migration_entry_to_page(entry); 1183 1184 if (PageAnon(page)) 1185 rss[MM_ANONPAGES]--; 1186 else 1187 rss[MM_FILEPAGES]--; 1188 } 1189 if (unlikely(!free_swap_and_cache(entry))) 1190 print_bad_pte(vma, addr, ptent, NULL); 1191 } 1192 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); 1193 } while (pte++, addr += PAGE_SIZE, addr != end); 1194 1195 add_mm_rss_vec(mm, rss); 1196 arch_leave_lazy_mmu_mode(); 1197 pte_unmap_unlock(start_pte, ptl); 1198 1199 /* 1200 * mmu_gather ran out of room to batch pages, we break out of 1201 * the PTE lock to avoid doing the potential expensive TLB invalidate 1202 * and page-free while holding it. 1203 */ 1204 if (force_flush) { 1205 force_flush = 0; 1206 1207 #ifdef HAVE_GENERIC_MMU_GATHER 1208 tlb->start = addr; 1209 tlb->end = end; 1210 #endif 1211 tlb_flush_mmu(tlb); 1212 if (addr != end) 1213 goto again; 1214 } 1215 1216 return addr; 1217 } 1218 1219 static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, 1220 struct vm_area_struct *vma, pud_t *pud, 1221 unsigned long addr, unsigned long end, 1222 struct zap_details *details) 1223 { 1224 pmd_t *pmd; 1225 unsigned long next; 1226 1227 pmd = pmd_offset(pud, addr); 1228 do { 1229 next = pmd_addr_end(addr, end); 1230 if (pmd_trans_huge(*pmd)) { 1231 if (next - addr != HPAGE_PMD_SIZE) { 1232 #ifdef CONFIG_DEBUG_VM 1233 if (!rwsem_is_locked(&tlb->mm->mmap_sem)) { 1234 pr_err("%s: mmap_sem is unlocked! addr=0x%lx end=0x%lx vma->vm_start=0x%lx vma->vm_end=0x%lx\n", 1235 __func__, addr, end, 1236 vma->vm_start, 1237 vma->vm_end); 1238 BUG(); 1239 } 1240 #endif 1241 split_huge_page_pmd(vma, addr, pmd); 1242 } else if (zap_huge_pmd(tlb, vma, pmd, addr)) 1243 goto next; 1244 /* fall through */ 1245 } 1246 /* 1247 * Here there can be other concurrent MADV_DONTNEED or 1248 * trans huge page faults running, and if the pmd is 1249 * none or trans huge it can change under us. This is 1250 * because MADV_DONTNEED holds the mmap_sem in read 1251 * mode. 1252 */ 1253 if (pmd_none_or_trans_huge_or_clear_bad(pmd)) 1254 goto next; 1255 next = zap_pte_range(tlb, vma, pmd, addr, next, details); 1256 next: 1257 cond_resched(); 1258 } while (pmd++, addr = next, addr != end); 1259 1260 return addr; 1261 } 1262 1263 static inline unsigned long zap_pud_range(struct mmu_gather *tlb, 1264 struct vm_area_struct *vma, pgd_t *pgd, 1265 unsigned long addr, unsigned long end, 1266 struct zap_details *details) 1267 { 1268 pud_t *pud; 1269 unsigned long next; 1270 1271 pud = pud_offset(pgd, addr); 1272 do { 1273 next = pud_addr_end(addr, end); 1274 if (pud_none_or_clear_bad(pud)) 1275 continue; 1276 next = zap_pmd_range(tlb, vma, pud, addr, next, details); 1277 } while (pud++, addr = next, addr != end); 1278 1279 return addr; 1280 } 1281 1282 static void unmap_page_range(struct mmu_gather *tlb, 1283 struct vm_area_struct *vma, 1284 unsigned long addr, unsigned long end, 1285 struct zap_details *details) 1286 { 1287 pgd_t *pgd; 1288 unsigned long next; 1289 1290 if (details && !details->check_mapping && !details->nonlinear_vma) 1291 details = NULL; 1292 1293 BUG_ON(addr >= end); 1294 mem_cgroup_uncharge_start(); 1295 tlb_start_vma(tlb, vma); 1296 pgd = pgd_offset(vma->vm_mm, addr); 1297 do { 1298 next = pgd_addr_end(addr, end); 1299 if (pgd_none_or_clear_bad(pgd)) 1300 continue; 1301 next = zap_pud_range(tlb, vma, pgd, addr, next, details); 1302 } while (pgd++, addr = next, addr != end); 1303 tlb_end_vma(tlb, vma); 1304 mem_cgroup_uncharge_end(); 1305 } 1306 1307 1308 static void unmap_single_vma(struct mmu_gather *tlb, 1309 struct vm_area_struct *vma, unsigned long start_addr, 1310 unsigned long end_addr, 1311 struct zap_details *details) 1312 { 1313 unsigned long start = max(vma->vm_start, start_addr); 1314 unsigned long end; 1315 1316 if (start >= vma->vm_end) 1317 return; 1318 end = min(vma->vm_end, end_addr); 1319 if (end <= vma->vm_start) 1320 return; 1321 1322 if (vma->vm_file) 1323 uprobe_munmap(vma, start, end); 1324 1325 if (unlikely(vma->vm_flags & VM_PFNMAP)) 1326 untrack_pfn(vma, 0, 0); 1327 1328 if (start != end) { 1329 if (unlikely(is_vm_hugetlb_page(vma))) { 1330 /* 1331 * It is undesirable to test vma->vm_file as it 1332 * should be non-null for valid hugetlb area. 1333 * However, vm_file will be NULL in the error 1334 * cleanup path of do_mmap_pgoff. When 1335 * hugetlbfs ->mmap method fails, 1336 * do_mmap_pgoff() nullifies vma->vm_file 1337 * before calling this function to clean up. 1338 * Since no pte has actually been setup, it is 1339 * safe to do nothing in this case. 1340 */ 1341 if (vma->vm_file) { 1342 mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex); 1343 __unmap_hugepage_range_final(tlb, vma, start, end, NULL); 1344 mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex); 1345 } 1346 } else 1347 unmap_page_range(tlb, vma, start, end, details); 1348 } 1349 } 1350 1351 /** 1352 * unmap_vmas - unmap a range of memory covered by a list of vma's 1353 * @tlb: address of the caller's struct mmu_gather 1354 * @vma: the starting vma 1355 * @start_addr: virtual address at which to start unmapping 1356 * @end_addr: virtual address at which to end unmapping 1357 * 1358 * Unmap all pages in the vma list. 1359 * 1360 * Only addresses between `start' and `end' will be unmapped. 1361 * 1362 * The VMA list must be sorted in ascending virtual address order. 1363 * 1364 * unmap_vmas() assumes that the caller will flush the whole unmapped address 1365 * range after unmap_vmas() returns. So the only responsibility here is to 1366 * ensure that any thus-far unmapped pages are flushed before unmap_vmas() 1367 * drops the lock and schedules. 1368 */ 1369 void unmap_vmas(struct mmu_gather *tlb, 1370 struct vm_area_struct *vma, unsigned long start_addr, 1371 unsigned long end_addr) 1372 { 1373 struct mm_struct *mm = vma->vm_mm; 1374 1375 mmu_notifier_invalidate_range_start(mm, start_addr, end_addr); 1376 for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) 1377 unmap_single_vma(tlb, vma, start_addr, end_addr, NULL); 1378 mmu_notifier_invalidate_range_end(mm, start_addr, end_addr); 1379 } 1380 1381 /** 1382 * zap_page_range - remove user pages in a given range 1383 * @vma: vm_area_struct holding the applicable pages 1384 * @start: starting address of pages to zap 1385 * @size: number of bytes to zap 1386 * @details: details of nonlinear truncation or shared cache invalidation 1387 * 1388 * Caller must protect the VMA list 1389 */ 1390 void zap_page_range(struct vm_area_struct *vma, unsigned long start, 1391 unsigned long size, struct zap_details *details) 1392 { 1393 struct mm_struct *mm = vma->vm_mm; 1394 struct mmu_gather tlb; 1395 unsigned long end = start + size; 1396 1397 lru_add_drain(); 1398 tlb_gather_mmu(&tlb, mm, 0); 1399 update_hiwater_rss(mm); 1400 mmu_notifier_invalidate_range_start(mm, start, end); 1401 for ( ; vma && vma->vm_start < end; vma = vma->vm_next) 1402 unmap_single_vma(&tlb, vma, start, end, details); 1403 mmu_notifier_invalidate_range_end(mm, start, end); 1404 tlb_finish_mmu(&tlb, start, end); 1405 } 1406 1407 /** 1408 * zap_page_range_single - remove user pages in a given range 1409 * @vma: vm_area_struct holding the applicable pages 1410 * @address: starting address of pages to zap 1411 * @size: number of bytes to zap 1412 * @details: details of nonlinear truncation or shared cache invalidation 1413 * 1414 * The range must fit into one VMA. 1415 */ 1416 static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, 1417 unsigned long size, struct zap_details *details) 1418 { 1419 struct mm_struct *mm = vma->vm_mm; 1420 struct mmu_gather tlb; 1421 unsigned long end = address + size; 1422 1423 lru_add_drain(); 1424 tlb_gather_mmu(&tlb, mm, 0); 1425 update_hiwater_rss(mm); 1426 mmu_notifier_invalidate_range_start(mm, address, end); 1427 unmap_single_vma(&tlb, vma, address, end, details); 1428 mmu_notifier_invalidate_range_end(mm, address, end); 1429 tlb_finish_mmu(&tlb, address, end); 1430 } 1431 1432 /** 1433 * zap_vma_ptes - remove ptes mapping the vma 1434 * @vma: vm_area_struct holding ptes to be zapped 1435 * @address: starting address of pages to zap 1436 * @size: number of bytes to zap 1437 * 1438 * This function only unmaps ptes assigned to VM_PFNMAP vmas. 1439 * 1440 * The entire address range must be fully contained within the vma. 1441 * 1442 * Returns 0 if successful. 1443 */ 1444 int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 1445 unsigned long size) 1446 { 1447 if (address < vma->vm_start || address + size > vma->vm_end || 1448 !(vma->vm_flags & VM_PFNMAP)) 1449 return -1; 1450 zap_page_range_single(vma, address, size, NULL); 1451 return 0; 1452 } 1453 EXPORT_SYMBOL_GPL(zap_vma_ptes); 1454 1455 /** 1456 * follow_page - look up a page descriptor from a user-virtual address 1457 * @vma: vm_area_struct mapping @address 1458 * @address: virtual address to look up 1459 * @flags: flags modifying lookup behaviour 1460 * 1461 * @flags can have FOLL_ flags set, defined in <linux/mm.h> 1462 * 1463 * Returns the mapped (struct page *), %NULL if no mapping exists, or 1464 * an error pointer if there is a mapping to something not represented 1465 * by a page descriptor (see also vm_normal_page()). 1466 */ 1467 struct page *follow_page(struct vm_area_struct *vma, unsigned long address, 1468 unsigned int flags) 1469 { 1470 pgd_t *pgd; 1471 pud_t *pud; 1472 pmd_t *pmd; 1473 pte_t *ptep, pte; 1474 spinlock_t *ptl; 1475 struct page *page; 1476 struct mm_struct *mm = vma->vm_mm; 1477 1478 page = follow_huge_addr(mm, address, flags & FOLL_WRITE); 1479 if (!IS_ERR(page)) { 1480 BUG_ON(flags & FOLL_GET); 1481 goto out; 1482 } 1483 1484 page = NULL; 1485 pgd = pgd_offset(mm, address); 1486 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 1487 goto no_page_table; 1488 1489 pud = pud_offset(pgd, address); 1490 if (pud_none(*pud)) 1491 goto no_page_table; 1492 if (pud_huge(*pud) && vma->vm_flags & VM_HUGETLB) { 1493 BUG_ON(flags & FOLL_GET); 1494 page = follow_huge_pud(mm, address, pud, flags & FOLL_WRITE); 1495 goto out; 1496 } 1497 if (unlikely(pud_bad(*pud))) 1498 goto no_page_table; 1499 1500 pmd = pmd_offset(pud, address); 1501 if (pmd_none(*pmd)) 1502 goto no_page_table; 1503 if (pmd_huge(*pmd) && vma->vm_flags & VM_HUGETLB) { 1504 BUG_ON(flags & FOLL_GET); 1505 page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE); 1506 goto out; 1507 } 1508 if ((flags & FOLL_NUMA) && pmd_numa(*pmd)) 1509 goto no_page_table; 1510 if (pmd_trans_huge(*pmd)) { 1511 if (flags & FOLL_SPLIT) { 1512 split_huge_page_pmd(vma, address, pmd); 1513 goto split_fallthrough; 1514 } 1515 spin_lock(&mm->page_table_lock); 1516 if (likely(pmd_trans_huge(*pmd))) { 1517 if (unlikely(pmd_trans_splitting(*pmd))) { 1518 spin_unlock(&mm->page_table_lock); 1519 wait_split_huge_page(vma->anon_vma, pmd); 1520 } else { 1521 page = follow_trans_huge_pmd(vma, address, 1522 pmd, flags); 1523 spin_unlock(&mm->page_table_lock); 1524 goto out; 1525 } 1526 } else 1527 spin_unlock(&mm->page_table_lock); 1528 /* fall through */ 1529 } 1530 split_fallthrough: 1531 if (unlikely(pmd_bad(*pmd))) 1532 goto no_page_table; 1533 1534 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 1535 1536 pte = *ptep; 1537 if (!pte_present(pte)) 1538 goto no_page; 1539 if ((flags & FOLL_NUMA) && pte_numa(pte)) 1540 goto no_page; 1541 if ((flags & FOLL_WRITE) && !pte_write(pte)) 1542 goto unlock; 1543 1544 page = vm_normal_page(vma, address, pte); 1545 if (unlikely(!page)) { 1546 if ((flags & FOLL_DUMP) || 1547 !is_zero_pfn(pte_pfn(pte))) 1548 goto bad_page; 1549 page = pte_page(pte); 1550 } 1551 1552 if (flags & FOLL_GET) 1553 get_page_foll(page); 1554 if (flags & FOLL_TOUCH) { 1555 if ((flags & FOLL_WRITE) && 1556 !pte_dirty(pte) && !PageDirty(page)) 1557 set_page_dirty(page); 1558 /* 1559 * pte_mkyoung() would be more correct here, but atomic care 1560 * is needed to avoid losing the dirty bit: it is easier to use 1561 * mark_page_accessed(). 1562 */ 1563 mark_page_accessed(page); 1564 } 1565 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 1566 /* 1567 * The preliminary mapping check is mainly to avoid the 1568 * pointless overhead of lock_page on the ZERO_PAGE 1569 * which might bounce very badly if there is contention. 1570 * 1571 * If the page is already locked, we don't need to 1572 * handle it now - vmscan will handle it later if and 1573 * when it attempts to reclaim the page. 1574 */ 1575 if (page->mapping && trylock_page(page)) { 1576 lru_add_drain(); /* push cached pages to LRU */ 1577 /* 1578 * Because we lock page here, and migration is 1579 * blocked by the pte's page reference, and we 1580 * know the page is still mapped, we don't even 1581 * need to check for file-cache page truncation. 1582 */ 1583 mlock_vma_page(page); 1584 unlock_page(page); 1585 } 1586 } 1587 unlock: 1588 pte_unmap_unlock(ptep, ptl); 1589 out: 1590 return page; 1591 1592 bad_page: 1593 pte_unmap_unlock(ptep, ptl); 1594 return ERR_PTR(-EFAULT); 1595 1596 no_page: 1597 pte_unmap_unlock(ptep, ptl); 1598 if (!pte_none(pte)) 1599 return page; 1600 1601 no_page_table: 1602 /* 1603 * When core dumping an enormous anonymous area that nobody 1604 * has touched so far, we don't want to allocate unnecessary pages or 1605 * page tables. Return error instead of NULL to skip handle_mm_fault, 1606 * then get_dump_page() will return NULL to leave a hole in the dump. 1607 * But we can only make this optimization where a hole would surely 1608 * be zero-filled if handle_mm_fault() actually did handle it. 1609 */ 1610 if ((flags & FOLL_DUMP) && 1611 (!vma->vm_ops || !vma->vm_ops->fault)) 1612 return ERR_PTR(-EFAULT); 1613 return page; 1614 } 1615 1616 static inline int stack_guard_page(struct vm_area_struct *vma, unsigned long addr) 1617 { 1618 return stack_guard_page_start(vma, addr) || 1619 stack_guard_page_end(vma, addr+PAGE_SIZE); 1620 } 1621 1622 /** 1623 * __get_user_pages() - pin user pages in memory 1624 * @tsk: task_struct of target task 1625 * @mm: mm_struct of target mm 1626 * @start: starting user address 1627 * @nr_pages: number of pages from start to pin 1628 * @gup_flags: flags modifying pin behaviour 1629 * @pages: array that receives pointers to the pages pinned. 1630 * Should be at least nr_pages long. Or NULL, if caller 1631 * only intends to ensure the pages are faulted in. 1632 * @vmas: array of pointers to vmas corresponding to each page. 1633 * Or NULL if the caller does not require them. 1634 * @nonblocking: whether waiting for disk IO or mmap_sem contention 1635 * 1636 * Returns number of pages pinned. This may be fewer than the number 1637 * requested. If nr_pages is 0 or negative, returns 0. If no pages 1638 * were pinned, returns -errno. Each page returned must be released 1639 * with a put_page() call when it is finished with. vmas will only 1640 * remain valid while mmap_sem is held. 1641 * 1642 * Must be called with mmap_sem held for read or write. 1643 * 1644 * __get_user_pages walks a process's page tables and takes a reference to 1645 * each struct page that each user address corresponds to at a given 1646 * instant. That is, it takes the page that would be accessed if a user 1647 * thread accesses the given user virtual address at that instant. 1648 * 1649 * This does not guarantee that the page exists in the user mappings when 1650 * __get_user_pages returns, and there may even be a completely different 1651 * page there in some cases (eg. if mmapped pagecache has been invalidated 1652 * and subsequently re faulted). However it does guarantee that the page 1653 * won't be freed completely. And mostly callers simply care that the page 1654 * contains data that was valid *at some point in time*. Typically, an IO 1655 * or similar operation cannot guarantee anything stronger anyway because 1656 * locks can't be held over the syscall boundary. 1657 * 1658 * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If 1659 * the page is written to, set_page_dirty (or set_page_dirty_lock, as 1660 * appropriate) must be called after the page is finished with, and 1661 * before put_page is called. 1662 * 1663 * If @nonblocking != NULL, __get_user_pages will not wait for disk IO 1664 * or mmap_sem contention, and if waiting is needed to pin all pages, 1665 * *@nonblocking will be set to 0. 1666 * 1667 * In most cases, get_user_pages or get_user_pages_fast should be used 1668 * instead of __get_user_pages. __get_user_pages should be used only if 1669 * you need some special @gup_flags. 1670 */ 1671 int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1672 unsigned long start, int nr_pages, unsigned int gup_flags, 1673 struct page **pages, struct vm_area_struct **vmas, 1674 int *nonblocking) 1675 { 1676 int i; 1677 unsigned long vm_flags; 1678 1679 if (nr_pages <= 0) 1680 return 0; 1681 1682 VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET)); 1683 1684 /* 1685 * Require read or write permissions. 1686 * If FOLL_FORCE is set, we only require the "MAY" flags. 1687 */ 1688 vm_flags = (gup_flags & FOLL_WRITE) ? 1689 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); 1690 vm_flags &= (gup_flags & FOLL_FORCE) ? 1691 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); 1692 1693 /* 1694 * If FOLL_FORCE and FOLL_NUMA are both set, handle_mm_fault 1695 * would be called on PROT_NONE ranges. We must never invoke 1696 * handle_mm_fault on PROT_NONE ranges or the NUMA hinting 1697 * page faults would unprotect the PROT_NONE ranges if 1698 * _PAGE_NUMA and _PAGE_PROTNONE are sharing the same pte/pmd 1699 * bitflag. So to avoid that, don't set FOLL_NUMA if 1700 * FOLL_FORCE is set. 1701 */ 1702 if (!(gup_flags & FOLL_FORCE)) 1703 gup_flags |= FOLL_NUMA; 1704 1705 i = 0; 1706 1707 do { 1708 struct vm_area_struct *vma; 1709 1710 vma = find_extend_vma(mm, start); 1711 if (!vma && in_gate_area(mm, start)) { 1712 unsigned long pg = start & PAGE_MASK; 1713 pgd_t *pgd; 1714 pud_t *pud; 1715 pmd_t *pmd; 1716 pte_t *pte; 1717 1718 /* user gate pages are read-only */ 1719 if (gup_flags & FOLL_WRITE) 1720 return i ? : -EFAULT; 1721 if (pg > TASK_SIZE) 1722 pgd = pgd_offset_k(pg); 1723 else 1724 pgd = pgd_offset_gate(mm, pg); 1725 BUG_ON(pgd_none(*pgd)); 1726 pud = pud_offset(pgd, pg); 1727 BUG_ON(pud_none(*pud)); 1728 pmd = pmd_offset(pud, pg); 1729 if (pmd_none(*pmd)) 1730 return i ? : -EFAULT; 1731 VM_BUG_ON(pmd_trans_huge(*pmd)); 1732 pte = pte_offset_map(pmd, pg); 1733 if (pte_none(*pte)) { 1734 pte_unmap(pte); 1735 return i ? : -EFAULT; 1736 } 1737 vma = get_gate_vma(mm); 1738 if (pages) { 1739 struct page *page; 1740 1741 page = vm_normal_page(vma, start, *pte); 1742 if (!page) { 1743 if (!(gup_flags & FOLL_DUMP) && 1744 is_zero_pfn(pte_pfn(*pte))) 1745 page = pte_page(*pte); 1746 else { 1747 pte_unmap(pte); 1748 return i ? : -EFAULT; 1749 } 1750 } 1751 pages[i] = page; 1752 get_page(page); 1753 } 1754 pte_unmap(pte); 1755 goto next_page; 1756 } 1757 1758 if (!vma || 1759 (vma->vm_flags & (VM_IO | VM_PFNMAP)) || 1760 !(vm_flags & vma->vm_flags)) 1761 return i ? : -EFAULT; 1762 1763 if (is_vm_hugetlb_page(vma)) { 1764 i = follow_hugetlb_page(mm, vma, pages, vmas, 1765 &start, &nr_pages, i, gup_flags); 1766 continue; 1767 } 1768 1769 do { 1770 struct page *page; 1771 unsigned int foll_flags = gup_flags; 1772 1773 /* 1774 * If we have a pending SIGKILL, don't keep faulting 1775 * pages and potentially allocating memory. 1776 */ 1777 if (unlikely(fatal_signal_pending(current))) 1778 return i ? i : -ERESTARTSYS; 1779 1780 cond_resched(); 1781 while (!(page = follow_page(vma, start, foll_flags))) { 1782 int ret; 1783 unsigned int fault_flags = 0; 1784 1785 /* For mlock, just skip the stack guard page. */ 1786 if (foll_flags & FOLL_MLOCK) { 1787 if (stack_guard_page(vma, start)) 1788 goto next_page; 1789 } 1790 if (foll_flags & FOLL_WRITE) 1791 fault_flags |= FAULT_FLAG_WRITE; 1792 if (nonblocking) 1793 fault_flags |= FAULT_FLAG_ALLOW_RETRY; 1794 if (foll_flags & FOLL_NOWAIT) 1795 fault_flags |= (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT); 1796 1797 ret = handle_mm_fault(mm, vma, start, 1798 fault_flags); 1799 1800 if (ret & VM_FAULT_ERROR) { 1801 if (ret & VM_FAULT_OOM) 1802 return i ? i : -ENOMEM; 1803 if (ret & (VM_FAULT_HWPOISON | 1804 VM_FAULT_HWPOISON_LARGE)) { 1805 if (i) 1806 return i; 1807 else if (gup_flags & FOLL_HWPOISON) 1808 return -EHWPOISON; 1809 else 1810 return -EFAULT; 1811 } 1812 if (ret & VM_FAULT_SIGBUS) 1813 return i ? i : -EFAULT; 1814 BUG(); 1815 } 1816 1817 if (tsk) { 1818 if (ret & VM_FAULT_MAJOR) 1819 tsk->maj_flt++; 1820 else 1821 tsk->min_flt++; 1822 } 1823 1824 if (ret & VM_FAULT_RETRY) { 1825 if (nonblocking) 1826 *nonblocking = 0; 1827 return i; 1828 } 1829 1830 /* 1831 * The VM_FAULT_WRITE bit tells us that 1832 * do_wp_page has broken COW when necessary, 1833 * even if maybe_mkwrite decided not to set 1834 * pte_write. We can thus safely do subsequent 1835 * page lookups as if they were reads. But only 1836 * do so when looping for pte_write is futile: 1837 * in some cases userspace may also be wanting 1838 * to write to the gotten user page, which a 1839 * read fault here might prevent (a readonly 1840 * page might get reCOWed by userspace write). 1841 */ 1842 if ((ret & VM_FAULT_WRITE) && 1843 !(vma->vm_flags & VM_WRITE)) 1844 foll_flags &= ~FOLL_WRITE; 1845 1846 cond_resched(); 1847 } 1848 if (IS_ERR(page)) 1849 return i ? i : PTR_ERR(page); 1850 if (pages) { 1851 pages[i] = page; 1852 1853 flush_anon_page(vma, page, start); 1854 flush_dcache_page(page); 1855 } 1856 next_page: 1857 if (vmas) 1858 vmas[i] = vma; 1859 i++; 1860 start += PAGE_SIZE; 1861 nr_pages--; 1862 } while (nr_pages && start < vma->vm_end); 1863 } while (nr_pages); 1864 return i; 1865 } 1866 EXPORT_SYMBOL(__get_user_pages); 1867 1868 /* 1869 * fixup_user_fault() - manually resolve a user page fault 1870 * @tsk: the task_struct to use for page fault accounting, or 1871 * NULL if faults are not to be recorded. 1872 * @mm: mm_struct of target mm 1873 * @address: user address 1874 * @fault_flags:flags to pass down to handle_mm_fault() 1875 * 1876 * This is meant to be called in the specific scenario where for locking reasons 1877 * we try to access user memory in atomic context (within a pagefault_disable() 1878 * section), this returns -EFAULT, and we want to resolve the user fault before 1879 * trying again. 1880 * 1881 * Typically this is meant to be used by the futex code. 1882 * 1883 * The main difference with get_user_pages() is that this function will 1884 * unconditionally call handle_mm_fault() which will in turn perform all the 1885 * necessary SW fixup of the dirty and young bits in the PTE, while 1886 * handle_mm_fault() only guarantees to update these in the struct page. 1887 * 1888 * This is important for some architectures where those bits also gate the 1889 * access permission to the page because they are maintained in software. On 1890 * such architectures, gup() will not be enough to make a subsequent access 1891 * succeed. 1892 * 1893 * This should be called with the mm_sem held for read. 1894 */ 1895 int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, 1896 unsigned long address, unsigned int fault_flags) 1897 { 1898 struct vm_area_struct *vma; 1899 int ret; 1900 1901 vma = find_extend_vma(mm, address); 1902 if (!vma || address < vma->vm_start) 1903 return -EFAULT; 1904 1905 ret = handle_mm_fault(mm, vma, address, fault_flags); 1906 if (ret & VM_FAULT_ERROR) { 1907 if (ret & VM_FAULT_OOM) 1908 return -ENOMEM; 1909 if (ret & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) 1910 return -EHWPOISON; 1911 if (ret & VM_FAULT_SIGBUS) 1912 return -EFAULT; 1913 BUG(); 1914 } 1915 if (tsk) { 1916 if (ret & VM_FAULT_MAJOR) 1917 tsk->maj_flt++; 1918 else 1919 tsk->min_flt++; 1920 } 1921 return 0; 1922 } 1923 1924 /* 1925 * get_user_pages() - pin user pages in memory 1926 * @tsk: the task_struct to use for page fault accounting, or 1927 * NULL if faults are not to be recorded. 1928 * @mm: mm_struct of target mm 1929 * @start: starting user address 1930 * @nr_pages: number of pages from start to pin 1931 * @write: whether pages will be written to by the caller 1932 * @force: whether to force write access even if user mapping is 1933 * readonly. This will result in the page being COWed even 1934 * in MAP_SHARED mappings. You do not want this. 1935 * @pages: array that receives pointers to the pages pinned. 1936 * Should be at least nr_pages long. Or NULL, if caller 1937 * only intends to ensure the pages are faulted in. 1938 * @vmas: array of pointers to vmas corresponding to each page. 1939 * Or NULL if the caller does not require them. 1940 * 1941 * Returns number of pages pinned. This may be fewer than the number 1942 * requested. If nr_pages is 0 or negative, returns 0. If no pages 1943 * were pinned, returns -errno. Each page returned must be released 1944 * with a put_page() call when it is finished with. vmas will only 1945 * remain valid while mmap_sem is held. 1946 * 1947 * Must be called with mmap_sem held for read or write. 1948 * 1949 * get_user_pages walks a process's page tables and takes a reference to 1950 * each struct page that each user address corresponds to at a given 1951 * instant. That is, it takes the page that would be accessed if a user 1952 * thread accesses the given user virtual address at that instant. 1953 * 1954 * This does not guarantee that the page exists in the user mappings when 1955 * get_user_pages returns, and there may even be a completely different 1956 * page there in some cases (eg. if mmapped pagecache has been invalidated 1957 * and subsequently re faulted). However it does guarantee that the page 1958 * won't be freed completely. And mostly callers simply care that the page 1959 * contains data that was valid *at some point in time*. Typically, an IO 1960 * or similar operation cannot guarantee anything stronger anyway because 1961 * locks can't be held over the syscall boundary. 1962 * 1963 * If write=0, the page must not be written to. If the page is written to, 1964 * set_page_dirty (or set_page_dirty_lock, as appropriate) must be called 1965 * after the page is finished with, and before put_page is called. 1966 * 1967 * get_user_pages is typically used for fewer-copy IO operations, to get a 1968 * handle on the memory by some means other than accesses via the user virtual 1969 * addresses. The pages may be submitted for DMA to devices or accessed via 1970 * their kernel linear mapping (via the kmap APIs). Care should be taken to 1971 * use the correct cache flushing APIs. 1972 * 1973 * See also get_user_pages_fast, for performance critical applications. 1974 */ 1975 int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1976 unsigned long start, int nr_pages, int write, int force, 1977 struct page **pages, struct vm_area_struct **vmas) 1978 { 1979 int flags = FOLL_TOUCH; 1980 1981 if (pages) 1982 flags |= FOLL_GET; 1983 if (write) 1984 flags |= FOLL_WRITE; 1985 if (force) 1986 flags |= FOLL_FORCE; 1987 1988 return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas, 1989 NULL); 1990 } 1991 EXPORT_SYMBOL(get_user_pages); 1992 1993 /** 1994 * get_dump_page() - pin user page in memory while writing it to core dump 1995 * @addr: user address 1996 * 1997 * Returns struct page pointer of user page pinned for dump, 1998 * to be freed afterwards by page_cache_release() or put_page(). 1999 * 2000 * Returns NULL on any kind of failure - a hole must then be inserted into 2001 * the corefile, to preserve alignment with its headers; and also returns 2002 * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found - 2003 * allowing a hole to be left in the corefile to save diskspace. 2004 * 2005 * Called without mmap_sem, but after all other threads have been killed. 2006 */ 2007 #ifdef CONFIG_ELF_CORE 2008 struct page *get_dump_page(unsigned long addr) 2009 { 2010 struct vm_area_struct *vma; 2011 struct page *page; 2012 2013 if (__get_user_pages(current, current->mm, addr, 1, 2014 FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma, 2015 NULL) < 1) 2016 return NULL; 2017 flush_cache_page(vma, addr, page_to_pfn(page)); 2018 return page; 2019 } 2020 #endif /* CONFIG_ELF_CORE */ 2021 2022 pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr, 2023 spinlock_t **ptl) 2024 { 2025 pgd_t * pgd = pgd_offset(mm, addr); 2026 pud_t * pud = pud_alloc(mm, pgd, addr); 2027 if (pud) { 2028 pmd_t * pmd = pmd_alloc(mm, pud, addr); 2029 if (pmd) { 2030 VM_BUG_ON(pmd_trans_huge(*pmd)); 2031 return pte_alloc_map_lock(mm, pmd, addr, ptl); 2032 } 2033 } 2034 return NULL; 2035 } 2036 2037 /* 2038 * This is the old fallback for page remapping. 2039 * 2040 * For historical reasons, it only allows reserved pages. Only 2041 * old drivers should use this, and they needed to mark their 2042 * pages reserved for the old functions anyway. 2043 */ 2044 static int insert_page(struct vm_area_struct *vma, unsigned long addr, 2045 struct page *page, pgprot_t prot) 2046 { 2047 struct mm_struct *mm = vma->vm_mm; 2048 int retval; 2049 pte_t *pte; 2050 spinlock_t *ptl; 2051 2052 retval = -EINVAL; 2053 if (PageAnon(page)) 2054 goto out; 2055 retval = -ENOMEM; 2056 flush_dcache_page(page); 2057 pte = get_locked_pte(mm, addr, &ptl); 2058 if (!pte) 2059 goto out; 2060 retval = -EBUSY; 2061 if (!pte_none(*pte)) 2062 goto out_unlock; 2063 2064 /* Ok, finally just insert the thing.. */ 2065 get_page(page); 2066 inc_mm_counter_fast(mm, MM_FILEPAGES); 2067 page_add_file_rmap(page); 2068 set_pte_at(mm, addr, pte, mk_pte(page, prot)); 2069 2070 retval = 0; 2071 pte_unmap_unlock(pte, ptl); 2072 return retval; 2073 out_unlock: 2074 pte_unmap_unlock(pte, ptl); 2075 out: 2076 return retval; 2077 } 2078 2079 /** 2080 * vm_insert_page - insert single page into user vma 2081 * @vma: user vma to map to 2082 * @addr: target user address of this page 2083 * @page: source kernel page 2084 * 2085 * This allows drivers to insert individual pages they've allocated 2086 * into a user vma. 2087 * 2088 * The page has to be a nice clean _individual_ kernel allocation. 2089 * If you allocate a compound page, you need to have marked it as 2090 * such (__GFP_COMP), or manually just split the page up yourself 2091 * (see split_page()). 2092 * 2093 * NOTE! Traditionally this was done with "remap_pfn_range()" which 2094 * took an arbitrary page protection parameter. This doesn't allow 2095 * that. Your vma protection will have to be set up correctly, which 2096 * means that if you want a shared writable mapping, you'd better 2097 * ask for a shared writable mapping! 2098 * 2099 * The page does not need to be reserved. 2100 * 2101 * Usually this function is called from f_op->mmap() handler 2102 * under mm->mmap_sem write-lock, so it can change vma->vm_flags. 2103 * Caller must set VM_MIXEDMAP on vma if it wants to call this 2104 * function from other places, for example from page-fault handler. 2105 */ 2106 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, 2107 struct page *page) 2108 { 2109 if (addr < vma->vm_start || addr >= vma->vm_end) 2110 return -EFAULT; 2111 if (!page_count(page)) 2112 return -EINVAL; 2113 if (!(vma->vm_flags & VM_MIXEDMAP)) { 2114 BUG_ON(down_read_trylock(&vma->vm_mm->mmap_sem)); 2115 BUG_ON(vma->vm_flags & VM_PFNMAP); 2116 vma->vm_flags |= VM_MIXEDMAP; 2117 } 2118 return insert_page(vma, addr, page, vma->vm_page_prot); 2119 } 2120 EXPORT_SYMBOL(vm_insert_page); 2121 2122 static int insert_pfn(struct vm_area_struct *vma, unsigned long addr, 2123 unsigned long pfn, pgprot_t prot) 2124 { 2125 struct mm_struct *mm = vma->vm_mm; 2126 int retval; 2127 pte_t *pte, entry; 2128 spinlock_t *ptl; 2129 2130 retval = -ENOMEM; 2131 pte = get_locked_pte(mm, addr, &ptl); 2132 if (!pte) 2133 goto out; 2134 retval = -EBUSY; 2135 if (!pte_none(*pte)) 2136 goto out_unlock; 2137 2138 /* Ok, finally just insert the thing.. */ 2139 entry = pte_mkspecial(pfn_pte(pfn, prot)); 2140 set_pte_at(mm, addr, pte, entry); 2141 update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */ 2142 2143 retval = 0; 2144 out_unlock: 2145 pte_unmap_unlock(pte, ptl); 2146 out: 2147 return retval; 2148 } 2149 2150 /** 2151 * vm_insert_pfn - insert single pfn into user vma 2152 * @vma: user vma to map to 2153 * @addr: target user address of this page 2154 * @pfn: source kernel pfn 2155 * 2156 * Similar to vm_insert_page, this allows drivers to insert individual pages 2157 * they've allocated into a user vma. Same comments apply. 2158 * 2159 * This function should only be called from a vm_ops->fault handler, and 2160 * in that case the handler should return NULL. 2161 * 2162 * vma cannot be a COW mapping. 2163 * 2164 * As this is called only for pages that do not currently exist, we 2165 * do not need to flush old virtual caches or the TLB. 2166 */ 2167 int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr, 2168 unsigned long pfn) 2169 { 2170 int ret; 2171 pgprot_t pgprot = vma->vm_page_prot; 2172 /* 2173 * Technically, architectures with pte_special can avoid all these 2174 * restrictions (same for remap_pfn_range). However we would like 2175 * consistency in testing and feature parity among all, so we should 2176 * try to keep these invariants in place for everybody. 2177 */ 2178 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 2179 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 2180 (VM_PFNMAP|VM_MIXEDMAP)); 2181 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 2182 BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); 2183 2184 if (addr < vma->vm_start || addr >= vma->vm_end) 2185 return -EFAULT; 2186 if (track_pfn_insert(vma, &pgprot, pfn)) 2187 return -EINVAL; 2188 2189 ret = insert_pfn(vma, addr, pfn, pgprot); 2190 2191 return ret; 2192 } 2193 EXPORT_SYMBOL(vm_insert_pfn); 2194 2195 int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 2196 unsigned long pfn) 2197 { 2198 BUG_ON(!(vma->vm_flags & VM_MIXEDMAP)); 2199 2200 if (addr < vma->vm_start || addr >= vma->vm_end) 2201 return -EFAULT; 2202 2203 /* 2204 * If we don't have pte special, then we have to use the pfn_valid() 2205 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must* 2206 * refcount the page if pfn_valid is true (hence insert_page rather 2207 * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP 2208 * without pte special, it would there be refcounted as a normal page. 2209 */ 2210 if (!HAVE_PTE_SPECIAL && pfn_valid(pfn)) { 2211 struct page *page; 2212 2213 page = pfn_to_page(pfn); 2214 return insert_page(vma, addr, page, vma->vm_page_prot); 2215 } 2216 return insert_pfn(vma, addr, pfn, vma->vm_page_prot); 2217 } 2218 EXPORT_SYMBOL(vm_insert_mixed); 2219 2220 /* 2221 * maps a range of physical memory into the requested pages. the old 2222 * mappings are removed. any references to nonexistent pages results 2223 * in null mappings (currently treated as "copy-on-access") 2224 */ 2225 static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, 2226 unsigned long addr, unsigned long end, 2227 unsigned long pfn, pgprot_t prot) 2228 { 2229 pte_t *pte; 2230 spinlock_t *ptl; 2231 2232 pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); 2233 if (!pte) 2234 return -ENOMEM; 2235 arch_enter_lazy_mmu_mode(); 2236 do { 2237 BUG_ON(!pte_none(*pte)); 2238 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); 2239 pfn++; 2240 } while (pte++, addr += PAGE_SIZE, addr != end); 2241 arch_leave_lazy_mmu_mode(); 2242 pte_unmap_unlock(pte - 1, ptl); 2243 return 0; 2244 } 2245 2246 static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, 2247 unsigned long addr, unsigned long end, 2248 unsigned long pfn, pgprot_t prot) 2249 { 2250 pmd_t *pmd; 2251 unsigned long next; 2252 2253 pfn -= addr >> PAGE_SHIFT; 2254 pmd = pmd_alloc(mm, pud, addr); 2255 if (!pmd) 2256 return -ENOMEM; 2257 VM_BUG_ON(pmd_trans_huge(*pmd)); 2258 do { 2259 next = pmd_addr_end(addr, end); 2260 if (remap_pte_range(mm, pmd, addr, next, 2261 pfn + (addr >> PAGE_SHIFT), prot)) 2262 return -ENOMEM; 2263 } while (pmd++, addr = next, addr != end); 2264 return 0; 2265 } 2266 2267 static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd, 2268 unsigned long addr, unsigned long end, 2269 unsigned long pfn, pgprot_t prot) 2270 { 2271 pud_t *pud; 2272 unsigned long next; 2273 2274 pfn -= addr >> PAGE_SHIFT; 2275 pud = pud_alloc(mm, pgd, addr); 2276 if (!pud) 2277 return -ENOMEM; 2278 do { 2279 next = pud_addr_end(addr, end); 2280 if (remap_pmd_range(mm, pud, addr, next, 2281 pfn + (addr >> PAGE_SHIFT), prot)) 2282 return -ENOMEM; 2283 } while (pud++, addr = next, addr != end); 2284 return 0; 2285 } 2286 2287 /** 2288 * remap_pfn_range - remap kernel memory to userspace 2289 * @vma: user vma to map to 2290 * @addr: target user address to start at 2291 * @pfn: physical address of kernel memory 2292 * @size: size of map area 2293 * @prot: page protection flags for this mapping 2294 * 2295 * Note: this is only safe if the mm semaphore is held when called. 2296 */ 2297 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 2298 unsigned long pfn, unsigned long size, pgprot_t prot) 2299 { 2300 pgd_t *pgd; 2301 unsigned long next; 2302 unsigned long end = addr + PAGE_ALIGN(size); 2303 struct mm_struct *mm = vma->vm_mm; 2304 int err; 2305 2306 /* 2307 * Physically remapped pages are special. Tell the 2308 * rest of the world about it: 2309 * VM_IO tells people not to look at these pages 2310 * (accesses can have side effects). 2311 * VM_PFNMAP tells the core MM that the base pages are just 2312 * raw PFN mappings, and do not have a "struct page" associated 2313 * with them. 2314 * VM_DONTEXPAND 2315 * Disable vma merging and expanding with mremap(). 2316 * VM_DONTDUMP 2317 * Omit vma from core dump, even when VM_IO turned off. 2318 * 2319 * There's a horrible special case to handle copy-on-write 2320 * behaviour that some programs depend on. We mark the "original" 2321 * un-COW'ed pages by matching them up with "vma->vm_pgoff". 2322 * See vm_normal_page() for details. 2323 */ 2324 if (is_cow_mapping(vma->vm_flags)) { 2325 if (addr != vma->vm_start || end != vma->vm_end) 2326 return -EINVAL; 2327 vma->vm_pgoff = pfn; 2328 } 2329 2330 err = track_pfn_remap(vma, &prot, pfn, addr, PAGE_ALIGN(size)); 2331 if (err) 2332 return -EINVAL; 2333 2334 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 2335 2336 BUG_ON(addr >= end); 2337 pfn -= addr >> PAGE_SHIFT; 2338 pgd = pgd_offset(mm, addr); 2339 flush_cache_range(vma, addr, end); 2340 do { 2341 next = pgd_addr_end(addr, end); 2342 err = remap_pud_range(mm, pgd, addr, next, 2343 pfn + (addr >> PAGE_SHIFT), prot); 2344 if (err) 2345 break; 2346 } while (pgd++, addr = next, addr != end); 2347 2348 if (err) 2349 untrack_pfn(vma, pfn, PAGE_ALIGN(size)); 2350 2351 return err; 2352 } 2353 EXPORT_SYMBOL(remap_pfn_range); 2354 2355 static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, 2356 unsigned long addr, unsigned long end, 2357 pte_fn_t fn, void *data) 2358 { 2359 pte_t *pte; 2360 int err; 2361 pgtable_t token; 2362 spinlock_t *uninitialized_var(ptl); 2363 2364 pte = (mm == &init_mm) ? 2365 pte_alloc_kernel(pmd, addr) : 2366 pte_alloc_map_lock(mm, pmd, addr, &ptl); 2367 if (!pte) 2368 return -ENOMEM; 2369 2370 BUG_ON(pmd_huge(*pmd)); 2371 2372 arch_enter_lazy_mmu_mode(); 2373 2374 token = pmd_pgtable(*pmd); 2375 2376 do { 2377 err = fn(pte++, token, addr, data); 2378 if (err) 2379 break; 2380 } while (addr += PAGE_SIZE, addr != end); 2381 2382 arch_leave_lazy_mmu_mode(); 2383 2384 if (mm != &init_mm) 2385 pte_unmap_unlock(pte-1, ptl); 2386 return err; 2387 } 2388 2389 static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, 2390 unsigned long addr, unsigned long end, 2391 pte_fn_t fn, void *data) 2392 { 2393 pmd_t *pmd; 2394 unsigned long next; 2395 int err; 2396 2397 BUG_ON(pud_huge(*pud)); 2398 2399 pmd = pmd_alloc(mm, pud, addr); 2400 if (!pmd) 2401 return -ENOMEM; 2402 do { 2403 next = pmd_addr_end(addr, end); 2404 err = apply_to_pte_range(mm, pmd, addr, next, fn, data); 2405 if (err) 2406 break; 2407 } while (pmd++, addr = next, addr != end); 2408 return err; 2409 } 2410 2411 static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd, 2412 unsigned long addr, unsigned long end, 2413 pte_fn_t fn, void *data) 2414 { 2415 pud_t *pud; 2416 unsigned long next; 2417 int err; 2418 2419 pud = pud_alloc(mm, pgd, addr); 2420 if (!pud) 2421 return -ENOMEM; 2422 do { 2423 next = pud_addr_end(addr, end); 2424 err = apply_to_pmd_range(mm, pud, addr, next, fn, data); 2425 if (err) 2426 break; 2427 } while (pud++, addr = next, addr != end); 2428 return err; 2429 } 2430 2431 /* 2432 * Scan a region of virtual memory, filling in page tables as necessary 2433 * and calling a provided function on each leaf page table. 2434 */ 2435 int apply_to_page_range(struct mm_struct *mm, unsigned long addr, 2436 unsigned long size, pte_fn_t fn, void *data) 2437 { 2438 pgd_t *pgd; 2439 unsigned long next; 2440 unsigned long end = addr + size; 2441 int err; 2442 2443 BUG_ON(addr >= end); 2444 pgd = pgd_offset(mm, addr); 2445 do { 2446 next = pgd_addr_end(addr, end); 2447 err = apply_to_pud_range(mm, pgd, addr, next, fn, data); 2448 if (err) 2449 break; 2450 } while (pgd++, addr = next, addr != end); 2451 2452 return err; 2453 } 2454 EXPORT_SYMBOL_GPL(apply_to_page_range); 2455 2456 /* 2457 * handle_pte_fault chooses page fault handler according to an entry 2458 * which was read non-atomically. Before making any commitment, on 2459 * those architectures or configurations (e.g. i386 with PAE) which 2460 * might give a mix of unmatched parts, do_swap_page and do_nonlinear_fault 2461 * must check under lock before unmapping the pte and proceeding 2462 * (but do_wp_page is only called after already making such a check; 2463 * and do_anonymous_page can safely check later on). 2464 */ 2465 static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd, 2466 pte_t *page_table, pte_t orig_pte) 2467 { 2468 int same = 1; 2469 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT) 2470 if (sizeof(pte_t) > sizeof(unsigned long)) { 2471 spinlock_t *ptl = pte_lockptr(mm, pmd); 2472 spin_lock(ptl); 2473 same = pte_same(*page_table, orig_pte); 2474 spin_unlock(ptl); 2475 } 2476 #endif 2477 pte_unmap(page_table); 2478 return same; 2479 } 2480 2481 static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma) 2482 { 2483 /* 2484 * If the source page was a PFN mapping, we don't have 2485 * a "struct page" for it. We do a best-effort copy by 2486 * just copying from the original user address. If that 2487 * fails, we just zero-fill it. Live with it. 2488 */ 2489 if (unlikely(!src)) { 2490 void *kaddr = kmap_atomic(dst); 2491 void __user *uaddr = (void __user *)(va & PAGE_MASK); 2492 2493 /* 2494 * This really shouldn't fail, because the page is there 2495 * in the page tables. But it might just be unreadable, 2496 * in which case we just give up and fill the result with 2497 * zeroes. 2498 */ 2499 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) 2500 clear_page(kaddr); 2501 kunmap_atomic(kaddr); 2502 flush_dcache_page(dst); 2503 } else 2504 copy_user_highpage(dst, src, va, vma); 2505 } 2506 2507 /* 2508 * This routine handles present pages, when users try to write 2509 * to a shared page. It is done by copying the page to a new address 2510 * and decrementing the shared-page counter for the old page. 2511 * 2512 * Note that this routine assumes that the protection checks have been 2513 * done by the caller (the low-level page fault routine in most cases). 2514 * Thus we can safely just mark it writable once we've done any necessary 2515 * COW. 2516 * 2517 * We also mark the page dirty at this point even though the page will 2518 * change only once the write actually happens. This avoids a few races, 2519 * and potentially makes it more efficient. 2520 * 2521 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2522 * but allow concurrent faults), with pte both mapped and locked. 2523 * We return with mmap_sem still held, but pte unmapped and unlocked. 2524 */ 2525 static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 2526 unsigned long address, pte_t *page_table, pmd_t *pmd, 2527 spinlock_t *ptl, pte_t orig_pte) 2528 __releases(ptl) 2529 { 2530 struct page *old_page, *new_page = NULL; 2531 pte_t entry; 2532 int ret = 0; 2533 int page_mkwrite = 0; 2534 struct page *dirty_page = NULL; 2535 unsigned long mmun_start = 0; /* For mmu_notifiers */ 2536 unsigned long mmun_end = 0; /* For mmu_notifiers */ 2537 2538 old_page = vm_normal_page(vma, address, orig_pte); 2539 if (!old_page) { 2540 /* 2541 * VM_MIXEDMAP !pfn_valid() case 2542 * 2543 * We should not cow pages in a shared writeable mapping. 2544 * Just mark the pages writable as we can't do any dirty 2545 * accounting on raw pfn maps. 2546 */ 2547 if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 2548 (VM_WRITE|VM_SHARED)) 2549 goto reuse; 2550 goto gotten; 2551 } 2552 2553 /* 2554 * Take out anonymous pages first, anonymous shared vmas are 2555 * not dirty accountable. 2556 */ 2557 if (PageAnon(old_page) && !PageKsm(old_page)) { 2558 if (!trylock_page(old_page)) { 2559 page_cache_get(old_page); 2560 pte_unmap_unlock(page_table, ptl); 2561 lock_page(old_page); 2562 page_table = pte_offset_map_lock(mm, pmd, address, 2563 &ptl); 2564 if (!pte_same(*page_table, orig_pte)) { 2565 unlock_page(old_page); 2566 goto unlock; 2567 } 2568 page_cache_release(old_page); 2569 } 2570 if (reuse_swap_page(old_page)) { 2571 /* 2572 * The page is all ours. Move it to our anon_vma so 2573 * the rmap code will not search our parent or siblings. 2574 * Protected against the rmap code by the page lock. 2575 */ 2576 page_move_anon_rmap(old_page, vma, address); 2577 unlock_page(old_page); 2578 goto reuse; 2579 } 2580 unlock_page(old_page); 2581 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) == 2582 (VM_WRITE|VM_SHARED))) { 2583 /* 2584 * Only catch write-faults on shared writable pages, 2585 * read-only shared pages can get COWed by 2586 * get_user_pages(.write=1, .force=1). 2587 */ 2588 if (vma->vm_ops && vma->vm_ops->page_mkwrite) { 2589 struct vm_fault vmf; 2590 int tmp; 2591 2592 vmf.virtual_address = (void __user *)(address & 2593 PAGE_MASK); 2594 vmf.pgoff = old_page->index; 2595 vmf.flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE; 2596 vmf.page = old_page; 2597 2598 /* 2599 * Notify the address space that the page is about to 2600 * become writable so that it can prohibit this or wait 2601 * for the page to get into an appropriate state. 2602 * 2603 * We do this without the lock held, so that it can 2604 * sleep if it needs to. 2605 */ 2606 page_cache_get(old_page); 2607 pte_unmap_unlock(page_table, ptl); 2608 2609 tmp = vma->vm_ops->page_mkwrite(vma, &vmf); 2610 if (unlikely(tmp & 2611 (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) { 2612 ret = tmp; 2613 goto unwritable_page; 2614 } 2615 if (unlikely(!(tmp & VM_FAULT_LOCKED))) { 2616 lock_page(old_page); 2617 if (!old_page->mapping) { 2618 ret = 0; /* retry the fault */ 2619 unlock_page(old_page); 2620 goto unwritable_page; 2621 } 2622 } else 2623 VM_BUG_ON(!PageLocked(old_page)); 2624 2625 /* 2626 * Since we dropped the lock we need to revalidate 2627 * the PTE as someone else may have changed it. If 2628 * they did, we just return, as we can count on the 2629 * MMU to tell us if they didn't also make it writable. 2630 */ 2631 page_table = pte_offset_map_lock(mm, pmd, address, 2632 &ptl); 2633 if (!pte_same(*page_table, orig_pte)) { 2634 unlock_page(old_page); 2635 goto unlock; 2636 } 2637 2638 page_mkwrite = 1; 2639 } 2640 dirty_page = old_page; 2641 get_page(dirty_page); 2642 2643 reuse: 2644 flush_cache_page(vma, address, pte_pfn(orig_pte)); 2645 entry = pte_mkyoung(orig_pte); 2646 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2647 if (ptep_set_access_flags(vma, address, page_table, entry,1)) 2648 update_mmu_cache(vma, address, page_table); 2649 pte_unmap_unlock(page_table, ptl); 2650 ret |= VM_FAULT_WRITE; 2651 2652 if (!dirty_page) 2653 return ret; 2654 2655 /* 2656 * Yes, Virginia, this is actually required to prevent a race 2657 * with clear_page_dirty_for_io() from clearing the page dirty 2658 * bit after it clear all dirty ptes, but before a racing 2659 * do_wp_page installs a dirty pte. 2660 * 2661 * __do_fault is protected similarly. 2662 */ 2663 if (!page_mkwrite) { 2664 wait_on_page_locked(dirty_page); 2665 set_page_dirty_balance(dirty_page, page_mkwrite); 2666 /* file_update_time outside page_lock */ 2667 if (vma->vm_file) 2668 file_update_time(vma->vm_file); 2669 } 2670 put_page(dirty_page); 2671 if (page_mkwrite) { 2672 struct address_space *mapping = dirty_page->mapping; 2673 2674 set_page_dirty(dirty_page); 2675 unlock_page(dirty_page); 2676 page_cache_release(dirty_page); 2677 if (mapping) { 2678 /* 2679 * Some device drivers do not set page.mapping 2680 * but still dirty their pages 2681 */ 2682 balance_dirty_pages_ratelimited(mapping); 2683 } 2684 } 2685 2686 return ret; 2687 } 2688 2689 /* 2690 * Ok, we need to copy. Oh, well.. 2691 */ 2692 page_cache_get(old_page); 2693 gotten: 2694 pte_unmap_unlock(page_table, ptl); 2695 2696 if (unlikely(anon_vma_prepare(vma))) 2697 goto oom; 2698 2699 if (is_zero_pfn(pte_pfn(orig_pte))) { 2700 new_page = alloc_zeroed_user_highpage_movable(vma, address); 2701 if (!new_page) 2702 goto oom; 2703 } else { 2704 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); 2705 if (!new_page) 2706 goto oom; 2707 cow_user_page(new_page, old_page, address, vma); 2708 } 2709 __SetPageUptodate(new_page); 2710 2711 if (mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL)) 2712 goto oom_free_new; 2713 2714 mmun_start = address & PAGE_MASK; 2715 mmun_end = mmun_start + PAGE_SIZE; 2716 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 2717 2718 /* 2719 * Re-check the pte - we dropped the lock 2720 */ 2721 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2722 if (likely(pte_same(*page_table, orig_pte))) { 2723 if (old_page) { 2724 if (!PageAnon(old_page)) { 2725 dec_mm_counter_fast(mm, MM_FILEPAGES); 2726 inc_mm_counter_fast(mm, MM_ANONPAGES); 2727 } 2728 } else 2729 inc_mm_counter_fast(mm, MM_ANONPAGES); 2730 flush_cache_page(vma, address, pte_pfn(orig_pte)); 2731 entry = mk_pte(new_page, vma->vm_page_prot); 2732 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 2733 /* 2734 * Clear the pte entry and flush it first, before updating the 2735 * pte with the new entry. This will avoid a race condition 2736 * seen in the presence of one thread doing SMC and another 2737 * thread doing COW. 2738 */ 2739 ptep_clear_flush(vma, address, page_table); 2740 page_add_new_anon_rmap(new_page, vma, address); 2741 /* 2742 * We call the notify macro here because, when using secondary 2743 * mmu page tables (such as kvm shadow page tables), we want the 2744 * new page to be mapped directly into the secondary page table. 2745 */ 2746 set_pte_at_notify(mm, address, page_table, entry); 2747 update_mmu_cache(vma, address, page_table); 2748 if (old_page) { 2749 /* 2750 * Only after switching the pte to the new page may 2751 * we remove the mapcount here. Otherwise another 2752 * process may come and find the rmap count decremented 2753 * before the pte is switched to the new page, and 2754 * "reuse" the old page writing into it while our pte 2755 * here still points into it and can be read by other 2756 * threads. 2757 * 2758 * The critical issue is to order this 2759 * page_remove_rmap with the ptp_clear_flush above. 2760 * Those stores are ordered by (if nothing else,) 2761 * the barrier present in the atomic_add_negative 2762 * in page_remove_rmap. 2763 * 2764 * Then the TLB flush in ptep_clear_flush ensures that 2765 * no process can access the old page before the 2766 * decremented mapcount is visible. And the old page 2767 * cannot be reused until after the decremented 2768 * mapcount is visible. So transitively, TLBs to 2769 * old page will be flushed before it can be reused. 2770 */ 2771 page_remove_rmap(old_page); 2772 } 2773 2774 /* Free the old page.. */ 2775 new_page = old_page; 2776 ret |= VM_FAULT_WRITE; 2777 } else 2778 mem_cgroup_uncharge_page(new_page); 2779 2780 if (new_page) 2781 page_cache_release(new_page); 2782 unlock: 2783 pte_unmap_unlock(page_table, ptl); 2784 if (mmun_end > mmun_start) 2785 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2786 if (old_page) { 2787 /* 2788 * Don't let another task, with possibly unlocked vma, 2789 * keep the mlocked page. 2790 */ 2791 if ((ret & VM_FAULT_WRITE) && (vma->vm_flags & VM_LOCKED)) { 2792 lock_page(old_page); /* LRU manipulation */ 2793 munlock_vma_page(old_page); 2794 unlock_page(old_page); 2795 } 2796 page_cache_release(old_page); 2797 } 2798 return ret; 2799 oom_free_new: 2800 page_cache_release(new_page); 2801 oom: 2802 if (old_page) 2803 page_cache_release(old_page); 2804 return VM_FAULT_OOM; 2805 2806 unwritable_page: 2807 page_cache_release(old_page); 2808 return ret; 2809 } 2810 2811 static void unmap_mapping_range_vma(struct vm_area_struct *vma, 2812 unsigned long start_addr, unsigned long end_addr, 2813 struct zap_details *details) 2814 { 2815 zap_page_range_single(vma, start_addr, end_addr - start_addr, details); 2816 } 2817 2818 static inline void unmap_mapping_range_tree(struct rb_root *root, 2819 struct zap_details *details) 2820 { 2821 struct vm_area_struct *vma; 2822 pgoff_t vba, vea, zba, zea; 2823 2824 vma_interval_tree_foreach(vma, root, 2825 details->first_index, details->last_index) { 2826 2827 vba = vma->vm_pgoff; 2828 vea = vba + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) - 1; 2829 /* Assume for now that PAGE_CACHE_SHIFT == PAGE_SHIFT */ 2830 zba = details->first_index; 2831 if (zba < vba) 2832 zba = vba; 2833 zea = details->last_index; 2834 if (zea > vea) 2835 zea = vea; 2836 2837 unmap_mapping_range_vma(vma, 2838 ((zba - vba) << PAGE_SHIFT) + vma->vm_start, 2839 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start, 2840 details); 2841 } 2842 } 2843 2844 static inline void unmap_mapping_range_list(struct list_head *head, 2845 struct zap_details *details) 2846 { 2847 struct vm_area_struct *vma; 2848 2849 /* 2850 * In nonlinear VMAs there is no correspondence between virtual address 2851 * offset and file offset. So we must perform an exhaustive search 2852 * across *all* the pages in each nonlinear VMA, not just the pages 2853 * whose virtual address lies outside the file truncation point. 2854 */ 2855 list_for_each_entry(vma, head, shared.nonlinear) { 2856 details->nonlinear_vma = vma; 2857 unmap_mapping_range_vma(vma, vma->vm_start, vma->vm_end, details); 2858 } 2859 } 2860 2861 /** 2862 * unmap_mapping_range - unmap the portion of all mmaps in the specified address_space corresponding to the specified page range in the underlying file. 2863 * @mapping: the address space containing mmaps to be unmapped. 2864 * @holebegin: byte in first page to unmap, relative to the start of 2865 * the underlying file. This will be rounded down to a PAGE_SIZE 2866 * boundary. Note that this is different from truncate_pagecache(), which 2867 * must keep the partial page. In contrast, we must get rid of 2868 * partial pages. 2869 * @holelen: size of prospective hole in bytes. This will be rounded 2870 * up to a PAGE_SIZE boundary. A holelen of zero truncates to the 2871 * end of the file. 2872 * @even_cows: 1 when truncating a file, unmap even private COWed pages; 2873 * but 0 when invalidating pagecache, don't throw away private data. 2874 */ 2875 void unmap_mapping_range(struct address_space *mapping, 2876 loff_t const holebegin, loff_t const holelen, int even_cows) 2877 { 2878 struct zap_details details; 2879 pgoff_t hba = holebegin >> PAGE_SHIFT; 2880 pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2881 2882 /* Check for overflow. */ 2883 if (sizeof(holelen) > sizeof(hlen)) { 2884 long long holeend = 2885 (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT; 2886 if (holeend & ~(long long)ULONG_MAX) 2887 hlen = ULONG_MAX - hba + 1; 2888 } 2889 2890 details.check_mapping = even_cows? NULL: mapping; 2891 details.nonlinear_vma = NULL; 2892 details.first_index = hba; 2893 details.last_index = hba + hlen - 1; 2894 if (details.last_index < details.first_index) 2895 details.last_index = ULONG_MAX; 2896 2897 2898 mutex_lock(&mapping->i_mmap_mutex); 2899 if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap))) 2900 unmap_mapping_range_tree(&mapping->i_mmap, &details); 2901 if (unlikely(!list_empty(&mapping->i_mmap_nonlinear))) 2902 unmap_mapping_range_list(&mapping->i_mmap_nonlinear, &details); 2903 mutex_unlock(&mapping->i_mmap_mutex); 2904 } 2905 EXPORT_SYMBOL(unmap_mapping_range); 2906 2907 /* 2908 * We enter with non-exclusive mmap_sem (to exclude vma changes, 2909 * but allow concurrent faults), and pte mapped but not yet locked. 2910 * We return with mmap_sem still held, but pte unmapped and unlocked. 2911 */ 2912 static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma, 2913 unsigned long address, pte_t *page_table, pmd_t *pmd, 2914 unsigned int flags, pte_t orig_pte) 2915 { 2916 spinlock_t *ptl; 2917 struct page *page, *swapcache = NULL; 2918 swp_entry_t entry; 2919 pte_t pte; 2920 int locked; 2921 struct mem_cgroup *ptr; 2922 int exclusive = 0; 2923 int ret = 0; 2924 2925 if (!pte_unmap_same(mm, pmd, page_table, orig_pte)) 2926 goto out; 2927 2928 entry = pte_to_swp_entry(orig_pte); 2929 if (unlikely(non_swap_entry(entry))) { 2930 if (is_migration_entry(entry)) { 2931 migration_entry_wait(mm, pmd, address); 2932 } else if (is_hwpoison_entry(entry)) { 2933 ret = VM_FAULT_HWPOISON; 2934 } else { 2935 print_bad_pte(vma, address, orig_pte, NULL); 2936 ret = VM_FAULT_SIGBUS; 2937 } 2938 goto out; 2939 } 2940 delayacct_set_flag(DELAYACCT_PF_SWAPIN); 2941 page = lookup_swap_cache(entry); 2942 if (!page) { 2943 page = swapin_readahead(entry, 2944 GFP_HIGHUSER_MOVABLE, vma, address); 2945 if (!page) { 2946 /* 2947 * Back out if somebody else faulted in this pte 2948 * while we released the pte lock. 2949 */ 2950 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 2951 if (likely(pte_same(*page_table, orig_pte))) 2952 ret = VM_FAULT_OOM; 2953 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2954 goto unlock; 2955 } 2956 2957 /* Had to read the page from swap area: Major fault */ 2958 ret = VM_FAULT_MAJOR; 2959 count_vm_event(PGMAJFAULT); 2960 mem_cgroup_count_vm_event(mm, PGMAJFAULT); 2961 } else if (PageHWPoison(page)) { 2962 /* 2963 * hwpoisoned dirty swapcache pages are kept for killing 2964 * owner processes (which may be unknown at hwpoison time) 2965 */ 2966 ret = VM_FAULT_HWPOISON; 2967 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2968 goto out_release; 2969 } 2970 2971 locked = lock_page_or_retry(page, mm, flags); 2972 2973 delayacct_clear_flag(DELAYACCT_PF_SWAPIN); 2974 if (!locked) { 2975 ret |= VM_FAULT_RETRY; 2976 goto out_release; 2977 } 2978 2979 /* 2980 * Make sure try_to_free_swap or reuse_swap_page or swapoff did not 2981 * release the swapcache from under us. The page pin, and pte_same 2982 * test below, are not enough to exclude that. Even if it is still 2983 * swapcache, we need to check that the page's swap has not changed. 2984 */ 2985 if (unlikely(!PageSwapCache(page) || page_private(page) != entry.val)) 2986 goto out_page; 2987 2988 if (ksm_might_need_to_copy(page, vma, address)) { 2989 swapcache = page; 2990 page = ksm_does_need_to_copy(page, vma, address); 2991 2992 if (unlikely(!page)) { 2993 ret = VM_FAULT_OOM; 2994 page = swapcache; 2995 swapcache = NULL; 2996 goto out_page; 2997 } 2998 } 2999 3000 if (mem_cgroup_try_charge_swapin(mm, page, GFP_KERNEL, &ptr)) { 3001 ret = VM_FAULT_OOM; 3002 goto out_page; 3003 } 3004 3005 /* 3006 * Back out if somebody else already faulted in this pte. 3007 */ 3008 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 3009 if (unlikely(!pte_same(*page_table, orig_pte))) 3010 goto out_nomap; 3011 3012 if (unlikely(!PageUptodate(page))) { 3013 ret = VM_FAULT_SIGBUS; 3014 goto out_nomap; 3015 } 3016 3017 /* 3018 * The page isn't present yet, go ahead with the fault. 3019 * 3020 * Be careful about the sequence of operations here. 3021 * To get its accounting right, reuse_swap_page() must be called 3022 * while the page is counted on swap but not yet in mapcount i.e. 3023 * before page_add_anon_rmap() and swap_free(); try_to_free_swap() 3024 * must be called after the swap_free(), or it will never succeed. 3025 * Because delete_from_swap_page() may be called by reuse_swap_page(), 3026 * mem_cgroup_commit_charge_swapin() may not be able to find swp_entry 3027 * in page->private. In this case, a record in swap_cgroup is silently 3028 * discarded at swap_free(). 3029 */ 3030 3031 inc_mm_counter_fast(mm, MM_ANONPAGES); 3032 dec_mm_counter_fast(mm, MM_SWAPENTS); 3033 pte = mk_pte(page, vma->vm_page_prot); 3034 if ((flags & FAULT_FLAG_WRITE) && reuse_swap_page(page)) { 3035 pte = maybe_mkwrite(pte_mkdirty(pte), vma); 3036 flags &= ~FAULT_FLAG_WRITE; 3037 ret |= VM_FAULT_WRITE; 3038 exclusive = 1; 3039 } 3040 flush_icache_page(vma, page); 3041 set_pte_at(mm, address, page_table, pte); 3042 do_page_add_anon_rmap(page, vma, address, exclusive); 3043 /* It's better to call commit-charge after rmap is established */ 3044 mem_cgroup_commit_charge_swapin(page, ptr); 3045 3046 swap_free(entry); 3047 if (vm_swap_full() || (vma->vm_flags & VM_LOCKED) || PageMlocked(page)) 3048 try_to_free_swap(page); 3049 unlock_page(page); 3050 if (swapcache) { 3051 /* 3052 * Hold the lock to avoid the swap entry to be reused 3053 * until we take the PT lock for the pte_same() check 3054 * (to avoid false positives from pte_same). For 3055 * further safety release the lock after the swap_free 3056 * so that the swap count won't change under a 3057 * parallel locked swapcache. 3058 */ 3059 unlock_page(swapcache); 3060 page_cache_release(swapcache); 3061 } 3062 3063 if (flags & FAULT_FLAG_WRITE) { 3064 ret |= do_wp_page(mm, vma, address, page_table, pmd, ptl, pte); 3065 if (ret & VM_FAULT_ERROR) 3066 ret &= VM_FAULT_ERROR; 3067 goto out; 3068 } 3069 3070 /* No need to invalidate - it was non-present before */ 3071 update_mmu_cache(vma, address, page_table); 3072 unlock: 3073 pte_unmap_unlock(page_table, ptl); 3074 out: 3075 return ret; 3076 out_nomap: 3077 mem_cgroup_cancel_charge_swapin(ptr); 3078 pte_unmap_unlock(page_table, ptl); 3079 out_page: 3080 unlock_page(page); 3081 out_release: 3082 page_cache_release(page); 3083 if (swapcache) { 3084 unlock_page(swapcache); 3085 page_cache_release(swapcache); 3086 } 3087 return ret; 3088 } 3089 3090 /* 3091 * This is like a special single-page "expand_{down|up}wards()", 3092 * except we must first make sure that 'address{-|+}PAGE_SIZE' 3093 * doesn't hit another vma. 3094 */ 3095 static inline int check_stack_guard_page(struct vm_area_struct *vma, unsigned long address) 3096 { 3097 address &= PAGE_MASK; 3098 if ((vma->vm_flags & VM_GROWSDOWN) && address == vma->vm_start) { 3099 struct vm_area_struct *prev = vma->vm_prev; 3100 3101 /* 3102 * Is there a mapping abutting this one below? 3103 * 3104 * That's only ok if it's the same stack mapping 3105 * that has gotten split.. 3106 */ 3107 if (prev && prev->vm_end == address) 3108 return prev->vm_flags & VM_GROWSDOWN ? 0 : -ENOMEM; 3109 3110 expand_downwards(vma, address - PAGE_SIZE); 3111 } 3112 if ((vma->vm_flags & VM_GROWSUP) && address + PAGE_SIZE == vma->vm_end) { 3113 struct vm_area_struct *next = vma->vm_next; 3114 3115 /* As VM_GROWSDOWN but s/below/above/ */ 3116 if (next && next->vm_start == address + PAGE_SIZE) 3117 return next->vm_flags & VM_GROWSUP ? 0 : -ENOMEM; 3118 3119 expand_upwards(vma, address + PAGE_SIZE); 3120 } 3121 return 0; 3122 } 3123 3124 /* 3125 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3126 * but allow concurrent faults), and pte mapped but not yet locked. 3127 * We return with mmap_sem still held, but pte unmapped and unlocked. 3128 */ 3129 static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, 3130 unsigned long address, pte_t *page_table, pmd_t *pmd, 3131 unsigned int flags) 3132 { 3133 struct page *page; 3134 spinlock_t *ptl; 3135 pte_t entry; 3136 3137 pte_unmap(page_table); 3138 3139 /* Check if we need to add a guard page to the stack */ 3140 if (check_stack_guard_page(vma, address) < 0) 3141 return VM_FAULT_SIGBUS; 3142 3143 /* Use the zero-page for reads */ 3144 if (!(flags & FAULT_FLAG_WRITE)) { 3145 entry = pte_mkspecial(pfn_pte(my_zero_pfn(address), 3146 vma->vm_page_prot)); 3147 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 3148 if (!pte_none(*page_table)) 3149 goto unlock; 3150 goto setpte; 3151 } 3152 3153 /* Allocate our own private page. */ 3154 if (unlikely(anon_vma_prepare(vma))) 3155 goto oom; 3156 page = alloc_zeroed_user_highpage_movable(vma, address); 3157 if (!page) 3158 goto oom; 3159 __SetPageUptodate(page); 3160 3161 if (mem_cgroup_newpage_charge(page, mm, GFP_KERNEL)) 3162 goto oom_free_page; 3163 3164 entry = mk_pte(page, vma->vm_page_prot); 3165 if (vma->vm_flags & VM_WRITE) 3166 entry = pte_mkwrite(pte_mkdirty(entry)); 3167 3168 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 3169 if (!pte_none(*page_table)) 3170 goto release; 3171 3172 inc_mm_counter_fast(mm, MM_ANONPAGES); 3173 page_add_new_anon_rmap(page, vma, address); 3174 setpte: 3175 set_pte_at(mm, address, page_table, entry); 3176 3177 /* No need to invalidate - it was non-present before */ 3178 update_mmu_cache(vma, address, page_table); 3179 unlock: 3180 pte_unmap_unlock(page_table, ptl); 3181 return 0; 3182 release: 3183 mem_cgroup_uncharge_page(page); 3184 page_cache_release(page); 3185 goto unlock; 3186 oom_free_page: 3187 page_cache_release(page); 3188 oom: 3189 return VM_FAULT_OOM; 3190 } 3191 3192 /* 3193 * __do_fault() tries to create a new page mapping. It aggressively 3194 * tries to share with existing pages, but makes a separate copy if 3195 * the FAULT_FLAG_WRITE is set in the flags parameter in order to avoid 3196 * the next page fault. 3197 * 3198 * As this is called only for pages that do not currently exist, we 3199 * do not need to flush old virtual caches or the TLB. 3200 * 3201 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3202 * but allow concurrent faults), and pte neither mapped nor locked. 3203 * We return with mmap_sem still held, but pte unmapped and unlocked. 3204 */ 3205 static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3206 unsigned long address, pmd_t *pmd, 3207 pgoff_t pgoff, unsigned int flags, pte_t orig_pte) 3208 { 3209 pte_t *page_table; 3210 spinlock_t *ptl; 3211 struct page *page; 3212 struct page *cow_page; 3213 pte_t entry; 3214 int anon = 0; 3215 struct page *dirty_page = NULL; 3216 struct vm_fault vmf; 3217 int ret; 3218 int page_mkwrite = 0; 3219 3220 /* 3221 * If we do COW later, allocate page befor taking lock_page() 3222 * on the file cache page. This will reduce lock holding time. 3223 */ 3224 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { 3225 3226 if (unlikely(anon_vma_prepare(vma))) 3227 return VM_FAULT_OOM; 3228 3229 cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); 3230 if (!cow_page) 3231 return VM_FAULT_OOM; 3232 3233 if (mem_cgroup_newpage_charge(cow_page, mm, GFP_KERNEL)) { 3234 page_cache_release(cow_page); 3235 return VM_FAULT_OOM; 3236 } 3237 } else 3238 cow_page = NULL; 3239 3240 vmf.virtual_address = (void __user *)(address & PAGE_MASK); 3241 vmf.pgoff = pgoff; 3242 vmf.flags = flags; 3243 vmf.page = NULL; 3244 3245 ret = vma->vm_ops->fault(vma, &vmf); 3246 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | 3247 VM_FAULT_RETRY))) 3248 goto uncharge_out; 3249 3250 if (unlikely(PageHWPoison(vmf.page))) { 3251 if (ret & VM_FAULT_LOCKED) 3252 unlock_page(vmf.page); 3253 ret = VM_FAULT_HWPOISON; 3254 goto uncharge_out; 3255 } 3256 3257 /* 3258 * For consistency in subsequent calls, make the faulted page always 3259 * locked. 3260 */ 3261 if (unlikely(!(ret & VM_FAULT_LOCKED))) 3262 lock_page(vmf.page); 3263 else 3264 VM_BUG_ON(!PageLocked(vmf.page)); 3265 3266 /* 3267 * Should we do an early C-O-W break? 3268 */ 3269 page = vmf.page; 3270 if (flags & FAULT_FLAG_WRITE) { 3271 if (!(vma->vm_flags & VM_SHARED)) { 3272 page = cow_page; 3273 anon = 1; 3274 copy_user_highpage(page, vmf.page, address, vma); 3275 __SetPageUptodate(page); 3276 } else { 3277 /* 3278 * If the page will be shareable, see if the backing 3279 * address space wants to know that the page is about 3280 * to become writable 3281 */ 3282 if (vma->vm_ops->page_mkwrite) { 3283 int tmp; 3284 3285 unlock_page(page); 3286 vmf.flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE; 3287 tmp = vma->vm_ops->page_mkwrite(vma, &vmf); 3288 if (unlikely(tmp & 3289 (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) { 3290 ret = tmp; 3291 goto unwritable_page; 3292 } 3293 if (unlikely(!(tmp & VM_FAULT_LOCKED))) { 3294 lock_page(page); 3295 if (!page->mapping) { 3296 ret = 0; /* retry the fault */ 3297 unlock_page(page); 3298 goto unwritable_page; 3299 } 3300 } else 3301 VM_BUG_ON(!PageLocked(page)); 3302 page_mkwrite = 1; 3303 } 3304 } 3305 3306 } 3307 3308 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 3309 3310 /* 3311 * This silly early PAGE_DIRTY setting removes a race 3312 * due to the bad i386 page protection. But it's valid 3313 * for other architectures too. 3314 * 3315 * Note that if FAULT_FLAG_WRITE is set, we either now have 3316 * an exclusive copy of the page, or this is a shared mapping, 3317 * so we can make it writable and dirty to avoid having to 3318 * handle that later. 3319 */ 3320 /* Only go through if we didn't race with anybody else... */ 3321 if (likely(pte_same(*page_table, orig_pte))) { 3322 flush_icache_page(vma, page); 3323 entry = mk_pte(page, vma->vm_page_prot); 3324 if (flags & FAULT_FLAG_WRITE) 3325 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 3326 if (anon) { 3327 inc_mm_counter_fast(mm, MM_ANONPAGES); 3328 page_add_new_anon_rmap(page, vma, address); 3329 } else { 3330 inc_mm_counter_fast(mm, MM_FILEPAGES); 3331 page_add_file_rmap(page); 3332 if (flags & FAULT_FLAG_WRITE) { 3333 dirty_page = page; 3334 get_page(dirty_page); 3335 } 3336 } 3337 set_pte_at(mm, address, page_table, entry); 3338 3339 /* no need to invalidate: a not-present page won't be cached */ 3340 update_mmu_cache(vma, address, page_table); 3341 } else { 3342 if (cow_page) 3343 mem_cgroup_uncharge_page(cow_page); 3344 if (anon) 3345 page_cache_release(page); 3346 else 3347 anon = 1; /* no anon but release faulted_page */ 3348 } 3349 3350 pte_unmap_unlock(page_table, ptl); 3351 3352 if (dirty_page) { 3353 struct address_space *mapping = page->mapping; 3354 int dirtied = 0; 3355 3356 if (set_page_dirty(dirty_page)) 3357 dirtied = 1; 3358 unlock_page(dirty_page); 3359 put_page(dirty_page); 3360 if ((dirtied || page_mkwrite) && mapping) { 3361 /* 3362 * Some device drivers do not set page.mapping but still 3363 * dirty their pages 3364 */ 3365 balance_dirty_pages_ratelimited(mapping); 3366 } 3367 3368 /* file_update_time outside page_lock */ 3369 if (vma->vm_file && !page_mkwrite) 3370 file_update_time(vma->vm_file); 3371 } else { 3372 unlock_page(vmf.page); 3373 if (anon) 3374 page_cache_release(vmf.page); 3375 } 3376 3377 return ret; 3378 3379 unwritable_page: 3380 page_cache_release(page); 3381 return ret; 3382 uncharge_out: 3383 /* fs's fault handler get error */ 3384 if (cow_page) { 3385 mem_cgroup_uncharge_page(cow_page); 3386 page_cache_release(cow_page); 3387 } 3388 return ret; 3389 } 3390 3391 static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3392 unsigned long address, pte_t *page_table, pmd_t *pmd, 3393 unsigned int flags, pte_t orig_pte) 3394 { 3395 pgoff_t pgoff = (((address & PAGE_MASK) 3396 - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 3397 3398 pte_unmap(page_table); 3399 return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); 3400 } 3401 3402 /* 3403 * Fault of a previously existing named mapping. Repopulate the pte 3404 * from the encoded file_pte if possible. This enables swappable 3405 * nonlinear vmas. 3406 * 3407 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3408 * but allow concurrent faults), and pte mapped but not yet locked. 3409 * We return with mmap_sem still held, but pte unmapped and unlocked. 3410 */ 3411 static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3412 unsigned long address, pte_t *page_table, pmd_t *pmd, 3413 unsigned int flags, pte_t orig_pte) 3414 { 3415 pgoff_t pgoff; 3416 3417 flags |= FAULT_FLAG_NONLINEAR; 3418 3419 if (!pte_unmap_same(mm, pmd, page_table, orig_pte)) 3420 return 0; 3421 3422 if (unlikely(!(vma->vm_flags & VM_NONLINEAR))) { 3423 /* 3424 * Page table corrupted: show pte and kill process. 3425 */ 3426 print_bad_pte(vma, address, orig_pte, NULL); 3427 return VM_FAULT_SIGBUS; 3428 } 3429 3430 pgoff = pte_to_pgoff(orig_pte); 3431 return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); 3432 } 3433 3434 int numa_migrate_prep(struct page *page, struct vm_area_struct *vma, 3435 unsigned long addr, int current_nid) 3436 { 3437 get_page(page); 3438 3439 count_vm_numa_event(NUMA_HINT_FAULTS); 3440 if (current_nid == numa_node_id()) 3441 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); 3442 3443 return mpol_misplaced(page, vma, addr); 3444 } 3445 3446 int do_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, 3447 unsigned long addr, pte_t pte, pte_t *ptep, pmd_t *pmd) 3448 { 3449 struct page *page = NULL; 3450 spinlock_t *ptl; 3451 int current_nid = -1; 3452 int target_nid; 3453 bool migrated = false; 3454 3455 /* 3456 * The "pte" at this point cannot be used safely without 3457 * validation through pte_unmap_same(). It's of NUMA type but 3458 * the pfn may be screwed if the read is non atomic. 3459 * 3460 * ptep_modify_prot_start is not called as this is clearing 3461 * the _PAGE_NUMA bit and it is not really expected that there 3462 * would be concurrent hardware modifications to the PTE. 3463 */ 3464 ptl = pte_lockptr(mm, pmd); 3465 spin_lock(ptl); 3466 if (unlikely(!pte_same(*ptep, pte))) { 3467 pte_unmap_unlock(ptep, ptl); 3468 goto out; 3469 } 3470 3471 pte = pte_mknonnuma(pte); 3472 set_pte_at(mm, addr, ptep, pte); 3473 update_mmu_cache(vma, addr, ptep); 3474 3475 page = vm_normal_page(vma, addr, pte); 3476 if (!page) { 3477 pte_unmap_unlock(ptep, ptl); 3478 return 0; 3479 } 3480 3481 current_nid = page_to_nid(page); 3482 target_nid = numa_migrate_prep(page, vma, addr, current_nid); 3483 pte_unmap_unlock(ptep, ptl); 3484 if (target_nid == -1) { 3485 /* 3486 * Account for the fault against the current node if it not 3487 * being replaced regardless of where the page is located. 3488 */ 3489 current_nid = numa_node_id(); 3490 put_page(page); 3491 goto out; 3492 } 3493 3494 /* Migrate to the requested node */ 3495 migrated = migrate_misplaced_page(page, target_nid); 3496 if (migrated) 3497 current_nid = target_nid; 3498 3499 out: 3500 if (current_nid != -1) 3501 task_numa_fault(current_nid, 1, migrated); 3502 return 0; 3503 } 3504 3505 /* NUMA hinting page fault entry point for regular pmds */ 3506 #ifdef CONFIG_NUMA_BALANCING 3507 static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, 3508 unsigned long addr, pmd_t *pmdp) 3509 { 3510 pmd_t pmd; 3511 pte_t *pte, *orig_pte; 3512 unsigned long _addr = addr & PMD_MASK; 3513 unsigned long offset; 3514 spinlock_t *ptl; 3515 bool numa = false; 3516 int local_nid = numa_node_id(); 3517 3518 spin_lock(&mm->page_table_lock); 3519 pmd = *pmdp; 3520 if (pmd_numa(pmd)) { 3521 set_pmd_at(mm, _addr, pmdp, pmd_mknonnuma(pmd)); 3522 numa = true; 3523 } 3524 spin_unlock(&mm->page_table_lock); 3525 3526 if (!numa) 3527 return 0; 3528 3529 /* we're in a page fault so some vma must be in the range */ 3530 BUG_ON(!vma); 3531 BUG_ON(vma->vm_start >= _addr + PMD_SIZE); 3532 offset = max(_addr, vma->vm_start) & ~PMD_MASK; 3533 VM_BUG_ON(offset >= PMD_SIZE); 3534 orig_pte = pte = pte_offset_map_lock(mm, pmdp, _addr, &ptl); 3535 pte += offset >> PAGE_SHIFT; 3536 for (addr = _addr + offset; addr < _addr + PMD_SIZE; pte++, addr += PAGE_SIZE) { 3537 pte_t pteval = *pte; 3538 struct page *page; 3539 int curr_nid = local_nid; 3540 int target_nid; 3541 bool migrated; 3542 if (!pte_present(pteval)) 3543 continue; 3544 if (!pte_numa(pteval)) 3545 continue; 3546 if (addr >= vma->vm_end) { 3547 vma = find_vma(mm, addr); 3548 /* there's a pte present so there must be a vma */ 3549 BUG_ON(!vma); 3550 BUG_ON(addr < vma->vm_start); 3551 } 3552 if (pte_numa(pteval)) { 3553 pteval = pte_mknonnuma(pteval); 3554 set_pte_at(mm, addr, pte, pteval); 3555 } 3556 page = vm_normal_page(vma, addr, pteval); 3557 if (unlikely(!page)) 3558 continue; 3559 /* only check non-shared pages */ 3560 if (unlikely(page_mapcount(page) != 1)) 3561 continue; 3562 3563 /* 3564 * Note that the NUMA fault is later accounted to either 3565 * the node that is currently running or where the page is 3566 * migrated to. 3567 */ 3568 curr_nid = local_nid; 3569 target_nid = numa_migrate_prep(page, vma, addr, 3570 page_to_nid(page)); 3571 if (target_nid == -1) { 3572 put_page(page); 3573 continue; 3574 } 3575 3576 /* Migrate to the requested node */ 3577 pte_unmap_unlock(pte, ptl); 3578 migrated = migrate_misplaced_page(page, target_nid); 3579 if (migrated) 3580 curr_nid = target_nid; 3581 task_numa_fault(curr_nid, 1, migrated); 3582 3583 pte = pte_offset_map_lock(mm, pmdp, addr, &ptl); 3584 } 3585 pte_unmap_unlock(orig_pte, ptl); 3586 3587 return 0; 3588 } 3589 #else 3590 static int do_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, 3591 unsigned long addr, pmd_t *pmdp) 3592 { 3593 BUG(); 3594 return 0; 3595 } 3596 #endif /* CONFIG_NUMA_BALANCING */ 3597 3598 /* 3599 * These routines also need to handle stuff like marking pages dirty 3600 * and/or accessed for architectures that don't do it in hardware (most 3601 * RISC architectures). The early dirtying is also good on the i386. 3602 * 3603 * There is also a hook called "update_mmu_cache()" that architectures 3604 * with external mmu caches can use to update those (ie the Sparc or 3605 * PowerPC hashed page tables that act as extended TLBs). 3606 * 3607 * We enter with non-exclusive mmap_sem (to exclude vma changes, 3608 * but allow concurrent faults), and pte mapped but not yet locked. 3609 * We return with mmap_sem still held, but pte unmapped and unlocked. 3610 */ 3611 int handle_pte_fault(struct mm_struct *mm, 3612 struct vm_area_struct *vma, unsigned long address, 3613 pte_t *pte, pmd_t *pmd, unsigned int flags) 3614 { 3615 pte_t entry; 3616 spinlock_t *ptl; 3617 3618 entry = *pte; 3619 if (!pte_present(entry)) { 3620 if (pte_none(entry)) { 3621 if (vma->vm_ops) { 3622 if (likely(vma->vm_ops->fault)) 3623 return do_linear_fault(mm, vma, address, 3624 pte, pmd, flags, entry); 3625 } 3626 return do_anonymous_page(mm, vma, address, 3627 pte, pmd, flags); 3628 } 3629 if (pte_file(entry)) 3630 return do_nonlinear_fault(mm, vma, address, 3631 pte, pmd, flags, entry); 3632 return do_swap_page(mm, vma, address, 3633 pte, pmd, flags, entry); 3634 } 3635 3636 if (pte_numa(entry)) 3637 return do_numa_page(mm, vma, address, entry, pte, pmd); 3638 3639 ptl = pte_lockptr(mm, pmd); 3640 spin_lock(ptl); 3641 if (unlikely(!pte_same(*pte, entry))) 3642 goto unlock; 3643 if (flags & FAULT_FLAG_WRITE) { 3644 if (!pte_write(entry)) 3645 return do_wp_page(mm, vma, address, 3646 pte, pmd, ptl, entry); 3647 entry = pte_mkdirty(entry); 3648 } 3649 entry = pte_mkyoung(entry); 3650 if (ptep_set_access_flags(vma, address, pte, entry, flags & FAULT_FLAG_WRITE)) { 3651 update_mmu_cache(vma, address, pte); 3652 } else { 3653 /* 3654 * This is needed only for protection faults but the arch code 3655 * is not yet telling us if this is a protection fault or not. 3656 * This still avoids useless tlb flushes for .text page faults 3657 * with threads. 3658 */ 3659 if (flags & FAULT_FLAG_WRITE) 3660 flush_tlb_fix_spurious_fault(vma, address); 3661 } 3662 unlock: 3663 pte_unmap_unlock(pte, ptl); 3664 return 0; 3665 } 3666 3667 /* 3668 * By the time we get here, we already hold the mm semaphore 3669 */ 3670 int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, 3671 unsigned long address, unsigned int flags) 3672 { 3673 pgd_t *pgd; 3674 pud_t *pud; 3675 pmd_t *pmd; 3676 pte_t *pte; 3677 3678 __set_current_state(TASK_RUNNING); 3679 3680 count_vm_event(PGFAULT); 3681 mem_cgroup_count_vm_event(mm, PGFAULT); 3682 3683 /* do counter updates before entering really critical section. */ 3684 check_sync_rss_stat(current); 3685 3686 if (unlikely(is_vm_hugetlb_page(vma))) 3687 return hugetlb_fault(mm, vma, address, flags); 3688 3689 retry: 3690 pgd = pgd_offset(mm, address); 3691 pud = pud_alloc(mm, pgd, address); 3692 if (!pud) 3693 return VM_FAULT_OOM; 3694 pmd = pmd_alloc(mm, pud, address); 3695 if (!pmd) 3696 return VM_FAULT_OOM; 3697 if (pmd_none(*pmd) && transparent_hugepage_enabled(vma)) { 3698 if (!vma->vm_ops) 3699 return do_huge_pmd_anonymous_page(mm, vma, address, 3700 pmd, flags); 3701 } else { 3702 pmd_t orig_pmd = *pmd; 3703 int ret; 3704 3705 barrier(); 3706 if (pmd_trans_huge(orig_pmd)) { 3707 unsigned int dirty = flags & FAULT_FLAG_WRITE; 3708 3709 if (pmd_numa(orig_pmd)) 3710 return do_huge_pmd_numa_page(mm, vma, address, 3711 orig_pmd, pmd); 3712 3713 if (dirty && !pmd_write(orig_pmd)) { 3714 ret = do_huge_pmd_wp_page(mm, vma, address, pmd, 3715 orig_pmd); 3716 /* 3717 * If COW results in an oom, the huge pmd will 3718 * have been split, so retry the fault on the 3719 * pte for a smaller charge. 3720 */ 3721 if (unlikely(ret & VM_FAULT_OOM)) 3722 goto retry; 3723 return ret; 3724 } else { 3725 huge_pmd_set_accessed(mm, vma, address, pmd, 3726 orig_pmd, dirty); 3727 } 3728 3729 return 0; 3730 } 3731 } 3732 3733 if (pmd_numa(*pmd)) 3734 return do_pmd_numa_page(mm, vma, address, pmd); 3735 3736 /* 3737 * Use __pte_alloc instead of pte_alloc_map, because we can't 3738 * run pte_offset_map on the pmd, if an huge pmd could 3739 * materialize from under us from a different thread. 3740 */ 3741 if (unlikely(pmd_none(*pmd)) && 3742 unlikely(__pte_alloc(mm, vma, pmd, address))) 3743 return VM_FAULT_OOM; 3744 /* if an huge pmd materialized from under us just retry later */ 3745 if (unlikely(pmd_trans_huge(*pmd))) 3746 return 0; 3747 /* 3748 * A regular pmd is established and it can't morph into a huge pmd 3749 * from under us anymore at this point because we hold the mmap_sem 3750 * read mode and khugepaged takes it in write mode. So now it's 3751 * safe to run pte_offset_map(). 3752 */ 3753 pte = pte_offset_map(pmd, address); 3754 3755 return handle_pte_fault(mm, vma, address, pte, pmd, flags); 3756 } 3757 3758 #ifndef __PAGETABLE_PUD_FOLDED 3759 /* 3760 * Allocate page upper directory. 3761 * We've already handled the fast-path in-line. 3762 */ 3763 int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) 3764 { 3765 pud_t *new = pud_alloc_one(mm, address); 3766 if (!new) 3767 return -ENOMEM; 3768 3769 smp_wmb(); /* See comment in __pte_alloc */ 3770 3771 spin_lock(&mm->page_table_lock); 3772 if (pgd_present(*pgd)) /* Another has populated it */ 3773 pud_free(mm, new); 3774 else 3775 pgd_populate(mm, pgd, new); 3776 spin_unlock(&mm->page_table_lock); 3777 return 0; 3778 } 3779 #endif /* __PAGETABLE_PUD_FOLDED */ 3780 3781 #ifndef __PAGETABLE_PMD_FOLDED 3782 /* 3783 * Allocate page middle directory. 3784 * We've already handled the fast-path in-line. 3785 */ 3786 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 3787 { 3788 pmd_t *new = pmd_alloc_one(mm, address); 3789 if (!new) 3790 return -ENOMEM; 3791 3792 smp_wmb(); /* See comment in __pte_alloc */ 3793 3794 spin_lock(&mm->page_table_lock); 3795 #ifndef __ARCH_HAS_4LEVEL_HACK 3796 if (pud_present(*pud)) /* Another has populated it */ 3797 pmd_free(mm, new); 3798 else 3799 pud_populate(mm, pud, new); 3800 #else 3801 if (pgd_present(*pud)) /* Another has populated it */ 3802 pmd_free(mm, new); 3803 else 3804 pgd_populate(mm, pud, new); 3805 #endif /* __ARCH_HAS_4LEVEL_HACK */ 3806 spin_unlock(&mm->page_table_lock); 3807 return 0; 3808 } 3809 #endif /* __PAGETABLE_PMD_FOLDED */ 3810 3811 int make_pages_present(unsigned long addr, unsigned long end) 3812 { 3813 int ret, len, write; 3814 struct vm_area_struct * vma; 3815 3816 vma = find_vma(current->mm, addr); 3817 if (!vma) 3818 return -ENOMEM; 3819 /* 3820 * We want to touch writable mappings with a write fault in order 3821 * to break COW, except for shared mappings because these don't COW 3822 * and we would not want to dirty them for nothing. 3823 */ 3824 write = (vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE; 3825 BUG_ON(addr >= end); 3826 BUG_ON(end > vma->vm_end); 3827 len = DIV_ROUND_UP(end, PAGE_SIZE) - addr/PAGE_SIZE; 3828 ret = get_user_pages(current, current->mm, addr, 3829 len, write, 0, NULL, NULL); 3830 if (ret < 0) 3831 return ret; 3832 return ret == len ? 0 : -EFAULT; 3833 } 3834 3835 #if !defined(__HAVE_ARCH_GATE_AREA) 3836 3837 #if defined(AT_SYSINFO_EHDR) 3838 static struct vm_area_struct gate_vma; 3839 3840 static int __init gate_vma_init(void) 3841 { 3842 gate_vma.vm_mm = NULL; 3843 gate_vma.vm_start = FIXADDR_USER_START; 3844 gate_vma.vm_end = FIXADDR_USER_END; 3845 gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC; 3846 gate_vma.vm_page_prot = __P101; 3847 3848 return 0; 3849 } 3850 __initcall(gate_vma_init); 3851 #endif 3852 3853 struct vm_area_struct *get_gate_vma(struct mm_struct *mm) 3854 { 3855 #ifdef AT_SYSINFO_EHDR 3856 return &gate_vma; 3857 #else 3858 return NULL; 3859 #endif 3860 } 3861 3862 int in_gate_area_no_mm(unsigned long addr) 3863 { 3864 #ifdef AT_SYSINFO_EHDR 3865 if ((addr >= FIXADDR_USER_START) && (addr < FIXADDR_USER_END)) 3866 return 1; 3867 #endif 3868 return 0; 3869 } 3870 3871 #endif /* __HAVE_ARCH_GATE_AREA */ 3872 3873 static int __follow_pte(struct mm_struct *mm, unsigned long address, 3874 pte_t **ptepp, spinlock_t **ptlp) 3875 { 3876 pgd_t *pgd; 3877 pud_t *pud; 3878 pmd_t *pmd; 3879 pte_t *ptep; 3880 3881 pgd = pgd_offset(mm, address); 3882 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 3883 goto out; 3884 3885 pud = pud_offset(pgd, address); 3886 if (pud_none(*pud) || unlikely(pud_bad(*pud))) 3887 goto out; 3888 3889 pmd = pmd_offset(pud, address); 3890 VM_BUG_ON(pmd_trans_huge(*pmd)); 3891 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) 3892 goto out; 3893 3894 /* We cannot handle huge page PFN maps. Luckily they don't exist. */ 3895 if (pmd_huge(*pmd)) 3896 goto out; 3897 3898 ptep = pte_offset_map_lock(mm, pmd, address, ptlp); 3899 if (!ptep) 3900 goto out; 3901 if (!pte_present(*ptep)) 3902 goto unlock; 3903 *ptepp = ptep; 3904 return 0; 3905 unlock: 3906 pte_unmap_unlock(ptep, *ptlp); 3907 out: 3908 return -EINVAL; 3909 } 3910 3911 static inline int follow_pte(struct mm_struct *mm, unsigned long address, 3912 pte_t **ptepp, spinlock_t **ptlp) 3913 { 3914 int res; 3915 3916 /* (void) is needed to make gcc happy */ 3917 (void) __cond_lock(*ptlp, 3918 !(res = __follow_pte(mm, address, ptepp, ptlp))); 3919 return res; 3920 } 3921 3922 /** 3923 * follow_pfn - look up PFN at a user virtual address 3924 * @vma: memory mapping 3925 * @address: user virtual address 3926 * @pfn: location to store found PFN 3927 * 3928 * Only IO mappings and raw PFN mappings are allowed. 3929 * 3930 * Returns zero and the pfn at @pfn on success, -ve otherwise. 3931 */ 3932 int follow_pfn(struct vm_area_struct *vma, unsigned long address, 3933 unsigned long *pfn) 3934 { 3935 int ret = -EINVAL; 3936 spinlock_t *ptl; 3937 pte_t *ptep; 3938 3939 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 3940 return ret; 3941 3942 ret = follow_pte(vma->vm_mm, address, &ptep, &ptl); 3943 if (ret) 3944 return ret; 3945 *pfn = pte_pfn(*ptep); 3946 pte_unmap_unlock(ptep, ptl); 3947 return 0; 3948 } 3949 EXPORT_SYMBOL(follow_pfn); 3950 3951 #ifdef CONFIG_HAVE_IOREMAP_PROT 3952 int follow_phys(struct vm_area_struct *vma, 3953 unsigned long address, unsigned int flags, 3954 unsigned long *prot, resource_size_t *phys) 3955 { 3956 int ret = -EINVAL; 3957 pte_t *ptep, pte; 3958 spinlock_t *ptl; 3959 3960 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 3961 goto out; 3962 3963 if (follow_pte(vma->vm_mm, address, &ptep, &ptl)) 3964 goto out; 3965 pte = *ptep; 3966 3967 if ((flags & FOLL_WRITE) && !pte_write(pte)) 3968 goto unlock; 3969 3970 *prot = pgprot_val(pte_pgprot(pte)); 3971 *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT; 3972 3973 ret = 0; 3974 unlock: 3975 pte_unmap_unlock(ptep, ptl); 3976 out: 3977 return ret; 3978 } 3979 3980 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, 3981 void *buf, int len, int write) 3982 { 3983 resource_size_t phys_addr; 3984 unsigned long prot = 0; 3985 void __iomem *maddr; 3986 int offset = addr & (PAGE_SIZE-1); 3987 3988 if (follow_phys(vma, addr, write, &prot, &phys_addr)) 3989 return -EINVAL; 3990 3991 maddr = ioremap_prot(phys_addr, PAGE_SIZE, prot); 3992 if (write) 3993 memcpy_toio(maddr + offset, buf, len); 3994 else 3995 memcpy_fromio(buf, maddr + offset, len); 3996 iounmap(maddr); 3997 3998 return len; 3999 } 4000 #endif 4001 4002 /* 4003 * Access another process' address space as given in mm. If non-NULL, use the 4004 * given task for page fault accounting. 4005 */ 4006 static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, 4007 unsigned long addr, void *buf, int len, int write) 4008 { 4009 struct vm_area_struct *vma; 4010 void *old_buf = buf; 4011 4012 down_read(&mm->mmap_sem); 4013 /* ignore errors, just check how much was successfully transferred */ 4014 while (len) { 4015 int bytes, ret, offset; 4016 void *maddr; 4017 struct page *page = NULL; 4018 4019 ret = get_user_pages(tsk, mm, addr, 1, 4020 write, 1, &page, &vma); 4021 if (ret <= 0) { 4022 /* 4023 * Check if this is a VM_IO | VM_PFNMAP VMA, which 4024 * we can access using slightly different code. 4025 */ 4026 #ifdef CONFIG_HAVE_IOREMAP_PROT 4027 vma = find_vma(mm, addr); 4028 if (!vma || vma->vm_start > addr) 4029 break; 4030 if (vma->vm_ops && vma->vm_ops->access) 4031 ret = vma->vm_ops->access(vma, addr, buf, 4032 len, write); 4033 if (ret <= 0) 4034 #endif 4035 break; 4036 bytes = ret; 4037 } else { 4038 bytes = len; 4039 offset = addr & (PAGE_SIZE-1); 4040 if (bytes > PAGE_SIZE-offset) 4041 bytes = PAGE_SIZE-offset; 4042 4043 maddr = kmap(page); 4044 if (write) { 4045 copy_to_user_page(vma, page, addr, 4046 maddr + offset, buf, bytes); 4047 set_page_dirty_lock(page); 4048 } else { 4049 copy_from_user_page(vma, page, addr, 4050 buf, maddr + offset, bytes); 4051 } 4052 kunmap(page); 4053 page_cache_release(page); 4054 } 4055 len -= bytes; 4056 buf += bytes; 4057 addr += bytes; 4058 } 4059 up_read(&mm->mmap_sem); 4060 4061 return buf - old_buf; 4062 } 4063 4064 /** 4065 * access_remote_vm - access another process' address space 4066 * @mm: the mm_struct of the target address space 4067 * @addr: start address to access 4068 * @buf: source or destination buffer 4069 * @len: number of bytes to transfer 4070 * @write: whether the access is a write 4071 * 4072 * The caller must hold a reference on @mm. 4073 */ 4074 int access_remote_vm(struct mm_struct *mm, unsigned long addr, 4075 void *buf, int len, int write) 4076 { 4077 return __access_remote_vm(NULL, mm, addr, buf, len, write); 4078 } 4079 4080 /* 4081 * Access another process' address space. 4082 * Source/target buffer must be kernel space, 4083 * Do not walk the page table directly, use get_user_pages 4084 */ 4085 int access_process_vm(struct task_struct *tsk, unsigned long addr, 4086 void *buf, int len, int write) 4087 { 4088 struct mm_struct *mm; 4089 int ret; 4090 4091 mm = get_task_mm(tsk); 4092 if (!mm) 4093 return 0; 4094 4095 ret = __access_remote_vm(tsk, mm, addr, buf, len, write); 4096 mmput(mm); 4097 4098 return ret; 4099 } 4100 4101 /* 4102 * Print the name of a VMA. 4103 */ 4104 void print_vma_addr(char *prefix, unsigned long ip) 4105 { 4106 struct mm_struct *mm = current->mm; 4107 struct vm_area_struct *vma; 4108 4109 /* 4110 * Do not print if we are in atomic 4111 * contexts (in exception stacks, etc.): 4112 */ 4113 if (preempt_count()) 4114 return; 4115 4116 down_read(&mm->mmap_sem); 4117 vma = find_vma(mm, ip); 4118 if (vma && vma->vm_file) { 4119 struct file *f = vma->vm_file; 4120 char *buf = (char *)__get_free_page(GFP_KERNEL); 4121 if (buf) { 4122 char *p; 4123 4124 p = d_path(&f->f_path, buf, PAGE_SIZE); 4125 if (IS_ERR(p)) 4126 p = "?"; 4127 printk("%s%s[%lx+%lx]", prefix, kbasename(p), 4128 vma->vm_start, 4129 vma->vm_end - vma->vm_start); 4130 free_page((unsigned long)buf); 4131 } 4132 } 4133 up_read(&mm->mmap_sem); 4134 } 4135 4136 #ifdef CONFIG_PROVE_LOCKING 4137 void might_fault(void) 4138 { 4139 /* 4140 * Some code (nfs/sunrpc) uses socket ops on kernel memory while 4141 * holding the mmap_sem, this is safe because kernel memory doesn't 4142 * get paged out, therefore we'll never actually fault, and the 4143 * below annotations will generate false positives. 4144 */ 4145 if (segment_eq(get_fs(), KERNEL_DS)) 4146 return; 4147 4148 might_sleep(); 4149 /* 4150 * it would be nicer only to annotate paths which are not under 4151 * pagefault_disable, however that requires a larger audit and 4152 * providing helpers like get_user_atomic. 4153 */ 4154 if (!in_atomic() && current->mm) 4155 might_lock_read(¤t->mm->mmap_sem); 4156 } 4157 EXPORT_SYMBOL(might_fault); 4158 #endif 4159 4160 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) 4161 static void clear_gigantic_page(struct page *page, 4162 unsigned long addr, 4163 unsigned int pages_per_huge_page) 4164 { 4165 int i; 4166 struct page *p = page; 4167 4168 might_sleep(); 4169 for (i = 0; i < pages_per_huge_page; 4170 i++, p = mem_map_next(p, page, i)) { 4171 cond_resched(); 4172 clear_user_highpage(p, addr + i * PAGE_SIZE); 4173 } 4174 } 4175 void clear_huge_page(struct page *page, 4176 unsigned long addr, unsigned int pages_per_huge_page) 4177 { 4178 int i; 4179 4180 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { 4181 clear_gigantic_page(page, addr, pages_per_huge_page); 4182 return; 4183 } 4184 4185 might_sleep(); 4186 for (i = 0; i < pages_per_huge_page; i++) { 4187 cond_resched(); 4188 clear_user_highpage(page + i, addr + i * PAGE_SIZE); 4189 } 4190 } 4191 4192 static void copy_user_gigantic_page(struct page *dst, struct page *src, 4193 unsigned long addr, 4194 struct vm_area_struct *vma, 4195 unsigned int pages_per_huge_page) 4196 { 4197 int i; 4198 struct page *dst_base = dst; 4199 struct page *src_base = src; 4200 4201 for (i = 0; i < pages_per_huge_page; ) { 4202 cond_resched(); 4203 copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma); 4204 4205 i++; 4206 dst = mem_map_next(dst, dst_base, i); 4207 src = mem_map_next(src, src_base, i); 4208 } 4209 } 4210 4211 void copy_user_huge_page(struct page *dst, struct page *src, 4212 unsigned long addr, struct vm_area_struct *vma, 4213 unsigned int pages_per_huge_page) 4214 { 4215 int i; 4216 4217 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) { 4218 copy_user_gigantic_page(dst, src, addr, vma, 4219 pages_per_huge_page); 4220 return; 4221 } 4222 4223 might_sleep(); 4224 for (i = 0; i < pages_per_huge_page; i++) { 4225 cond_resched(); 4226 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma); 4227 } 4228 } 4229 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ 4230